use jinja more, parse date & remove, parse tags & remove, fix up config

main
Anish Lakhwara 2022-01-29 18:08:10 +10:00
parent 65f0b2e600
commit 920fd42f6a
5 changed files with 70 additions and 76 deletions

View File

@ -67,7 +67,7 @@ def generate_index_pages(template=config.template_index, template_nav=config.tem
file_loader = FileSystemLoader('')
env = Environment(loader=file_loader)
template = env.get_template(template)
current_index_template = template.render(page_body=subPages, page_date=None, page_tags=None, naem_of_site="Kitaab", page_navigation=generate_navigation(template_nav), page_title='index')
current_index_template = template.render(page_body=subPages, page_date=None, page_tags=None, page_navigation=generate_navigation(template_nav), page_title='index', config=config)
slugfile = config.build_folder + index + '_index.html'
print(slugfile)
with open(slugfile, 'w', encoding=encoding) as fobj:
@ -76,44 +76,15 @@ def generate_index_pages(template=config.template_index, template_nav=config.tem
# Generates html files in the site folder, using the entries and the template.
def generate_html_pages(site_folder, entries, template, sub_pages_list, template_nav):
for entry in entries:
page_template = template.replace('page_title', entry['title'])
page_template = template.replace('page_tags', str(entry['tags']))
# If content page
# Replaces "page_body" with the page content in the template
# Replaces "page_date" with the page date in the template
page_template = page_template.replace(
'page_body', entry['pageContent'])
page_template = page_template.replace(
'page_date', "<date>%s</date>" % (entry['date']))
# Creating navigation
url_link = build_url
url_text = entry['parent_text']
nav_html = generate_navigation(template_nav)
#if not entry["parent_url"]:
# # If index page, return to the home
# url_link = build_url
#else:
# # If content page, return to parent page
# url_link = build_url + entry['parent_url']
#nav_template = open(template_nav, 'r').read()
#nav_html = nav_template.replace("link_url", url_link)
#nav_html = nav_html.replace("text_url", url_text)
#nav_html = nav_html.replace("go_back", config.go_back)
page_template = page_template.replace('page_navigation', nav_html)
# Replaces all occurrences of build_url in the template files (assets, urls, etc)
page_template = page_template.replace('build_url', build_url)
page_template = page_template.replace(
'name_of_site', config.name_of_site)
page_template = page_template.replace(
'site_meta_description', config.site_meta_description)
page_template = page_template.replace(
'twitter_name', config.twitter_name)
file_loader = FileSystemLoader('')
env = Environment(loader=file_loader)
template = env.get_template(template)
page_template = template.render(page_body=entry['pageContent'], page_date="<date>%s</date>"%entry['date'], page_tags=entry['tags'], naem_of_site="Kitaab", page_navigation=nav_html, page_title=entry['title'], build_url=entry['parent_text'], config=config)
print(page_template)
# Checking if content folder exists
folderExists = os.path.exists(site_folder+entry['folder'])
@ -124,14 +95,6 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template
# Write the HTML file
slug_file = site_folder + entry['slug']
if entry["file"] == 'index':
print("")
print("")
print("FOUND INDEX")
print(slug_file)
print("")
print("")
with open(slug_file, 'w', encoding=encoding) as fobj:
fobj.write(page_template)
@ -140,8 +103,9 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template
# Get title by parsing and cleaning the first line of the markdown file
def get_entry_title(page):
textContent = None
try:
pageContent = open(page, 'r')
pageContent = open(page, 'r', errors='ignore')
textContent = pageContent.read()
textContent = textContent.splitlines()
textContent = textContent[0]
@ -159,8 +123,6 @@ def get_entry_title(page):
return None
except IndexError as e:
print(f"{page} could not be parsed: {e}")
return textContent
@ -244,7 +206,7 @@ def remove_tags(page):
textContent = page.splitlines()
tags = textContent[1]
if ':' in tags:
textContent = [textContent[0]] + textContent[1:]
textContent = [textContent[0]] + textContent[2:]
return '\n'.join(textContent)
except:
print(f'Unexpected error in removing tags in {page}')
@ -262,10 +224,35 @@ def remove_title(page):
print(f'Unexpected error in removing title in {page}')
return page
def remove_date(page):
try:
regex = r"%date [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}"
page = re.sub(regex, '', page)
return page
except Exception as e:
print(e)
print(f'Unexpected error in removing title in {page}')
return page
def get_date(page):
try:
textContent = page.splitlines()
date = textContent[2]
if '%date' in date:
textContent.strip('%date')
date = str(datetime.strptime(textContent, '%y%m%d-%H%M'))
return date
return None
except:
#print(f"Could not parse date")
return None
# From the list of files, creates the main array of entries that will be processed later
def create_entries(pages):
fullContent = []
noTitle = []
for page in pages:
tempPage = {}
@ -274,6 +261,7 @@ def create_entries(pages):
title = get_entry_title(page)
if title is None:
print(f'Did not find title for {page}')
noTitle.append(page)
continue
markdown_text = open(page, 'r').read()
@ -281,8 +269,10 @@ def create_entries(pages):
markdown_text = fix_images_urls(markdown_text)
markdown_text = fix_wiki_links(markdown_text, path["folder"])
tags = get_tags(markdown_text)
date = get_date(markdown_text)
markdown_text = remove_tags(markdown_text)
markdown_text = remove_title(markdown_text)
markdown_text = remove_date(markdown_text)
pageContent = markdown(markdown_text)
# Create the page object with all the informations we need
@ -292,7 +282,10 @@ def create_entries(pages):
tempPage['parent_url'] = path['parent_url']
tempPage['parent_text'] = path['parent_text']
tempPage['date'] = path['date']
tempPage['iso_date'] = path['iso_date']
if date:
tempPage['iso_date'] = date
else:
tempPage['iso_date'] = path['iso_date']
tempPage['title'] = fix_amp(title)
tempPage['pageContent'] = pageContent
tempPage['tags'] = tags
@ -341,7 +334,8 @@ def clean_path(path):
if match:
has_date = True
else:
print(f"{path} does not have a date")
pass
#print(f"{path} does not have a date")
if has_date:
if match[0] != path_items["file"]:
@ -489,8 +483,6 @@ def create_rss_feed(rss_entries, rss_template, rss_item_template, site_folder):
def generate_website():
print('Welcome to the builder!')
# If build folder exists delete it
if os.path.exists(config.build_folder):
shutil.rmtree(config.build_folder)
@ -518,7 +510,7 @@ def generate_website():
"content_list")
home_page = home_page.replace('page_date', "")
generate_html_pages(config.build_folder, entries, template,
generate_html_pages(config.build_folder, entries, config.template_file,
sub_pages_list, config.template_nav)
for entry in entries:

View File

@ -11,7 +11,7 @@ template_nav = "partial/nav.html"
template_index = "partial/main.html"
rss_template = "partial/rss.xml"
rss_item_template = "partial/item.xml"
name_of_site = "itaab"
name_of_site = "Kitaab"
site_meta_description = ""
twitter_name = "@aynish@merveilles.town"
home_name = "Home"

View File

@ -1,8 +1,8 @@
import pypandoc
def convert_file(file):
filename = file.split('.')[0]
pypandoc.convert_file(filename + '.wiki', '.md', outputfile=filename + '.md')
filename = file.strip('wiki')[:-1] # filenames can contain ../ ./ etc, so we strip only the final wiki, and then drop the last character which is a '.'
pypandoc.convert_file(filename + '.wiki', 'md',format='vimwiki', outputfile=filename + '.md')
def convert_files(files):
for file in files:

View File

@ -3,7 +3,7 @@
# censor - burn all mentions of this word wherever it appears in Kitaab
# filter - leave names in tact but do not link to relevant documents
# create - list in an index file for website
from os import listdir, mkdir, path
from os import listdir, mkdir, path, remove
import shutil
import random
import json
@ -77,15 +77,13 @@ def copyFiles(fileList):
except FileNotFoundError:
print("Could not copy file {} to {}".format(baseFilePath + f, outputFilePath + f))
def censorFiles():
outputBasePath = '../raw/' # TODO relative file path
def censorFiles(outputBasePath='../raw/'):
for fileName in listdir(outputBasePath):
if not path.isfile(outputBasePath + fileName):
continue
fileName = outputBasePath + fileName
newFile = ""
print("Censoring: {}".format(fileName))
with open(fileName, 'r') as f:
with open(fileName, 'r', errors='ignore') as f:
for line in f.readlines():
for c in censor:
line = line.replace(c, generateBlocks(len(c)))
@ -94,15 +92,19 @@ def censorFiles():
for line in newFile:
f.write(line)
def convertFiles():
outputBasePath = '../raw/'
def convertFiles(outputBasePath='../raw/'):
for filename in listdir(outputBasePath):
if not path.isfile(outputBasePath + filename):
continue
filename = outputBasePath + filename
convert.convert_file(filename)
print(filename)
shutil.rmtree(filename)
if '.md' == filename[-3:]:
print(f"{filename} file already converted, skipping")
continue
try:
convert.convert_file(outputBasePath + filename)
remove(outputBasePath + filename)
except RuntimeError as e:
print(f"Could not convert file {outputBasePath + filename} because {e}")
if __name__=='__main__':
@ -114,5 +116,5 @@ if __name__=='__main__':
copyFiles(FilteredFiles)
print("Censoring files...")
censorFiles()
print("Converting files...")
convertFiles()
#print("Converting files...")
#convertFiles()

View File

@ -4,22 +4,22 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>page_title | name_of_site</title>
<meta property="og:title" content="{{page_title}} | {{name_of_site}}">
<title>page_title | config.name_of_site</title>
<meta property="og:title" content="{{page_title}} | {{config.name_of_site}}">
<meta property="og:type" content="website">
<meta property="og:description"
content="site_meta_description">
<meta property="og:site_name" content="{{name_of_site}}">
content="config.site_meta_description">
<meta property="og:site_name" content="{{config.name_of_site}}">
<meta property="og:url" content="build_url">
<meta property="og:image" content="build_urlassets/social.png">
<meta property="og:image:height" content="300">
<meta property="og:image:width" content="300">
<meta property="og:image:type" content="image/jpeg">
<!-- Mastodon stuff should go here
<meta name="twitter:site" content="twitter_name">
<meta name="twitter:title" content="page_title | name_of_site">
<meta name="twitter:site" content="config.twitter_name">
<meta name="twitter:title" content="page_title | config.name_of_site">
<meta name="twitter:image" content="build_urlassets/social.png">
<meta name="twitter:image:alt" content="page_title | name_of_site">
<meta name="twitter:image:alt" content="page_title | config.name_of_site">
--!>
<link rel="apple-touch-icon" sizes="57x57" href="build_urlassets/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="build_urlassets/apple-icon-60x60.png">
@ -43,7 +43,7 @@
<body class="flow-large container">
<header>
<a href="index.html" aria-label="{{page_title}}">{{name_of_site}}</a>
<a href="index.html" aria-label="{{page_title}}">{{config.name_of_site}}</a>
</header>
<main>
{{ page_navigation }}