From 920fd42f6ac5d432f8c39e852c44764b746d35ad Mon Sep 17 00:00:00 2001 From: Anish Lakhwara Date: Sat, 29 Jan 2022 18:08:10 +1000 Subject: [PATCH] use jinja more, parse date & remove, parse tags & remove, fix up config --- src/kitaab-public/build/build.py | 96 +++++++++++++--------------- src/kitaab-public/build/config.py | 2 +- src/kitaab-public/grab/convert.py | 4 +- src/kitaab-public/grab/makePublic.py | 28 ++++---- src/kitaab-public/partial/main.html | 16 ++--- 5 files changed, 70 insertions(+), 76 deletions(-) diff --git a/src/kitaab-public/build/build.py b/src/kitaab-public/build/build.py index e4d0915..72e366e 100644 --- a/src/kitaab-public/build/build.py +++ b/src/kitaab-public/build/build.py @@ -67,7 +67,7 @@ def generate_index_pages(template=config.template_index, template_nav=config.tem file_loader = FileSystemLoader('') env = Environment(loader=file_loader) template = env.get_template(template) - current_index_template = template.render(page_body=subPages, page_date=None, page_tags=None, naem_of_site="Kitaab", page_navigation=generate_navigation(template_nav), page_title='index') + current_index_template = template.render(page_body=subPages, page_date=None, page_tags=None, page_navigation=generate_navigation(template_nav), page_title='index', config=config) slugfile = config.build_folder + index + '_index.html' print(slugfile) with open(slugfile, 'w', encoding=encoding) as fobj: @@ -76,44 +76,15 @@ def generate_index_pages(template=config.template_index, template_nav=config.tem # Generates html files in the site folder, using the entries and the template. def generate_html_pages(site_folder, entries, template, sub_pages_list, template_nav): for entry in entries: - page_template = template.replace('page_title', entry['title']) - page_template = template.replace('page_tags', str(entry['tags'])) - - # If content page - # Replaces "page_body" with the page content in the template - # Replaces "page_date" with the page date in the template - page_template = page_template.replace( - 'page_body', entry['pageContent']) - page_template = page_template.replace( - 'page_date', "%s" % (entry['date'])) - # Creating navigation - url_link = build_url - url_text = entry['parent_text'] - nav_html = generate_navigation(template_nav) - #if not entry["parent_url"]: - # # If index page, return to the home - # url_link = build_url - #else: - # # If content page, return to parent page - # url_link = build_url + entry['parent_url'] - - #nav_template = open(template_nav, 'r').read() - #nav_html = nav_template.replace("link_url", url_link) - #nav_html = nav_html.replace("text_url", url_text) - #nav_html = nav_html.replace("go_back", config.go_back) - page_template = page_template.replace('page_navigation', nav_html) - # Replaces all occurrences of build_url in the template files (assets, urls, etc) - page_template = page_template.replace('build_url', build_url) - page_template = page_template.replace( - 'name_of_site', config.name_of_site) - page_template = page_template.replace( - 'site_meta_description', config.site_meta_description) - page_template = page_template.replace( - 'twitter_name', config.twitter_name) + file_loader = FileSystemLoader('') + env = Environment(loader=file_loader) + template = env.get_template(template) + page_template = template.render(page_body=entry['pageContent'], page_date="%s"%entry['date'], page_tags=entry['tags'], naem_of_site="Kitaab", page_navigation=nav_html, page_title=entry['title'], build_url=entry['parent_text'], config=config) + print(page_template) # Checking if content folder exists folderExists = os.path.exists(site_folder+entry['folder']) @@ -124,14 +95,6 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template # Write the HTML file slug_file = site_folder + entry['slug'] - if entry["file"] == 'index': - print("") - print("") - print("FOUND INDEX") - print(slug_file) - print("") - print("") - with open(slug_file, 'w', encoding=encoding) as fobj: fobj.write(page_template) @@ -140,8 +103,9 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template # Get title by parsing and cleaning the first line of the markdown file def get_entry_title(page): + textContent = None try: - pageContent = open(page, 'r') + pageContent = open(page, 'r', errors='ignore') textContent = pageContent.read() textContent = textContent.splitlines() textContent = textContent[0] @@ -159,8 +123,6 @@ def get_entry_title(page): return None except IndexError as e: print(f"{page} could not be parsed: {e}") - - return textContent @@ -244,7 +206,7 @@ def remove_tags(page): textContent = page.splitlines() tags = textContent[1] if ':' in tags: - textContent = [textContent[0]] + textContent[1:] + textContent = [textContent[0]] + textContent[2:] return '\n'.join(textContent) except: print(f'Unexpected error in removing tags in {page}') @@ -262,10 +224,35 @@ def remove_title(page): print(f'Unexpected error in removing title in {page}') return page +def remove_date(page): + try: + regex = r"%date [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}" + page = re.sub(regex, '', page) + return page + except Exception as e: + print(e) + print(f'Unexpected error in removing title in {page}') + return page + + +def get_date(page): + try: + textContent = page.splitlines() + date = textContent[2] + if '%date' in date: + textContent.strip('%date') + date = str(datetime.strptime(textContent, '%y%m%d-%H%M')) + return date + return None + except: + #print(f"Could not parse date") + return None + # From the list of files, creates the main array of entries that will be processed later def create_entries(pages): fullContent = [] + noTitle = [] for page in pages: tempPage = {} @@ -274,6 +261,7 @@ def create_entries(pages): title = get_entry_title(page) if title is None: print(f'Did not find title for {page}') + noTitle.append(page) continue markdown_text = open(page, 'r').read() @@ -281,8 +269,10 @@ def create_entries(pages): markdown_text = fix_images_urls(markdown_text) markdown_text = fix_wiki_links(markdown_text, path["folder"]) tags = get_tags(markdown_text) + date = get_date(markdown_text) markdown_text = remove_tags(markdown_text) markdown_text = remove_title(markdown_text) + markdown_text = remove_date(markdown_text) pageContent = markdown(markdown_text) # Create the page object with all the informations we need @@ -292,7 +282,10 @@ def create_entries(pages): tempPage['parent_url'] = path['parent_url'] tempPage['parent_text'] = path['parent_text'] tempPage['date'] = path['date'] - tempPage['iso_date'] = path['iso_date'] + if date: + tempPage['iso_date'] = date + else: + tempPage['iso_date'] = path['iso_date'] tempPage['title'] = fix_amp(title) tempPage['pageContent'] = pageContent tempPage['tags'] = tags @@ -341,7 +334,8 @@ def clean_path(path): if match: has_date = True else: - print(f"{path} does not have a date") + pass + #print(f"{path} does not have a date") if has_date: if match[0] != path_items["file"]: @@ -489,8 +483,6 @@ def create_rss_feed(rss_entries, rss_template, rss_item_template, site_folder): def generate_website(): - print('Welcome to the builder!') - # If build folder exists delete it if os.path.exists(config.build_folder): shutil.rmtree(config.build_folder) @@ -518,7 +510,7 @@ def generate_website(): "content_list") home_page = home_page.replace('page_date', "") - generate_html_pages(config.build_folder, entries, template, + generate_html_pages(config.build_folder, entries, config.template_file, sub_pages_list, config.template_nav) for entry in entries: diff --git a/src/kitaab-public/build/config.py b/src/kitaab-public/build/config.py index 614b6c8..c76c386 100644 --- a/src/kitaab-public/build/config.py +++ b/src/kitaab-public/build/config.py @@ -11,7 +11,7 @@ template_nav = "partial/nav.html" template_index = "partial/main.html" rss_template = "partial/rss.xml" rss_item_template = "partial/item.xml" -name_of_site = "itaab" +name_of_site = "Kitaab" site_meta_description = "" twitter_name = "@aynish@merveilles.town" home_name = "Home" diff --git a/src/kitaab-public/grab/convert.py b/src/kitaab-public/grab/convert.py index 992e9df..dce391b 100644 --- a/src/kitaab-public/grab/convert.py +++ b/src/kitaab-public/grab/convert.py @@ -1,8 +1,8 @@ import pypandoc def convert_file(file): - filename = file.split('.')[0] - pypandoc.convert_file(filename + '.wiki', '.md', outputfile=filename + '.md') + filename = file.strip('wiki')[:-1] # filenames can contain ../ ./ etc, so we strip only the final wiki, and then drop the last character which is a '.' + pypandoc.convert_file(filename + '.wiki', 'md',format='vimwiki', outputfile=filename + '.md') def convert_files(files): for file in files: diff --git a/src/kitaab-public/grab/makePublic.py b/src/kitaab-public/grab/makePublic.py index 30653e3..ba29373 100644 --- a/src/kitaab-public/grab/makePublic.py +++ b/src/kitaab-public/grab/makePublic.py @@ -3,7 +3,7 @@ # censor - burn all mentions of this word wherever it appears in Kitaab # filter - leave names in tact but do not link to relevant documents # create - list in an index file for website -from os import listdir, mkdir, path +from os import listdir, mkdir, path, remove import shutil import random import json @@ -77,15 +77,13 @@ def copyFiles(fileList): except FileNotFoundError: print("Could not copy file {} to {}".format(baseFilePath + f, outputFilePath + f)) -def censorFiles(): - outputBasePath = '../raw/' # TODO relative file path +def censorFiles(outputBasePath='../raw/'): for fileName in listdir(outputBasePath): if not path.isfile(outputBasePath + fileName): continue fileName = outputBasePath + fileName newFile = "" - print("Censoring: {}".format(fileName)) - with open(fileName, 'r') as f: + with open(fileName, 'r', errors='ignore') as f: for line in f.readlines(): for c in censor: line = line.replace(c, generateBlocks(len(c))) @@ -94,15 +92,19 @@ def censorFiles(): for line in newFile: f.write(line) -def convertFiles(): - outputBasePath = '../raw/' +def convertFiles(outputBasePath='../raw/'): for filename in listdir(outputBasePath): if not path.isfile(outputBasePath + filename): continue - filename = outputBasePath + filename - convert.convert_file(filename) - print(filename) - shutil.rmtree(filename) + if '.md' == filename[-3:]: + print(f"{filename} file already converted, skipping") + continue + try: + convert.convert_file(outputBasePath + filename) + remove(outputBasePath + filename) + except RuntimeError as e: + print(f"Could not convert file {outputBasePath + filename} because {e}") + if __name__=='__main__': @@ -114,5 +116,5 @@ if __name__=='__main__': copyFiles(FilteredFiles) print("Censoring files...") censorFiles() - print("Converting files...") - convertFiles() + #print("Converting files...") + #convertFiles() diff --git a/src/kitaab-public/partial/main.html b/src/kitaab-public/partial/main.html index cd94bdc..b784008 100644 --- a/src/kitaab-public/partial/main.html +++ b/src/kitaab-public/partial/main.html @@ -4,22 +4,22 @@ - page_title | name_of_site - + page_title | config.name_of_site + - + content="config.site_meta_description"> + @@ -43,7 +43,7 @@
- {{name_of_site}} + {{config.name_of_site}}
{{ page_navigation }}