use jinja more, parse date & remove, parse tags & remove, fix up config

src/kitaab-public/build/build.py

@@ -67,7 +67,7 @@ def generate_index_pages(template=config.template_index, template_nav=config.tem
         file_loader = FileSystemLoader('')
         env = Environment(loader=file_loader)
         template = env.get_template(template)
-        current_index_template = template.render(page_body=subPages, page_date=None, page_tags=None, naem_of_site="Kitaab", page_navigation=generate_navigation(template_nav), page_title='index')
+        current_index_template = template.render(page_body=subPages, page_date=None, page_tags=None, page_navigation=generate_navigation(template_nav), page_title='index', config=config)
         slugfile = config.build_folder + index + '_index.html'
         print(slugfile)
         with open(slugfile, 'w', encoding=encoding) as fobj:
@@ -76,44 +76,15 @@ def generate_index_pages(template=config.template_index, template_nav=config.tem
 # Generates html files in the site folder, using the entries and the template.
 def generate_html_pages(site_folder, entries, template, sub_pages_list, template_nav):
     for entry in entries:
-        page_template = template.replace('page_title', entry['title'])
-        page_template = template.replace('page_tags', str(entry['tags']))
-
-        # If content page
-        # Replaces "page_body" with the page content in the template
-        # Replaces "page_date" with the page date in the template
-        page_template = page_template.replace(
-            'page_body', entry['pageContent'])
-        page_template = page_template.replace(
-            'page_date', "<date>%s</date>" % (entry['date']))
-
         # Creating navigation
-        url_link = build_url
-        url_text = entry['parent_text']
-        
         nav_html = generate_navigation(template_nav)
-        #if not entry["parent_url"]:
-        #    # If index page, return to the home
-        #    url_link = build_url
-        #else:
-        #    # If content page, return to parent page
-        #    url_link = build_url + entry['parent_url']
-
-        #nav_template = open(template_nav, 'r').read()
-        #nav_html = nav_template.replace("link_url", url_link)
-        #nav_html = nav_html.replace("text_url", url_text)
-        #nav_html = nav_html.replace("go_back", config.go_back)
-        page_template = page_template.replace('page_navigation', nav_html)
-
         # Replaces all occurrences of build_url in the template files (assets, urls, etc)
-        page_template = page_template.replace('build_url', build_url)
 
-        page_template = page_template.replace(
+        file_loader = FileSystemLoader('')
-            'name_of_site', config.name_of_site)
+        env = Environment(loader=file_loader)
-        page_template = page_template.replace(
+        template = env.get_template(template)
-            'site_meta_description', config.site_meta_description)
+        page_template = template.render(page_body=entry['pageContent'], page_date="<date>%s</date>"%entry['date'], page_tags=entry['tags'], naem_of_site="Kitaab", page_navigation=nav_html, page_title=entry['title'], build_url=entry['parent_text'], config=config)
-        page_template = page_template.replace(
+        print(page_template)
-            'twitter_name', config.twitter_name)
 
         # Checking if content folder exists
         folderExists = os.path.exists(site_folder+entry['folder'])
@@ -124,14 +95,6 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template
 
         # Write the HTML file
         slug_file = site_folder + entry['slug']
-        if entry["file"] == 'index':
-            print("")
-            print("")
-            print("FOUND INDEX")
-            print(slug_file)
-            print("")
-            print("")
-
         with open(slug_file, 'w', encoding=encoding) as fobj:
             fobj.write(page_template)
 
@@ -140,8 +103,9 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template
 
 # Get title by parsing and cleaning the first line of the markdown file
 def get_entry_title(page):
+    textContent = None
     try:
-        pageContent = open(page, 'r')
+        pageContent = open(page, 'r', errors='ignore')
         textContent = pageContent.read()
         textContent = textContent.splitlines()
         textContent = textContent[0]
@@ -159,8 +123,6 @@ def get_entry_title(page):
         return None
     except IndexError as e:
         print(f"{page} could not be parsed: {e}")
-
-
     return textContent
 
 
@@ -244,7 +206,7 @@ def remove_tags(page):
         textContent = page.splitlines()
         tags = textContent[1]
         if ':' in tags:
-            textContent = [textContent[0]] + textContent[1:]
+            textContent = [textContent[0]] + textContent[2:]
         return '\n'.join(textContent)
     except:
         print(f'Unexpected error in removing tags in {page}')
@@ -262,10 +224,35 @@ def remove_title(page):
         print(f'Unexpected error in removing title in {page}')
         return page
 
+def remove_date(page):
+    try:
+        regex = r"%date [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}"
+        page = re.sub(regex, '', page)
+        return page
+    except Exception as e:
+        print(e)
+        print(f'Unexpected error in removing title in {page}')
+        return page
+
+
+def get_date(page):
+    try:
+        textContent = page.splitlines()
+        date = textContent[2]
+        if '%date' in date:
+            textContent.strip('%date')
+            date = str(datetime.strptime(textContent, '%y%m%d-%H%M'))
+            return date
+        return None
+    except:
+        #print(f"Could not parse date")
+        return None
+
 
 # From the list of files, creates the main array of entries that will be processed later
 def create_entries(pages):
     fullContent = []
+    noTitle = []
     for page in pages:
         tempPage = {}
 
@@ -274,6 +261,7 @@ def create_entries(pages):
         title = get_entry_title(page)
         if title is None:
             print(f'Did not find title for {page}')
+            noTitle.append(page)
             continue
 
         markdown_text = open(page, 'r').read()
@@ -281,8 +269,10 @@ def create_entries(pages):
         markdown_text = fix_images_urls(markdown_text)
         markdown_text = fix_wiki_links(markdown_text, path["folder"])
         tags = get_tags(markdown_text) 
+        date = get_date(markdown_text)
         markdown_text = remove_tags(markdown_text)
         markdown_text = remove_title(markdown_text)
+        markdown_text = remove_date(markdown_text)
         pageContent = markdown(markdown_text)
 
         # Create the page object with all the informations we need
@@ -292,6 +282,9 @@ def create_entries(pages):
         tempPage['parent_url'] = path['parent_url']
         tempPage['parent_text'] = path['parent_text']
         tempPage['date'] = path['date']
+        if date:
+            tempPage['iso_date'] = date
+        else:
             tempPage['iso_date'] = path['iso_date']
         tempPage['title'] = fix_amp(title)
         tempPage['pageContent'] = pageContent
@@ -341,7 +334,8 @@ def clean_path(path):
     if match:
         has_date = True
     else:
-        print(f"{path} does not have a date")
+        pass
+        #print(f"{path} does not have a date")
 
     if has_date:
         if match[0] != path_items["file"]:
@@ -489,8 +483,6 @@ def create_rss_feed(rss_entries, rss_template, rss_item_template, site_folder):
 
 
 def generate_website():
-    print('Welcome to the builder!')
-
     # If build folder exists delete it
     if os.path.exists(config.build_folder):
         shutil.rmtree(config.build_folder)
@@ -518,7 +510,7 @@ def generate_website():
                                       "content_list")
         home_page = home_page.replace('page_date', "")
 
-        generate_html_pages(config.build_folder, entries, template,
+        generate_html_pages(config.build_folder, entries, config.template_file,
                             sub_pages_list, config.template_nav)
 
         for entry in entries:

src/kitaab-public/build/config.py

@@ -11,7 +11,7 @@ template_nav = "partial/nav.html"
 template_index = "partial/main.html"
 rss_template = "partial/rss.xml"
 rss_item_template = "partial/item.xml"
-name_of_site = "itaab"
+name_of_site = "Kitaab"
 site_meta_description = ""
 twitter_name = "@aynish@merveilles.town"
 home_name = "Home"

src/kitaab-public/grab/convert.py

@@ -1,8 +1,8 @@
 import pypandoc
 
 def convert_file(file):
-    filename = file.split('.')[0]
+    filename = file.strip('wiki')[:-1] # filenames can contain ../ ./ etc, so we strip only the final wiki, and then drop the last character which is a '.'
-    pypandoc.convert_file(filename + '.wiki', '.md', outputfile=filename + '.md')
+    pypandoc.convert_file(filename + '.wiki', 'md',format='vimwiki', outputfile=filename + '.md')
 
 def convert_files(files):
     for file in files:

src/kitaab-public/grab/makePublic.py

@@ -3,7 +3,7 @@
 # censor - burn all mentions of this word wherever it appears in Kitaab
 # filter - leave names in tact but do not link to relevant documents
 # create - list in an index file for website
-from os import listdir, mkdir, path
+from os import listdir, mkdir, path, remove
 import shutil
 import random
 import json
@@ -77,15 +77,13 @@ def copyFiles(fileList):
         except FileNotFoundError:
             print("Could not copy file {} to {}".format(baseFilePath + f, outputFilePath + f))
 
-def censorFiles():
+def censorFiles(outputBasePath='../raw/'):
-    outputBasePath = '../raw/' # TODO relative file path
     for fileName in listdir(outputBasePath):
         if not path.isfile(outputBasePath + fileName):
             continue
         fileName = outputBasePath + fileName
         newFile = ""
-        print("Censoring: {}".format(fileName))
+        with open(fileName, 'r', errors='ignore') as f:
-        with open(fileName, 'r') as f:
             for line in f.readlines():
                 for c in censor:
                     line = line.replace(c, generateBlocks(len(c)))
@@ -94,15 +92,19 @@ def censorFiles():
             for line in newFile:
                 f.write(line)
 
-def convertFiles():
+def convertFiles(outputBasePath='../raw/'):
-    outputBasePath = '../raw/'
     for filename in listdir(outputBasePath):
         if not path.isfile(outputBasePath + filename):
             continue
-        filename = outputBasePath + filename
+        if '.md' == filename[-3:]:
-        convert.convert_file(filename)
+            print(f"{filename} file already converted, skipping")
-        print(filename)
+            continue
-        shutil.rmtree(filename)
+        try:
+            convert.convert_file(outputBasePath + filename)
+            remove(outputBasePath + filename)
+        except RuntimeError as e:
+            print(f"Could not convert file {outputBasePath + filename} because {e}")
+
 
 
 if __name__=='__main__':
@@ -114,5 +116,5 @@ if __name__=='__main__':
     copyFiles(FilteredFiles)
     print("Censoring files...")
     censorFiles()
-    print("Converting files...")
+    #print("Converting files...")
-    convertFiles()
+    #convertFiles()

src/kitaab-public/partial/main.html

@@ -4,22 +4,22 @@
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>page_title | name_of_site</title>
+    <title>page_title | config.name_of_site</title>
-    <meta property="og:title" content="{{page_title}} | {{name_of_site}}">
+    <meta property="og:title" content="{{page_title}} | {{config.name_of_site}}">
     <meta property="og:type" content="website">
     <meta property="og:description"
-        content="site_meta_description">
+        content="config.site_meta_description">
-    <meta property="og:site_name" content="{{name_of_site}}">
+    <meta property="og:site_name" content="{{config.name_of_site}}">
     <meta property="og:url" content="build_url">
     <meta property="og:image" content="build_urlassets/social.png">
     <meta property="og:image:height" content="300">
     <meta property="og:image:width" content="300">
     <meta property="og:image:type" content="image/jpeg">
 		<!-- Mastodon stuff should go here
-    <meta name="twitter:site" content="twitter_name">
+    <meta name="twitter:site" content="config.twitter_name">
-    <meta name="twitter:title" content="page_title | name_of_site">
+    <meta name="twitter:title" content="page_title | config.name_of_site">
     <meta name="twitter:image" content="build_urlassets/social.png">
-    <meta name="twitter:image:alt" content="page_title | name_of_site">
+    <meta name="twitter:image:alt" content="page_title | config.name_of_site">
 			--!>
     <link rel="apple-touch-icon" sizes="57x57" href="build_urlassets/apple-icon-57x57.png">
     <link rel="apple-touch-icon" sizes="60x60" href="build_urlassets/apple-icon-60x60.png">
@@ -43,7 +43,7 @@
 
 <body class="flow-large container">
     <header>
-        <a href="index.html" aria-label="{{page_title}}">{{name_of_site}}</a>
+        <a href="index.html" aria-label="{{page_title}}">{{config.name_of_site}}</a>
     </header>
     <main>
 			{{ page_navigation }}