starting to convert files into md so parsing is easier (maybe), fixed filtering (sorta?)

shell.nix

@@ -6,12 +6,15 @@ let
     icecream
     python-ctags3
     jinja2
+    click
+    pypandoc
     # other python packages you want
   ]);
 in
 pkgs.mkShell {
   buildInputs = [
     python-with-my-packages
+    pkgs.pandoc
     # other dependencies
   ];
   shellHook = ''

src/__main__.py

@@ -0,0 +1,15 @@
+import click
+import kitaab_public
+
+@click.command()
+def grab():
+	pass
+
+@click.command()
+def build():
+    pass
+
+@click.command()
+def main():
+    grab()
+    build()

src/kitaab-public/build/build.py

@@ -32,7 +32,10 @@ build_url = config.relative_build_url
 if args.prod:
     build_url = config.absolute_build_url
 
-def generate_page():
+def generate_config():
+    pass
+
+def generate_page(template_file=config.template_file, options=generate_config()):
     pass
 
 def generate_navigation(template_nav=config.template_nav):
@@ -76,18 +79,6 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template
         page_template = template.replace('page_title', entry['title'])
         page_template = template.replace('page_tags', str(entry['tags']))
 
-        # Checking if the page is root of a folder
-        if entry["file"] == "index":
-
-            # If root page
-            # Concatenate the content of the index page with the listing of sub-pages
-            # Remove "page_date" from template
-            # Replaces "page_body" with the page content in the template
-            entry["pageContent"] += sub_pages_list
-            page_template = page_template.replace('page_date', "")
-            page_template = page_template.replace(
-                'page_body', entry['pageContent'])
-        else:
         # If content page
         # Replaces "page_body" with the page content in the template
         # Replaces "page_date" with the page date in the template
@@ -133,6 +124,14 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template
 
         # Write the HTML file
         slug_file = site_folder + entry['slug']
+        if entry["file"] == 'index':
+            print("")
+            print("")
+            print("FOUND INDEX")
+            print(slug_file)
+            print("")
+            print("")
+
         with open(slug_file, 'w', encoding=encoding) as fobj:
             fobj.write(page_template)
 
@@ -155,6 +154,12 @@ def get_entry_title(page):
     except FileNotFoundError:
         print(f"Page not found, but present in tags: {page}")
         return None
+    except UnicodeDecodeError:
+        print(f"Page could not be read: {page}")
+        return None
+    except IndexError as e:
+        print(f"{page} could not be parsed: {e}")
+
 
     return textContent
 
@@ -210,8 +215,12 @@ def fix_images_urls(page):
 
 
 def fix_amp(page):
+    try:
         page = page.replace(" & ", " & ")
         return page
+    except AttributeError as e:
+        print(f"{page} is not a string")
+        return ""
 
 def fix_wiki_links(page, path):
     regex = r"\[\[([\s\S]+?\|?[\s\S]+?)\]\]"
@@ -330,8 +339,9 @@ def clean_path(path):
     has_date = False
 
     if match:
-        print(f"Match found for {path_items['file']}")
         has_date = True
+    else:
+        print(f"{path} does not have a date")
 
     if has_date:
         if match[0] != path_items["file"]:
@@ -368,7 +378,7 @@ def clean_path(path):
     if config.flat_build == False:
         path_items["slug"] = path_items["folder"] + "/" + path_items["slug"]
 
-    if path_items["file"] == "index":
+    if "index" in path_items["file"]:
         path_items["parent_url"] = ""
         path_items["parent_text"] = config.home_name
         if config.flat_build:

src/kitaab-public/grab/TagHelper.py

@@ -1,10 +1,7 @@
-import ctags
 from ctags import CTags, TagEntry
 import sys
 from icecream import ic
-import os
-import shutil
-import fileinput
+from os import listdir, path
 
 # TODO don't hardcode the path
 def loadTagFile(path='/home/anish/vimwiki/.vimwiki_tags'):
@@ -59,10 +56,15 @@ def makeDictFromTags(tagFile, key='name', value='file'):
         tag = tagFile.next(entry)
     return ret
 
-def getAllFiles():
+def getAllFilesWithTgas():
     # Returns a set containing all the files we wish to make public
     return makeSetFromTags(loadTagFile())
 
+def getAllFiles():
+    files = [f for f in listdir('/home/anish/vimwiki') if '.wiki' in f and path.isfile('/home/anish/vimwiki/' + f)]
+    files = set(files)
+    return files
+
 def getAllTags():
     # Returns a dictionary with:
     # - key as tag titles

src/kitaab-public/grab/convert.py

@@ -0,0 +1,10 @@
+import pypandoc
+
+def convert_file(file):
+    filename = file.split('.')[0]
+    pypandoc.convert_file(filename + '.wiki', '.md', outputfile=filename + '.md')
+
+def convert_files(files):
+    for file in files:
+        convert_file(file)
+

src/kitaab-public/grab/makePublic.py

@@ -3,19 +3,20 @@
 # censor - burn all mentions of this word wherever it appears in Kitaab
 # filter - leave names in tact but do not link to relevant documents
 # create - list in an index file for website
-from os import listdir, mkdir
-from os.path import isfile, join
+from os import listdir, mkdir, path
 import shutil
 import random
 import json
 import TagHelper
+import convert
 from icecream import ic
 from censor import privateList
 
 ## TODO: generalize based on headings(?) in index.wiki
 
 censor = privateList
-filter = ["Dream", "Gratitude", "Review", "jrnl", "piracy", "work", "job", "debt", "people"]
+# TODO put this in config too...
+filter = ["dream", "gratitude", "review", "jrnl", "piracy", "work", "job", "debt", "people", "priv"]
 #TODO create = ... 
 
 def loadNetwork(path='network.json'):
@@ -40,7 +41,6 @@ def filterFiles(AllFileList):
             doNotTakeFiles.update(tagList[mask])
         except KeyError:
             print("Key not found: {}".format(mask))
-    print(AllFileList)
     return AllFileList - doNotTakeFiles
 
 def censorNetwork(network):
@@ -66,19 +66,22 @@ def createIndexNode(network):
 
 def copyFiles(fileList):
     baseFilePath = '/home/anish/vimwiki/'
+    outputFilePath = '../raw/' # TODO relative path.... 
     try:
-        mkdir('output')
+        mkdir(outputFilePath) 
     except FileExistsError:
         pass
     for f in fileList:
         try:
-            shutil.copy(baseFilePath + f, './raw/')
+            shutil.copy(baseFilePath + f, outputFilePath + f)
         except FileNotFoundError:
-            print("Could not copy file {}:".format(baseFilePath + f))
+            print("Could not copy file {} to {}".format(baseFilePath + f, outputFilePath + f))
 
 def censorFiles():
-    outputBasePath = './output/'
+    outputBasePath = '../raw/' # TODO relative file path
     for fileName in listdir(outputBasePath):
+        if not path.isfile(outputBasePath + fileName):
+            continue
         fileName = outputBasePath + fileName
         newFile = ""
         print("Censoring: {}".format(fileName))
@@ -91,6 +94,17 @@ def censorFiles():
             for line in newFile:
                 f.write(line)
 
+def convertFiles():
+    outputBasePath = '../raw/'
+    for filename in listdir(outputBasePath):
+        if not path.isfile(outputBasePath + filename):
+            continue
+        filename = outputBasePath + filename
+        convert.convert_file(filename)
+        print(filename)
+        shutil.rmtree(filename)
+
+
 if __name__=='__main__':
     
     #network = loadNetwork()
@@ -100,3 +114,5 @@ if __name__=='__main__':
     copyFiles(FilteredFiles)
     print("Censoring files...")
     censorFiles()
+    print("Converting files...")
+    convertFiles()

src/kitaab-public/partial/nav.html

@@ -1,6 +1,6 @@
 <div id='navigation' style="float: right;">
 <nav>
-	<h2> <a href='first.html'> Index </a> </h2>
+	<h2> <a href='raw.html'> Index </a> </h2>
     <ul>
 			{% for link_url in index_urls %}
 			<li> <a href='{{link_url}}_index.html'>{{link_url}}</a></li>