From 65f0b2e600c394def317843b5477567fec06814a Mon Sep 17 00:00:00 2001 From: Anish Lakhwara Date: Sat, 29 Jan 2022 04:47:10 +1000 Subject: [PATCH] starting to convert files into md so parsing is easier (maybe), fixed filtering (sorta?) --- .envrc | 0 shell.nix | 3 ++ src/__main__.py | 15 +++++++ src/kitaab-public/build/build.py | 58 ++++++++++++++++------------ src/kitaab-public/grab/TagHelper.py | 12 +++--- src/kitaab-public/grab/convert.py | 10 +++++ src/kitaab-public/grab/makePublic.py | 32 +++++++++++---- src/kitaab-public/partial/nav.html | 2 +- 8 files changed, 94 insertions(+), 38 deletions(-) create mode 100644 .envrc create mode 100644 src/__main__.py create mode 100644 src/kitaab-public/grab/convert.py diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..e69de29 diff --git a/shell.nix b/shell.nix index 0183ac9..10e14a4 100644 --- a/shell.nix +++ b/shell.nix @@ -6,12 +6,15 @@ let icecream python-ctags3 jinja2 + click + pypandoc # other python packages you want ]); in pkgs.mkShell { buildInputs = [ python-with-my-packages + pkgs.pandoc # other dependencies ]; shellHook = '' diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..d81fd63 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,15 @@ +import click +import kitaab_public + +@click.command() +def grab(): + pass + +@click.command() +def build(): + pass + +@click.command() +def main(): + grab() + build() diff --git a/src/kitaab-public/build/build.py b/src/kitaab-public/build/build.py index dfdc1ff..e4d0915 100644 --- a/src/kitaab-public/build/build.py +++ b/src/kitaab-public/build/build.py @@ -32,7 +32,10 @@ build_url = config.relative_build_url if args.prod: build_url = config.absolute_build_url -def generate_page(): +def generate_config(): + pass + +def generate_page(template_file=config.template_file, options=generate_config()): pass def generate_navigation(template_nav=config.template_nav): @@ -76,25 +79,13 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template page_template = template.replace('page_title', entry['title']) page_template = template.replace('page_tags', str(entry['tags'])) - # Checking if the page is root of a folder - if entry["file"] == "index": - - # If root page - # Concatenate the content of the index page with the listing of sub-pages - # Remove "page_date" from template - # Replaces "page_body" with the page content in the template - entry["pageContent"] += sub_pages_list - page_template = page_template.replace('page_date', "") - page_template = page_template.replace( - 'page_body', entry['pageContent']) - else: - # If content page - # Replaces "page_body" with the page content in the template - # Replaces "page_date" with the page date in the template - page_template = page_template.replace( - 'page_body', entry['pageContent']) - page_template = page_template.replace( - 'page_date', "%s" % (entry['date'])) + # If content page + # Replaces "page_body" with the page content in the template + # Replaces "page_date" with the page date in the template + page_template = page_template.replace( + 'page_body', entry['pageContent']) + page_template = page_template.replace( + 'page_date', "%s" % (entry['date'])) # Creating navigation url_link = build_url @@ -133,6 +124,14 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template # Write the HTML file slug_file = site_folder + entry['slug'] + if entry["file"] == 'index': + print("") + print("") + print("FOUND INDEX") + print(slug_file) + print("") + print("") + with open(slug_file, 'w', encoding=encoding) as fobj: fobj.write(page_template) @@ -155,6 +154,12 @@ def get_entry_title(page): except FileNotFoundError: print(f"Page not found, but present in tags: {page}") return None + except UnicodeDecodeError: + print(f"Page could not be read: {page}") + return None + except IndexError as e: + print(f"{page} could not be parsed: {e}") + return textContent @@ -210,8 +215,12 @@ def fix_images_urls(page): def fix_amp(page): - page = page.replace(" & ", " & ") - return page + try: + page = page.replace(" & ", " & ") + return page + except AttributeError as e: + print(f"{page} is not a string") + return "" def fix_wiki_links(page, path): regex = r"\[\[([\s\S]+?\|?[\s\S]+?)\]\]" @@ -330,8 +339,9 @@ def clean_path(path): has_date = False if match: - print(f"Match found for {path_items['file']}") has_date = True + else: + print(f"{path} does not have a date") if has_date: if match[0] != path_items["file"]: @@ -368,7 +378,7 @@ def clean_path(path): if config.flat_build == False: path_items["slug"] = path_items["folder"] + "/" + path_items["slug"] - if path_items["file"] == "index": + if "index" in path_items["file"]: path_items["parent_url"] = "" path_items["parent_text"] = config.home_name if config.flat_build: diff --git a/src/kitaab-public/grab/TagHelper.py b/src/kitaab-public/grab/TagHelper.py index 11b5519..f393a9c 100644 --- a/src/kitaab-public/grab/TagHelper.py +++ b/src/kitaab-public/grab/TagHelper.py @@ -1,10 +1,7 @@ -import ctags from ctags import CTags, TagEntry import sys from icecream import ic -import os -import shutil -import fileinput +from os import listdir, path # TODO don't hardcode the path def loadTagFile(path='/home/anish/vimwiki/.vimwiki_tags'): @@ -59,10 +56,15 @@ def makeDictFromTags(tagFile, key='name', value='file'): tag = tagFile.next(entry) return ret -def getAllFiles(): +def getAllFilesWithTgas(): # Returns a set containing all the files we wish to make public return makeSetFromTags(loadTagFile()) +def getAllFiles(): + files = [f for f in listdir('/home/anish/vimwiki') if '.wiki' in f and path.isfile('/home/anish/vimwiki/' + f)] + files = set(files) + return files + def getAllTags(): # Returns a dictionary with: # - key as tag titles diff --git a/src/kitaab-public/grab/convert.py b/src/kitaab-public/grab/convert.py new file mode 100644 index 0000000..992e9df --- /dev/null +++ b/src/kitaab-public/grab/convert.py @@ -0,0 +1,10 @@ +import pypandoc + +def convert_file(file): + filename = file.split('.')[0] + pypandoc.convert_file(filename + '.wiki', '.md', outputfile=filename + '.md') + +def convert_files(files): + for file in files: + convert_file(file) + diff --git a/src/kitaab-public/grab/makePublic.py b/src/kitaab-public/grab/makePublic.py index 6a803a8..30653e3 100644 --- a/src/kitaab-public/grab/makePublic.py +++ b/src/kitaab-public/grab/makePublic.py @@ -3,19 +3,20 @@ # censor - burn all mentions of this word wherever it appears in Kitaab # filter - leave names in tact but do not link to relevant documents # create - list in an index file for website -from os import listdir, mkdir -from os.path import isfile, join +from os import listdir, mkdir, path import shutil import random import json import TagHelper +import convert from icecream import ic from censor import privateList ## TODO: generalize based on headings(?) in index.wiki censor = privateList -filter = ["Dream", "Gratitude", "Review", "jrnl", "piracy", "work", "job", "debt", "people"] +# TODO put this in config too... +filter = ["dream", "gratitude", "review", "jrnl", "piracy", "work", "job", "debt", "people", "priv"] #TODO create = ... def loadNetwork(path='network.json'): @@ -40,7 +41,6 @@ def filterFiles(AllFileList): doNotTakeFiles.update(tagList[mask]) except KeyError: print("Key not found: {}".format(mask)) - print(AllFileList) return AllFileList - doNotTakeFiles def censorNetwork(network): @@ -66,19 +66,22 @@ def createIndexNode(network): def copyFiles(fileList): baseFilePath = '/home/anish/vimwiki/' + outputFilePath = '../raw/' # TODO relative path.... try: - mkdir('output') + mkdir(outputFilePath) except FileExistsError: pass for f in fileList: try: - shutil.copy(baseFilePath + f, './raw/') + shutil.copy(baseFilePath + f, outputFilePath + f) except FileNotFoundError: - print("Could not copy file {}:".format(baseFilePath + f)) + print("Could not copy file {} to {}".format(baseFilePath + f, outputFilePath + f)) def censorFiles(): - outputBasePath = './output/' + outputBasePath = '../raw/' # TODO relative file path for fileName in listdir(outputBasePath): + if not path.isfile(outputBasePath + fileName): + continue fileName = outputBasePath + fileName newFile = "" print("Censoring: {}".format(fileName)) @@ -91,6 +94,17 @@ def censorFiles(): for line in newFile: f.write(line) +def convertFiles(): + outputBasePath = '../raw/' + for filename in listdir(outputBasePath): + if not path.isfile(outputBasePath + filename): + continue + filename = outputBasePath + filename + convert.convert_file(filename) + print(filename) + shutil.rmtree(filename) + + if __name__=='__main__': #network = loadNetwork() @@ -100,3 +114,5 @@ if __name__=='__main__': copyFiles(FilteredFiles) print("Censoring files...") censorFiles() + print("Converting files...") + convertFiles() diff --git a/src/kitaab-public/partial/nav.html b/src/kitaab-public/partial/nav.html index fb16d98..29367ab 100644 --- a/src/kitaab-public/partial/nav.html +++ b/src/kitaab-public/partial/nav.html @@ -1,6 +1,6 @@