From 85b27502046dedaae54ab4f822ed74d9d3cf014d Mon Sep 17 00:00:00 2001 From: Anish Lakhwara Date: Thu, 14 Apr 2022 22:33:40 +1000 Subject: [PATCH] trying to get links working again? --- basant/build/kitaab.py | 109 ++++++++++++++--------------- basant/grab/__init__.py | 0 basant/grab/convert.py | 10 --- basant/grab/kitaab.py | 101 --------------------------- basant/grab/network-builder.py | 25 ------- basant/grab/network.py | 35 ---------- basant/grab/tree.py | 120 -------------------------------- basant/partial/head.html | 42 +++++++++++ basant/partial/index_table.html | 43 +----------- basant/partial/main.html | 43 +----------- basant/shared/TagHelper.py | 22 +----- basant/shared/censor.py | 2 - basant/shared/config.py | 6 +- basant/shared/tag-fixer.py | 106 ---------------------------- flake.lock | 8 +-- 15 files changed, 108 insertions(+), 564 deletions(-) delete mode 100644 basant/grab/__init__.py delete mode 100644 basant/grab/convert.py delete mode 100644 basant/grab/kitaab.py delete mode 100644 basant/grab/network-builder.py delete mode 100644 basant/grab/network.py delete mode 100644 basant/grab/tree.py create mode 100644 basant/partial/head.html delete mode 100644 basant/shared/censor.py delete mode 100644 basant/shared/tag-fixer.py diff --git a/basant/build/kitaab.py b/basant/build/kitaab.py index 2d2b47f..96927ed 100644 --- a/basant/build/kitaab.py +++ b/basant/build/kitaab.py @@ -30,14 +30,6 @@ if args.prod: build_url = config.absolute_build_url -def generate_config(): - pass - - -def generate_page(template_file=config.template_main, options=generate_config()): - pass - - def generate_navigation(template_nav=config.template_nav): file_loader = FileSystemLoader('') env = Environment(loader=file_loader) @@ -48,6 +40,14 @@ def generate_navigation(template_nav=config.template_nav): return output +def generate_head(template_head=config.template_head, page_title=config.home_name): + file_loader = FileSystemLoader('') + env = Environment(loader=file_loader) + template = env.get_template(template_head) + output = template.render(build_url=build_url, page_title=page_title, name_of_site=config.name_of_site) + return output + + def add_file_path(pages, root_dir): ret = set() for page in pages: @@ -55,25 +55,30 @@ def add_file_path(pages, root_dir): return ret -def generate_index_pages(template=config.template_main, template_nav=config.template_nav): +def generate_index_pages(template=config.template_main, template_nav=config.template_nav, template_head=config.template_head): template = template all_tags = TagHelper.getAllTags() - index_tags = config.nav_tags - metaIndexSubPages = [] for index in all_tags.keys(): pages = all_tags[index] - pages = add_file_path(pages, config.content_folder[0]) + pages = add_file_path(pages, config.files_folder) entries = create_entries(pages) # TODO this broke as well, not sure what happened to entries, but the whole page is blank? subPages = generate_sub_pages(entries, len( - entries), config.content_folder[0], False) + entries), config.files_folder, False) # Build the files by filling out the template file_loader = FileSystemLoader('') env = Environment(loader=file_loader) # regular index pagse use main template template = env.get_template(template) - current_index_template = template.render(page_body=subPages, page_date=None, page_tags=None, page_navigation=generate_navigation( - template_nav), page_title=index.capitalize() + ' index', config=config) + page_title = index.capitalize() + ' index' + current_index_template = template.render( + head=generate_head(template_head, page_title), + page_body=subPages, + page_date=None, + page_tags=None, + page_navigation=generate_navigation(template_nav), + page_title=page_title, + config=config) slugfile = config.build_folder + index + '_index.html' with open(slugfile, 'w', encoding=encoding) as fobj: fobj.write(current_index_template) @@ -115,8 +120,15 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template template = env.get_template(template) if len(entry['tags']) > 1: entry['tags'] = entry['tags'].split(' ') - page_template = template.render(page_body=entry['pageContent'], page_date="%s" % entry['date'], page_tags=entry['tags'], - name_of_site="Kitaab", page_navigation=nav_html, page_title=entry['title'], build_url=entry['parent_text'], config=config) + page_template = template.render( + page_body=entry['pageContent'], + page_date="%s" % entry['date'], + page_tags=entry['tags'], + name_of_site="Kitaab", + page_navigation=nav_html, + page_title=entry['title'], + build_url=entry['parent_text'], + config=config) # Checking if content folder exists folderExists = os.path.exists(site_folder + entry['folder']) @@ -261,7 +273,6 @@ def remove_title(page): textContent = textContent[1:] return '\n'.join(textContent) except Exception as e: - print(e) print(f'Unexpected error in removing title in {page}') return page @@ -273,7 +284,7 @@ def remove_date(page): return page except Exception as e: print(e) - print(f'Unexpected error in removing title in {page}') + print(f'Unexpected error in removing date in {page}') return page @@ -284,7 +295,7 @@ def replacement_function(replace): # TODO gross hardcoded lines = [] # this file should probably be an asset?? - with open(config.content_folder[0] + '210124-0033.wiki') as f: + with open(config.files_folder + '210124-0033.wiki') as f: lines = f.readlines() lines = lines[-2:] line1 = lines[0].split(',') @@ -496,12 +507,6 @@ def generate_sub_pages(entries, num, folder, title, title_name=""): return sub_page_list -def fill_template_defaults(template): - template = template.replace('page_title', config.home_name) - template = template.replace('build_url', build_url) - return template - - # Creates the home page using home.md def create_home_page(template, site_folder): @@ -528,8 +533,8 @@ def create_rss_feed(rss_entries, rss_template, rss_item_template, site_folder): entry_template = itemTemplate entry_template = entry_template.replace( 'rssItemTitle', rss_entry["title"]) - entry_template = entry_template.replace('rssItemUrl', build_url + - rss_entry["slug"]) + entry_template = entry_template.replace('rssItemUrl', build_url + + rss_entry["slug"]) entry_template = entry_template.replace( 'rssItemDate', rss_entry["iso_date"]) entry_template = entry_template.replace( @@ -563,47 +568,41 @@ def generate_website(): # Get main html template file_loader = FileSystemLoader('') env = Environment(loader=file_loader) - home_page = env.get_template( - config.template_main) # TODO maybe not hardcode - # home_page = create_home_page(template, config.build_folder) + home_page = env.get_template(config.template_main) generate_index_pages(config.template_main, config.template_nav) rss_entries = [] home_pageSubList = [] - for folder in config.content_folder: - pages = glob.glob(folder + '**/*.wiki', recursive=True) - entries = create_entries(pages) - sub_pages_list = generate_sub_pages( - entries, len(entries), folder, False) - # For each section, create a short listing of sub pages - # and add it to the home page - home_pageSubList = generate_sub_pages(entries, 50, folder, False) - # home_page = home_page.replace('content_list', home_pageSubList + - # "content_list") - # home_page = home_page.replace('page_date', "") + pages = glob.glob(config.files_folder + '**/*.wiki', recursive=True) + entries = create_entries(pages) + sub_pages_list = generate_sub_pages( + entries, len(entries), config.files_folder, False) - generate_html_pages(config.build_folder, entries, config.template_main, - sub_pages_list, config.template_nav) + # For each section, create a short listing of sub pages + # and add it to the home page + home_pageSubList = generate_sub_pages(entries, 50, config.files_folder, False) - for entry in entries: - rss_entries.append(entry) + generate_html_pages(config.build_folder, entries, config.template_main, + sub_pages_list, config.template_nav) + + for entry in entries: + rss_entries.append(entry) # Move the assets move_files(config.build_folder, config.assets_folder) move_files(config.build_folder, config.medias_folder) # Once all sections have been processed, finish the home page - # Removes the "content_list" in the partial - # home_page = home_page.replace('content_list', "") - # home_page = home_page.replace('name_of_site', config.name_of_site) - # home_page = home_page.replace('site_meta_description', - # config.site_meta_description) - # home_page = home_page.replace( - # 'twitter_name', config.twitter_name) - home_page = home_page.render(page_body=home_pageSubList, page_date=None, page_tags=None, - page_navigation=generate_navigation(), page_title="Recent Files", config=config) + home_page = home_page.render( + head=generate_head(page_title="Recent Files"), + page_body=home_pageSubList, + page_date=None, + page_tags=None, + page_navigation=generate_navigation(), + page_title="Recent Files", + config=config) slug_file = config.build_folder + "recents.html" with open(slug_file, 'w', encoding=encoding) as fobj: diff --git a/basant/grab/__init__.py b/basant/grab/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/basant/grab/convert.py b/basant/grab/convert.py deleted file mode 100644 index dce391b..0000000 --- a/basant/grab/convert.py +++ /dev/null @@ -1,10 +0,0 @@ -import pypandoc - -def convert_file(file): - filename = file.strip('wiki')[:-1] # filenames can contain ../ ./ etc, so we strip only the final wiki, and then drop the last character which is a '.' - pypandoc.convert_file(filename + '.wiki', 'md',format='vimwiki', outputfile=filename + '.md') - -def convert_files(files): - for file in files: - convert_file(file) - diff --git a/basant/grab/kitaab.py b/basant/grab/kitaab.py deleted file mode 100644 index ddf8ece..0000000 --- a/basant/grab/kitaab.py +++ /dev/null @@ -1,101 +0,0 @@ -#! /usr/bin/env python3 - -# Various methods of grabbing files from Kitaab -# based on their ctags as generated by vimwiki -# censor - burn all mentions of this word wherever it appears in Kitaab -# filter - leave names in tact but do not link to relevant documents -# create - list in an index file for website -from basant.shared import TagHelper, config, helpers -from basant.shared import censor as c -from os import listdir, mkdir, path, remove -import shutil -from . import convert -import os - -# TODO: generalize based on headings(?) in index.wiki - -censor = c.privateList -# TODO put this in config too... -filter = ["dream", "gratitude", "review", "jrnl", "work", "job", "debt", - "people", "priv", "threatdefence", "info"] -# TODO create = ... -outputBasePath = config.raw_folder - - -def filterFiles(AllFileList): - toFilter = filter + censor - tagList = TagHelper.getAllTags() - doNotTakeFiles = set() - for mask in toFilter: - try: - doNotTakeFiles.update(tagList[mask]) - except KeyError: - print("Key not found: {}".format(mask)) - return AllFileList - doNotTakeFiles - - -def copyFiles(fileList): - baseFilePath = config.files_folder - outputFilePath = outputBasePath - try: - mkdir(outputFilePath) - except FileExistsError: - pass - for f in fileList: - try: - shutil.copy(baseFilePath + f, outputFilePath + f) - except FileNotFoundError: - print("Could not copy file {} to {}".format(baseFilePath+f, - outputFilePath+f)) - - -def censorFiles(outputBasePath=outputBasePath): - for fileName in listdir(outputBasePath): - if not path.isfile(outputBasePath + fileName): - continue - fileName = outputBasePath + fileName - newFile = "" - with open(fileName, 'r', errors='ignore') as f: - for line in f.readlines(): - for c in censor: - line = line.replace(c, helpers.generateBlocks(len(c))) - newFile = newFile + line - with open(fileName, 'w') as f: - for line in newFile: - f.write(line) - - -def convertFiles(outputBasePath=outputBasePath): - for filename in listdir(outputBasePath): - if not path.isfile(outputBasePath + filename): - continue - if '.md' == filename[-3:]: - print(f"{filename} file already converted, skipping") - continue - try: - convert.convert_file(outputBasePath + filename) - remove(outputBasePath + filename) - except RuntimeError as e: - print(f"Could not convert file {outputBasePath+filename}: {e}") - - -def deleteOldRaw(): - if os.path.exists(config.raw_folder): - shutil.rmtree(config.raw_folder) - - -def grabFiles(): - deleteOldRaw() - AllFiles = TagHelper.getAllFiles() - FilteredFiles = filterFiles(AllFiles) - copyFiles(FilteredFiles) - print("Censoring files...") - censorFiles() - - -if __name__ == '__main__': - # network = loadNetwork() - # network = censorNetwork(network) - # print("Converting files...") - # convertFiles() - grabFiles() diff --git a/basant/grab/network-builder.py b/basant/grab/network-builder.py deleted file mode 100644 index 4d2cde1..0000000 --- a/basant/grab/network-builder.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -import networkx as nx -import pygraphviz - -def loadJson(file): - with open(file, 'r') as f: - return json.load(f) - -def makeNetwork(json): - G = nx.DiGraph() - for (key, value) in json.items(): - G.add_nodes_from([key] + value) - G.add_edges_from([(key, v) for v in value]) - return G - -def makeAGraph(graph): - G = nx.nx_agraph.to_agraph(graph) - G.draw("nop.png", prog="nop") - -if __name__ == "__main__": - OUTFILE = 'network.json' - json = loadJson('network.json') - graph = makeNetwork(json) - with open(OUTFILE, 'w') as f: - json.dump(f, makeAGraph(graph)) diff --git a/basant/grab/network.py b/basant/grab/network.py deleted file mode 100644 index 79b4a31..0000000 --- a/basant/grab/network.py +++ /dev/null @@ -1,35 +0,0 @@ -import json -import random -from basant.shared import helpers -import basant.shared.censor - -censor = basant.shared.censor.privateList - - -def loadNetwork(path='network.json'): - # Loads the network.json file - with open(path, 'r') as f: - everything = json.load(f) - return everything - - -def setNetwork(everything): - # Everything is loaded from the network.json file - # keys = set(everything.network()) - network = "" - values = set(everything.values()) - everything = network + values - return everything - - -def censorNetwork(network): - # Takes network as a set - for c in censor: - if c in network: - network.add(helpers.generateBlocks(len(c) + random.randint(0, 5))) - network.remove(c) - return network - - -def createIndexNode(network): - pass diff --git a/basant/grab/tree.py b/basant/grab/tree.py deleted file mode 100644 index 605ab45..0000000 --- a/basant/grab/tree.py +++ /dev/null @@ -1,120 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- -from icecream import ic -import json - -"""network-vimwiki.py -Create site map for vimwiki. -Must run script under vimwiki directory. - -Example: - output network map to network.json - dfsvw.py - -Python version: 3""" -import sys -import re - -linkRE = re.compile(r'\[\[([^\\\]|#]+)(?:#[^\\\]|]+)?(?:\|([^\\\]]+))?\]\]', re.UNICODE) - -def probeLink(s): - """ - >>> probeLink(' abc') - [] - >>> probeLink(' [[]]') - [] - >>> probeLink(' [[abc]] [[abc|123]] ') - [('abc', ''), ('abc', '123')] - >>> a, b = probeLink(' [[中文1|中文2]]')[0]; print a, b - 中文1 中文2 - >>> probeLink(' [[abc|abc 123]]') - [('abc', 'abc 123')] - """ - ret = linkRE.findall(s) - return [(a, b) for a, b in ret] - - -def probeAllLink(fn): - """extract all link in a wiki file""" - try: - lst = [] - print(fn) - fp = open(fn) - for line in fp.readlines(): - l = probeLink(line) - lst.extend(l) - except IOError: - print("cannot open",sys.stderr) - except UnicodeDecodeError: - lst = [] - fp = open(fn, encoding="utf-16BE") - for line in fp.readlines(): - l = probeLink(line) - lst.extend(l) - return lst - - -def outputNode(link, name, level): - """output one wiki node, level for vimwiki tree hierarchy""" - """ Unused """ - prefix = " " * 4 * level - if not name: - name = link - dct = {"prefix":prefix, "link":link, "name":name} - if sys.stdout.isatty(): - fmt = "{prefix}{name}" - else: - if link == name: - fmt = "{prefix}[[{name}]]" - else: - fmt = "{prefix}[[{link}|{name}]]" - print(fmt.format(**dct)) - -def addNode(link, node, name, graph): - """ adds a node to the dictionary that tracks edges""" - if name: - node = name - if link in graph.keys(): - graph[link].add(node) - else: - graph[link] = {node} - -def bfs(index): - """breath first search on vimwiki - keep seen and unseen to track which files to search - ugly loops instead of recursive setup """ - seen = set() - unseen = [index] - graph = {} - filecount = 0 - while 1: - try: - link = unseen.pop(0) - except IndexError: - print("Search complete...") - ic(filecount) - return graph - seen.add(link) - ret = probeAllLink(link + '.wiki') - for (node, name) in ret: - addNode(link, node, name, graph) - if not node in seen: - unseen.append(node) - filecount = filecount + 1 - ic(filecount) - ic(seen) - -def transformGraph(graph): - for key, value in graph.items(): - graph[key] = list(value) - return graph - -if __name__ == "__main__": - graph = bfs('index') - print("Transforming Graph...") - graph = transformGraph(graph) - print("Transform Complete...") - ic(graph) - print("Outputting file") - with open('network.json', 'w+') as out: - json.dump(graph, out) diff --git a/basant/partial/head.html b/basant/partial/head.html new file mode 100644 index 0000000..aee347c --- /dev/null +++ b/basant/partial/head.html @@ -0,0 +1,42 @@ + + + + {{page_title}} | {{config.name_of_site}} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/basant/partial/index_table.html b/basant/partial/index_table.html index e244fb3..916985d 100644 --- a/basant/partial/index_table.html +++ b/basant/partial/index_table.html @@ -1,48 +1,7 @@ - - - - {{page_title}} | {{config.name_of_site}} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +{{head}}
diff --git a/basant/partial/main.html b/basant/partial/main.html index e232a7d..b1817fa 100644 --- a/basant/partial/main.html +++ b/basant/partial/main.html @@ -1,48 +1,7 @@ - - - - {{page_title}} | {{config.name_of_site}} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +{{head}}
diff --git a/basant/shared/TagHelper.py b/basant/shared/TagHelper.py index f4af00f..39271c8 100644 --- a/basant/shared/TagHelper.py +++ b/basant/shared/TagHelper.py @@ -1,9 +1,7 @@ from ctags import CTags, TagEntry import sys from os import listdir, path -from . import config, censor as c - -privateList = c.privateList +from . import config def getAllFiles(): @@ -13,20 +11,6 @@ def getAllFiles(): return files -def untaggedFiles(AllFileList=getAllFiles()): - # TODO broken - censor = [] - toFilter = filter + censor - tagList = getAllTags() - doNotTakeFiles = set() - for mask in toFilter: - try: - doNotTakeFiles.update(tagList[mask]) - except KeyError: - print("Key not found: {}".format(mask)) - return set.difference(AllFileList, doNotTakeFiles) - - def loadTagFile(path=config.files_folder + '.vimwiki_tags'): # Available file information keys: # opened - was the tag file successfully opened? @@ -90,6 +74,8 @@ def getAllFilesWithTags(): def getUntaggedFiles(): allFiles = getAllFiles() allTags = getAllFilesWithTags() + print("All tags from untagged") + print(allTags) return allFiles - allTags @@ -99,6 +85,4 @@ def getAllTags(): # - values as files that have the key in their tags all_tags = makeDictFromTags(loadTagFile()) all_tags['untagged'] = getUntaggedFiles() - for tag in privateList: - all_tags.pop(tag, None) return all_tags diff --git a/basant/shared/censor.py b/basant/shared/censor.py deleted file mode 100644 index 985d4fe..0000000 --- a/basant/shared/censor.py +++ /dev/null @@ -1,2 +0,0 @@ -privateList = ['y0x3y', 'piracy', 'weed', 'drugs', 'gel capsules', - 'no weed', 'hackres town', 'hackers.town', 'radicalEd'] diff --git a/basant/shared/config.py b/basant/shared/config.py index 228d909..2266ab8 100644 --- a/basant/shared/config.py +++ b/basant/shared/config.py @@ -1,17 +1,17 @@ -content_folder = ['public/'] nav_tags = ['dev', 'way', 'recipe', 'ops', 'blog', 'rock', 'phil', 'story', 'blog', 'untagged'] relative_build_url = "" absolute_build_url = "https://anish.lakhwara.com/" files_folder = 'public/' build_folder = "output/" -assets_folder = "basant/assets/" -medias_folder = "basant/media/" +assets_folder = "./basant/assets/" +medias_folder = "./basant/media/" raw_folder = "raw/" template_main = "./basant/partial/main.html" template_nav = "./basant/partial/nav.html" template_index = "./basant/partial/index_table.html" template_list = "./basant/partial/list.html" +template_head = './basant/partial/head.html' rss_template = "./basant/partial/rss.xml" rss_item_template = "./basant/partial/item.xml" name_of_site = "Kitaab" diff --git a/basant/shared/tag-fixer.py b/basant/shared/tag-fixer.py deleted file mode 100644 index a8a71cc..0000000 --- a/basant/shared/tag-fixer.py +++ /dev/null @@ -1,106 +0,0 @@ -#! /usr/bin/env python3 - -import ctags -from ctags import CTags, TagEntry -import sys -from icecream import ic -import shutil -from . import config - - -def loadTagFile(path='.vimwiki_tags'): - # Available file information keys: - # opened - was the tag file successfully opened? - # error_number - errno value when 'opened' is false - # format - format of tag file (1 = original, 2 = extended) - # sort - how is the tag file sorted? - # author - name of author of generating program (may be empy string) - # name - name of program (may be empy string) - # url - URL of distribution (may be empy string) - # version - program version (may be empty string) - try: - tagFile = CTags(path) - return tagFile - except: - print("No tagfile at {}".farmat(path)) - sys.exit(1) - - -def loopThrough(tagFile): - # Available TagEntry keys: - # name - name of tag - # file - path of source file containing definition of tag - # pattern - pattern for locating source line (None if no pattern) - # lineNumber - line number in source file of tag definition (may be zero if not known) - # kind - kind of tag (none if not known) - # fileScope - is tag of file-limited scope? - - # Note: other keys will be assumed as an extension key and will - # return None if no such key is found - ''' Entries that are less than 1 character in length are errors - Returns a set containing of all the files with errors in their tags''' - entry = TagEntry() - tag = tagFile.next(entry) - tagErrors = set() - errorTagNames = set() - while tag: - if len(entry['name']) <= 1: - tagErrors.add(entry['file']) - errorTagNames.add(entry['name']) - else: - print("Good tag found: {}".format(entry['name'])) - tag = tagFile.next(entry) - ic(errorTagNames) - return tagErrors - - -def fixErrors(errors): - ''' Replaces :: with :, which seems to be the error for many of the tags ''' - fileBase = config.files_folder - for v in errors: - print("Fixing {}...".format(v)) - fileName = fileBase + getFileName(v) - newFile = "" - with open(fileName, 'r') as f: - for line in f.readlines(): - line = line.replace("::", ":") - newFile = newFile + line - with open(fileName, 'w') as f: - for line in newFile: - f.write(line) - print("Written new file...") - print() - print(newFile) - - -def printErrors(errors): - fileBase = config.files_folder - for v in errors: - fileName = fileBase + getFileName(v) - print("Reading {}...".format(v)) - with open(fileName, 'r') as f: - f.readline() - ic(f.readline()) - - -def getFileName(f): - return str(f, encoding="utf-8") - - -def getBackupFiles(errors): - fileBase = config.files_folder - backupBase = config.files_folder + '/jrnl/' - for files in errors.values(): - for f in files: - f = getFileName(f) - print("Moving {} from backup".format(f)) - shutil.copy(backupBase + f, fileBase + f) - - -if __name__ == '__main__': - tagFile = loadTagFile() - errors = loopThrough(tagFile) - printErrors(errors) - # ic(errors) - # getBackupFiles(errors) - # fixErrors(errors) diff --git a/flake.lock b/flake.lock index a8eb15c..e97d066 100644 --- a/flake.lock +++ b/flake.lock @@ -50,11 +50,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1649834414, - "narHash": "sha256-Wu/PJQ+4KBL7GkAJWDLY6CG7n/GIP9keB1nLvFsEpAw=", + "lastModified": 1649938384, + "narHash": "sha256-VZUS7k1NFszRpvvpZKgswF3dlv72DvkRyNrir7RLlus=", "ref": "main", - "rev": "5e217728465fdd0617c8496dd4059d631b7394ae", - "revCount": 302, + "rev": "b96c3ee8a8d1b9b83b2a27e6fc6100f3c2cf8bcc", + "revCount": 304, "type": "git", "url": "ssh://gitea@git.sealight.xyz/aynish/kitaab" },