commit 7a08a7fef2d809564040b5dd1a5057a397b85e89 Author: Anish Lakhwara Date: Mon Nov 29 22:26:21 2021 +1000 initial diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4a0b09c --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +output/* +graphs/* +network.json +__pycache__/* +.tags +.vimwiki_tags +.vim/* +censor.py diff --git a/TagHelper.py b/TagHelper.py new file mode 100644 index 0000000..589b52b --- /dev/null +++ b/TagHelper.py @@ -0,0 +1,68 @@ +import ctags +from ctags import CTags, TagEntry +import sys +from icecream import ic +import os +import shutil +import fileinput + +def loadTagFile(path='.vimwiki_tags'): + # Available file information keys: + # opened - was the tag file successfully opened? + # error_number - errno value when 'opened' is false + # format - format of tag file (1 = original, 2 = extended) + # sort - how is the tag file sorted? + # author - name of author of generating program (may be empy string) + # name - name of program (may be empy string) + # url - URL of distribution (may be empy string) + # version - program version (may be empty string) + try: + tagFile = CTags(path) + return tagFile + except: + print("No tagfile at {}".farmat(path)) + sys.exit(1) + +def makeSetFromTags(tagFile, key='file'): + # Available TagEntry keys: + # name - name of tag + # file - path of source file containing definition of tag + # pattern - pattern for locating source line (None if no pattern) + # lineNumber - line number in source file of tag definition (may be zero if not known) + # kind - kind of tag (none if not known) + # fileScope - is tag of file-limited scope? + + # Note: other keys will be assumed as an extension key and will + # return None if no such key is found + entry = TagEntry() + tag = tagFile.next(entry) + AllFileList = set() + while tag: + AllFileList.add(entry[key].decode()) + tag = tagFile.next(entry) + return AllFileList + +def makeDictFromTags(tagFile, key='name', value='file'): + entry = TagEntry() + tag = tagFile.next(entry) + ret = {} + while tag: + # Not sure why they're being read as bytes now? + strKey = entry[key].decode() + strValue = entry[value].decode() + if strKey not in ret.keys(): + ret[strKey] = set(strValue) + else: + ret[strKey].add(strValue) + tag = tagFile.next(entry) + return ret + +def getAllFiles(): + # Returns a set containing all the files we wish to make public + return makeSetFromTags(loadTagFile()) + +def getAllTags(): + # Returns a dictionary with: + # - key as tag titles + # - values as files that have the key in their tags + return makeDictFromTags(loadTagFile()) diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..57437a1 --- /dev/null +++ b/flake.nix @@ -0,0 +1,21 @@ +{ + description = "Sketches in Quil"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let pkgs = nixpkgs.legacyPackages.${system}; + in { + devShell = pkgs.mkShell { + buildInputs = [ + pkgs.leiningen + pkgs.jdk11 + pkgs.clojure + ]; + }; + }); +} diff --git a/makePublic.py b/makePublic.py new file mode 100644 index 0000000..b95a043 --- /dev/null +++ b/makePublic.py @@ -0,0 +1,100 @@ +#! /usr/bin/env python3 +# Various methods of grabbing files from Kitaab based on their ctags as generated by vimwiki +# censor - burn all mentions of this word wherever it appears in Kitaab +# filter - leave names in tact but do not link to relevant documents +# create - list in an index file for website +from os import listdir, mkdir +from os.path import isfile, join +import shutil +import random +import TagHelper +import censor as cen +from icecream import ic + +## TODO: generalize based on headings(?) in index.wiki + +censor = cen.listBad +filter = ["Dream", "Gratitude", "Review", "jrnl", "piracy", "work", "job", "debt", "people"] +#TODO create = ... + +def loadNetwork(path='network.json'): + # Loads the network.json file + with open(path, 'r') as f: + everything = json.load(f) + return everything + +def setNetwork(everything): + # Everything is loaded from the network.json file + keys = set(everything.network()) + values = set(everything.values()) + everything = network + values + return everything + +def filterFiles(AllFileList): + toFilter = filter + censor + tagList = TagHelper.getAllTags() + doNotTakeFiles = set() + for mask in toFilter: + try: + doNotTakeFiles.update(tagList[mask]) + except KeyError: + print("Key not found: {}".format(mask)) + return AllFileList - doNotTakeFiles + +def censorNetwork(network): + # Takes network as a set + for c in censor: + if c in network: + network.add(generateBlocks(len(c) + random.randint(0, 5))) + network.remove(c) + return network + +def generateBlocks(size=1): + # Generates a length size unicode string that is formed from blocks: + charList = ['▐', '░', '▒', '▓', '▔', '▕', '▖▗', '▘▙', '▚', '▛', '▜', '▝', '▞', '▟'] + ret = "" + while size > 0: + size = size - 1 + rand = random.randint(0,13) + ret = ret + charList[rand] + return ret + +def createIndexNode(network): + pass + +def copyFiles(fileList): + baseFilePath = '/home/anish/vimwiki/' + try: + mkdir('output') + except FileExistsError: + pass + for f in fileList: + try: + shutil.copy(baseFilePath + f, './output/') + except FileNotFoundError: + print("Could not copy file {}:".format(baseFilePath + f)) + +def censorFiles(): + outputBasePath = './output/' + for fileName in listdir(outputBasePath): + fileName = outputBasePath + fileName + newFile = "" + print("Censoring: {}".format(fileName)) + with open(fileName, 'r') as f: + for line in f.readlines(): + for c in censor: + line = line.replace(c, generateBlocks(len(c))) + newFile = newFile + line + with open(fileName, 'w') as f: + for line in newFile: + f.write(line) + +if __name__=='__main__': + + #network = loadNetwork() + #network = censorNetwork(network) + AllFiles = TagHelper.getAllFiles() + FilteredFiles = filterFiles(AllFiles) + copyFiles(FilteredFiles) + print("Censoring files...") + censorFiles() diff --git a/network-builder.py b/network-builder.py new file mode 100644 index 0000000..4d2cde1 --- /dev/null +++ b/network-builder.py @@ -0,0 +1,25 @@ +import json +import networkx as nx +import pygraphviz + +def loadJson(file): + with open(file, 'r') as f: + return json.load(f) + +def makeNetwork(json): + G = nx.DiGraph() + for (key, value) in json.items(): + G.add_nodes_from([key] + value) + G.add_edges_from([(key, v) for v in value]) + return G + +def makeAGraph(graph): + G = nx.nx_agraph.to_agraph(graph) + G.draw("nop.png", prog="nop") + +if __name__ == "__main__": + OUTFILE = 'network.json' + json = loadJson('network.json') + graph = makeNetwork(json) + with open(OUTFILE, 'w') as f: + json.dump(f, makeAGraph(graph)) diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..8627f69 --- /dev/null +++ b/readme.md @@ -0,0 +1,21 @@ +# Kitaab helper + +Helps make my Kitaab public by grabbing Kitaab, filtering & censoring it, and then passing it through to an SSG to make HTML out of a markdown page. + +### Files +makePublic.py +- the whole filter pipeline + +TagHelper.py +- Reads the .vimwiki_tags file that is generated by vimwiki + +tree.py +- builds a tree out of all the links using dfs + +network-builder.py +- uses the network from tree.py to visualize links + +tag-fixer.py +- old script used to fix problem tags (1 character tags) +- (TODO) tags that are case sensitive + diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..f63e030 --- /dev/null +++ b/shell.nix @@ -0,0 +1,20 @@ +# shell.nix +{ pkgs ? import {} }: +let + my-python = pkgs.python3; + python-with-my-packages = my-python.withPackages (p: with p; [ + icecream + # other python packages you want + ]); +in +pkgs.mkShell { + buildInputs = [ + python-with-my-packages + # other dependencies + ]; + shellHook = '' + PYTHONPATH=${python-with-my-packages}/${python-with-my-packages.sitePackages} + # maybe set more env-vars + ''; +} + diff --git a/tag-fixer.py b/tag-fixer.py new file mode 100644 index 0000000..21d985d --- /dev/null +++ b/tag-fixer.py @@ -0,0 +1,100 @@ +#! /usr/bin/env python3 + +import ctags +from ctags import CTags, TagEntry +import sys +from icecream import ic +import os +import shutil +import fileinput + +def loadTagFile(path='.vimwiki_tags'): + # Available file information keys: + # opened - was the tag file successfully opened? + # error_number - errno value when 'opened' is false + # format - format of tag file (1 = original, 2 = extended) + # sort - how is the tag file sorted? + # author - name of author of generating program (may be empy string) + # name - name of program (may be empy string) + # url - URL of distribution (may be empy string) + # version - program version (may be empty string) + try: + tagFile = CTags(path) + return tagFile + except: + print("No tagfile at {}".farmat(path)) + sys.exit(1) + +def loopThrough(tagFile): + # Available TagEntry keys: + # name - name of tag + # file - path of source file containing definition of tag + # pattern - pattern for locating source line (None if no pattern) + # lineNumber - line number in source file of tag definition (may be zero if not known) + # kind - kind of tag (none if not known) + # fileScope - is tag of file-limited scope? + + # Note: other keys will be assumed as an extension key and will + # return None if no such key is found + ''' Entries that are less than 1 character in length are errors + Returns a set containing of all the files with errors in their tags''' + entry = TagEntry() + tag = tagFile.next(entry) + tagErrors = set() + errorTagNames = set() + while tag: + if len(entry['name']) <= 1: + tagErrors.add(entry['file']) + errorTagNames.add(entry['name']) + else: + print("Good tag found: {}".format(entry['name'])) + tag = tagFile.next(entry) + ic(errorTagNames) + return tagErrors + +def fixErrors(errors): + ''' Replaces :: with :, which seems to be the error for many of the tags ''' + fileBase = '/home/anish/vimwiki/' + for v in errors: + print("Fixing {}...".format(v)) + fileName = fileBase + getFileName(v) + newFile = "" + with open(fileName, 'r') as f: + for line in f.readlines(): + line = line.replace("::", ":") + newFile = newFile + line + with open(fileName, 'w') as f: + for line in newFile: + f.write(line) + print("Written new file...") + print() + print(newFile) + +def printErrors(errors): + fileBase = '/home/anish/vimwiki/' + for v in errors: + fileName = fileBase + getFileName(v) + print("Reading {}...".format(v)) + with open(fileName, 'r') as f: + f.readline() + ic(f.readline()) + +def getFileName(f): + return str(f, encoding="utf-8") + +def getBackupFiles(errors): + fileBase = '/home/anish/vimwiki/' + backupBase = '/home/anish/vimwiki/jrnl/' + for files in errors.values(): + for f in files: + f = getFileName(f) + print("Moving {} from backup".format(f)) + shutil.copy(backupBase + f, fileBase + f) + +if __name__=='__main__': + tagFile = loadTagFile() + errors = loopThrough(tagFile) + printErrors(errors) + #ic(errors) + #getBackupFiles(errors) + #fixErrors(errors) diff --git a/tree.py b/tree.py new file mode 100644 index 0000000..605ab45 --- /dev/null +++ b/tree.py @@ -0,0 +1,120 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +from icecream import ic +import json + +"""network-vimwiki.py +Create site map for vimwiki. +Must run script under vimwiki directory. + +Example: + output network map to network.json + dfsvw.py + +Python version: 3""" +import sys +import re + +linkRE = re.compile(r'\[\[([^\\\]|#]+)(?:#[^\\\]|]+)?(?:\|([^\\\]]+))?\]\]', re.UNICODE) + +def probeLink(s): + """ + >>> probeLink(' abc') + [] + >>> probeLink(' [[]]') + [] + >>> probeLink(' [[abc]] [[abc|123]] ') + [('abc', ''), ('abc', '123')] + >>> a, b = probeLink(' [[中文1|中文2]]')[0]; print a, b + 中文1 中文2 + >>> probeLink(' [[abc|abc 123]]') + [('abc', 'abc 123')] + """ + ret = linkRE.findall(s) + return [(a, b) for a, b in ret] + + +def probeAllLink(fn): + """extract all link in a wiki file""" + try: + lst = [] + print(fn) + fp = open(fn) + for line in fp.readlines(): + l = probeLink(line) + lst.extend(l) + except IOError: + print("cannot open",sys.stderr) + except UnicodeDecodeError: + lst = [] + fp = open(fn, encoding="utf-16BE") + for line in fp.readlines(): + l = probeLink(line) + lst.extend(l) + return lst + + +def outputNode(link, name, level): + """output one wiki node, level for vimwiki tree hierarchy""" + """ Unused """ + prefix = " " * 4 * level + if not name: + name = link + dct = {"prefix":prefix, "link":link, "name":name} + if sys.stdout.isatty(): + fmt = "{prefix}{name}" + else: + if link == name: + fmt = "{prefix}[[{name}]]" + else: + fmt = "{prefix}[[{link}|{name}]]" + print(fmt.format(**dct)) + +def addNode(link, node, name, graph): + """ adds a node to the dictionary that tracks edges""" + if name: + node = name + if link in graph.keys(): + graph[link].add(node) + else: + graph[link] = {node} + +def bfs(index): + """breath first search on vimwiki + keep seen and unseen to track which files to search + ugly loops instead of recursive setup """ + seen = set() + unseen = [index] + graph = {} + filecount = 0 + while 1: + try: + link = unseen.pop(0) + except IndexError: + print("Search complete...") + ic(filecount) + return graph + seen.add(link) + ret = probeAllLink(link + '.wiki') + for (node, name) in ret: + addNode(link, node, name, graph) + if not node in seen: + unseen.append(node) + filecount = filecount + 1 + ic(filecount) + ic(seen) + +def transformGraph(graph): + for key, value in graph.items(): + graph[key] = list(value) + return graph + +if __name__ == "__main__": + graph = bfs('index') + print("Transforming Graph...") + graph = transformGraph(graph) + print("Transform Complete...") + ic(graph) + print("Outputting file") + with open('network.json', 'w+') as out: + json.dump(graph, out)