initial

.gitignore

@@ -0,0 +1,8 @@
+output/*
+graphs/*
+network.json
+__pycache__/*
+.tags
+.vimwiki_tags
+.vim/*
+censor.py

TagHelper.py

@@ -0,0 +1,68 @@
+import ctags
+from ctags import CTags, TagEntry
+import sys
+from icecream import ic
+import os
+import shutil
+import fileinput
+
+def loadTagFile(path='.vimwiki_tags'):
+    # Available file information keys:
+    #  opened -  was the tag file successfully opened?
+    #  error_number - errno value when 'opened' is false
+    #  format - format of tag file (1 = original, 2 = extended)
+    #  sort - how is the tag file sorted? 
+    #  author - name of author of generating program (may be empy string)
+    #  name - name of program (may be empy string)
+    #  url - URL of distribution (may be empy string)
+    #  version - program version (may be empty string)
+    try:
+        tagFile = CTags(path)
+        return tagFile
+    except:
+        print("No tagfile at {}".farmat(path))
+        sys.exit(1)
+
+def makeSetFromTags(tagFile, key='file'):
+    # Available TagEntry keys:
+    #  name - name of tag
+    #  file - path of source file containing definition of tag
+    #  pattern - pattern for locating source line (None if no pattern)
+    #  lineNumber - line number in source file of tag definition (may be zero if not known)
+    #  kind - kind of tag (none if not known)
+    #  fileScope - is tag of file-limited scope?
+    
+    # Note: other keys will be assumed as an extension key and will 
+    # return None if no such key is found 
+    entry = TagEntry()
+    tag = tagFile.next(entry)
+    AllFileList = set()
+    while tag:
+        AllFileList.add(entry[key].decode())
+        tag = tagFile.next(entry)
+    return AllFileList
+
+def makeDictFromTags(tagFile, key='name', value='file'):
+    entry = TagEntry()
+    tag = tagFile.next(entry)
+    ret = {}
+    while tag:
+        # Not sure why they're being read as bytes now?
+        strKey = entry[key].decode()
+        strValue = entry[value].decode()
+        if strKey not in ret.keys():
+            ret[strKey] = set(strValue)
+        else:
+            ret[strKey].add(strValue)
+        tag = tagFile.next(entry)
+    return ret
+
+def getAllFiles():
+    # Returns a set containing all the files we wish to make public
+    return makeSetFromTags(loadTagFile())
+
+def getAllTags():
+    # Returns a dictionary with:
+    # - key as tag titles
+    # - values as files that have the key in their tags
+    return makeDictFromTags(loadTagFile())

flake.nix

@@ -0,0 +1,21 @@
+{
+  description = "Sketches in Quil";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, flake-utils }: 
+    flake-utils.lib.eachDefaultSystem (system:
+    let pkgs = nixpkgs.legacyPackages.${system};
+    in {
+      devShell = pkgs.mkShell { 
+        buildInputs = [ 
+          pkgs.leiningen 
+          pkgs.jdk11
+          pkgs.clojure
+        ]; 
+      };
+    });
+}

makePublic.py

@@ -0,0 +1,100 @@
+#! /usr/bin/env python3
+# Various methods of grabbing files from Kitaab based on their ctags as generated by vimwiki
+# censor - burn all mentions of this word wherever it appears in Kitaab
+# filter - leave names in tact but do not link to relevant documents
+# create - list in an index file for website
+from os import listdir, mkdir
+from os.path import isfile, join
+import shutil
+import random
+import TagHelper
+import censor as cen
+from icecream import ic
+
+## TODO: generalize based on headings(?) in index.wiki
+
+censor = cen.listBad
+filter = ["Dream", "Gratitude", "Review", "jrnl", "piracy", "work", "job", "debt", "people"]
+#TODO create = ... 
+
+def loadNetwork(path='network.json'):
+    # Loads the network.json file
+    with open(path, 'r') as f:
+        everything = json.load(f)
+    return everything
+
+def setNetwork(everything):
+    # Everything is loaded from the network.json file
+    keys = set(everything.network())
+    values = set(everything.values())
+    everything = network + values
+    return everything
+
+def filterFiles(AllFileList):
+    toFilter = filter + censor
+    tagList = TagHelper.getAllTags()
+    doNotTakeFiles = set()
+    for mask in toFilter:
+        try:
+            doNotTakeFiles.update(tagList[mask])
+        except KeyError:
+            print("Key not found: {}".format(mask))
+    return AllFileList - doNotTakeFiles
+
+def censorNetwork(network):
+    # Takes network as a set
+    for c in censor:
+        if c in network:
+            network.add(generateBlocks(len(c) + random.randint(0, 5)))
+            network.remove(c)
+    return network
+
+def generateBlocks(size=1):
+    # Generates a length size unicode string that is formed from blocks:
+    charList = ['▐', '░', '▒', '▓', '▔', '▕', '▖▗', '▘▙', '▚', '▛', '▜', '▝', '▞', '▟']
+    ret = ""
+    while size > 0:
+        size = size - 1
+        rand = random.randint(0,13)
+        ret = ret + charList[rand]
+    return ret
+
+def createIndexNode(network):
+    pass
+
+def copyFiles(fileList):
+    baseFilePath = '/home/anish/vimwiki/'
+    try:
+        mkdir('output')
+    except FileExistsError:
+        pass
+    for f in fileList:
+        try:
+            shutil.copy(baseFilePath + f, './output/')
+        except FileNotFoundError:
+            print("Could not copy file {}:".format(baseFilePath + f))
+
+def censorFiles():
+    outputBasePath = './output/'
+    for fileName in listdir(outputBasePath):
+        fileName = outputBasePath + fileName
+        newFile = ""
+        print("Censoring: {}".format(fileName))
+        with open(fileName, 'r') as f:
+            for line in f.readlines():
+                for c in censor:
+                    line = line.replace(c, generateBlocks(len(c)))
+                newFile = newFile + line
+        with open(fileName, 'w') as f:
+            for line in newFile:
+                f.write(line)
+
+if __name__=='__main__':
+    
+    #network = loadNetwork()
+    #network = censorNetwork(network)
+    AllFiles = TagHelper.getAllFiles()
+    FilteredFiles = filterFiles(AllFiles)
+    copyFiles(FilteredFiles)
+    print("Censoring files...")
+    censorFiles()

network-builder.py

@@ -0,0 +1,25 @@
+import json
+import networkx as nx
+import pygraphviz
+
+def loadJson(file):
+    with open(file, 'r') as f:
+        return json.load(f)
+
+def makeNetwork(json):
+    G = nx.DiGraph()
+    for (key, value) in json.items():
+        G.add_nodes_from([key] + value)
+        G.add_edges_from([(key, v) for v in value])
+    return G
+
+def makeAGraph(graph):
+    G = nx.nx_agraph.to_agraph(graph)
+    G.draw("nop.png", prog="nop")
+    
+if __name__ == "__main__":
+    OUTFILE = 'network.json'
+    json = loadJson('network.json')
+    graph = makeNetwork(json)
+    with open(OUTFILE, 'w') as f:
+        json.dump(f, makeAGraph(graph))

readme.md

@@ -0,0 +1,21 @@
+# Kitaab helper
+
+Helps make my Kitaab public by grabbing Kitaab, filtering & censoring it, and then passing it through to an SSG to make HTML out of a markdown page. 
+
+### Files 
+makePublic.py
+- the whole filter pipeline 
+
+TagHelper.py
+- Reads the .vimwiki_tags file that is generated by vimwiki
+
+tree.py
+- builds a tree out of all the links using dfs 
+
+network-builder.py
+- uses the network from tree.py to visualize links
+
+tag-fixer.py
+- old script used to fix problem tags (1 character tags) 
+- (TODO) tags that are case sensitive 
+

shell.nix

@@ -0,0 +1,20 @@
+# shell.nix
+{ pkgs ? import <nixpkgs> {} }:
+let
+  my-python = pkgs.python3;
+  python-with-my-packages = my-python.withPackages (p: with p; [
+    icecream
+    # other python packages you want
+  ]);
+in
+pkgs.mkShell {
+  buildInputs = [
+    python-with-my-packages
+    # other dependencies
+  ];
+  shellHook = ''
+    PYTHONPATH=${python-with-my-packages}/${python-with-my-packages.sitePackages}
+    # maybe set more env-vars
+  '';
+}
+

tag-fixer.py

@@ -0,0 +1,100 @@
+#! /usr/bin/env python3 
+
+import ctags
+from ctags import CTags, TagEntry
+import sys
+from icecream import ic
+import os
+import shutil
+import fileinput
+ 
+def loadTagFile(path='.vimwiki_tags'):
+    # Available file information keys:
+    #  opened -  was the tag file successfully opened?
+    #  error_number - errno value when 'opened' is false
+    #  format - format of tag file (1 = original, 2 = extended)
+    #  sort - how is the tag file sorted? 
+    #  author - name of author of generating program (may be empy string)
+    #  name - name of program (may be empy string)
+    #  url - URL of distribution (may be empy string)
+    #  version - program version (may be empty string)
+    try:
+        tagFile = CTags(path)
+        return tagFile
+    except:
+        print("No tagfile at {}".farmat(path))
+        sys.exit(1)
+
+def loopThrough(tagFile):
+    # Available TagEntry keys:
+    #  name - name of tag
+    #  file - path of source file containing definition of tag
+    #  pattern - pattern for locating source line (None if no pattern)
+    #  lineNumber - line number in source file of tag definition (may be zero if not known)
+    #  kind - kind of tag (none if not known)
+    #  fileScope - is tag of file-limited scope?
+    
+    # Note: other keys will be assumed as an extension key and will 
+    # return None if no such key is found 
+    ''' Entries that are less than 1 character in length are errors
+        Returns a set containing of all the files with errors in their tags''' 
+    entry = TagEntry()
+    tag = tagFile.next(entry)
+    tagErrors = set()
+    errorTagNames = set()
+    while tag:
+        if len(entry['name']) <= 1:
+            tagErrors.add(entry['file'])
+            errorTagNames.add(entry['name'])
+        else:
+            print("Good tag found: {}".format(entry['name']))
+        tag = tagFile.next(entry)
+    ic(errorTagNames)
+    return tagErrors
+
+def fixErrors(errors):
+    ''' Replaces :: with :, which seems to be the error for many of the tags '''
+    fileBase = '/home/anish/vimwiki/'
+    for v in errors:
+        print("Fixing {}...".format(v))
+        fileName = fileBase + getFileName(v)
+        newFile = ""
+        with open(fileName, 'r') as f:
+            for line in f.readlines():
+                line = line.replace("::", ":")
+                newFile = newFile + line
+        with open(fileName, 'w') as f:
+            for line in newFile:
+                f.write(line)
+        print("Written new file...")
+        print()
+        print(newFile)
+
+def printErrors(errors):
+    fileBase = '/home/anish/vimwiki/'
+    for v in errors:
+        fileName = fileBase + getFileName(v)
+        print("Reading {}...".format(v))
+        with open(fileName, 'r') as f:
+            f.readline()
+            ic(f.readline())
+
+def getFileName(f):
+    return str(f, encoding="utf-8")
+
+def getBackupFiles(errors):
+    fileBase = '/home/anish/vimwiki/'
+    backupBase = '/home/anish/vimwiki/jrnl/'
+    for files in errors.values():
+        for f in files:
+            f = getFileName(f)
+            print("Moving {} from backup".format(f))
+            shutil.copy(backupBase + f, fileBase + f)
+
+if __name__=='__main__':
+    tagFile = loadTagFile()
+    errors = loopThrough(tagFile)
+    printErrors(errors)
+    #ic(errors)
+    #getBackupFiles(errors)
+    #fixErrors(errors)

tree.py

@@ -0,0 +1,120 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+from icecream import ic
+import json
+
+"""network-vimwiki.py
+Create site map for vimwiki.
+Must run script under vimwiki directory.
+
+Example:
+    output network map to network.json
+        dfsvw.py
+
+Python version: 3"""
+import sys
+import re
+
+linkRE = re.compile(r'\[\[([^\\\]|#]+)(?:#[^\\\]|]+)?(?:\|([^\\\]]+))?\]\]', re.UNICODE)
+
+def probeLink(s):
+    """
+    >>> probeLink('   abc')
+    []
+    >>> probeLink('   [[]]')
+    []
+    >>> probeLink('   [[abc]]   [[abc|123]]  ')
+    [('abc', ''), ('abc', '123')]
+    >>> a, b = probeLink(' [[中文1|中文2]]')[0]; print a, b
+    中文1 中文2
+    >>> probeLink(' [[abc|abc 123]]')
+    [('abc', 'abc 123')]
+    """
+    ret = linkRE.findall(s)
+    return [(a, b) for a, b in ret]
+
+
+def probeAllLink(fn):
+    """extract all link in a wiki file"""
+    try:
+        lst = []
+        print(fn)
+        fp = open(fn)
+        for line in fp.readlines():
+            l = probeLink(line)
+            lst.extend(l)
+    except IOError:
+        print("cannot open",sys.stderr)
+    except UnicodeDecodeError:
+        lst = []
+        fp = open(fn, encoding="utf-16BE")
+        for line in fp.readlines():
+            l = probeLink(line)
+            lst.extend(l)
+    return lst
+
+
+def outputNode(link, name, level):
+    """output one wiki node, level for vimwiki tree hierarchy"""
+    """ Unused """
+    prefix = " " * 4 * level
+    if not name:
+        name = link
+    dct = {"prefix":prefix, "link":link, "name":name}
+    if sys.stdout.isatty():
+        fmt = "{prefix}{name}"
+    else:
+        if link == name:
+            fmt = "{prefix}[[{name}]]"
+        else:
+            fmt = "{prefix}[[{link}|{name}]]"
+    print(fmt.format(**dct))
+
+def addNode(link, node, name, graph):
+    """ adds a node to the dictionary that tracks edges"""
+    if name:
+        node = name
+    if link in graph.keys():
+        graph[link].add(node)
+    else:
+        graph[link] = {node}
+
+def bfs(index):
+    """breath first search on vimwiki
+    keep seen and unseen to track which files to search
+    ugly loops instead of recursive setup """
+    seen = set()
+    unseen = [index]
+    graph = {}
+    filecount = 0
+    while 1:
+        try:
+            link = unseen.pop(0)
+        except IndexError:
+            print("Search complete...")
+            ic(filecount)
+            return graph
+        seen.add(link)
+        ret = probeAllLink(link + '.wiki')
+        for (node, name) in ret:
+            addNode(link, node, name, graph)
+            if not node in seen:
+                unseen.append(node)
+        filecount = filecount + 1
+        ic(filecount)
+        ic(seen)
+
+def transformGraph(graph):
+    for key, value in graph.items():
+        graph[key] = list(value)
+    return graph
+
+if __name__ == "__main__":
+    graph = bfs('index')
+    print("Transforming Graph...")
+    graph = transformGraph(graph)
+    print("Transform Complete...")
+    ic(graph)
+    print("Outputting file")
+    with open('network.json', 'w+') as out:
+        json.dump(graph, out)