main
Anish Lakhwara 2021-11-29 22:26:21 +10:00
commit 7a08a7fef2
9 changed files with 483 additions and 0 deletions

8
.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
output/*
graphs/*
network.json
__pycache__/*
.tags
.vimwiki_tags
.vim/*
censor.py

68
TagHelper.py Normal file
View File

@ -0,0 +1,68 @@
import ctags
from ctags import CTags, TagEntry
import sys
from icecream import ic
import os
import shutil
import fileinput
def loadTagFile(path='.vimwiki_tags'):
# Available file information keys:
# opened - was the tag file successfully opened?
# error_number - errno value when 'opened' is false
# format - format of tag file (1 = original, 2 = extended)
# sort - how is the tag file sorted?
# author - name of author of generating program (may be empy string)
# name - name of program (may be empy string)
# url - URL of distribution (may be empy string)
# version - program version (may be empty string)
try:
tagFile = CTags(path)
return tagFile
except:
print("No tagfile at {}".farmat(path))
sys.exit(1)
def makeSetFromTags(tagFile, key='file'):
# Available TagEntry keys:
# name - name of tag
# file - path of source file containing definition of tag
# pattern - pattern for locating source line (None if no pattern)
# lineNumber - line number in source file of tag definition (may be zero if not known)
# kind - kind of tag (none if not known)
# fileScope - is tag of file-limited scope?
# Note: other keys will be assumed as an extension key and will
# return None if no such key is found
entry = TagEntry()
tag = tagFile.next(entry)
AllFileList = set()
while tag:
AllFileList.add(entry[key].decode())
tag = tagFile.next(entry)
return AllFileList
def makeDictFromTags(tagFile, key='name', value='file'):
entry = TagEntry()
tag = tagFile.next(entry)
ret = {}
while tag:
# Not sure why they're being read as bytes now?
strKey = entry[key].decode()
strValue = entry[value].decode()
if strKey not in ret.keys():
ret[strKey] = set(strValue)
else:
ret[strKey].add(strValue)
tag = tagFile.next(entry)
return ret
def getAllFiles():
# Returns a set containing all the files we wish to make public
return makeSetFromTags(loadTagFile())
def getAllTags():
# Returns a dictionary with:
# - key as tag titles
# - values as files that have the key in their tags
return makeDictFromTags(loadTagFile())

21
flake.nix Normal file
View File

@ -0,0 +1,21 @@
{
description = "Sketches in Quil";
inputs = {
nixpkgs.url = "github:nixos/nixpkgs";
flake-utils.url = "github:numtide/flake-utils";
};
outputs = { self, nixpkgs, flake-utils }:
flake-utils.lib.eachDefaultSystem (system:
let pkgs = nixpkgs.legacyPackages.${system};
in {
devShell = pkgs.mkShell {
buildInputs = [
pkgs.leiningen
pkgs.jdk11
pkgs.clojure
];
};
});
}

100
makePublic.py Normal file
View File

@ -0,0 +1,100 @@
#! /usr/bin/env python3
# Various methods of grabbing files from Kitaab based on their ctags as generated by vimwiki
# censor - burn all mentions of this word wherever it appears in Kitaab
# filter - leave names in tact but do not link to relevant documents
# create - list in an index file for website
from os import listdir, mkdir
from os.path import isfile, join
import shutil
import random
import TagHelper
import censor as cen
from icecream import ic
## TODO: generalize based on headings(?) in index.wiki
censor = cen.listBad
filter = ["Dream", "Gratitude", "Review", "jrnl", "piracy", "work", "job", "debt", "people"]
#TODO create = ...
def loadNetwork(path='network.json'):
# Loads the network.json file
with open(path, 'r') as f:
everything = json.load(f)
return everything
def setNetwork(everything):
# Everything is loaded from the network.json file
keys = set(everything.network())
values = set(everything.values())
everything = network + values
return everything
def filterFiles(AllFileList):
toFilter = filter + censor
tagList = TagHelper.getAllTags()
doNotTakeFiles = set()
for mask in toFilter:
try:
doNotTakeFiles.update(tagList[mask])
except KeyError:
print("Key not found: {}".format(mask))
return AllFileList - doNotTakeFiles
def censorNetwork(network):
# Takes network as a set
for c in censor:
if c in network:
network.add(generateBlocks(len(c) + random.randint(0, 5)))
network.remove(c)
return network
def generateBlocks(size=1):
# Generates a length size unicode string that is formed from blocks:
charList = ['', '', '', '', '', '', '▖▗', '▘▙', '', '', '', '', '', '']
ret = ""
while size > 0:
size = size - 1
rand = random.randint(0,13)
ret = ret + charList[rand]
return ret
def createIndexNode(network):
pass
def copyFiles(fileList):
baseFilePath = '/home/anish/vimwiki/'
try:
mkdir('output')
except FileExistsError:
pass
for f in fileList:
try:
shutil.copy(baseFilePath + f, './output/')
except FileNotFoundError:
print("Could not copy file {}:".format(baseFilePath + f))
def censorFiles():
outputBasePath = './output/'
for fileName in listdir(outputBasePath):
fileName = outputBasePath + fileName
newFile = ""
print("Censoring: {}".format(fileName))
with open(fileName, 'r') as f:
for line in f.readlines():
for c in censor:
line = line.replace(c, generateBlocks(len(c)))
newFile = newFile + line
with open(fileName, 'w') as f:
for line in newFile:
f.write(line)
if __name__=='__main__':
#network = loadNetwork()
#network = censorNetwork(network)
AllFiles = TagHelper.getAllFiles()
FilteredFiles = filterFiles(AllFiles)
copyFiles(FilteredFiles)
print("Censoring files...")
censorFiles()

25
network-builder.py Normal file
View File

@ -0,0 +1,25 @@
import json
import networkx as nx
import pygraphviz
def loadJson(file):
with open(file, 'r') as f:
return json.load(f)
def makeNetwork(json):
G = nx.DiGraph()
for (key, value) in json.items():
G.add_nodes_from([key] + value)
G.add_edges_from([(key, v) for v in value])
return G
def makeAGraph(graph):
G = nx.nx_agraph.to_agraph(graph)
G.draw("nop.png", prog="nop")
if __name__ == "__main__":
OUTFILE = 'network.json'
json = loadJson('network.json')
graph = makeNetwork(json)
with open(OUTFILE, 'w') as f:
json.dump(f, makeAGraph(graph))

21
readme.md Normal file
View File

@ -0,0 +1,21 @@
# Kitaab helper
Helps make my Kitaab public by grabbing Kitaab, filtering & censoring it, and then passing it through to an SSG to make HTML out of a markdown page.
### Files
makePublic.py
- the whole filter pipeline
TagHelper.py
- Reads the .vimwiki_tags file that is generated by vimwiki
tree.py
- builds a tree out of all the links using dfs
network-builder.py
- uses the network from tree.py to visualize links
tag-fixer.py
- old script used to fix problem tags (1 character tags)
- (TODO) tags that are case sensitive

20
shell.nix Normal file
View File

@ -0,0 +1,20 @@
# shell.nix
{ pkgs ? import <nixpkgs> {} }:
let
my-python = pkgs.python3;
python-with-my-packages = my-python.withPackages (p: with p; [
icecream
# other python packages you want
]);
in
pkgs.mkShell {
buildInputs = [
python-with-my-packages
# other dependencies
];
shellHook = ''
PYTHONPATH=${python-with-my-packages}/${python-with-my-packages.sitePackages}
# maybe set more env-vars
'';
}

100
tag-fixer.py Normal file
View File

@ -0,0 +1,100 @@
#! /usr/bin/env python3
import ctags
from ctags import CTags, TagEntry
import sys
from icecream import ic
import os
import shutil
import fileinput
def loadTagFile(path='.vimwiki_tags'):
# Available file information keys:
# opened - was the tag file successfully opened?
# error_number - errno value when 'opened' is false
# format - format of tag file (1 = original, 2 = extended)
# sort - how is the tag file sorted?
# author - name of author of generating program (may be empy string)
# name - name of program (may be empy string)
# url - URL of distribution (may be empy string)
# version - program version (may be empty string)
try:
tagFile = CTags(path)
return tagFile
except:
print("No tagfile at {}".farmat(path))
sys.exit(1)
def loopThrough(tagFile):
# Available TagEntry keys:
# name - name of tag
# file - path of source file containing definition of tag
# pattern - pattern for locating source line (None if no pattern)
# lineNumber - line number in source file of tag definition (may be zero if not known)
# kind - kind of tag (none if not known)
# fileScope - is tag of file-limited scope?
# Note: other keys will be assumed as an extension key and will
# return None if no such key is found
''' Entries that are less than 1 character in length are errors
Returns a set containing of all the files with errors in their tags'''
entry = TagEntry()
tag = tagFile.next(entry)
tagErrors = set()
errorTagNames = set()
while tag:
if len(entry['name']) <= 1:
tagErrors.add(entry['file'])
errorTagNames.add(entry['name'])
else:
print("Good tag found: {}".format(entry['name']))
tag = tagFile.next(entry)
ic(errorTagNames)
return tagErrors
def fixErrors(errors):
''' Replaces :: with :, which seems to be the error for many of the tags '''
fileBase = '/home/anish/vimwiki/'
for v in errors:
print("Fixing {}...".format(v))
fileName = fileBase + getFileName(v)
newFile = ""
with open(fileName, 'r') as f:
for line in f.readlines():
line = line.replace("::", ":")
newFile = newFile + line
with open(fileName, 'w') as f:
for line in newFile:
f.write(line)
print("Written new file...")
print()
print(newFile)
def printErrors(errors):
fileBase = '/home/anish/vimwiki/'
for v in errors:
fileName = fileBase + getFileName(v)
print("Reading {}...".format(v))
with open(fileName, 'r') as f:
f.readline()
ic(f.readline())
def getFileName(f):
return str(f, encoding="utf-8")
def getBackupFiles(errors):
fileBase = '/home/anish/vimwiki/'
backupBase = '/home/anish/vimwiki/jrnl/'
for files in errors.values():
for f in files:
f = getFileName(f)
print("Moving {} from backup".format(f))
shutil.copy(backupBase + f, fileBase + f)
if __name__=='__main__':
tagFile = loadTagFile()
errors = loopThrough(tagFile)
printErrors(errors)
#ic(errors)
#getBackupFiles(errors)
#fixErrors(errors)

120
tree.py Normal file
View File

@ -0,0 +1,120 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from icecream import ic
import json
"""network-vimwiki.py
Create site map for vimwiki.
Must run script under vimwiki directory.
Example:
output network map to network.json
dfsvw.py
Python version: 3"""
import sys
import re
linkRE = re.compile(r'\[\[([^\\\]|#]+)(?:#[^\\\]|]+)?(?:\|([^\\\]]+))?\]\]', re.UNICODE)
def probeLink(s):
"""
>>> probeLink(' abc')
[]
>>> probeLink(' [[]]')
[]
>>> probeLink(' [[abc]] [[abc|123]] ')
[('abc', ''), ('abc', '123')]
>>> a, b = probeLink(' [[中文1|中文2]]')[0]; print a, b
中文1 中文2
>>> probeLink(' [[abc|abc 123]]')
[('abc', 'abc 123')]
"""
ret = linkRE.findall(s)
return [(a, b) for a, b in ret]
def probeAllLink(fn):
"""extract all link in a wiki file"""
try:
lst = []
print(fn)
fp = open(fn)
for line in fp.readlines():
l = probeLink(line)
lst.extend(l)
except IOError:
print("cannot open",sys.stderr)
except UnicodeDecodeError:
lst = []
fp = open(fn, encoding="utf-16BE")
for line in fp.readlines():
l = probeLink(line)
lst.extend(l)
return lst
def outputNode(link, name, level):
"""output one wiki node, level for vimwiki tree hierarchy"""
""" Unused """
prefix = " " * 4 * level
if not name:
name = link
dct = {"prefix":prefix, "link":link, "name":name}
if sys.stdout.isatty():
fmt = "{prefix}{name}"
else:
if link == name:
fmt = "{prefix}[[{name}]]"
else:
fmt = "{prefix}[[{link}|{name}]]"
print(fmt.format(**dct))
def addNode(link, node, name, graph):
""" adds a node to the dictionary that tracks edges"""
if name:
node = name
if link in graph.keys():
graph[link].add(node)
else:
graph[link] = {node}
def bfs(index):
"""breath first search on vimwiki
keep seen and unseen to track which files to search
ugly loops instead of recursive setup """
seen = set()
unseen = [index]
graph = {}
filecount = 0
while 1:
try:
link = unseen.pop(0)
except IndexError:
print("Search complete...")
ic(filecount)
return graph
seen.add(link)
ret = probeAllLink(link + '.wiki')
for (node, name) in ret:
addNode(link, node, name, graph)
if not node in seen:
unseen.append(node)
filecount = filecount + 1
ic(filecount)
ic(seen)
def transformGraph(graph):
for key, value in graph.items():
graph[key] = list(value)
return graph
if __name__ == "__main__":
graph = bfs('index')
print("Transforming Graph...")
graph = transformGraph(graph)
print("Transform Complete...")
ic(graph)
print("Outputting file")
with open('network.json', 'w+') as out:
json.dump(graph, out)