154 lines
4.4 KiB
Python
154 lines
4.4 KiB
Python
#! /usr/bin/env python3
|
|
|
|
# Various methods of grabbing files from Kitaab
|
|
# based on their ctags as generated by vimwiki
|
|
# censor - burn all mentions of this word wherever it appears in Kitaab
|
|
# filter - leave names in tact but do not link to relevant documents
|
|
# create - list in an index file for website
|
|
from shared.censor import privateList
|
|
from shared import TagHelper, config
|
|
from os import listdir, mkdir, path, remove
|
|
import shutil
|
|
import random
|
|
import json
|
|
from . import convert
|
|
import os
|
|
import sys
|
|
sys.path.append("../")
|
|
|
|
# TODO: generalize based on headings(?) in index.wiki
|
|
|
|
censor = privateList
|
|
# TODO put this in config too...
|
|
filter = ["dream", "gratitude", "review", "jrnl", "work", "job", "debt",
|
|
"people", "priv", "threatdefence", "info"]
|
|
# TODO create = ...
|
|
|
|
|
|
def loadNetwork(path='network.json'):
|
|
# Loads the network.json file
|
|
with open(path, 'r') as f:
|
|
everything = json.load(f)
|
|
return everything
|
|
|
|
|
|
def setNetwork(everything):
|
|
# Everything is loaded from the network.json file
|
|
# keys = set(everything.network())
|
|
network = ""
|
|
values = set(everything.values())
|
|
everything = network + values
|
|
return everything
|
|
|
|
|
|
def filterFiles(AllFileList):
|
|
toFilter = filter + censor
|
|
tagList = TagHelper.getAllTags()
|
|
doNotTakeFiles = set()
|
|
for mask in toFilter:
|
|
try:
|
|
doNotTakeFiles.update(tagList[mask])
|
|
except KeyError:
|
|
print("Key not found: {}".format(mask))
|
|
return AllFileList - doNotTakeFiles
|
|
|
|
|
|
def untaggedFiles(AllFileList=TagHelper.getAllFiles()):
|
|
toFilter = filter + censor
|
|
tagList = TagHelper.getAllTags()
|
|
doNotTakeFiles = set()
|
|
for mask in toFilter:
|
|
try:
|
|
doNotTakeFiles.update(tagList[mask])
|
|
except KeyError:
|
|
print("Key not found: {}".format(mask))
|
|
return set.difference(AllFileList, doNotTakeFiles)
|
|
|
|
|
|
def censorNetwork(network):
|
|
# Takes network as a set
|
|
for c in censor:
|
|
if c in network:
|
|
network.add(generateBlocks(len(c) + random.randint(0, 5)))
|
|
network.remove(c)
|
|
return network
|
|
|
|
|
|
def generateBlocks(size=1):
|
|
# Generates a length size unicode string that is formed from blocks
|
|
charList = ['▐', '░', '▒', '▓', '▔', '▕', '▖▗',
|
|
'▘▙', '▚', '▛', '▜', '▝', '▞', '▟']
|
|
ret = ""
|
|
while size > 0:
|
|
size = size - 1
|
|
rand = random.randint(0, 13)
|
|
ret = ret + charList[rand]
|
|
return ret
|
|
|
|
|
|
def createIndexNode(network):
|
|
pass
|
|
|
|
|
|
def copyFiles(fileList):
|
|
baseFilePath = '/home/anish/vimwiki/'
|
|
outputFilePath = '../raw/' # TODO relative path....
|
|
try:
|
|
mkdir(outputFilePath)
|
|
except FileExistsError:
|
|
pass
|
|
for f in fileList:
|
|
try:
|
|
shutil.copy(baseFilePath + f, outputFilePath + f)
|
|
except FileNotFoundError:
|
|
print("Could not copy file {} to {}".format(baseFilePath+f,
|
|
outputFilePath+f))
|
|
|
|
|
|
def censorFiles(outputBasePath='../raw/'):
|
|
for fileName in listdir(outputBasePath):
|
|
if not path.isfile(outputBasePath + fileName):
|
|
continue
|
|
fileName = outputBasePath + fileName
|
|
newFile = ""
|
|
with open(fileName, 'r', errors='ignore') as f:
|
|
for line in f.readlines():
|
|
for c in censor:
|
|
line = line.replace(c, generateBlocks(len(c)))
|
|
newFile = newFile + line
|
|
with open(fileName, 'w') as f:
|
|
for line in newFile:
|
|
f.write(line)
|
|
|
|
|
|
def convertFiles(outputBasePath='../raw/'):
|
|
for filename in listdir(outputBasePath):
|
|
if not path.isfile(outputBasePath + filename):
|
|
continue
|
|
if '.md' == filename[-3:]:
|
|
print(f"{filename} file already converted, skipping")
|
|
continue
|
|
try:
|
|
convert.convert_file(outputBasePath + filename)
|
|
remove(outputBasePath + filename)
|
|
except RuntimeError as e:
|
|
print(f"Could not convert file {outputBasePath+filename}: {e}")
|
|
|
|
|
|
def deleteOldRaw():
|
|
if os.path.exists(config.raw_folder):
|
|
shutil.rmtree(config.raw_folder)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# network = loadNetwork()
|
|
# network = censorNetwork(network)
|
|
deleteOldRaw()
|
|
AllFiles = TagHelper.getAllFiles()
|
|
FilteredFiles = filterFiles(AllFiles)
|
|
copyFiles(FilteredFiles)
|
|
print("Censoring files...")
|
|
censorFiles()
|
|
# print("Converting files...")
|
|
# convertFiles()
|