basant/basant/grab/kitaab.py

102 lines
3.1 KiB
Python

#! /usr/bin/env python3
# Various methods of grabbing files from Kitaab
# based on their ctags as generated by vimwiki
# censor - burn all mentions of this word wherever it appears in Kitaab
# filter - leave names in tact but do not link to relevant documents
# create - list in an index file for website
from basant.shared import TagHelper, config, helpers
from basant.shared import censor as c
from os import listdir, mkdir, path, remove
import shutil
from . import convert
import os
# TODO: generalize based on headings(?) in index.wiki
censor = c.privateList
# TODO put this in config too...
filter = ["dream", "gratitude", "review", "jrnl", "work", "job", "debt",
"people", "priv", "threatdefence", "info"]
# TODO create = ...
outputBasePath = config.raw_folder
def filterFiles(AllFileList):
toFilter = filter + censor
tagList = TagHelper.getAllTags()
doNotTakeFiles = set()
for mask in toFilter:
try:
doNotTakeFiles.update(tagList[mask])
except KeyError:
print("Key not found: {}".format(mask))
return AllFileList - doNotTakeFiles
def copyFiles(fileList):
baseFilePath = config.files_folder
outputFilePath = outputBasePath
try:
mkdir(outputFilePath)
except FileExistsError:
pass
for f in fileList:
try:
shutil.copy(baseFilePath + f, outputFilePath + f)
except FileNotFoundError:
print("Could not copy file {} to {}".format(baseFilePath+f,
outputFilePath+f))
def censorFiles(outputBasePath=outputBasePath):
for fileName in listdir(outputBasePath):
if not path.isfile(outputBasePath + fileName):
continue
fileName = outputBasePath + fileName
newFile = ""
with open(fileName, 'r', errors='ignore') as f:
for line in f.readlines():
for c in censor:
line = line.replace(c, helpers.generateBlocks(len(c)))
newFile = newFile + line
with open(fileName, 'w') as f:
for line in newFile:
f.write(line)
def convertFiles(outputBasePath=outputBasePath):
for filename in listdir(outputBasePath):
if not path.isfile(outputBasePath + filename):
continue
if '.md' == filename[-3:]:
print(f"{filename} file already converted, skipping")
continue
try:
convert.convert_file(outputBasePath + filename)
remove(outputBasePath + filename)
except RuntimeError as e:
print(f"Could not convert file {outputBasePath+filename}: {e}")
def deleteOldRaw():
if os.path.exists(config.raw_folder):
shutil.rmtree(config.raw_folder)
def grabFiles():
deleteOldRaw()
AllFiles = TagHelper.getAllFiles()
FilteredFiles = filterFiles(AllFiles)
copyFiles(FilteredFiles)
print("Censoring files...")
censorFiles()
if __name__ == '__main__':
# network = loadNetwork()
# network = censorNetwork(network)
# print("Converting files...")
# convertFiles()
grabFiles()