102 lines
3.1 KiB
Python
102 lines
3.1 KiB
Python
#! /usr/bin/env python3
|
|
|
|
# Various methods of grabbing files from Kitaab
|
|
# based on their ctags as generated by vimwiki
|
|
# censor - burn all mentions of this word wherever it appears in Kitaab
|
|
# filter - leave names in tact but do not link to relevant documents
|
|
# create - list in an index file for website
|
|
from basant.shared import TagHelper, config, helpers
|
|
from basant.shared import censor as c
|
|
from os import listdir, mkdir, path, remove
|
|
import shutil
|
|
from . import convert
|
|
import os
|
|
|
|
# TODO: generalize based on headings(?) in index.wiki
|
|
|
|
censor = c.privateList
|
|
# TODO put this in config too...
|
|
filter = ["dream", "gratitude", "review", "jrnl", "work", "job", "debt",
|
|
"people", "priv", "threatdefence", "info"]
|
|
# TODO create = ...
|
|
outputBasePath = config.raw_folder
|
|
|
|
|
|
def filterFiles(AllFileList):
|
|
toFilter = filter + censor
|
|
tagList = TagHelper.getAllTags()
|
|
doNotTakeFiles = set()
|
|
for mask in toFilter:
|
|
try:
|
|
doNotTakeFiles.update(tagList[mask])
|
|
except KeyError:
|
|
print("Key not found: {}".format(mask))
|
|
return AllFileList - doNotTakeFiles
|
|
|
|
|
|
def copyFiles(fileList):
|
|
baseFilePath = config.files_folder
|
|
outputFilePath = outputBasePath
|
|
try:
|
|
mkdir(outputFilePath)
|
|
except FileExistsError:
|
|
pass
|
|
for f in fileList:
|
|
try:
|
|
shutil.copy(baseFilePath + f, outputFilePath + f)
|
|
except FileNotFoundError:
|
|
print("Could not copy file {} to {}".format(baseFilePath+f,
|
|
outputFilePath+f))
|
|
|
|
|
|
def censorFiles(outputBasePath=outputBasePath):
|
|
for fileName in listdir(outputBasePath):
|
|
if not path.isfile(outputBasePath + fileName):
|
|
continue
|
|
fileName = outputBasePath + fileName
|
|
newFile = ""
|
|
with open(fileName, 'r', errors='ignore') as f:
|
|
for line in f.readlines():
|
|
for c in censor:
|
|
line = line.replace(c, helpers.generateBlocks(len(c)))
|
|
newFile = newFile + line
|
|
with open(fileName, 'w') as f:
|
|
for line in newFile:
|
|
f.write(line)
|
|
|
|
|
|
def convertFiles(outputBasePath=outputBasePath):
|
|
for filename in listdir(outputBasePath):
|
|
if not path.isfile(outputBasePath + filename):
|
|
continue
|
|
if '.md' == filename[-3:]:
|
|
print(f"{filename} file already converted, skipping")
|
|
continue
|
|
try:
|
|
convert.convert_file(outputBasePath + filename)
|
|
remove(outputBasePath + filename)
|
|
except RuntimeError as e:
|
|
print(f"Could not convert file {outputBasePath+filename}: {e}")
|
|
|
|
|
|
def deleteOldRaw():
|
|
if os.path.exists(config.raw_folder):
|
|
shutil.rmtree(config.raw_folder)
|
|
|
|
|
|
def grabFiles():
|
|
deleteOldRaw()
|
|
AllFiles = TagHelper.getAllFiles()
|
|
FilteredFiles = filterFiles(AllFiles)
|
|
copyFiles(FilteredFiles)
|
|
print("Censoring files...")
|
|
censorFiles()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# network = loadNetwork()
|
|
# network = censorNetwork(network)
|
|
# print("Converting files...")
|
|
# convertFiles()
|
|
grabFiles()
|