#! /usr/bin/env python3 # Various methods of grabbing files from Kitaab # based on their ctags as generated by vimwiki # censor - burn all mentions of this word wherever it appears in Kitaab # filter - leave names in tact but do not link to relevant documents # create - list in an index file for website from basant.shared import TagHelper, config, helpers from basant.shared import censor as c from os import listdir, mkdir, path, remove import shutil from . import convert import os # TODO: generalize based on headings(?) in index.wiki censor = c.privateList # TODO put this in config too... filter = ["dream", "gratitude", "review", "jrnl", "work", "job", "debt", "people", "priv", "threatdefence", "info"] # TODO create = ... outputBasePath = config.raw_folder def filterFiles(AllFileList): toFilter = filter + censor tagList = TagHelper.getAllTags() doNotTakeFiles = set() for mask in toFilter: try: doNotTakeFiles.update(tagList[mask]) except KeyError: print("Key not found: {}".format(mask)) return AllFileList - doNotTakeFiles def copyFiles(fileList): baseFilePath = config.files_folder outputFilePath = outputBasePath try: mkdir(outputFilePath) except FileExistsError: pass for f in fileList: try: shutil.copy(baseFilePath + f, outputFilePath + f) except FileNotFoundError: print("Could not copy file {} to {}".format(baseFilePath+f, outputFilePath+f)) def censorFiles(outputBasePath=outputBasePath): for fileName in listdir(outputBasePath): if not path.isfile(outputBasePath + fileName): continue fileName = outputBasePath + fileName newFile = "" with open(fileName, 'r', errors='ignore') as f: for line in f.readlines(): for c in censor: line = line.replace(c, helpers.generateBlocks(len(c))) newFile = newFile + line with open(fileName, 'w') as f: for line in newFile: f.write(line) def convertFiles(outputBasePath=outputBasePath): for filename in listdir(outputBasePath): if not path.isfile(outputBasePath + filename): continue if '.md' == filename[-3:]: print(f"{filename} file already converted, skipping") continue try: convert.convert_file(outputBasePath + filename) remove(outputBasePath + filename) except RuntimeError as e: print(f"Could not convert file {outputBasePath+filename}: {e}") def deleteOldRaw(): if os.path.exists(config.raw_folder): shutil.rmtree(config.raw_folder) def grabFiles(): deleteOldRaw() AllFiles = TagHelper.getAllFiles() FilteredFiles = filterFiles(AllFiles) copyFiles(FilteredFiles) print("Censoring files...") censorFiles() if __name__ == '__main__': # network = loadNetwork() # network = censorNetwork(network) # print("Converting files...") # convertFiles() grabFiles()