basant/basant/grab/kitaab.py

102 lines
3.1 KiB
Python
Raw Normal View History

2022-01-17 09:14:17 -08:00
#! /usr/bin/env python3
2022-02-26 03:57:34 -08:00
# Various methods of grabbing files from Kitaab
# based on their ctags as generated by vimwiki
2022-01-17 09:14:17 -08:00
# censor - burn all mentions of this word wherever it appears in Kitaab
# filter - leave names in tact but do not link to relevant documents
# create - list in an index file for website
2022-04-03 05:57:45 -07:00
from basant.shared import TagHelper, config, helpers
2022-04-03 06:22:35 -07:00
from basant.shared import censor as c
from os import listdir, mkdir, path, remove
2022-01-17 09:14:17 -08:00
import shutil
2022-02-14 04:09:35 -08:00
from . import convert
import os
2022-01-17 09:14:17 -08:00
2022-02-26 03:57:34 -08:00
# TODO: generalize based on headings(?) in index.wiki
2022-01-17 09:14:17 -08:00
2022-04-03 06:22:35 -07:00
censor = c.privateList
# TODO put this in config too...
2022-02-26 03:57:34 -08:00
filter = ["dream", "gratitude", "review", "jrnl", "work", "job", "debt",
"people", "priv", "threatdefence", "info"]
# TODO create = ...
2022-03-24 08:48:14 -07:00
outputBasePath = config.raw_folder
2022-01-17 09:14:17 -08:00
2022-02-26 03:57:34 -08:00
2022-01-17 09:14:17 -08:00
def filterFiles(AllFileList):
toFilter = filter + censor
tagList = TagHelper.getAllTags()
doNotTakeFiles = set()
for mask in toFilter:
try:
doNotTakeFiles.update(tagList[mask])
except KeyError:
print("Key not found: {}".format(mask))
return AllFileList - doNotTakeFiles
2022-02-26 03:57:34 -08:00
2022-01-17 09:14:17 -08:00
def copyFiles(fileList):
2022-04-03 05:57:45 -07:00
baseFilePath = config.files_folder
outputFilePath = outputBasePath
2022-01-17 09:14:17 -08:00
try:
2022-02-26 03:57:34 -08:00
mkdir(outputFilePath)
2022-01-17 09:14:17 -08:00
except FileExistsError:
pass
for f in fileList:
try:
shutil.copy(baseFilePath + f, outputFilePath + f)
2022-01-17 09:14:17 -08:00
except FileNotFoundError:
2022-02-26 03:57:34 -08:00
print("Could not copy file {} to {}".format(baseFilePath+f,
outputFilePath+f))
2022-01-17 09:14:17 -08:00
2022-03-24 08:48:14 -07:00
def censorFiles(outputBasePath=outputBasePath):
2022-01-17 09:14:17 -08:00
for fileName in listdir(outputBasePath):
if not path.isfile(outputBasePath + fileName):
continue
2022-01-17 09:14:17 -08:00
fileName = outputBasePath + fileName
newFile = ""
with open(fileName, 'r', errors='ignore') as f:
2022-01-17 09:14:17 -08:00
for line in f.readlines():
for c in censor:
2022-03-24 08:48:14 -07:00
line = line.replace(c, helpers.generateBlocks(len(c)))
2022-01-17 09:14:17 -08:00
newFile = newFile + line
with open(fileName, 'w') as f:
for line in newFile:
f.write(line)
2022-02-26 03:57:34 -08:00
2022-03-24 08:48:14 -07:00
def convertFiles(outputBasePath=outputBasePath):
for filename in listdir(outputBasePath):
if not path.isfile(outputBasePath + filename):
continue
if '.md' == filename[-3:]:
print(f"{filename} file already converted, skipping")
continue
try:
convert.convert_file(outputBasePath + filename)
remove(outputBasePath + filename)
except RuntimeError as e:
2022-02-26 03:57:34 -08:00
print(f"Could not convert file {outputBasePath+filename}: {e}")
2022-02-14 04:09:35 -08:00
def deleteOldRaw():
if os.path.exists(config.raw_folder):
shutil.rmtree(config.raw_folder)
2022-04-11 02:57:04 -07:00
def grabFiles():
2022-02-14 04:09:35 -08:00
deleteOldRaw()
2022-01-17 09:14:17 -08:00
AllFiles = TagHelper.getAllFiles()
FilteredFiles = filterFiles(AllFiles)
copyFiles(FilteredFiles)
print("Censoring files...")
censorFiles()
2022-04-11 02:57:04 -07:00
if __name__ == '__main__':
# network = loadNetwork()
# network = censorNetwork(network)
2022-02-26 03:57:34 -08:00
# print("Converting files...")
# convertFiles()
2022-04-11 02:57:04 -07:00
grabFiles()