fix linting issues
This commit is contained in:
parent
def2c82a56
commit
55094a3367
|
@ -1,8 +1,12 @@
|
||||||
#! /usr/bin/env python3
|
#! /usr/bin/env python3
|
||||||
# Various methods of grabbing files from Kitaab based on their ctags as generated by vimwiki
|
|
||||||
|
# Various methods of grabbing files from Kitaab
|
||||||
|
# based on their ctags as generated by vimwiki
|
||||||
# censor - burn all mentions of this word wherever it appears in Kitaab
|
# censor - burn all mentions of this word wherever it appears in Kitaab
|
||||||
# filter - leave names in tact but do not link to relevant documents
|
# filter - leave names in tact but do not link to relevant documents
|
||||||
# create - list in an index file for website
|
# create - list in an index file for website
|
||||||
|
from shared.censor import privateList
|
||||||
|
from shared import TagHelper, config
|
||||||
from os import listdir, mkdir, path, remove
|
from os import listdir, mkdir, path, remove
|
||||||
import shutil
|
import shutil
|
||||||
import random
|
import random
|
||||||
|
@ -11,15 +15,15 @@ from . import convert
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
sys.path.append("../")
|
sys.path.append("../")
|
||||||
from shared.censor import privateList
|
|
||||||
from shared import TagHelper, config
|
|
||||||
|
|
||||||
## TODO: generalize based on headings(?) in index.wiki
|
# TODO: generalize based on headings(?) in index.wiki
|
||||||
|
|
||||||
censor = privateList
|
censor = privateList
|
||||||
# TODO put this in config too...
|
# TODO put this in config too...
|
||||||
filter = ["dream", "gratitude", "review", "jrnl", "work", "job", "debt", "people", "priv", "threatdefence", "info"]
|
filter = ["dream", "gratitude", "review", "jrnl", "work", "job", "debt",
|
||||||
#TODO create = ...
|
"people", "priv", "threatdefence", "info"]
|
||||||
|
# TODO create = ...
|
||||||
|
|
||||||
|
|
||||||
def loadNetwork(path='network.json'):
|
def loadNetwork(path='network.json'):
|
||||||
# Loads the network.json file
|
# Loads the network.json file
|
||||||
|
@ -27,13 +31,16 @@ def loadNetwork(path='network.json'):
|
||||||
everything = json.load(f)
|
everything = json.load(f)
|
||||||
return everything
|
return everything
|
||||||
|
|
||||||
|
|
||||||
def setNetwork(everything):
|
def setNetwork(everything):
|
||||||
# Everything is loaded from the network.json file
|
# Everything is loaded from the network.json file
|
||||||
keys = set(everything.network())
|
# keys = set(everything.network())
|
||||||
|
network = ""
|
||||||
values = set(everything.values())
|
values = set(everything.values())
|
||||||
everything = network + values
|
everything = network + values
|
||||||
return everything
|
return everything
|
||||||
|
|
||||||
|
|
||||||
def filterFiles(AllFileList):
|
def filterFiles(AllFileList):
|
||||||
toFilter = filter + censor
|
toFilter = filter + censor
|
||||||
tagList = TagHelper.getAllTags()
|
tagList = TagHelper.getAllTags()
|
||||||
|
@ -45,6 +52,7 @@ def filterFiles(AllFileList):
|
||||||
print("Key not found: {}".format(mask))
|
print("Key not found: {}".format(mask))
|
||||||
return AllFileList - doNotTakeFiles
|
return AllFileList - doNotTakeFiles
|
||||||
|
|
||||||
|
|
||||||
def untaggedFiles(AllFileList=TagHelper.getAllFiles()):
|
def untaggedFiles(AllFileList=TagHelper.getAllFiles()):
|
||||||
toFilter = filter + censor
|
toFilter = filter + censor
|
||||||
tagList = TagHelper.getAllTags()
|
tagList = TagHelper.getAllTags()
|
||||||
|
@ -54,7 +62,8 @@ def untaggedFiles(AllFileList=TagHelper.getAllFiles()):
|
||||||
doNotTakeFiles.update(tagList[mask])
|
doNotTakeFiles.update(tagList[mask])
|
||||||
except KeyError:
|
except KeyError:
|
||||||
print("Key not found: {}".format(mask))
|
print("Key not found: {}".format(mask))
|
||||||
return set.difference(AllFileList, doNotTakeFiles)
|
return set.difference(AllFileList, doNotTakeFiles)
|
||||||
|
|
||||||
|
|
||||||
def censorNetwork(network):
|
def censorNetwork(network):
|
||||||
# Takes network as a set
|
# Takes network as a set
|
||||||
|
@ -64,31 +73,37 @@ def censorNetwork(network):
|
||||||
network.remove(c)
|
network.remove(c)
|
||||||
return network
|
return network
|
||||||
|
|
||||||
|
|
||||||
def generateBlocks(size=1):
|
def generateBlocks(size=1):
|
||||||
# Generates a length size unicode string that is formed from blocks:
|
# Generates a length size unicode string that is formed from blocks
|
||||||
charList = ['▐', '░', '▒', '▓', '▔', '▕', '▖▗', '▘▙', '▚', '▛', '▜', '▝', '▞', '▟']
|
charList = ['▐', '░', '▒', '▓', '▔', '▕', '▖▗',
|
||||||
|
'▘▙', '▚', '▛', '▜', '▝', '▞', '▟']
|
||||||
ret = ""
|
ret = ""
|
||||||
while size > 0:
|
while size > 0:
|
||||||
size = size - 1
|
size = size - 1
|
||||||
rand = random.randint(0,13)
|
rand = random.randint(0, 13)
|
||||||
ret = ret + charList[rand]
|
ret = ret + charList[rand]
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
|
|
||||||
def createIndexNode(network):
|
def createIndexNode(network):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def copyFiles(fileList):
|
def copyFiles(fileList):
|
||||||
baseFilePath = '/home/anish/vimwiki/'
|
baseFilePath = '/home/anish/vimwiki/'
|
||||||
outputFilePath = '../raw/' # TODO relative path....
|
outputFilePath = '../raw/' # TODO relative path....
|
||||||
try:
|
try:
|
||||||
mkdir(outputFilePath)
|
mkdir(outputFilePath)
|
||||||
except FileExistsError:
|
except FileExistsError:
|
||||||
pass
|
pass
|
||||||
for f in fileList:
|
for f in fileList:
|
||||||
try:
|
try:
|
||||||
shutil.copy(baseFilePath + f, outputFilePath + f)
|
shutil.copy(baseFilePath + f, outputFilePath + f)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
print("Could not copy file {} to {}".format(baseFilePath + f, outputFilePath + f))
|
print("Could not copy file {} to {}".format(baseFilePath+f,
|
||||||
|
outputFilePath+f))
|
||||||
|
|
||||||
|
|
||||||
def censorFiles(outputBasePath='../raw/'):
|
def censorFiles(outputBasePath='../raw/'):
|
||||||
for fileName in listdir(outputBasePath):
|
for fileName in listdir(outputBasePath):
|
||||||
|
@ -105,6 +120,7 @@ def censorFiles(outputBasePath='../raw/'):
|
||||||
for line in newFile:
|
for line in newFile:
|
||||||
f.write(line)
|
f.write(line)
|
||||||
|
|
||||||
|
|
||||||
def convertFiles(outputBasePath='../raw/'):
|
def convertFiles(outputBasePath='../raw/'):
|
||||||
for filename in listdir(outputBasePath):
|
for filename in listdir(outputBasePath):
|
||||||
if not path.isfile(outputBasePath + filename):
|
if not path.isfile(outputBasePath + filename):
|
||||||
|
@ -116,22 +132,22 @@ def convertFiles(outputBasePath='../raw/'):
|
||||||
convert.convert_file(outputBasePath + filename)
|
convert.convert_file(outputBasePath + filename)
|
||||||
remove(outputBasePath + filename)
|
remove(outputBasePath + filename)
|
||||||
except RuntimeError as e:
|
except RuntimeError as e:
|
||||||
print(f"Could not convert file {outputBasePath + filename} because {e}")
|
print(f"Could not convert file {outputBasePath+filename}: {e}")
|
||||||
|
|
||||||
|
|
||||||
def deleteOldRaw():
|
def deleteOldRaw():
|
||||||
if os.path.exists(config.raw_folder):
|
if os.path.exists(config.raw_folder):
|
||||||
shutil.rmtree(config.raw_folder)
|
shutil.rmtree(config.raw_folder)
|
||||||
|
|
||||||
|
|
||||||
if __name__=='__main__':
|
if __name__ == '__main__':
|
||||||
|
# network = loadNetwork()
|
||||||
#network = loadNetwork()
|
# network = censorNetwork(network)
|
||||||
#network = censorNetwork(network)
|
|
||||||
deleteOldRaw()
|
deleteOldRaw()
|
||||||
AllFiles = TagHelper.getAllFiles()
|
AllFiles = TagHelper.getAllFiles()
|
||||||
FilteredFiles = filterFiles(AllFiles)
|
FilteredFiles = filterFiles(AllFiles)
|
||||||
copyFiles(FilteredFiles)
|
copyFiles(FilteredFiles)
|
||||||
print("Censoring files...")
|
print("Censoring files...")
|
||||||
censorFiles()
|
censorFiles()
|
||||||
#print("Converting files...")
|
# print("Converting files...")
|
||||||
#convertFiles()
|
# convertFiles()
|
||||||
|
|
Loading…
Reference in a new issue