diff --git a/.envrc b/.envrc
new file mode 100644
index 0000000..e69de29
diff --git a/shell.nix b/shell.nix
index 0183ac9..10e14a4 100644
--- a/shell.nix
+++ b/shell.nix
@@ -6,12 +6,15 @@ let
icecream
python-ctags3
jinja2
+ click
+ pypandoc
# other python packages you want
]);
in
pkgs.mkShell {
buildInputs = [
python-with-my-packages
+ pkgs.pandoc
# other dependencies
];
shellHook = ''
diff --git a/src/__main__.py b/src/__main__.py
new file mode 100644
index 0000000..d81fd63
--- /dev/null
+++ b/src/__main__.py
@@ -0,0 +1,15 @@
+import click
+import kitaab_public
+
+@click.command()
+def grab():
+ pass
+
+@click.command()
+def build():
+ pass
+
+@click.command()
+def main():
+ grab()
+ build()
diff --git a/src/kitaab-public/build/build.py b/src/kitaab-public/build/build.py
index dfdc1ff..e4d0915 100644
--- a/src/kitaab-public/build/build.py
+++ b/src/kitaab-public/build/build.py
@@ -32,7 +32,10 @@ build_url = config.relative_build_url
if args.prod:
build_url = config.absolute_build_url
-def generate_page():
+def generate_config():
+ pass
+
+def generate_page(template_file=config.template_file, options=generate_config()):
pass
def generate_navigation(template_nav=config.template_nav):
@@ -76,25 +79,13 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template
page_template = template.replace('page_title', entry['title'])
page_template = template.replace('page_tags', str(entry['tags']))
- # Checking if the page is root of a folder
- if entry["file"] == "index":
-
- # If root page
- # Concatenate the content of the index page with the listing of sub-pages
- # Remove "page_date" from template
- # Replaces "page_body" with the page content in the template
- entry["pageContent"] += sub_pages_list
- page_template = page_template.replace('page_date', "")
- page_template = page_template.replace(
- 'page_body', entry['pageContent'])
- else:
- # If content page
- # Replaces "page_body" with the page content in the template
- # Replaces "page_date" with the page date in the template
- page_template = page_template.replace(
- 'page_body', entry['pageContent'])
- page_template = page_template.replace(
- 'page_date', "%s" % (entry['date']))
+ # If content page
+ # Replaces "page_body" with the page content in the template
+ # Replaces "page_date" with the page date in the template
+ page_template = page_template.replace(
+ 'page_body', entry['pageContent'])
+ page_template = page_template.replace(
+ 'page_date', "%s" % (entry['date']))
# Creating navigation
url_link = build_url
@@ -133,6 +124,14 @@ def generate_html_pages(site_folder, entries, template, sub_pages_list, template
# Write the HTML file
slug_file = site_folder + entry['slug']
+ if entry["file"] == 'index':
+ print("")
+ print("")
+ print("FOUND INDEX")
+ print(slug_file)
+ print("")
+ print("")
+
with open(slug_file, 'w', encoding=encoding) as fobj:
fobj.write(page_template)
@@ -155,6 +154,12 @@ def get_entry_title(page):
except FileNotFoundError:
print(f"Page not found, but present in tags: {page}")
return None
+ except UnicodeDecodeError:
+ print(f"Page could not be read: {page}")
+ return None
+ except IndexError as e:
+ print(f"{page} could not be parsed: {e}")
+
return textContent
@@ -210,8 +215,12 @@ def fix_images_urls(page):
def fix_amp(page):
- page = page.replace(" & ", " & ")
- return page
+ try:
+ page = page.replace(" & ", " & ")
+ return page
+ except AttributeError as e:
+ print(f"{page} is not a string")
+ return ""
def fix_wiki_links(page, path):
regex = r"\[\[([\s\S]+?\|?[\s\S]+?)\]\]"
@@ -330,8 +339,9 @@ def clean_path(path):
has_date = False
if match:
- print(f"Match found for {path_items['file']}")
has_date = True
+ else:
+ print(f"{path} does not have a date")
if has_date:
if match[0] != path_items["file"]:
@@ -368,7 +378,7 @@ def clean_path(path):
if config.flat_build == False:
path_items["slug"] = path_items["folder"] + "/" + path_items["slug"]
- if path_items["file"] == "index":
+ if "index" in path_items["file"]:
path_items["parent_url"] = ""
path_items["parent_text"] = config.home_name
if config.flat_build:
diff --git a/src/kitaab-public/grab/TagHelper.py b/src/kitaab-public/grab/TagHelper.py
index 11b5519..f393a9c 100644
--- a/src/kitaab-public/grab/TagHelper.py
+++ b/src/kitaab-public/grab/TagHelper.py
@@ -1,10 +1,7 @@
-import ctags
from ctags import CTags, TagEntry
import sys
from icecream import ic
-import os
-import shutil
-import fileinput
+from os import listdir, path
# TODO don't hardcode the path
def loadTagFile(path='/home/anish/vimwiki/.vimwiki_tags'):
@@ -59,10 +56,15 @@ def makeDictFromTags(tagFile, key='name', value='file'):
tag = tagFile.next(entry)
return ret
-def getAllFiles():
+def getAllFilesWithTgas():
# Returns a set containing all the files we wish to make public
return makeSetFromTags(loadTagFile())
+def getAllFiles():
+ files = [f for f in listdir('/home/anish/vimwiki') if '.wiki' in f and path.isfile('/home/anish/vimwiki/' + f)]
+ files = set(files)
+ return files
+
def getAllTags():
# Returns a dictionary with:
# - key as tag titles
diff --git a/src/kitaab-public/grab/convert.py b/src/kitaab-public/grab/convert.py
new file mode 100644
index 0000000..992e9df
--- /dev/null
+++ b/src/kitaab-public/grab/convert.py
@@ -0,0 +1,10 @@
+import pypandoc
+
+def convert_file(file):
+ filename = file.split('.')[0]
+ pypandoc.convert_file(filename + '.wiki', '.md', outputfile=filename + '.md')
+
+def convert_files(files):
+ for file in files:
+ convert_file(file)
+
diff --git a/src/kitaab-public/grab/makePublic.py b/src/kitaab-public/grab/makePublic.py
index 6a803a8..30653e3 100644
--- a/src/kitaab-public/grab/makePublic.py
+++ b/src/kitaab-public/grab/makePublic.py
@@ -3,19 +3,20 @@
# censor - burn all mentions of this word wherever it appears in Kitaab
# filter - leave names in tact but do not link to relevant documents
# create - list in an index file for website
-from os import listdir, mkdir
-from os.path import isfile, join
+from os import listdir, mkdir, path
import shutil
import random
import json
import TagHelper
+import convert
from icecream import ic
from censor import privateList
## TODO: generalize based on headings(?) in index.wiki
censor = privateList
-filter = ["Dream", "Gratitude", "Review", "jrnl", "piracy", "work", "job", "debt", "people"]
+# TODO put this in config too...
+filter = ["dream", "gratitude", "review", "jrnl", "piracy", "work", "job", "debt", "people", "priv"]
#TODO create = ...
def loadNetwork(path='network.json'):
@@ -40,7 +41,6 @@ def filterFiles(AllFileList):
doNotTakeFiles.update(tagList[mask])
except KeyError:
print("Key not found: {}".format(mask))
- print(AllFileList)
return AllFileList - doNotTakeFiles
def censorNetwork(network):
@@ -66,19 +66,22 @@ def createIndexNode(network):
def copyFiles(fileList):
baseFilePath = '/home/anish/vimwiki/'
+ outputFilePath = '../raw/' # TODO relative path....
try:
- mkdir('output')
+ mkdir(outputFilePath)
except FileExistsError:
pass
for f in fileList:
try:
- shutil.copy(baseFilePath + f, './raw/')
+ shutil.copy(baseFilePath + f, outputFilePath + f)
except FileNotFoundError:
- print("Could not copy file {}:".format(baseFilePath + f))
+ print("Could not copy file {} to {}".format(baseFilePath + f, outputFilePath + f))
def censorFiles():
- outputBasePath = './output/'
+ outputBasePath = '../raw/' # TODO relative file path
for fileName in listdir(outputBasePath):
+ if not path.isfile(outputBasePath + fileName):
+ continue
fileName = outputBasePath + fileName
newFile = ""
print("Censoring: {}".format(fileName))
@@ -91,6 +94,17 @@ def censorFiles():
for line in newFile:
f.write(line)
+def convertFiles():
+ outputBasePath = '../raw/'
+ for filename in listdir(outputBasePath):
+ if not path.isfile(outputBasePath + filename):
+ continue
+ filename = outputBasePath + filename
+ convert.convert_file(filename)
+ print(filename)
+ shutil.rmtree(filename)
+
+
if __name__=='__main__':
#network = loadNetwork()
@@ -100,3 +114,5 @@ if __name__=='__main__':
copyFiles(FilteredFiles)
print("Censoring files...")
censorFiles()
+ print("Converting files...")
+ convertFiles()
diff --git a/src/kitaab-public/partial/nav.html b/src/kitaab-public/partial/nav.html
index fb16d98..29367ab 100644
--- a/src/kitaab-public/partial/nav.html
+++ b/src/kitaab-public/partial/nav.html
@@ -1,6 +1,6 @@