from . import mistune from poonam import TagHelper from basant.shared import config from jinja2 import Environment, FileSystemLoader import argparse import glob import os import random import platform import re import shutil from datetime import datetime, timezone, timedelta, tzinfo # Default encoding encoding = 'utf-8' # Activating the renderer renderer = mistune.Renderer() markdown = mistune.Markdown(renderer=renderer) # Checking for dev command parser = argparse.ArgumentParser() parser.add_argument( "-p", "--prod", help="Prints the supplied argument.", action="store_true") args = parser.parse_args() # Declaring build url that will be used in several parts of the app build_url = config.relative_build_url if args.prod: build_url = config.absolute_build_url def generate_navigation(template_nav=config.template_nav): file_loader = FileSystemLoader('') env = Environment(loader=file_loader) template = env.get_template(template_nav) index_tags = config.nav_tags output = template.render(index_urls=[f.split(".")[0] for f in index_tags]) return output def generate_head(template_head=config.template_head, page_title=config.home_name): file_loader = FileSystemLoader('') env = Environment(loader=file_loader) template = env.get_template(template_head) output = template.render(build_url=build_url, page_title=page_title, name_of_site=config.name_of_site) return output def add_file_path(pages, root_dir): ret = set() for page in pages: ret.add(root_dir + '/' + page) return ret def generate_index_pages(template=config.template_main, template_nav=config.template_nav, template_head=config.template_head): template = template all_tags = TagHelper.getAllTags(path=config.files_folder) for index in all_tags.keys(): pages = all_tags[index] pages = add_file_path(pages, config.files_folder) entries = create_entries(pages) subPages = generate_sub_pages(entries, len( entries), config.files_folder, False) # Build the files by filling out the template file_loader = FileSystemLoader('') env = Environment(loader=file_loader) # regular index pagse use main template template = env.get_template(template) page_title = index.capitalize() + ' index' current_index_template = template.render( head=generate_head(template_head, page_title), page_body=subPages, page_date=None, page_tags=None, page_navigation=generate_navigation(template_nav), page_title=page_title, config=config) slugfile = config.build_folder + index + '_index.html' with open(slugfile, 'w', encoding=encoding) as fobj: fobj.write(current_index_template) generate_table_index() def generate_table_index(template=config.template_index, template_nav=config.template_nav): # build the metaIndex page # This function creates the tag_index.html page file_loader = FileSystemLoader('') env = Environment(loader=file_loader) template = env.get_template(template) all_tags = TagHelper.getAllTags(path=config.files_folder) parsed_set = {} for k, v in all_tags.items(): parsed_set[k] = len(v) current_index_template = template.render(index_urls=parsed_set, page_date=None, page_tags=None, page_navigation=generate_navigation( template_nav), page_title='Piles', head=generate_head(), config=config) slugfile = config.build_folder + 'tag_index.html' with open(slugfile, 'w', encoding=encoding) as fobj: fobj.write(current_index_template) # Generates html files in the site folder, using the entries and the template def generate_html_pages(site_folder, entries, template, sub_pages_list, template_nav): for entry in entries: # Creating navigation nav_html = generate_navigation(template_nav) # Replaces all occurrences of build_url in the template # (assets, urls, etc) file_loader = FileSystemLoader('') env = Environment(loader=file_loader) template = env.get_template(template) if len(entry['tags']) > 1: entry['tags'] = entry['tags'].split(' ') page_template = template.render( head=generate_head(page_title=entry['title']), page_body=entry['pageContent'], page_date="%s" % entry['date'], page_update="%s" % entry['update'], page_tags=entry['tags'], name_of_site="Kitaab", page_navigation=nav_html, page_title=entry['title'], build_url=entry['parent_text'], config=config) # Checking if content folder exists folderExists = os.path.exists(site_folder + entry['folder']) # If not, create it if config.flat_build: if not folderExists: os.mkdir(site_folder + entry['folder']) # Write the HTML file slug_file = site_folder + entry['slug'] with open(slug_file, 'w', encoding=encoding) as fobj: fobj.write(page_template) print("All pages created!") # Get title by parsing and cleaning the first line of the markdown file def get_entry_title(page): textContent = None try: pageContent = open(page, 'r', errors='ignore') textContent = pageContent.read() textContent = textContent.splitlines() textContent = textContent[0] textContent = textContent.replace('# ', '') if '%title' in textContent: textContent = textContent.replace('%title', '') else: # remove .wiki from the end of the file name textContent = page.split('/')[1][:-5] except FileNotFoundError: # print(f"Page not found, but present in tags: {page}") return None except UnicodeDecodeError: print(f"Page could not be read: {page}") return None except IndexError as e: print(f"{page} probably was'nt grabbed, cannot read file") return textContent # Get the slug from the markdown file name def get_entry_slug(page): slug = page.split("/")[-1] slug = re.sub('\.wiki$', '', slug) if slug: return slug else: return '' def style_iframes(page): regex = r"<\/iframe>" matches = re.finditer(regex, page, re.MULTILINE) for matchNum, match in enumerate(matches, start=1): match = match.group() iframe = "
%s
" % match page = page.replace(match, iframe) return page # Get a list of the tags in the page def get_tags(page): regex = r":[a-zA-Z:]*:" matches = re.finditer(regex, page, re.MULTILINE) ret = [] for match in matches: match = match.group() match = match.rstrip(":") match = match.lstrip(":") match = match.split(":") ret = ret + match ret = " ".join(ret) return ret # Checks for local images links in markdown and add the build_url and medias_folder url def fix_images_urls(page): regex = r"\!\[.*\]\((.*)\)" matches = re.finditer(regex, page, re.MULTILINE) media_folder = 'media/' # TODO bad hard code for matchNum, match in enumerate(matches, start=1): for groupNum in range(0, len(match.groups())): captured_group = match.group(groupNum + 1) if captured_group[:4] != "http": full_url = build_url + media_folder + captured_group page = page.replace(captured_group, full_url) return page def fix_amp(page): try: page = page.replace(" & ", " & ") return page except AttributeError as e: print(f"{page} is not a string") return "" def fix_wiki_links(page, path): regex = r"\[\[([\s\S]+?\|?[\s\S]+?)\]\]" matches = re.finditer(regex, page, re.MULTILINE) for matchNum, match in enumerate(matches, start=1): for groupNum in range(0, len(match.groups())): captured_group = match.group(groupNum + 1) link_elem = captured_group.split("|") if len(link_elem) > 1: full_url = "" + \ link_elem[1].strip() + "" else: if config.flat_build: full_url = "" + \ link_elem[0].strip() + "" else: full_url = "" + \ link_elem[0].strip() + "" page = page.replace(match[0], full_url) return page def remove_tags(page): try: regex = r":[a-zA-Z:]*:" page = re.sub(regex, '', page) return page except: print(f'Unexpected error in removing tags in {page}') return page def remove_title(page): try: textContent = page.splitlines() title = textContent[0] if '%title' in title: textContent = textContent[1:] return '\n'.join(textContent) except Exception as e: print(f'Unexpected error in removing title in {page}') return page def remove_date(page): try: regex = r"%date [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}" page = re.sub(regex, '', page) return page except Exception as e: print(f'Unexpected error in removing date in {page}') return page def remove_update(page): try: regex = r"%update [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}" page = re.sub(regex, '', page) return page except Exception as e: print(f'Unexpected error in removing date in {page}') return page def remove_links(page): try: regex = r"%link *" page = re.sub(regex, '', page) return page except Exception as e: print(f'Unexpected error in removing date in {page}') return page def replacement_function(replace): replace = replace.group() replace = replace.strip('@<>') if replace == 'name-generation': # TODO gross hardcoded lines = [] # this file should probably be an asset?? with open(config.files_folder + '210124-0033.wiki') as f: lines = f.readlines() lines = lines[-2:] line1 = lines[0].split(',') line2 = lines[1].split(',') line1len = len(line1) line2len = len(line2) first = line1[random.randrange(line1len)].strip('\n') second = line2[random.randrange(line2len)].strip('\n') return first + ' ' + second def replace_generated(page): # try: page = re.sub(r'@<[A-Za-z- ]*>', replacement_function, page) return page # except Exception as e: # print(e) # print(f"Could not replace generated in {page}") # return page def get_date(page): try: regex = r"%date [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}" match = re.findall(regex, page)[0].strip('%date') return match except: # print(f"Could not parse date") return None def get_update(page): try: regex = r"%update [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}" match = re.findall(regex, page)[0].strip('%update') return match except: # print(f"Could not parse date") return None def get_links(page): try: regex = r"%link *" match = re.findall(regex, page)[0].strip('%link') ret = [] for m in match: m = m.group() m = m.strip("%link") ret.append(m) return ret except: # print(f"Could not parse date") return None def parse_headings(page): try: page = re.sub(r'= [\[\]a-zA-Z0-9:|+& ]* =', lambda x: x.group().replace('=', '###', 1).strip('='), page) return page except Exception as e: print(f"Could not parse headers {e}") return page # From the list of files, creates the main array of entries that will be processed later def create_entries(pages): fullContent = [] noTitle = [] for page in pages: tempPage = {} # Process the page with dedicated functions path = clean_path(page) title = get_entry_title(page) if title is None: noTitle.append(page) continue try: markdown_text = open(page, 'r').read() except UnicodeDecodeError: print(f"Couldn't parse {title}, {page}") continue markdown_text = parse_headings(markdown_text) markdown_text = replace_generated(markdown_text) markdown_text = style_iframes(markdown_text) markdown_text = fix_images_urls(markdown_text) markdown_text = fix_wiki_links(markdown_text, path["folder"]) tags = get_tags(markdown_text) date = get_date(markdown_text) update = get_update(markdown_text) links = get_links(markdown_text) markdown_text = remove_tags(markdown_text) markdown_text = remove_title(markdown_text) markdown_text = remove_date(markdown_text) markdown_text = remove_update(markdown_text) markdown_text = remove_links(markdown_text) pageContent = markdown(markdown_text) # Create the page object with all the informations we need tempPage['slug'] = path["slug"] tempPage['file'] = path['file'] tempPage['folder'] = path["folder"] tempPage['parent_url'] = path['parent_url'] tempPage['parent_text'] = path['parent_text'] tempPage['iso_date'] = path['date'] if date: tempPage['date'] = date else: tempPage['date'] = path['iso_date'][:-3] if update: tempPage['update'] = update else: tempPage['update'] = tempPage['date'] tempPage['title'] = fix_amp(title) tempPage['pageContent'] = pageContent tempPage['tags'] = tags tempPage['links'] = links fullContent.append(tempPage) return fullContent # Copy assets to production folder def move_files(site_folder, path): assets = os.listdir(path) if 'basant/' in path: to_copy_to = path.split("basant/")[1] else: to_copy_to = path if assets: for asset in assets: asset = os.path.join(path, asset) if os.path.isfile(asset): shutil.copy(asset, site_folder + to_copy_to) else: print("No assets found!") # Transforms the file locations to an array of strings def clean_path(path): path_clean = re.sub('\.wiki$', '', path) items = [] if(platform.system() == 'Windows'): items = path_clean.split('\\') else: items = path_clean.split('/') # TODO ugly hack to handle poonam outputting public, but we want ./ print(items) if items[0] == 'public': items[0] = './' path_items = { "slug": None, "date": None, "folder": items[0], "file": items[len(items) - 1] } # xx-xx-xxxx = xx-xx-xxxx.html # xx-xx-xxxx-string or string = string.html or /folder/string.html # Checking if the file has a date regex = r"[0-9]{6}-[0-9]{4}" match = re.match(regex, path_items["file"]) has_date = False if match: has_date = True else: pass # print(f"{path} does not have a date") if has_date: if match[0] != path_items["file"]: path_items["date"] = match[0] path_items["slug"] = path_items["file"].replace( match[0] + "-", "") + ".html" else: path_items["date"] = match[0] path_items["slug"] = path_items["file"] + ".html" # Converts the EU date to US date to allow page sorting if config.date_format == "EU": path_items["iso_date"] = str( datetime.strptime(path_items["date"], '%y%m%d-%H%M')) if config.date_format == "ISO": path_items["iso_date"] = str( datetime.strptime(path_items["date"], '%Y-%m-%d')) else: path_items["slug"] = path_items["file"] + ".html" # last_edit = str(subprocess.check_output('git log -1 --format="%ci" ' + path, shell=True)).replace("b'", "").replace("\\n'", '') # get last edited time from system try: last_edit = os.path.getmtime('/home/anish/' + path) last_edit_iso = datetime.fromtimestamp(last_edit) except ValueError: last_edit_iso = datetime.fromtimestamp(0) except FileNotFoundError: last_edit_iso = datetime.fromtimestamp(0) if config.date_format == "EU": path_items["date"] = str(last_edit_iso.strftime("%d %b, %y")) else: path_items["date"] = str(last_edit_iso) path_items["iso_date"] = str(last_edit_iso) if config.flat_build: path_items["slug"] = path_items["folder"] + "/" + path_items["slug"] # TODO weird special case if "index" == path_items["file"]: path_items["parent_url"] = "" path_items["parent_text"] = config.home_name path_items["slug"] = "created_index" + ".html" else: if config.flat_build: path_items["parent_url"] = path_items["folder"] + ".html" path_items["parent_text"] = path_items["folder"].replace( "-", " ").capitalize() else: path_items["parent_url"] = path_items["folder"] path_items["parent_text"] = path_items["folder"].replace( "-", " ").capitalize() return path_items # Generate the list of sub pages for each section def generate_sub_pages(entries, num, folder, title, title_name=""): # Sort entries by date using the iso_date format entries.sort(key=lambda x: x["iso_date"], reverse=True) # Take n number of entries (5 for the home, all for the sub-section pages) selected_entries = entries[:num] # Create the list file_loader = FileSystemLoader('') env = Environment(loader=file_loader) template = env.get_template(config.template_list) sub_page_list = template.render( entries=selected_entries, title_name=title_name, title=title) return sub_page_list # Creates the home page using home.md def create_home_page(template, site_folder): # Read the file and add "content_list" as a future replacement point for sub page listing html = "content_list" # Replace template strings with content template = template.replace('page_title', config.home_name) template = template.replace('page_body', html) template = template.replace('build_url', build_url) template = template.replace('page_navigation', "") template = template.replace('footer', "") return template def create_rss_feed(rss_entries, rss_template, rss_item_template, site_folder): template = open(rss_template, 'r').read() itemTemplate = open(rss_item_template, 'r').read() rss_entries.sort(key=lambda x: x["iso_date"], reverse=True) rss_items = "" for rss_entry in rss_entries[:25]: entry_template = itemTemplate entry_template = entry_template.replace( 'rssItemTitle', rss_entry["title"]) entry_template = entry_template.replace('rssItemUrl', build_url + rss_entry["slug"]) date = datetime.strptime(rss_entry["iso_date"], "%y%m%d-%H%M") date = date.strftime("%a, %d %b %Y %H:%M") entry_template = entry_template.replace( 'rssItemDate', date) entry_template = entry_template.replace( 'rssItemContent', rss_entry["pageContent"]) rss_items += entry_template template = template.replace('name_of_site', config.name_of_site) template = template.replace('site_meta_description', config.site_meta_description) template = template.replace('build_url', build_url) template = template.replace('date_build', str( datetime.now().date())) template = template.replace('rss_content', rss_items) slug_file = site_folder + "feed.xml" with open(slug_file, 'w', encoding=encoding) as fobj: fobj.write(template) return def generate_website(): # If build folder exists delete it if os.path.exists(config.build_folder): shutil.rmtree(config.build_folder) # Make new folders os.makedirs(config.build_folder + 'assets/') os.makedirs(config.build_folder + 'media/') # Get main html template file_loader = FileSystemLoader('') env = Environment(loader=file_loader) home_page = env.get_template(config.template_main) generate_index_pages(config.template_main, config.template_nav) rss_entries = [] home_pageSubList = [] pages = glob.glob(config.files_folder + '**/*.wiki', recursive=True) entries = create_entries(pages) sub_pages_list = generate_sub_pages( entries, len(entries), config.files_folder, False) # For each section, create a short listing of sub pages # and add it to the home page home_pageSubList = generate_sub_pages(entries, 50, config.files_folder, False) generate_html_pages(config.build_folder, entries, config.template_main, sub_pages_list, config.template_nav) for entry in entries: rss_entries.append(entry) # Move the assets move_files(config.build_folder, config.assets_folder) move_files(config.build_folder, config.medias_folder) # Once all sections have been processed, finish the home page home_page = home_page.render( head=generate_head(page_title="Recent Files"), page_body=home_pageSubList, page_date=None, page_tags=None, page_navigation=generate_navigation(), page_title="Recent Files", config=config) slug_file = config.build_folder + "recents.html" with open(slug_file, 'w', encoding=encoding) as fobj: fobj.write(home_page) # Create RSS File create_rss_feed(rss_entries, config.rss_template, config.rss_item_template, config.build_folder) # Triggers the website build if __name__ == '__main__': generate_website()