From ead9839afeb8f28f63b47d01bad3710a0ac8635d Mon Sep 17 00:00:00 2001 From: Anish Lakhwara Date: Wed, 29 Nov 2023 19:55:05 -0800 Subject: [PATCH] feat: handle code blocks --- README.incd | 27 +++++++++++++++++++++++-- main.fnl | 57 ++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 71 insertions(+), 13 deletions(-) diff --git a/README.incd b/README.incd index c28dfc7..1a73d6c 100644 --- a/README.incd +++ b/README.incd @@ -3,17 +3,33 @@ Turn IncDown into HTML. First attempt at a parser, written in [Fennel](https://fennel-lang.org/) -Input -> Block Tokenizer (Lexxing?) -> Inline Lexer -> Rendered? +Input -> Block Lexxing -> Inline Lexer -> Rendered? I think that's how this will work = Status = -Block Tokenizer currently working, except for the code fences + +Block Lexer currently working_[#], but doesn't support footnotes, not sure how to address that. Thinking I might do what I've done here use `::` to denote footnotes, instead of using metadata attributes. Not decided +I'm also wondering how to do the inline lexer. Should I store the index start and end values of specific decorations / lexically meaningful parts? I suppose I could use that to enter whatever sequence was necessary to render them. There's also only a handful. +The other thing I need to work out is how to handle metadata and sections, and what piece of the pipeline would handle that. Because for rendering HTML, sections don't really matter, but for handling the metadata it does. So the question kind of is what sort of application needs the metadata, and how should that metadata be stored? If there just block tokens should it be the renderer's job to handle? Not sure yet. + +:: + +- Code fencse are now working, but it's ugly as fuck. I have previously been using the older markdown starnard that uses `{{{` to open a fence and `}}}` to close one. This meant I needed a seperate check to parse the end of the block, but all other blocks work by applying the same regex (which should match *every* line). Code fences, (even if using ``` instead of {{{) would not match every block, but instead begins a "mode", which even if the pattern doesn't match, we add it to the current block. + +:: + +----- +%status archived + += The previous problem with code fences = The code fence regex is wrong, since it looks for the entire block, instead of just the first line. Not quite sure how to use a different regex or "swap modes" into code fence mode such that everything until `}}}` is part of the codeblock. ----- +== Spec == + = Blocks = Each block is generally seperated by \n This means we require lookahead? @@ -51,3 +67,10 @@ If this goes well may attempt to write: - TreeSitter Grammar - NVim Theme (does this come free with treesitter?) - Pandoc Grammer? (Seems unlikely) + + +----- + += Things skipped because I don't use them (yet?) = +- Raw HTML entries +- Tables diff --git a/main.fnl b/main.fnl index c9b87c1..ccd0289 100644 --- a/main.fnl +++ b/main.fnl @@ -17,7 +17,9 @@ :regex "%%.-%s.-\n"} {:name "Code" :sp-chars ["{{{"] - :regex "{{{.-\n.-}}}\n"} + :regex "{{{.-\n" + :capture-mode true + :mode-regex "}}}"} {:name "Header" :sp-chars ["="] :end-also true @@ -35,6 +37,7 @@ (var blocks []) (var current-block-type nil) (var current-block "") + (var capture-mode false) (while (< current (- (length contents) 1)) (var matched false) (let [start (string.find contents "\n" current) @@ -43,28 +46,59 @@ (print (.. "Current: " current)) (print (.. "Start: " start)) (print (.. "Current-str: " current-str)) - (print (.. "checking regex: " (. g :regex))) - (print (string.match current-str (. g :regex))) - (let [m (string.match current-str (. g :regex))] + (print (.. "checking regex: " (if (not capture-mode) + (. g :regex) + (tostring (. g :mode-regex))))) + (let [m (if (not capture-mode) + (string.match current-str (. g :regex)) ; We're not in previous capture mode + (if (= (. g :name) current-block-type) + (string.match current-str (. g :mode-regex)) + nil))] ; Use the mode specific regex + (print (.. "M: " (tostring m))) (if (not (= nil m)) ; This is a match or move on (do (set matched true) - (if (or ; We append to the current block + (if + capture-mode ; We matched and it's capture-mode, so we end capture mode + (do + (print "Capture-mode completed") + (set current-block (.. current-block current-str "\n")) + (table.insert blocks {:type current-block-type + :block current-block}) + (print (.. "Inserted into table: " current-block)) + (set current-block "") + (print (.. "New current-block: " current-block)) + (set capture-mode nil) + (set current-block-type nil)) + (or ; We append to the current block (= nil current-block-type) ; If it's the first block (and (= (. g :name) current-block-type) ; Or it matches the previous block, (not (= (. g :name) "Paragraph")))) ; and it's not a paragraph - (do + (do ; Continueing adding things (set current-block-type (. g :name)) + (set capture-mode (. g :capture-mode)) (set current-block (.. current-block current-str "\n")) (print (.. "Set current-block: " current-block))) - (do ; Or we start a new block + (do ; Else we start a new block (table.insert blocks {:type current-block-type :block current-block}) (print (.. "Inserted into table: " current-block)) (set current-block current-str) - (print (.. "New current-block: " current-str)) + (print (.. "New current-block: " current-block)) + (set capture-mode (. g :capture-mode)) + (print (.. "New Capture-mode: " (tostring (. g :capture-mode)))) (set current-block-type (. g :name)))) - (set current (+ 1 start)))))))) + (set current (+ 1 start))) + (if (and + capture-mode ; Or we're in capture mode *and didn't match* + (= (. g :name) current-block-type)) ; And are testing the right regex + (do + (print (.. "Didnt match capture-mode regex, appending anyway")) + (set current-block (.. current-block current-str "\n")) + (print (.. "New current-block: " current-block)) + (set matched true) + (set current (+ 1 start))))))))) + (print "End of file reached, unloading current") (table.insert blocks {:type current-block-type :block current-block}) @@ -80,12 +114,13 @@ ; (let [start (string.find "This is a paragraph.\n- This\n- List\n- Should\n- Be One\n" "\n" 1)] ; (print start)) -(string.match "{{{python\n return arg + 1\n}}}\n" "{{{.-\n.-}}}\n") +; (string.match "{{{python\n return arg + 1\n}}}\n" "{{{.-\n.-}}}\n") ; (= [{:type "Paragraph" ; :block "This is a paragraph." ; {:type "List" ; :block "- This\n- List\n- Should\n- Be One"} (parse-blocks - {:contents "= Big Test File =\n%title Some Title\nThis is a paragraph.\n- This\n- List\n- Should\n- Be One\n-----\n= A new Section Title =\n![Has an Image](file:///link-to-image.png)\n> With some quoted matieral\n>Across multiple lines\n > and different indents\n\n{{{python \n def file(arg, other):\n rturn arg + other\n}}}\n\nMore Paragraphs\n" + {:contents "= Big Test File =\n%title Some Title\nThis is a paragraph.\n- This\n- List\n- Should\n- Be One\n-----\n= A new Section Title =\n![Has an Image](file:///link-to-image.png)\n> With some quoted matieral\n>Across multiple lines\n > and different indents\n\n{{{python \n def file(arg, other):\n rturn rg + other\n}}}\n\nMore Paragraphs\n" :grammar block-grammar}) +