feat: handle code blocks

README.incd

@@ -3,17 +3,33 @@
 Turn IncDown into HTML. 
 First attempt at a parser, written in [Fennel](https://fennel-lang.org/)
 
-Input -> Block Tokenizer (Lexxing?) -> Inline Lexer -> Rendered?
+Input -> Block Lexxing -> Inline Lexer -> Rendered?
 
 I think that's how this will work
 
 = Status =
-Block Tokenizer currently working, except for the code fences
+
+Block Lexer currently working_[#], but doesn't support footnotes, not sure how to address that. Thinking I might do what I've done here use `::` to denote footnotes, instead of using metadata attributes. Not decided
+I'm also wondering how to do the inline lexer. Should I store the index start and end values of specific decorations / lexically meaningful parts? I suppose I could use that to enter whatever sequence was necessary to render them. There's also only a handful. 
+The other thing I need to work out is how to handle metadata and sections, and what piece of the pipeline would handle that. Because for rendering HTML, sections don't really matter, but for handling the metadata it does. So the question kind of is what sort of application needs the metadata, and how should that metadata be stored? If there just block tokens should it be the renderer's job to handle? Not sure yet. 
+
+:: 
+
+- Code fencse are now working, but it's ugly as fuck. I have previously been using the older markdown starnard that uses `{{{` to open a fence and `}}}` to close one. This meant I needed a seperate check to parse the end of the block, but all other blocks work by applying the same regex (which should match *every* line). Code fences, (even if using ``` instead of {{{) would not match every block, but instead begins a "mode", which even if the pattern doesn't match, we add it to the current block. 
+
+::
+
+-----
+%status archived
+
+= The previous problem with code fences =
 
 The code fence regex is wrong, since it looks for the entire block, instead of just the first line. Not quite sure how to use a different regex or "swap modes" into code fence mode such that everything until `}}}` is part of the codeblock. 
 
 -----
 
+== Spec ==
+
 = Blocks =
 Each block is generally seperated by \n 
 This means we require lookahead?
@@ -51,3 +67,10 @@ If this goes well may attempt to write:
 - TreeSitter Grammar
 - NVim Theme (does this come free with treesitter?)
 - Pandoc Grammer? (Seems unlikely)
+
+
+-----
+
+= Things skipped because I don't use them (yet?) =
+- Raw HTML entries 
+- Tables

main.fnl

@@ -17,7 +17,9 @@
     :regex "%%.-%s.-\n"}
    {:name "Code"
     :sp-chars ["{{{"]
-    :regex "{{{.-\n.-}}}\n"}
+    :regex "{{{.-\n"
+    :capture-mode true
+    :mode-regex "}}}"}
    {:name "Header"
     :sp-chars ["="]
     :end-also true
@@ -35,6 +37,7 @@
   (var blocks [])
   (var current-block-type nil)
   (var current-block "")
+  (var capture-mode false)
   (while (< current (- (length contents) 1))
     (var matched false)
     (let [start (string.find contents "\n" current)
@@ -43,28 +46,59 @@
         (print (.. "Current: " current))
         (print (.. "Start: " start))
         (print (.. "Current-str: " current-str))
-        (print (.. "checking regex: " (. g :regex)))
+        (print (.. "checking regex: " (if (not capture-mode) 
-        (print (string.match current-str (. g :regex)))
+                                        (. g :regex)
-        (let [m (string.match current-str (. g :regex))]
+                                        (tostring (. g :mode-regex)))))
+        (let [m (if (not capture-mode)
+                  (string.match current-str (. g :regex)) ; We're not in previous capture mode
+                  (if (= (. g :name) current-block-type) 
+                    (string.match current-str (. g :mode-regex))
+                    nil))] ; Use the mode specific regex
+          (print (.. "M: " (tostring m)))
           (if (not (= nil m)) ; This is a match or move on
             (do
               (set matched true)
-              (if (or ; We append to the current block
+              (if 
+                capture-mode ; We matched and it's capture-mode, so we end capture mode
+                  (do
+                    (print "Capture-mode completed")
+                    (set current-block (.. current-block current-str "\n"))
+                    (table.insert blocks {:type current-block-type 
+                                          :block current-block})
+                    (print (.. "Inserted into table: " current-block))
+                    (set current-block "")
+                    (print (.. "New current-block: " current-block))
+                    (set capture-mode nil)
+                    (set current-block-type nil))
+                (or ; We append to the current block
                     (= nil current-block-type) ; If it's the first block
                     (and (= (. g :name) current-block-type) ; Or it matches the previous block,
                          (not (= (. g :name) "Paragraph")))) ; and it's not a paragraph
-                (do 
+                (do ; Continueing adding things
                   (set current-block-type (. g :name))
+                  (set capture-mode (. g :capture-mode))
                   (set current-block (.. current-block current-str "\n"))
                   (print (.. "Set current-block: " current-block)))
-                (do ; Or we start a new block
+                (do ; Else we start a new block
                   (table.insert blocks {:type current-block-type 
                                         :block current-block})
                   (print (.. "Inserted into table: " current-block))
                   (set current-block current-str)
-                  (print (.. "New current-block: " current-str))
+                  (print (.. "New current-block: " current-block))
+                  (set capture-mode (. g :capture-mode))
+                  (print (.. "New Capture-mode: " (tostring (. g :capture-mode))))
                   (set current-block-type (. g :name))))
-              (set current (+ 1 start))))))))
+              (set current (+ 1 start)))
+            (if (and 
+                  capture-mode ; Or we're in capture mode *and didn't match*
+                  (= (. g :name) current-block-type)) ; And are testing the right regex
+              (do 
+                (print (.. "Didnt match capture-mode regex, appending anyway"))
+                (set current-block (.. current-block current-str "\n"))
+                (print (.. "New current-block: " current-block))
+                (set matched true)
+                (set current (+ 1 start)))))))))
+               
   (print "End of file reached, unloading current")
   (table.insert blocks {:type current-block-type 
                         :block current-block})
@@ -80,12 +114,13 @@
 ; (let [start (string.find "This is a paragraph.\n- This\n- List\n- Should\n- Be One\n" "\n" 1)]
 ;   (print start))
 
-(string.match "{{{python\n return arg + 1\n}}}\n" "{{{.-\n.-}}}\n")
+; (string.match "{{{python\n return arg + 1\n}}}\n" "{{{.-\n.-}}}\n")
 
 ; (= [{:type "Paragraph"
 ;      :block "This is a paragraph."
 ;     {:type "List"
 ;      :block "- This\n- List\n- Should\n- Be One"} 
 (parse-blocks 
-  {:contents "= Big Test File =\n%title Some Title\nThis is a paragraph.\n- This\n- List\n- Should\n- Be One\n-----\n= A new Section Title =\n![Has an Image](file:///link-to-image.png)\n> With some quoted matieral\n>Across multiple lines\n      > and different indents\n\n{{{python \n def file(arg, other):\n  rturn arg + other\n}}}\n\nMore Paragraphs\n" 
+  {:contents "= Big Test File =\n%title Some Title\nThis is a paragraph.\n- This\n- List\n- Should\n- Be One\n-----\n= A new Section Title =\n![Has an Image](file:///link-to-image.png)\n> With some quoted matieral\n>Across multiple lines\n      > and different indents\n\n{{{python \n def file(arg, other):\n  rturn rg + other\n}}}\n\nMore Paragraphs\n" 
    :grammar block-grammar})
+