feat: inline-parser

README.incd

@@ -8,6 +8,14 @@ Input -> Block Lexer -> Inline Lexer -> Renderer
 I think that's how this will work
 
 = Status =
+Started writing the inline lexer sort of, but not sure I'm doing it correctly. Since we now know the "type" of each block, we can extract out semantically meaningful data. The one thing I'm still unsure about is how to apply decoration checks to *all* contents. Regardless of it's a header, a paragraph, or a list, I should still be able to apply any of the decorations I can define. Image alt descriptions should also be decorated, and so too should footnote content.
+
+I'm also thinking about whether I want to be able to link to specific sections. The simplest way to implement this wouold be to have a "reverse" counter: the last parsed section is 0 and we count upwards to the most recent. Since by convention we insert data at the "start" of the file. I could have a special character (I was thinking `#`) to refer to a section, but that means the filename cannot have that character. Personally, that seems okay?  
+
+I'm also currently unsure how I'm to link footnotes against their reference. 
+
+-----
+%date 23-12-01
 
 Block Lexer currently working_[#], but doesn't support footnotes, not sure how to address that. Thinking I might do what I've done here use `::` to denote footnotes, instead of using metadata attributes. Not decided
 I'm also wondering how to do the inline lexer. Should I store the index start and end values of specific decorations / lexically meaningful parts? I suppose I could use that to enter whatever sequence was necessary to render them. There's also only a handful. 
@@ -16,11 +24,11 @@ The other thing I need to work out is how to handle metadata and sections, and w
 :: 
 
 - Code fences are now working, but it's ugly as fuck. I have previously been using the older markdown starnard that uses `{{{` to open a fence and `}}}` to close one. This meant I needed a seperate check to parse the end of the block, but all other blocks work by applying the same regex (which should match *every* line). Code fences, (even if using ``` instead of {{{) would not match every block, but instead begins a "mode", which even if the pattern doesn't match, we add it to the current block. 
-
 ::
 
 -----
 %status archived
+%date 23-11-30
 
 = The previous problem with code fences =
 
@@ -41,6 +49,7 @@ Block parser breaks the text up into blocks:
   - Any character not claimed by the others?
 - Lists
   - # or - started lines (any number of spaces)
+	- Task lists?
 - Headings
   - = start and ending
 - Quotes
@@ -56,7 +65,14 @@ Block parser breaks the text up into blocks:
 
 = Inline =
 - Decorations
-- Links (Internal & External)
+  - Strikethrough ~
+  - Italics _
+  - Strong *
+	- Monospace `
+	- Spoiler? >! (This clashes with quotes?)
+	- Censor? >~  (This clashes with quotes?)
+- Links (Internal & External) [[]] or []()
+
 - Images? Maybe images can't be inlined? 
 - Footnotes
 
@@ -68,7 +84,6 @@ If this goes well may attempt to write:
 - NVim Theme (does this come free with treesitter?)
 - Pandoc Grammer? (Seems unlikely)
 
-
 -----
 
 = Things skipped because I don't use them (yet?) =

block.fnl

@@ -0,0 +1,134 @@
+(local fennel (require :fennel)) 
+(fn _G.pp [x] (print (fennel.view x)))
+
+(local block-grammar 
+  [{:name "Section"
+    :sp-chars ["-----"]
+    :regex "%-%-%-%-%-"}
+   {:name "Image"
+    :sp-chars ["!["]
+    :regex "!%[.-%]%(.-%)\n"}
+   {:name "Tag"
+    :sp-chars [":"]
+    :end-also true
+    :regex ":.-:\n"}
+   {:name "Quote"
+    :any-space true
+    :sp-chars [">"]
+    :continue-until-no-match true
+    :regex "%s*>.-\n"}
+   {:name "Metadata"
+    :sp-chars ["%"]
+    :regex "%%.-%s.-\n"}
+   {:name "Code"
+    :sp-chars ["{{{"]
+    :regex "{{{.-\n"
+    :capture-mode true
+    :mode-regex "}}}"}
+   {:name "Header"
+    :sp-chars ["="]
+    :end-also true
+    :regex "=+[^=]-=+\n"} 
+   {:name "List"
+    :sp-chars ["#" "-"]
+    :any-space true
+    :continue-until-no-match true
+    :regex "%s*[%-%#].-\n"}
+   {:name "Footnote"
+    :sp-chars ["::\n"]
+    :regex "%s*::%s-\n"
+    :capture-mode true
+    :mode-regex "%s*::%s-\n"}
+   {:name "Paragraph"
+    :regex ".-\n"}])
+
+(fn block-lexer [contents grammar]
+  (var current 1)
+  (var blocks [])
+  (var current-block-type nil)
+  (var current-block "")
+  (var capture-mode false)
+  (while (< current (- (length contents) 1))
+    (var matched false)
+    (let [start (string.find contents "\n" current)
+          current-str (string.sub contents current start)]
+      (each [_ g (ipairs grammar) &until matched]
+        (print (.. "Current: " current))
+        (print (.. "Start: " start))
+        (print (.. "Current-str: " current-str))
+        (print (.. "checking regex: " (if (not capture-mode) 
+                                        (. g :regex)
+                                        (tostring (. g :mode-regex)))))
+        (let [m (if (not capture-mode)
+                  (string.match current-str (. g :regex)) ; We're not in previous capture mode
+                  (if (= (. g :name) current-block-type)  ; And the regex we're checking is the one we want
+                    (string.match current-str (. g :mode-regex)) ; Use the mode specific regex
+                    nil))] ; Or don't check
+          (print (.. "M: " (tostring m)))
+          (if (not (= nil m)) ; This is a match or move on
+            (do
+              (set matched true)
+              (if 
+                capture-mode ; We matched and it's capture-mode, so we end capture mode
+                  (do
+                    (print "Capture-mode completed")
+                    (set current-block (.. current-block current-str "\n"))
+                    (table.insert blocks {:type current-block-type 
+                                          :block current-block})
+                    (print (.. "Inserted into table: " current-block))
+                    (set current-block "")
+                    (print (.. "New current-block: " current-block))
+                    (set capture-mode nil)
+                    (set current-block-type nil))
+                (or ; We append to the current block
+                    (= nil current-block-type) ; If it's the first block
+                    (and (= (. g :name) current-block-type) ; Or it matches the previous block,
+                         (not (= (. g :name) "Paragraph")))) ; and it's not a paragraph
+                (do ; Continueing adding things
+                  (set current-block-type (. g :name))
+                  (set capture-mode (. g :capture-mode))
+                  (set current-block (.. current-block current-str "\n"))
+                  (print (.. "Set current-block: " current-block)))
+                (do ; Else we start a new block, and insert our accumulation
+                  (table.insert blocks {:type current-block-type 
+                                        :block current-block})
+                  (print (.. "Inserted into table: " current-block))
+                  (set current-block current-str)
+                  (print (.. "New current-block: " current-block))
+                  (set capture-mode (. g :capture-mode))
+                  (print (.. "New Capture-mode: " (tostring (. g :capture-mode))))
+                  (set current-block-type (. g :name))))
+              (set current (+ 1 start)))
+            (if (and ; We didn't match, BUT
+                  capture-mode ; Or we're in capture mode 
+                  (= (. g :name) current-block-type)) ; And are testing the right regex (should be implicitly true if we reached here)
+              (do 
+                (print (.. "Didnt match capture-mode regex, appending anyway"))
+                (set current-block (.. current-block current-str "\n"))
+                (print (.. "New current-block: " current-block))
+                (set matched true) ; We don't need to check the others
+                (set current (+ 1 start)))))))))
+  (if current-block-type
+   (do 
+    (print "End of file reached, unloading current")
+    (table.insert blocks {:type current-block-type 
+                          :block current-block})))
+  blocks) 
+
+  ; Each \n point
+  ; Read contents from current to next \n
+  ; If it matches the regex
+  ; Either it's the same pattern as the last one (or the last one was nil), so we continue adding to the block-accumulator
+  ; Or it's a different pattern, which means the current block is complete add it to the blocks list, and start a new accumulator filled with our current-str
+  ; If not, continue with block descent
+
+; This string has weird behaviour in nvim conjure repl 
+; (string.match "{{{python\n return arg + 1\n}}}\n" "{{{.-\n.-}}}\n")
+
+; (= [{:type "Paragraph"
+;      :block "This is a paragraph."
+;     {:type "List"
+;      :block "- This\n- List\n- Should\n- Be One"} 
+
+{:grammar block-grammar
+ :lexer block-lexer}

inline.fnl

@@ -0,0 +1,159 @@
+(local fennel (require :fennel)) 
+(fn _G.pp [x] (print (fennel.view x)))
+
+(fn general-contents-fn [pattern typ str current]
+  (if (string.match str pattern)
+    (let [(start end contents) (string.find str pattern current)]
+      (values 
+        {:type typ
+         :contents contents}
+        end))))
+
+(local contents-grammar
+  {:match-first "[%*%_%~%[%>%`]"
+   :Italics (partial general-contents-fn "_(.-)_" "Italics")
+   :Bold (partial general-contents-fn "%*(.-)%*" "Bold")
+   :Strikethrough (partial general-contents-fn "%~(.-)%~" "Strikethrough")
+   :Monospace (partial general-contents-fn "%`(.-)%`" "Monospace")
+   :Link (fn link-fn [str current]
+           (let [patterns {:In-Link "%[%[(.-)%]%]"
+                           :Named-In-Link "%[%[(.-)%|(.-)%]%]" 
+                           :Out-Link "%[(.-)%]%((.-)%)"}
+                 next-char (string.sub str current (+ 1 current))]
+             (if (= next-char "[")
+               (if (not (= nil (string.match str (. patterns :Named-In-Link) current)))
+                 (let [(start end filename title) (string.find str (. patterns :Named-In-Link) current)]
+                   (values {:type "Link"
+                            :title title
+                            :destination filename}
+                           end))
+                 (let [(start end filename) (string.find str (. patterns :In-Link) current)]
+                   (values {:type "Link"
+                            :title filename
+                            :destination filename}
+                           end)))
+               (let [(start end title destination) (string.find str (. patterns :Out-Link) current)]
+                 (values {:type "Link"
+                          :title title
+                          :destination destination}
+                         end)))))})
+   ; :Spoiler
+   ; :Censor})
+
+(fn contents-lexer [contents ?grammar]
+  (let [grammar (or ?grammar contents-grammar)]
+    (var current 1)
+    (var lexed [])
+    (while (> (length contents) current)
+          (let [found (string.find contents (. grammar :match-first) current)]
+            (if (not (= nil found))
+              (let [char (string.sub contents found found)]
+                (table.insert lexed {:type "raw-text"
+                                     :contents (string.sub contents current (- found 1))})
+                (match char
+                 ; This would be better if we had a reverse lookup table
+                 "_" (let [(block new-end) ((. grammar :Italics) contents current)]
+                       (table.insert lexed block)
+                       (set current new-end)) ; Italics
+                 "*" (let [(block new-end) ((. grammar :Bold) contents current)]
+                       (table.insert lexed block)
+                       (set current new-end)) ; Bold
+                 "~" (let [(block new-end) ((. grammar :Strikethrough) contents current)]
+                       (table.insert lexed block)
+                       (set current new-end)) ; Strikethrough
+                 "`" (let [(block new-end) ((. grammar :Monospace) contents current)]
+                      (table.insert lexed block)
+                      (set current new-end)); Monospace
+                 "[" (let [(block new-end) ((. grammar :Link) contents current)]
+                      (table.insert lexed block)
+                      (set current new-end)))) ; Links
+              (do 
+                (table.insert lexed {:type "raw-text"
+                                     :contents contents})
+                (set current (length contents))))) ; No decoration at all
+        lexed)))
+             ; ">" (let [(block new-end) ((. grammar :Spoiler) contents current)]
+             ;      (table.insert lexed block)
+             ;      (set current new-end))))
+
+(fn list-rule [str ?regex]
+  (let [regex (or ?regex "(%s*)%-%s*(.-)\n")]
+   (var list-elems [])
+   (each [indents line (string.gmatch str regex)]
+     (table.insert list-elems {:indent-level (length indents)
+                               :contents line}))
+   {:type "List"
+    :elements list-elems}))
+
+(local inline-grammar
+  {:Metadata { :rule (fn metadata-rule [str ?regex]
+                      (let [regex (or ?regex "%%(.-)%s(.-)\n")
+                            (key val) (string.match str regex)]
+                        {:type "Metadata"
+                         : key
+                         : val}))}
+   :Code { :rule (fn code-rule [str ?regex]
+                  (let [regex (or ?regex "{{{(.-)\n(.-)}}}\n")
+                        (hint code) (string.match str regex)]
+                    {:type "Code"
+                     :codehint hint
+                     :contents code}))}
+   :Header { :rule (fn header-rule [str ?regex]
+                    (let [regex (or ?regex "(=*)%s*(.-)%s*=*\n")
+                          (header-size contents) (string.match str regex)]
+                      {:type "Header"
+                       :size (- 4 (length header-size))
+                       : contents}))}
+   :Image {:rule (fn image-rule [str ?regex]
+                  (let [regex (or ?regex "%!%[(.-)%]%((.-)%)\n")
+                        (alt link) (string.match str regex)]
+                    {:type "Image"
+                     :contents alt
+                     : link}))}
+   :List {:rule list-rule}
+   :Footnote {:rule (fn footnote-rule [str ?regex]
+                     (let [regex (or ?regex "::%s-\n%s*(.-)\n::\n")
+                           list (string.match str regex)]
+                       {:type "Footnotes"
+                        :elements (. (list-rule list) :elements)}))}
+   :Tag {:rule (fn tag-rule [str ?regex]
+                (let [regex (or ?regex "%:(.-):")]
+                 (var tags [])
+                 (each [tag (string.gmatch str regex)]
+                   (table.insert tags tag))
+                 {:type "Tag"
+                  :tags tags}))}
+   :Quote {:rule (fn quote-rule [str ?regex]
+                   (var total "")
+                   (let [regex (or ?regex "%s*>%s*(.-)\n")]
+                     (each [m (string.gmatch str regex)]
+                       (set total (.. total m "\n"))))
+                   {:type "Quote"
+                    :contents total})}
+   :Paragraph {:rule (fn paragraph-rule [str]
+                      {:type "Paragraph"
+                       :contents str})}}) 
+
+; TODO
+; All :contents entries must go through the contents parser 
+
+(fn inline-lexer [blocks grammar]
+  (var lexed [])
+  (var section [])
+  (each [_ block (ipairs blocks)]
+    (_G.pp block)
+    (let [rule-set (. grammar (. block :type))]
+     (if (= "Section" (. block :type))
+       (do ; Each section gets it's own list 
+         (table.insert lexed section)
+         (set section []))
+       (table.insert section ((. rule-set :rule) (. block :block))))))
+  ; Final section
+  (if (> (length section) 0)
+    (table.insert lexed section))
+  lexed)
+
+{: inline-grammar
+ : inline-lexer
+ : contents-lexer
+ : contents-grammar}

main.fnl

@@ -1,152 +1,13 @@
 (local fennel (require :fennel)) 
 (fn _G.pp [x] (print (fennel.view x)))
 
-(local block-grammar 
+(local blocks (require :block))
-  [{:name "Section"
+(local inline (require :inline))
-    :sp-chars ["-----"]
-    :regex "%-%-%-%-%-"}
-   {:name "Image"
-    :sp-chars ["!["]
-    :regex "!%[.-%]%(.-%)\n"}
-   {:name "Tag"
-    :sp-chars [":"]
-    :end-also true
-    :regex ":.-:\n"}
-   {:name "Quote"
-    :any-space true
-    :sp-chars [">"]
-    :continue-until-no-match true
-    :regex "%s*>.-\n"}
-   {:name "Metadata"
-    :sp-chars ["%"]
-    :regex "%%.-%s.-\n"}
-   {:name "Code"
-    :sp-chars ["{{{"]
-    :regex "{{{.-\n"
-    :capture-mode true
-    :mode-regex "}}}"}
-   {:name "Header"
-    :sp-chars ["="]
-    :end-also true
-    :regex "=+[^=]-=+\n"} 
-   {:name "List"
-    :sp-chars ["#" "-"]
-    :any-space true
-    :continue-until-no-match true
-    :regex "%s*[%-%#].-\n"}
-   {:name "Footnote"
-    :sp-chars ["::\n"]
-    :regex "%s*::%s-\n"
-    :capture-mode true
-    :mode-regex "%s*::%s-\n"}
-   {:name "Paragraph"
-    :regex ".-\n"}])
 
-(fn block-lexer [{: contents : grammar}]
+(local test-input "= Big Test File =\n%title Some Title\n:note:\n*This* is a paragraph.\n- This\n- List\n- Should\n- Be One\n-----\n= A new Section Title =\n![Has an Image](file:///link-to-image.png)\n> With some quoted matieral\n>Across multiple lines\n      > and different indents\n\n{{{python \n def file(arg, other):\n  rturn rg + other\n}}}\n\nMore Paragraphs\n-----\n::  \n\n- This is the first note\n-This is the second\n::\n") 
-  (var current 1)
-  (var blocks [])
-  (var current-block-type nil)
-  (var current-block "")
-  (var capture-mode false)
-  (while (< current (- (length contents) 1))
-    (var matched false)
-    (let [start (string.find contents "\n" current)
-          current-str (string.sub contents current start)]
-      (each [_ g (ipairs grammar) &until matched]
-        (print (.. "Current: " current))
-        (print (.. "Start: " start))
-        (print (.. "Current-str: " current-str))
-        (print (.. "checking regex: " (if (not capture-mode) 
-                                        (. g :regex)
-                                        (tostring (. g :mode-regex)))))
-        (let [m (if (not capture-mode)
-                  (string.match current-str (. g :regex)) ; We're not in previous capture mode
-                  (if (= (. g :name) current-block-type)  ; And the regex we're checking is the one we want
-                    (string.match current-str (. g :mode-regex)) ; Use the mode specific regex
-                    nil))] ; Or don't check
-          (print (.. "M: " (tostring m)))
-          (if (not (= nil m)) ; This is a match or move on
-            (do
-              (set matched true)
-              (if 
-                capture-mode ; We matched and it's capture-mode, so we end capture mode
-                  (do
-                    (print "Capture-mode completed")
-                    (set current-block (.. current-block current-str "\n"))
-                    (table.insert blocks {:type current-block-type 
-                                          :block current-block})
-                    (print (.. "Inserted into table: " current-block))
-                    (set current-block "")
-                    (print (.. "New current-block: " current-block))
-                    (set capture-mode nil)
-                    (set current-block-type nil))
-                (or ; We append to the current block
-                    (= nil current-block-type) ; If it's the first block
-                    (and (= (. g :name) current-block-type) ; Or it matches the previous block,
-                         (not (= (. g :name) "Paragraph")))) ; and it's not a paragraph
-                (do ; Continueing adding things
-                  (set current-block-type (. g :name))
-                  (set capture-mode (. g :capture-mode))
-                  (set current-block (.. current-block current-str "\n"))
-                  (print (.. "Set current-block: " current-block)))
-                (do ; Else we start a new block, and insert our accumulation
-                  (table.insert blocks {:type current-block-type 
-                                        :block current-block})
-                  (print (.. "Inserted into table: " current-block))
-                  (set current-block current-str)
-                  (print (.. "New current-block: " current-block))
-                  (set capture-mode (. g :capture-mode))
-                  (print (.. "New Capture-mode: " (tostring (. g :capture-mode))))
-                  (set current-block-type (. g :name))))
-              (set current (+ 1 start)))
-            (if (and ; We didn't match, BUT
-                  capture-mode ; Or we're in capture mode 
-                  (= (. g :name) current-block-type)) ; And are testing the right regex (should be implicitly true if we reached here)
-              (do 
-                (print (.. "Didnt match capture-mode regex, appending anyway"))
-                (set current-block (.. current-block current-str "\n"))
-                (print (.. "New current-block: " current-block))
-                (set matched true) ; We don't need to check the others
-                (set current (+ 1 start)))))))))
-  (print "End of file reached, unloading current")
-  (table.insert blocks {:type current-block-type 
-                        :block current-block})
-  blocks) 
 
-  ; Each \n point
+(-> test-input
-  ; Read contents from current to next \n
+    (blocks.lexer blocks.grammar)
-  ; If it matches the regex
+    (inline.inline-lexer inline.inline-grammar)
-  ; Either it's the same pattern as the last one (or the last one was nil), so we continue adding to the block-accumulator
+    (_G.pp))
-  ; Or it's a different pattern, which means the current block is complete add it to the blocks list, and start a new accumulator filled with our current-str
+    ; (inline.contents-lexer inline.contents-grammar))
-  ; If not, continue with block descent
-
-(fn inline-lexer [blocks grammar]
-  (var lexed [])
-  (each [_ block (ipairs blocks)]
-    (match (. block :type)
-      "Paragraph" 
-        (do)
-      "Footnote"
-        (do)
-      "List"
-        (do)
-      "Header" 
-        (do)
-      "Image"
-        (do)
-      "Metadata"
-        (do)
-      "Code"
-        (do))))
-
-; This string has weird behaviour in nvim conjure repl 
-; (string.match "{{{python\n return arg + 1\n}}}\n" "{{{.-\n.-}}}\n")
-
-; (= [{:type "Paragraph"
-;      :block "This is a paragraph."
-;     {:type "List"
-;      :block "- This\n- List\n- Should\n- Be One"} 
-
-(block-lexer 
-  {:contents "= Big Test File =\n%title Some Title\n:note:\nThis is a paragraph.\n- This\n- List\n- Should\n- Be One\n-----\n= A new Section Title =\n![Has an Image](file:///link-to-image.png)\n> With some quoted matieral\n>Across multiple lines\n      > and different indents\n\n{{{python \n def file(arg, other):\n  rturn rg + other\n}}}\n\nMore Paragraphs\n-----\n::  \n\n- This is the first note\n-This is the second\n" 
-   :grammar block-grammar})