feat: init block tokenier

README.incd

@@ -0,0 +1,53 @@
+=== IncDown Parser ===
+
+Turn IncDown into HTML. 
+First attempt at a parser, written in [Fennel](https://fennel-lang.org/)
+
+Input -> Block Tokenizer (Lexxing?) -> Inline Lexer -> Rendered?
+
+I think that's how this will work
+
+= Status =
+Block Tokenizer currently working, except for the code fences
+
+The code fence regex is wrong, since it looks for the entire block, instead of just the first line. Not quite sure how to use a different regex or "swap modes" into code fence mode such that everything until `}}}` is part of the codeblock. 
+
+-----
+
+= Blocks =
+Each block is generally seperated by \n 
+This means we require lookahead?
+
+Block parser breaks the text up into blocks:
+- Metadata
+  - %attribute
+- Paragraphs
+  - Any character not claimed by the others?
+- Lists
+  - # or - started lines (any number of spaces)
+- Headings
+  - = start and ending
+- Quotes
+  - > start (any number of spaces)
+- Images
+  - ![Alt](Link)
+- Foot notes
+  - _[#] and then %footnote ??? need to think about this one
+- Code blocks
+  - {{{code hint\n code}}}
+- Section breaks
+  - 5 continuous - on a single line
+
+= Inline =
+- Decorations
+- Links
+- Images? Maybe images can't be inlined? 
+- Footnotes
+
+== Future Work ==
+
+If this goes well may attempt to write:
+- Blog engine in Fennel on top of this
+- TreeSitter Grammar
+- NVim Theme (does this come free with treesitter?)
+- Pandoc Grammer? (Seems unlikely)

flake.lock

@@ -0,0 +1,26 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1700959420,
+        "narHash": "sha256-mzRwNDwt8A4Q1PPDPBk0wbv24kX7FqbSFFj3Mh4ykk4=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "b7a2c41788518c080ef3b51631f78985cfdd0228",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,33 @@
+{
+  description = "IncDown Parser written in Fennel";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs";
+  };
+
+  outputs = { self, nixpkgs }:
+  let 
+    pkgs = nixpkgs.legacyPackages.x86_64-linux;
+  in
+    {
+      defaultPackage.x86_64-linux = pkgs.stdenv.mkDerivation {
+        name = "incd";
+        src = ./.;
+        buildInputs = [ pkgs.luaPackages.fennel pkgs.fd ];
+        installPhase = ''
+          mkdir -p $out/
+
+          cp -r ./ $out
+        '';
+      };
+      devShell.x86_64-linux = pkgs.mkShell {
+        buildInputs = [
+          pkgs.lua
+          pkgs.luaPackages.fennel
+        ];
+        shellHook = ''
+        '';
+      };
+    };
+}
+

main.fnl

@@ -0,0 +1,91 @@
+(local fennel (require :fennel)) 
+(fn _G.pp [x] (print (fennel.view x)))
+
+(local block-grammar 
+  [{:name "Section"
+    :sp-chars ["-----"]
+    :regex "%-%-%-%-%-"}
+   {:name "Image"
+    :sp-chars ["!["]
+    :regex "!%[.-%]%(.-%)\n"}
+   {:name "Quote"
+    :any-space true
+    :sp-chars [">"]
+    :regex "%s*>.-\n"}
+   {:name "Metadata"
+    :sp-chars ["%"]
+    :regex "%%.-%s.-\n"}
+   {:name "Code"
+    :sp-chars ["{{{"]
+    :regex "{{{.-\n.-}}}\n"}
+   {:name "Header"
+    :sp-chars ["="]
+    :end-also true
+    :regex "=+[^=]-=+\n"} 
+   {:name "List"
+    :sp-chars ["#" "-"]
+    :any-space true
+    :continue-until-no-match true
+    :regex "%s*[%-%#].-\n"}
+   {:name "Paragraph"
+    :regex ".-\n"}])
+
+(fn parse-blocks [{: contents : grammar}]
+  (var current 1)
+  (var blocks [])
+  (var current-block-type nil)
+  (var current-block "")
+  (while (< current (- (length contents) 1))
+    (var matched false)
+    (let [start (string.find contents "\n" current)
+          current-str (string.sub contents current start)]
+      (each [_ g (ipairs grammar) &until matched]
+        (print (.. "Current: " current))
+        (print (.. "Start: " start))
+        (print (.. "Current-str: " current-str))
+        (print (.. "checking regex: " (. g :regex)))
+        (print (string.match current-str (. g :regex)))
+        (let [m (string.match current-str (. g :regex))]
+          (if (not (= nil m)) ; This is a match or move on
+            (do
+              (set matched true)
+              (if (or ; We append to the current block
+                    (= nil current-block-type) ; If it's the first block
+                    (and (= (. g :name) current-block-type) ; Or it matches the previous block,
+                         (not (= (. g :name) "Paragraph")))) ; and it's not a paragraph
+                (do 
+                  (set current-block-type (. g :name))
+                  (set current-block (.. current-block current-str "\n"))
+                  (print (.. "Set current-block: " current-block)))
+                (do ; Or we start a new block
+                  (table.insert blocks {:type current-block-type 
+                                        :block current-block})
+                  (print (.. "Inserted into table: " current-block))
+                  (set current-block current-str)
+                  (print (.. "New current-block: " current-str))
+                  (set current-block-type (. g :name))))
+              (set current (+ 1 start))))))))
+  (print "End of file reached, unloading current")
+  (table.insert blocks {:type current-block-type 
+                        :block current-block})
+  blocks) 
+
+  ; Each \n point
+  ; Read contents from current to next
+  ; If it matches any of the block structures
+  ; Add to that blocks current
+  ; If the block is complete, add it to the blocks list
+  ; If not, continue with block descent
+
+; (let [start (string.find "This is a paragraph.\n- This\n- List\n- Should\n- Be One\n" "\n" 1)]
+;   (print start))
+
+(string.match "{{{python\n return arg + 1\n}}}\n" "{{{.-\n.-}}}\n")
+
+; (= [{:type "Paragraph"
+;      :block "This is a paragraph."
+;     {:type "List"
+;      :block "- This\n- List\n- Should\n- Be One"} 
+(parse-blocks 
+  {:contents "= Big Test File =\n%title Some Title\nThis is a paragraph.\n- This\n- List\n- Should\n- Be One\n-----\n= A new Section Title =\n![Has an Image](file:///link-to-image.png)\n> With some quoted matieral\n>Across multiple lines\n      > and different indents\n\n{{{python \n def file(arg, other):\n  rturn arg + other\n}}}\n\nMore Paragraphs\n" 
+   :grammar block-grammar})