we had to start over because i leaked passwords...

2022-11-25 23:25:49 +10:00 · 2022-11-25 23:25:49 +10:00 · 2c820b5895
commit 2c820b5895
10 changed files with 567 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1 @@
+documents/**
--- a/bb.edn
+++ b/bb.edn
@ -0,0 +1,2 @@
+{:path ["src"]
+ :deps {local/deps {:local/root "."}}}
--- a/deps.edn
+++ b/deps.edn
@ -0,0 +1,30 @@
+{:deps
+ {selmer/selmer {:mvn/version "1.12.50"},
+  org.clojure/clojure {:mvn/version "1.11.1"},
+  org.clojure/tools.logging {:mvn/version "1.1.0"},
+  insn/insn {:mvn/version "0.5.2"},
+  org.clojure/tools.cli {:mvn/version "1.0.206"},
+  org.babashka/babashka.impl.reify {:mvn/version "0.1.1"},
+  rewrite-clj/rewrite-clj {:mvn/version "1.0.699-alpha"},
+  babashka/babashka.core
+  {:git/url "https://github.com/babashka/babashka.core",
+   :git/sha "52a6037bd4b632bffffb04394fb4efd0cdab6b1e"},
+  hiccup/hiccup {:mvn/version "2.0.0-alpha2"},
+  org.clojure/data.xml {:mvn/version "0.2.0-alpha6"},
+  org.clojure/data.csv {:mvn/version "1.0.0"},
+  com.taoensso/timbre {:mvn/version "5.2.1"},
+  com.cognitect/transit-clj {:mvn/version "1.0.329"},
+  babashka/babashka.curl {:mvn/version "0.1.1"},
+  clj-commons/clj-yaml {:mvn/version "0.7.108"},
+  org.clojure/core.rrb-vector {:mvn/version "0.1.2"},
+  org.babashka/sci.impl.types {:mvn/version "0.0.2"},
+  cheshire/cheshire {:mvn/version "5.11.0"},
+  org.clojure/core.match {:mvn/version "1.0.0"},
+  org.clojure/test.check {:mvn/version "1.1.1"},
+  babashka/fs {:mvn/version "0.1.2"},
+  cljc.java-time/cljc.java-time {:mvn/version "0.1.12"},
+  org.clojure/data.priority-map {:mvn/version "1.1.0"},
+  nrepl/bencode {:mvn/version "1.1.0"},
+  org.clojure/core.async {:mvn/version "1.5.648"},
+  http-kit/http-kit {:mvn/version "2.6.0-RC1"}
+  etaoin/etaoin {:mvn/version "1.0.38"}}}
--- a/docs/how-to-rules.rst
+++ b/docs/how-to-rules.rst
@ -0,0 +1,17 @@
+=====
+Rules
+=====
+
+A rule is a function that takes:
+
+Amount, Transaction Details, Category, Merchant Name, and f
+It is matched on Account Number + Transaction Type
+f is a map that returns the account-to of the transaction, based on it's applies-to and condition. 
+This means we can describe a set of rules as map containing::
+
+  { :applies-to (the Account No + Transaction Type combo)
+    :function   (the function which will return account-to)
+    :condition (whether this is the rule for the transaction)
+  }
+
+We then need a standard way to pass all the elements we care about to applies-to 
--- a/flake.lock
+++ b/flake.lock
@ -0,0 +1,26 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1665753223,
+        "narHash": "sha256-RsRUlQG3WaeqdWvuCaPvjptMEoq4yck2Gsj0lTklbk0=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "6594eb2fc86e22005f48aaeb1011b5a72d99e58e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/flake.nix
+++ b/flake.nix
@ -0,0 +1,38 @@
+{
+  description = "Beancount Importer";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs";
+  };
+
+  outputs = { self, nixpkgs }:
+    let
+      pkgs = nixpkgs.legacyPackages.x86_64-linux;
+    in
+    {
+
+      packages.x86_64-linux.muneem = pkgs.stdenv.mkDerivation {
+        name = "muneem";
+        src = ./.;
+        installPhase = ''
+          mkdir -p $out
+          cp -r src/ $out/
+          cp bb.edn $out/
+          cp deps.edn $out/
+        '';
+      };
+
+      defaultPackage.x86_64-linux = self.packages.x86_64-linux.muneem;
+      devShell.x86_64-linux = pkgs.mkShell {
+        buildInputs = with pkgs; [
+          jdk11
+          babashka
+          geckodriver
+          chromedriver
+          chromium
+        ];
+        shellHook = ''
+        '';
+      };
+    };
+}
--- a/readme.rst
+++ b/readme.rst
@ -0,0 +1,52 @@
+======
+Muneem
+======
+
+TODO
+----
+- [ ] Download Savings acct CSV
+- [ ] Import Savings acct CSV
+- [ ] RECONCILE
+- [ ] Download NAB trade transactions and value
+- [ ] Download spaceship returns
+- [ ] `pad` checking / savings accounts against some known value way in the past
+- [ ] MAYBE gonna have to write a rule engine that handles this
+- [ ] MAYBE Parse the transaction only once at the start, and then use the parsed map instead of passing around a vector of strings
+- [ ] Mention uncategorized transactions in matrix chat so that i can categorize it 
+- [ ] Post weekly spending in matrix chat
+
+
+DONE
+----
+- [x] Read the CSV(s)
+- [x] Figure out a way to download both CSVs from each account
+- [x] Figure out what columns match to what data we care about
+- [x] Marshall the data into a way that bean count understands
+- [x] Open accounts that previously didn't exist
+- [x] Write the transactions to the file
+
+beancount:: 
+
+  Beancount Format
+  2016-12-06 * "Biang!" "Dinner"
+    Liabilities:CreditCard   -47.23 USD
+    Expenses:Restaurants
+
+
+-----------
+Assumptions
+-----------
+
+We'll also be making assumptions about the way accounts are structured. Since NAB provides categories we can put that into the account
+
+=================== ==================================== =============== =======
+Transaction Type    To                                   From            Misc
+=================== ==================================== =============== =======
+EFTPOS DEBIT        Expenses:<Category>:<Merchant Name>  Assets:Checking
+TRANSFER CREDIT     Assets:Checking                      Assets:Savings
+MISCELLANEOUS DEBIT Expenses:<Category>:<Merchant Name>  Assets:Checking
+TRANSFER DEBIT      Expenses:Rent                        Assets:Checking IFF: (= amount 170) 
+
+Additionally: 
+- we'll may need to use account nunmber for distinguishing different CSVs
+- The actual date is stored in Transaction details, which we can use as the comment
--- a/src/muneem/etaoin.clj
+++ b/src/muneem/etaoin.clj
@ -0,0 +1,43 @@
+(ns muneem.etaoin
+  (:require [etaoin.api :as e]
+            [clojure.java.shell :refer [sh]]
+            [etaoin.keys :as k]))
+
+(def driver (e/chrome {:args ["--disable-blink-features=AutomationControlled" "--maximize"]}))
+                       ;:download-dir "./documents/"
+
+(defn download-csv
+  []
+  (e/js-execute driver "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
+  (e/go driver "https://ib.nab.com.au/nabib/index.jsp")
+  (e/wait-visible driver [{:id :username}])
+  ; Don't advertise that we're a bot
+  (e/fill driver {:id :username} (System/getenv "NAB_USER"))
+  (e/fill driver {:id :password} (System/getenv "NAB_PASSWORD"))
+  (e/fill driver {:id :password} k/enter)
+  (e/wait 4)
+  (e/click driver {:tag :button :id :hamburger-btn})
+  (e/click-el driver (e/query driver {:css "li.selected>button"}))
+  (e/click-el driver (e/query driver {:tag :a :fn/text "Transaction history"}))
+  (e/wait 4)
+  ; Update to last 7 days
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-toggle-button-DateDropdown\").click()")
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-listbox-DateDropdown-0\").click()")
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div.Containerstyle__StyledWrapper-sc-10ei1r0-0.dnudTa > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > form > div > div:nth-child(3) > div > button\").click()")
+  (e/wait 2)
+  ; Download checking file
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#export-button > div\").click()")
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div:nth-child(2) > div > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > div > div > div > div.StyledTopRow-cijGbF.StyledExportCol-fYxmhQ.ibndGm.cAjmIO.sc-dkPtRN.Colstyle__StyledCol-sc-1evc4kf-0.ketKXK.fqofCs > div > div > div > ul > li:nth-child(1) > button\").click()")
+  (e/wait 3)
+  ; Download savings file
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-toggle-button-accountsListDropdown > div > div > div\")")
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-listbox-accountsListDropdown-1 > div > div > div.AccountCardstyle__StyledAccountTypeContainer-sc-16p3i3w-1.cLXpwB\")")
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#export-button > div\").click()")
+  (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div:nth-child(2) > div > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > div > div > div > div.StyledTopRow-cijGbF.StyledExportCol-fYxmhQ.ibndGm.cAjmIO.sc-dkPtRN.Colstyle__StyledCol-sc-1evc4kf-0.ketKXK.fqofCs > div > div > div > ul > li:nth-child(1) > button\").click()")
+  (e/wait 3)
+  (e/quit driver))
+
+(defn move-download
+  []
+  (sh "mv" "~/Downloads/Transactions.csv" "./documents/checking.csv")
+  (sh "mv" "~/Downloads/Transactions\(1\).csv" "./documents/savings.csv"))
--- a/src/muneem/import.clj
+++ b/src/muneem/import.clj
@ -0,0 +1,346 @@
+(ns muneem.import
+  (:require [clojure.data.csv :as csv]
+            [clojure.java.io]
+            [cljc.java-time.local-date :as jt]
+            [cljc.java-time.format.date-time-formatter :as pt]
+            [clojure.string]))
+
+(def beancount-file "./documents/test.beancount")
+(def transaction-details (partial #(nth % 5)))
+(def category (partial #(nth % 7)))
+(defn merchant-name [trans]
+  (if (= (count trans) 9)
+    (nth trans 8)
+    ""))
+(def amount (partial #(nth % 1)))
+(def trans-date (partial #(nth % 0)))
+(def transaction-type (partial #(nth % 4)))
+
+(defn take-csv
+  "Takes file name and reads data."
+  [file]
+  (-> (csv/read-csv (slurp file))
+      ; We don't care about the header
+      (rest)))
+
+(defn make-date
+  "Reads date from the 5th column (Transaction Details)
+  The 2nd element split by space is DD/MM of the actal transaction
+  Add 2022 to that and use it as the transaction date"
+  [trans statement]
+  ; TODO this doesn't work for various transactions and other years
+  ; Parse real date -> check that it is in fact a date
+  ; Use normal date to determine year
+  ; If it's not a date, use regular date
+  (let [real-date (-> trans
+                      transaction-details
+                      (clojure.string/split #" ")
+                      second)
+        trans-date-year (-> (trans-date trans)
+                            (clojure.string/split #" ")
+                            (last)
+                            (->> (str "20")))
+        with-year (-> real-date
+                      (str "/" trans-date-year)
+                      (clojure.string/split #"/")
+                      reverse)
+        parse-year (-> (trans-date trans)
+                       (jt/parse (pt/of-pattern "dd MMM uu"))
+                       (jt/format (pt/of-pattern "YYYY-MM-dd")))]
+    (if (and (not (nil? real-date)) (re-matches #"../.." real-date))
+      (conj statement (clojure.string/join "-" with-year))
+      (conj statement parse-year))))
+
+(defn add-transaction-flag
+  "Bean count requires transaction type, we use * which means txn"
+  [statement]
+  (conj statement "*"))
+
+(defn add-narration
+  "Add some comments about the transaction from the 5th column (Transaction Details)"
+  [trans statement]
+  (let [narration (-> trans
+                      transaction-details
+                      (clojure.string/split #" ")
+                      ; first 2 are payment method and date
+                      rest
+                      rest
+                      (conj "\"")
+                      (cons "\""))]
+    (conj statement narration)))
+
+(defn sanitize
+  [string]
+  (-> string
+      (clojure.string/capitalize)
+      (clojure.string/replace #" " "")
+      (clojure.string/replace #"'" "")
+      (clojure.string/replace #"\." "")
+      (clojure.string/replace #"\(.+\)" "")
+      (clojure.string/replace #"&" "-")))
+
+; These should all be the same method tbh
+; Or they should be a map?
+; check out how-to-rules for more
+
+(defn add-account-from-and-amount
+  "We're only handling checking account for now
+  All amounts are from it's perspsective"
+  [trans statement]
+  (let [amount (amount trans)
+        account "Assets:Checking"
+        string (str "  " account "     " amount " AUD")]
+    (conj statement string)))
+
+(defn add-account-from-and-amount-savings
+  [trans statement]
+  (let [amount (amount trans)
+        account "Assets:Savings"
+        string (str "  " account "     " amount " AUD")]
+    (conj statement string)))
+
+(defn debit-account-to
+  [trans statement]
+  (let [category (sanitize (category trans))
+        merchant-name (sanitize (merchant-name trans))
+                   ; Uncategorized strings need to be reconciled
+                   ; TODO we'd probably want rules based on the transaction details
+                   ; to automatically categorize further ones
+        account-to (if (= category "Uncategorised")
+                     (if (> (count merchant-name) 0)
+                       (str "  ! Expenses:" category ":" merchant-name)
+                       (str "  ! Expenses:" category))
+                     (if (> (count merchant-name) 0)
+                       (str "  Expenses:" category ":" merchant-name)
+                       (str "  Expenses:" category)))]
+    (conj statement account-to)))
+
+(defn fees-account-to
+  [trans statement]
+  (let [category (sanitize (category trans))
+        merchant-name (sanitize (merchant-name trans))
+        account-to (str "  Expenses:Fees")]
+    (conj statement account-to)))
+
+(defn atm-debit-account-to
+  [trans statement]
+  (let [category (sanitize (category trans))
+        merchant-name (sanitize (merchant-name trans))
+        account-to (str "  Assets:Cash")]
+    (conj statement account-to)))
+
+(defn transfer-credit-account-to
+  [_ statement]
+  (let [account-to (str "  Assets:Savings")]
+    (conj statement account-to)))
+
+(defn bank-credit-account-to
+  [_ statement]
+  (let [account-to (str "  ! Income:ThreatDefence")]
+    (conj statement account-to)))
+
+(defn dividend-savings
+  [_ statement]
+  (let [account-to (str "  Income:Dividends")]
+    (conj statement account-to)))
+
+(defn salary-savings
+  [_ statement]
+  (let [account-to (str "  Income:Cubiko")]
+    (conj statement account-to)))
+
+(defn interest-account-to
+  [_ statement]
+  (let [account-to (str "  Expenses:NAB")]
+    (conj statement account-to)))
+
+(defn interest-savings
+  [_ statement]
+  (let [account-to (str "  Income:NAB")]
+    (conj statement account-to)))
+
+(defn automatic-drawing-account-to
+  [_ statement]
+  (let [account-to (str "  Assets:Spaceship")]
+    (conj statement account-to)))
+
+(defn misc-credit-account-to
+  [_ statement]
+  (let [account-to (str "  Income:Beem")]
+    (conj statement account-to)))
+
+(defn deposit-account-to
+  "This function is going to change a lot
+  It currently only works with Expenses
+  but in the future will need to handle determining account to"
+  [trans statement]
+  (let [account-to (str "  ! Expenses:OverCharged")]
+    (conj statement account-to)))
+
+(defn transfer-debit-account-to
+  "This function is going to change a lot
+  It currently only works with Expenses
+  but in the future will need to handle determining account to"
+  [trans statement]
+  (let [rent (str "  Expenses:Rent")
+        trans-type (transaction-type trans)
+        amt (amount trans)
+        else-account (str "  Expenses:" (-> trans-type
+                                            (clojure.string/split #" ")
+                                            (nth 1)))]
+    (if (= amt 170)
+      (conj statement rent)
+      (conj statement else-account))))
+
+(defn transfer-debit-savings
+  "This function is going to change a lot
+  It currently only works with Expenses
+  but in the future will need to handle determining account to"
+  [trans statement]
+  (let [account-to (str "  Assets:Checking")]
+    (conj statement account-to)))
+
+;; RULES
+; {:applies-to (fn [{trans-type :transaction-type 
+;                    amount :amount
+;                    category :category
+;                    merchant-name :merchant-name)))
+
+(defn determine-category
+  "We probably want some sort of rule engine 
+  that takes rules and applies the first match or something (see docs)
+  but this will do for now
+  
+  Writes out the account-to to the statement"
+  [trans statement]
+  (let [trans-type (transaction-type trans)]
+    (case trans-type
+      "EFTPOS DEBIT" (debit-account-to trans statement)
+      "TRANSFER CREDIT" (transfer-credit-account-to trans statement)
+      "MISCELLANEOUS DEBIT" (debit-account-to trans statement)
+      "TRANSFER DEBIT" (transfer-debit-account-to trans statement)
+      "INTEREST PAID" (interest-account-to trans statement)
+      "AUTOMATIC DRAWING" (automatic-drawing-account-to trans statement)
+      "FEES" (fees-account-to trans statement)
+      "REVERSAL CREDIT" (automatic-drawing-account-to trans statement)
+      "ATM DEBIT" (atm-debit-account-to trans statement)
+      "DEPOSIT" (deposit-account-to trans statement)
+      "INTER-BANK CREDIT" (bank-credit-account-to trans statement)
+      "MISCELLANEOUS CREDIT" (misc-credit-account-to trans statement))))
+
+(defn determine-category
+  "We probably want some sort of rule engine 
+  that takes rules and applies the first match or something (see docs)
+  but this will do for now
+  
+  Writes out the account-to to the statement"
+  [trans statement]
+  (let [trans-type (transaction-type trans)]
+    (case trans-type
+      "TRANSFER DEBIT" (transfer-debit-savings trans statement)
+      "INTEREST PAID" (interest-savings trans statement)
+      "INTER-BANK CREDIT" (bank-credit-account-to trans statement)
+      "DIVIDEND" (dividend-savings trans statement)
+      "SALARY" (salary-savings trans statement))))
+
+(defn write-statement
+  [statement]
+  (spit beancount-file (clojure.string/join "\n" (conj statement "\n")) :append true))
+
+(defn tap
+  [s]
+  (println s)
+  s)
+
+(defn organize-correctly
+  [statement]
+  (-> statement
+      (flatten)
+      (into [])
+      (->> (clojure.string/join " "))
+      (->> (conj []))))
+
+(def accounts (atom #{}))
+(def new-accounts (atom #{}))
+
+(defn create-account
+  [transaction]
+  (let [to-account (second (clojure.string/split (second transaction) #"  "))
+        from-account (second (clojure.string/split (nth transaction 2) #"  "))
+        marshalled-from-account (clojure.string/replace from-account #"! " "")]
+    (if (not (contains? (deref accounts) to-account))
+      (swap! new-accounts conj to-account))
+    (if (not (contains? (deref accounts) marshalled-from-account))
+      (swap! new-accounts conj marshalled-from-account))
+    transaction))
+
+(defn write-transaction-to-file
+  [transaction]
+  (->> []
+       (make-date transaction)
+       (add-transaction-flag)
+       (add-narration transaction)
+       (organize-correctly)
+       (add-account-from-and-amount transaction)
+       (determine-category transaction)
+       #_(tap)
+       (create-account)
+       (write-statement)))
+
+(defn write-transaction-to-file-savings
+  [transaction]
+  (->> []
+       (make-date transaction)
+       (add-transaction-flag)
+       (add-narration transaction)
+       (organize-correctly)
+       (add-account-from-and-amount-savings transaction)
+       (determine-category transaction)
+       #_(tap)
+       (create-account)
+       (write-statement)))
+
+(defn index-of [x coll]
+  (let [idx? (fn [i a] (when (= x a) i))]
+    (first (keep-indexed idx? coll))))
+
+(defn get-account
+  "Returns the account from statements where the account is opened"
+  [account-info]
+  (let [split (into [] (clojure.string/split account-info #" "))
+        account (last split)]
+    account))
+
+(defn read-accounts
+  [ledger]
+  (let [file-contents (slurp ledger)
+        split (into [] (rest (clojure.string/split file-contents #"\n")))
+        empty-index (index-of "" split)
+        only-accounts (if (not (nil? empty-index))
+                        (subvec split 0 empty-index)
+                        split)]
+    (->> (mapv get-account only-accounts)
+         (set)
+         (reset! accounts))))
+
+(defn open-account-string
+  [account]
+  (str "2016-01-01 open " account))
+
+(defn write-accounts
+  [ledger]
+  (let [file-contents (slurp ledger)
+        split (clojure.string/split file-contents #"\n")
+        empty-index (index-of "" split)
+        [before after] (split-at empty-index split)
+        middle (mapv open-account-string (into [] (deref new-accounts)))
+        full-file (vec (concat before middle after "\n"))]
+    (spit ledger (clojure.string/join "\n" full-file))))
+
+(defn main
+  [csv checking? & _args]
+  (read-accounts beancount-file)
+  (if checking?
+    (mapv write-transaction-to-file (drop-last (take-csv csv)))
+    (mapv write-transaction-to-file-savings (drop-last (take-csv csv))))
+  (write-accounts beancount-file))
--- a/src/muneem/main.clj
+++ b/src/muneem/main.clj
@ -0,0 +1,12 @@
+(ns muneem.main
+  (:require [muneem.import :as import]
+            [muneem.etaoin :as download]))
+
+(defn -main
+  [& _args]
+  (download/download-csv)
+  (download/move-download)
+  (import/main "./documents/checking.csv" true)
+  (import/main "./documents/savings.csv" true))
+
+(-main)