From 2c820b58951f00f3b48071a194a5274b75138d16 Mon Sep 17 00:00:00 2001 From: Anish Lakhwara Date: Fri, 25 Nov 2022 23:25:49 +1000 Subject: [PATCH] we had to start over because i leaked passwords... --- .gitignore | 1 + bb.edn | 2 + deps.edn | 30 ++++ docs/how-to-rules.rst | 17 +++ flake.lock | 26 ++++ flake.nix | 38 +++++ readme.rst | 52 +++++++ src/muneem/etaoin.clj | 43 ++++++ src/muneem/import.clj | 346 ++++++++++++++++++++++++++++++++++++++++++ src/muneem/main.clj | 12 ++ 10 files changed, 567 insertions(+) create mode 100644 .gitignore create mode 100644 bb.edn create mode 100644 deps.edn create mode 100644 docs/how-to-rules.rst create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 readme.rst create mode 100644 src/muneem/etaoin.clj create mode 100644 src/muneem/import.clj create mode 100644 src/muneem/main.clj diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2ddf3a1 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +documents/** diff --git a/bb.edn b/bb.edn new file mode 100644 index 0000000..80e98b4 --- /dev/null +++ b/bb.edn @@ -0,0 +1,2 @@ +{:path ["src"] + :deps {local/deps {:local/root "."}}} diff --git a/deps.edn b/deps.edn new file mode 100644 index 0000000..d41fc68 --- /dev/null +++ b/deps.edn @@ -0,0 +1,30 @@ +{:deps + {selmer/selmer {:mvn/version "1.12.50"}, + org.clojure/clojure {:mvn/version "1.11.1"}, + org.clojure/tools.logging {:mvn/version "1.1.0"}, + insn/insn {:mvn/version "0.5.2"}, + org.clojure/tools.cli {:mvn/version "1.0.206"}, + org.babashka/babashka.impl.reify {:mvn/version "0.1.1"}, + rewrite-clj/rewrite-clj {:mvn/version "1.0.699-alpha"}, + babashka/babashka.core + {:git/url "https://github.com/babashka/babashka.core", + :git/sha "52a6037bd4b632bffffb04394fb4efd0cdab6b1e"}, + hiccup/hiccup {:mvn/version "2.0.0-alpha2"}, + org.clojure/data.xml {:mvn/version "0.2.0-alpha6"}, + org.clojure/data.csv {:mvn/version "1.0.0"}, + com.taoensso/timbre {:mvn/version "5.2.1"}, + com.cognitect/transit-clj {:mvn/version "1.0.329"}, + babashka/babashka.curl {:mvn/version "0.1.1"}, + clj-commons/clj-yaml {:mvn/version "0.7.108"}, + org.clojure/core.rrb-vector {:mvn/version "0.1.2"}, + org.babashka/sci.impl.types {:mvn/version "0.0.2"}, + cheshire/cheshire {:mvn/version "5.11.0"}, + org.clojure/core.match {:mvn/version "1.0.0"}, + org.clojure/test.check {:mvn/version "1.1.1"}, + babashka/fs {:mvn/version "0.1.2"}, + cljc.java-time/cljc.java-time {:mvn/version "0.1.12"}, + org.clojure/data.priority-map {:mvn/version "1.1.0"}, + nrepl/bencode {:mvn/version "1.1.0"}, + org.clojure/core.async {:mvn/version "1.5.648"}, + http-kit/http-kit {:mvn/version "2.6.0-RC1"} + etaoin/etaoin {:mvn/version "1.0.38"}}} diff --git a/docs/how-to-rules.rst b/docs/how-to-rules.rst new file mode 100644 index 0000000..248a4b0 --- /dev/null +++ b/docs/how-to-rules.rst @@ -0,0 +1,17 @@ +===== +Rules +===== + +A rule is a function that takes: + +Amount, Transaction Details, Category, Merchant Name, and f +It is matched on Account Number + Transaction Type +f is a map that returns the account-to of the transaction, based on it's applies-to and condition. +This means we can describe a set of rules as map containing:: + + { :applies-to (the Account No + Transaction Type combo) + :function (the function which will return account-to) + :condition (whether this is the rule for the transaction) + } + +We then need a standard way to pass all the elements we care about to applies-to diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..e3ab8b4 --- /dev/null +++ b/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1665753223, + "narHash": "sha256-RsRUlQG3WaeqdWvuCaPvjptMEoq4yck2Gsj0lTklbk0=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "6594eb2fc86e22005f48aaeb1011b5a72d99e58e", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..44a32d8 --- /dev/null +++ b/flake.nix @@ -0,0 +1,38 @@ +{ + description = "Beancount Importer"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs"; + }; + + outputs = { self, nixpkgs }: + let + pkgs = nixpkgs.legacyPackages.x86_64-linux; + in + { + + packages.x86_64-linux.muneem = pkgs.stdenv.mkDerivation { + name = "muneem"; + src = ./.; + installPhase = '' + mkdir -p $out + cp -r src/ $out/ + cp bb.edn $out/ + cp deps.edn $out/ + ''; + }; + + defaultPackage.x86_64-linux = self.packages.x86_64-linux.muneem; + devShell.x86_64-linux = pkgs.mkShell { + buildInputs = with pkgs; [ + jdk11 + babashka + geckodriver + chromedriver + chromium + ]; + shellHook = '' + ''; + }; + }; +} diff --git a/readme.rst b/readme.rst new file mode 100644 index 0000000..ccb62e3 --- /dev/null +++ b/readme.rst @@ -0,0 +1,52 @@ +====== +Muneem +====== + +TODO +---- +- [ ] Download Savings acct CSV +- [ ] Import Savings acct CSV +- [ ] RECONCILE +- [ ] Download NAB trade transactions and value +- [ ] Download spaceship returns +- [ ] `pad` checking / savings accounts against some known value way in the past +- [ ] MAYBE gonna have to write a rule engine that handles this +- [ ] MAYBE Parse the transaction only once at the start, and then use the parsed map instead of passing around a vector of strings +- [ ] Mention uncategorized transactions in matrix chat so that i can categorize it +- [ ] Post weekly spending in matrix chat + + +DONE +---- +- [x] Read the CSV(s) +- [x] Figure out a way to download both CSVs from each account +- [x] Figure out what columns match to what data we care about +- [x] Marshall the data into a way that bean count understands +- [x] Open accounts that previously didn't exist +- [x] Write the transactions to the file + +beancount:: + + Beancount Format + 2016-12-06 * "Biang!" "Dinner" + Liabilities:CreditCard -47.23 USD + Expenses:Restaurants + + +----------- +Assumptions +----------- + +We'll also be making assumptions about the way accounts are structured. Since NAB provides categories we can put that into the account + +=================== ==================================== =============== ======= +Transaction Type To From Misc +=================== ==================================== =============== ======= +EFTPOS DEBIT Expenses:: Assets:Checking +TRANSFER CREDIT Assets:Checking Assets:Savings +MISCELLANEOUS DEBIT Expenses:: Assets:Checking +TRANSFER DEBIT Expenses:Rent Assets:Checking IFF: (= amount 170) + +Additionally: +- we'll may need to use account nunmber for distinguishing different CSVs +- The actual date is stored in Transaction details, which we can use as the comment diff --git a/src/muneem/etaoin.clj b/src/muneem/etaoin.clj new file mode 100644 index 0000000..dfb8ac4 --- /dev/null +++ b/src/muneem/etaoin.clj @@ -0,0 +1,43 @@ +(ns muneem.etaoin + (:require [etaoin.api :as e] + [clojure.java.shell :refer [sh]] + [etaoin.keys :as k])) + +(def driver (e/chrome {:args ["--disable-blink-features=AutomationControlled" "--maximize"]})) + ;:download-dir "./documents/" + +(defn download-csv + [] + (e/js-execute driver "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") + (e/go driver "https://ib.nab.com.au/nabib/index.jsp") + (e/wait-visible driver [{:id :username}]) + ; Don't advertise that we're a bot + (e/fill driver {:id :username} (System/getenv "NAB_USER")) + (e/fill driver {:id :password} (System/getenv "NAB_PASSWORD")) + (e/fill driver {:id :password} k/enter) + (e/wait 4) + (e/click driver {:tag :button :id :hamburger-btn}) + (e/click-el driver (e/query driver {:css "li.selected>button"})) + (e/click-el driver (e/query driver {:tag :a :fn/text "Transaction history"})) + (e/wait 4) + ; Update to last 7 days + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-toggle-button-DateDropdown\").click()") + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-listbox-DateDropdown-0\").click()") + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div.Containerstyle__StyledWrapper-sc-10ei1r0-0.dnudTa > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > form > div > div:nth-child(3) > div > button\").click()") + (e/wait 2) + ; Download checking file + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#export-button > div\").click()") + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div:nth-child(2) > div > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > div > div > div > div.StyledTopRow-cijGbF.StyledExportCol-fYxmhQ.ibndGm.cAjmIO.sc-dkPtRN.Colstyle__StyledCol-sc-1evc4kf-0.ketKXK.fqofCs > div > div > div > ul > li:nth-child(1) > button\").click()") + (e/wait 3) + ; Download savings file + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-toggle-button-accountsListDropdown > div > div > div\")") + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-listbox-accountsListDropdown-1 > div > div > div.AccountCardstyle__StyledAccountTypeContainer-sc-16p3i3w-1.cLXpwB\")") + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#export-button > div\").click()") + (e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div:nth-child(2) > div > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > div > div > div > div.StyledTopRow-cijGbF.StyledExportCol-fYxmhQ.ibndGm.cAjmIO.sc-dkPtRN.Colstyle__StyledCol-sc-1evc4kf-0.ketKXK.fqofCs > div > div > div > ul > li:nth-child(1) > button\").click()") + (e/wait 3) + (e/quit driver)) + +(defn move-download + [] + (sh "mv" "~/Downloads/Transactions.csv" "./documents/checking.csv") + (sh "mv" "~/Downloads/Transactions\(1\).csv" "./documents/savings.csv")) diff --git a/src/muneem/import.clj b/src/muneem/import.clj new file mode 100644 index 0000000..a9e0e85 --- /dev/null +++ b/src/muneem/import.clj @@ -0,0 +1,346 @@ +(ns muneem.import + (:require [clojure.data.csv :as csv] + [clojure.java.io] + [cljc.java-time.local-date :as jt] + [cljc.java-time.format.date-time-formatter :as pt] + [clojure.string])) + +(def beancount-file "./documents/test.beancount") +(def transaction-details (partial #(nth % 5))) +(def category (partial #(nth % 7))) +(defn merchant-name [trans] + (if (= (count trans) 9) + (nth trans 8) + "")) +(def amount (partial #(nth % 1))) +(def trans-date (partial #(nth % 0))) +(def transaction-type (partial #(nth % 4))) + +(defn take-csv + "Takes file name and reads data." + [file] + (-> (csv/read-csv (slurp file)) + ; We don't care about the header + (rest))) + +(defn make-date + "Reads date from the 5th column (Transaction Details) + The 2nd element split by space is DD/MM of the actal transaction + Add 2022 to that and use it as the transaction date" + [trans statement] + ; TODO this doesn't work for various transactions and other years + ; Parse real date -> check that it is in fact a date + ; Use normal date to determine year + ; If it's not a date, use regular date + (let [real-date (-> trans + transaction-details + (clojure.string/split #" ") + second) + trans-date-year (-> (trans-date trans) + (clojure.string/split #" ") + (last) + (->> (str "20"))) + with-year (-> real-date + (str "/" trans-date-year) + (clojure.string/split #"/") + reverse) + parse-year (-> (trans-date trans) + (jt/parse (pt/of-pattern "dd MMM uu")) + (jt/format (pt/of-pattern "YYYY-MM-dd")))] + (if (and (not (nil? real-date)) (re-matches #"../.." real-date)) + (conj statement (clojure.string/join "-" with-year)) + (conj statement parse-year)))) + +(defn add-transaction-flag + "Bean count requires transaction type, we use * which means txn" + [statement] + (conj statement "*")) + +(defn add-narration + "Add some comments about the transaction from the 5th column (Transaction Details)" + [trans statement] + (let [narration (-> trans + transaction-details + (clojure.string/split #" ") + ; first 2 are payment method and date + rest + rest + (conj "\"") + (cons "\""))] + (conj statement narration))) + +(defn sanitize + [string] + (-> string + (clojure.string/capitalize) + (clojure.string/replace #" " "") + (clojure.string/replace #"'" "") + (clojure.string/replace #"\." "") + (clojure.string/replace #"\(.+\)" "") + (clojure.string/replace #"&" "-"))) + +; These should all be the same method tbh +; Or they should be a map? +; check out how-to-rules for more + +(defn add-account-from-and-amount + "We're only handling checking account for now + All amounts are from it's perspsective" + [trans statement] + (let [amount (amount trans) + account "Assets:Checking" + string (str " " account " " amount " AUD")] + (conj statement string))) + +(defn add-account-from-and-amount-savings + [trans statement] + (let [amount (amount trans) + account "Assets:Savings" + string (str " " account " " amount " AUD")] + (conj statement string))) + +(defn debit-account-to + [trans statement] + (let [category (sanitize (category trans)) + merchant-name (sanitize (merchant-name trans)) + ; Uncategorized strings need to be reconciled + ; TODO we'd probably want rules based on the transaction details + ; to automatically categorize further ones + account-to (if (= category "Uncategorised") + (if (> (count merchant-name) 0) + (str " ! Expenses:" category ":" merchant-name) + (str " ! Expenses:" category)) + (if (> (count merchant-name) 0) + (str " Expenses:" category ":" merchant-name) + (str " Expenses:" category)))] + (conj statement account-to))) + +(defn fees-account-to + [trans statement] + (let [category (sanitize (category trans)) + merchant-name (sanitize (merchant-name trans)) + account-to (str " Expenses:Fees")] + (conj statement account-to))) + +(defn atm-debit-account-to + [trans statement] + (let [category (sanitize (category trans)) + merchant-name (sanitize (merchant-name trans)) + account-to (str " Assets:Cash")] + (conj statement account-to))) + +(defn transfer-credit-account-to + [_ statement] + (let [account-to (str " Assets:Savings")] + (conj statement account-to))) + +(defn bank-credit-account-to + [_ statement] + (let [account-to (str " ! Income:ThreatDefence")] + (conj statement account-to))) + +(defn dividend-savings + [_ statement] + (let [account-to (str " Income:Dividends")] + (conj statement account-to))) + +(defn salary-savings + [_ statement] + (let [account-to (str " Income:Cubiko")] + (conj statement account-to))) + +(defn interest-account-to + [_ statement] + (let [account-to (str " Expenses:NAB")] + (conj statement account-to))) + +(defn interest-savings + [_ statement] + (let [account-to (str " Income:NAB")] + (conj statement account-to))) + +(defn automatic-drawing-account-to + [_ statement] + (let [account-to (str " Assets:Spaceship")] + (conj statement account-to))) + +(defn misc-credit-account-to + [_ statement] + (let [account-to (str " Income:Beem")] + (conj statement account-to))) + +(defn deposit-account-to + "This function is going to change a lot + It currently only works with Expenses + but in the future will need to handle determining account to" + [trans statement] + (let [account-to (str " ! Expenses:OverCharged")] + (conj statement account-to))) + +(defn transfer-debit-account-to + "This function is going to change a lot + It currently only works with Expenses + but in the future will need to handle determining account to" + [trans statement] + (let [rent (str " Expenses:Rent") + trans-type (transaction-type trans) + amt (amount trans) + else-account (str " Expenses:" (-> trans-type + (clojure.string/split #" ") + (nth 1)))] + (if (= amt 170) + (conj statement rent) + (conj statement else-account)))) + +(defn transfer-debit-savings + "This function is going to change a lot + It currently only works with Expenses + but in the future will need to handle determining account to" + [trans statement] + (let [account-to (str " Assets:Checking")] + (conj statement account-to))) + +;; RULES +; {:applies-to (fn [{trans-type :transaction-type +; amount :amount +; category :category +; merchant-name :merchant-name))) + +(defn determine-category + "We probably want some sort of rule engine + that takes rules and applies the first match or something (see docs) + but this will do for now + + Writes out the account-to to the statement" + [trans statement] + (let [trans-type (transaction-type trans)] + (case trans-type + "EFTPOS DEBIT" (debit-account-to trans statement) + "TRANSFER CREDIT" (transfer-credit-account-to trans statement) + "MISCELLANEOUS DEBIT" (debit-account-to trans statement) + "TRANSFER DEBIT" (transfer-debit-account-to trans statement) + "INTEREST PAID" (interest-account-to trans statement) + "AUTOMATIC DRAWING" (automatic-drawing-account-to trans statement) + "FEES" (fees-account-to trans statement) + "REVERSAL CREDIT" (automatic-drawing-account-to trans statement) + "ATM DEBIT" (atm-debit-account-to trans statement) + "DEPOSIT" (deposit-account-to trans statement) + "INTER-BANK CREDIT" (bank-credit-account-to trans statement) + "MISCELLANEOUS CREDIT" (misc-credit-account-to trans statement)))) + +(defn determine-category + "We probably want some sort of rule engine + that takes rules and applies the first match or something (see docs) + but this will do for now + + Writes out the account-to to the statement" + [trans statement] + (let [trans-type (transaction-type trans)] + (case trans-type + "TRANSFER DEBIT" (transfer-debit-savings trans statement) + "INTEREST PAID" (interest-savings trans statement) + "INTER-BANK CREDIT" (bank-credit-account-to trans statement) + "DIVIDEND" (dividend-savings trans statement) + "SALARY" (salary-savings trans statement)))) + +(defn write-statement + [statement] + (spit beancount-file (clojure.string/join "\n" (conj statement "\n")) :append true)) + +(defn tap + [s] + (println s) + s) + +(defn organize-correctly + [statement] + (-> statement + (flatten) + (into []) + (->> (clojure.string/join " ")) + (->> (conj [])))) + +(def accounts (atom #{})) +(def new-accounts (atom #{})) + +(defn create-account + [transaction] + (let [to-account (second (clojure.string/split (second transaction) #" ")) + from-account (second (clojure.string/split (nth transaction 2) #" ")) + marshalled-from-account (clojure.string/replace from-account #"! " "")] + (if (not (contains? (deref accounts) to-account)) + (swap! new-accounts conj to-account)) + (if (not (contains? (deref accounts) marshalled-from-account)) + (swap! new-accounts conj marshalled-from-account)) + transaction)) + +(defn write-transaction-to-file + [transaction] + (->> [] + (make-date transaction) + (add-transaction-flag) + (add-narration transaction) + (organize-correctly) + (add-account-from-and-amount transaction) + (determine-category transaction) + #_(tap) + (create-account) + (write-statement))) + +(defn write-transaction-to-file-savings + [transaction] + (->> [] + (make-date transaction) + (add-transaction-flag) + (add-narration transaction) + (organize-correctly) + (add-account-from-and-amount-savings transaction) + (determine-category transaction) + #_(tap) + (create-account) + (write-statement))) + +(defn index-of [x coll] + (let [idx? (fn [i a] (when (= x a) i))] + (first (keep-indexed idx? coll)))) + +(defn get-account + "Returns the account from statements where the account is opened" + [account-info] + (let [split (into [] (clojure.string/split account-info #" ")) + account (last split)] + account)) + +(defn read-accounts + [ledger] + (let [file-contents (slurp ledger) + split (into [] (rest (clojure.string/split file-contents #"\n"))) + empty-index (index-of "" split) + only-accounts (if (not (nil? empty-index)) + (subvec split 0 empty-index) + split)] + (->> (mapv get-account only-accounts) + (set) + (reset! accounts)))) + +(defn open-account-string + [account] + (str "2016-01-01 open " account)) + +(defn write-accounts + [ledger] + (let [file-contents (slurp ledger) + split (clojure.string/split file-contents #"\n") + empty-index (index-of "" split) + [before after] (split-at empty-index split) + middle (mapv open-account-string (into [] (deref new-accounts))) + full-file (vec (concat before middle after "\n"))] + (spit ledger (clojure.string/join "\n" full-file)))) + +(defn main + [csv checking? & _args] + (read-accounts beancount-file) + (if checking? + (mapv write-transaction-to-file (drop-last (take-csv csv))) + (mapv write-transaction-to-file-savings (drop-last (take-csv csv)))) + (write-accounts beancount-file)) diff --git a/src/muneem/main.clj b/src/muneem/main.clj new file mode 100644 index 0000000..2dd581c --- /dev/null +++ b/src/muneem/main.clj @@ -0,0 +1,12 @@ +(ns muneem.main + (:require [muneem.import :as import] + [muneem.etaoin :as download])) + +(defn -main + [& _args] + (download/download-csv) + (download/move-download) + (import/main "./documents/checking.csv" true) + (import/main "./documents/savings.csv" true)) + +(-main)