we had to start over because i leaked passwords...

main
Anish Lakhwara 2022-11-25 23:25:49 +10:00
commit 2c820b5895
10 changed files with 567 additions and 0 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
documents/**

2
bb.edn Normal file
View File

@ -0,0 +1,2 @@
{:path ["src"]
:deps {local/deps {:local/root "."}}}

30
deps.edn Normal file
View File

@ -0,0 +1,30 @@
{:deps
{selmer/selmer {:mvn/version "1.12.50"},
org.clojure/clojure {:mvn/version "1.11.1"},
org.clojure/tools.logging {:mvn/version "1.1.0"},
insn/insn {:mvn/version "0.5.2"},
org.clojure/tools.cli {:mvn/version "1.0.206"},
org.babashka/babashka.impl.reify {:mvn/version "0.1.1"},
rewrite-clj/rewrite-clj {:mvn/version "1.0.699-alpha"},
babashka/babashka.core
{:git/url "https://github.com/babashka/babashka.core",
:git/sha "52a6037bd4b632bffffb04394fb4efd0cdab6b1e"},
hiccup/hiccup {:mvn/version "2.0.0-alpha2"},
org.clojure/data.xml {:mvn/version "0.2.0-alpha6"},
org.clojure/data.csv {:mvn/version "1.0.0"},
com.taoensso/timbre {:mvn/version "5.2.1"},
com.cognitect/transit-clj {:mvn/version "1.0.329"},
babashka/babashka.curl {:mvn/version "0.1.1"},
clj-commons/clj-yaml {:mvn/version "0.7.108"},
org.clojure/core.rrb-vector {:mvn/version "0.1.2"},
org.babashka/sci.impl.types {:mvn/version "0.0.2"},
cheshire/cheshire {:mvn/version "5.11.0"},
org.clojure/core.match {:mvn/version "1.0.0"},
org.clojure/test.check {:mvn/version "1.1.1"},
babashka/fs {:mvn/version "0.1.2"},
cljc.java-time/cljc.java-time {:mvn/version "0.1.12"},
org.clojure/data.priority-map {:mvn/version "1.1.0"},
nrepl/bencode {:mvn/version "1.1.0"},
org.clojure/core.async {:mvn/version "1.5.648"},
http-kit/http-kit {:mvn/version "2.6.0-RC1"}
etaoin/etaoin {:mvn/version "1.0.38"}}}

17
docs/how-to-rules.rst Normal file
View File

@ -0,0 +1,17 @@
=====
Rules
=====
A rule is a function that takes:
Amount, Transaction Details, Category, Merchant Name, and f
It is matched on Account Number + Transaction Type
f is a map that returns the account-to of the transaction, based on it's applies-to and condition.
This means we can describe a set of rules as map containing::
{ :applies-to (the Account No + Transaction Type combo)
:function (the function which will return account-to)
:condition (whether this is the rule for the transaction)
}
We then need a standard way to pass all the elements we care about to applies-to

26
flake.lock Normal file
View File

@ -0,0 +1,26 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1665753223,
"narHash": "sha256-RsRUlQG3WaeqdWvuCaPvjptMEoq4yck2Gsj0lTklbk0=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "6594eb2fc86e22005f48aaeb1011b5a72d99e58e",
"type": "github"
},
"original": {
"owner": "nixos",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

38
flake.nix Normal file
View File

@ -0,0 +1,38 @@
{
description = "Beancount Importer";
inputs = {
nixpkgs.url = "github:nixos/nixpkgs";
};
outputs = { self, nixpkgs }:
let
pkgs = nixpkgs.legacyPackages.x86_64-linux;
in
{
packages.x86_64-linux.muneem = pkgs.stdenv.mkDerivation {
name = "muneem";
src = ./.;
installPhase = ''
mkdir -p $out
cp -r src/ $out/
cp bb.edn $out/
cp deps.edn $out/
'';
};
defaultPackage.x86_64-linux = self.packages.x86_64-linux.muneem;
devShell.x86_64-linux = pkgs.mkShell {
buildInputs = with pkgs; [
jdk11
babashka
geckodriver
chromedriver
chromium
];
shellHook = ''
'';
};
};
}

52
readme.rst Normal file
View File

@ -0,0 +1,52 @@
======
Muneem
======
TODO
----
- [ ] Download Savings acct CSV
- [ ] Import Savings acct CSV
- [ ] RECONCILE
- [ ] Download NAB trade transactions and value
- [ ] Download spaceship returns
- [ ] `pad` checking / savings accounts against some known value way in the past
- [ ] MAYBE gonna have to write a rule engine that handles this
- [ ] MAYBE Parse the transaction only once at the start, and then use the parsed map instead of passing around a vector of strings
- [ ] Mention uncategorized transactions in matrix chat so that i can categorize it
- [ ] Post weekly spending in matrix chat
DONE
----
- [x] Read the CSV(s)
- [x] Figure out a way to download both CSVs from each account
- [x] Figure out what columns match to what data we care about
- [x] Marshall the data into a way that bean count understands
- [x] Open accounts that previously didn't exist
- [x] Write the transactions to the file
beancount::
Beancount Format
2016-12-06 * "Biang!" "Dinner"
Liabilities:CreditCard -47.23 USD
Expenses:Restaurants
-----------
Assumptions
-----------
We'll also be making assumptions about the way accounts are structured. Since NAB provides categories we can put that into the account
=================== ==================================== =============== =======
Transaction Type To From Misc
=================== ==================================== =============== =======
EFTPOS DEBIT Expenses:<Category>:<Merchant Name> Assets:Checking
TRANSFER CREDIT Assets:Checking Assets:Savings
MISCELLANEOUS DEBIT Expenses:<Category>:<Merchant Name> Assets:Checking
TRANSFER DEBIT Expenses:Rent Assets:Checking IFF: (= amount 170)
Additionally:
- we'll may need to use account nunmber for distinguishing different CSVs
- The actual date is stored in Transaction details, which we can use as the comment

43
src/muneem/etaoin.clj Normal file
View File

@ -0,0 +1,43 @@
(ns muneem.etaoin
(:require [etaoin.api :as e]
[clojure.java.shell :refer [sh]]
[etaoin.keys :as k]))
(def driver (e/chrome {:args ["--disable-blink-features=AutomationControlled" "--maximize"]}))
;:download-dir "./documents/"
(defn download-csv
[]
(e/js-execute driver "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
(e/go driver "https://ib.nab.com.au/nabib/index.jsp")
(e/wait-visible driver [{:id :username}])
; Don't advertise that we're a bot
(e/fill driver {:id :username} (System/getenv "NAB_USER"))
(e/fill driver {:id :password} (System/getenv "NAB_PASSWORD"))
(e/fill driver {:id :password} k/enter)
(e/wait 4)
(e/click driver {:tag :button :id :hamburger-btn})
(e/click-el driver (e/query driver {:css "li.selected>button"}))
(e/click-el driver (e/query driver {:tag :a :fn/text "Transaction history"}))
(e/wait 4)
; Update to last 7 days
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-toggle-button-DateDropdown\").click()")
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-listbox-DateDropdown-0\").click()")
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div.Containerstyle__StyledWrapper-sc-10ei1r0-0.dnudTa > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > form > div > div:nth-child(3) > div > button\").click()")
(e/wait 2)
; Download checking file
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#export-button > div\").click()")
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div:nth-child(2) > div > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > div > div > div > div.StyledTopRow-cijGbF.StyledExportCol-fYxmhQ.ibndGm.cAjmIO.sc-dkPtRN.Colstyle__StyledCol-sc-1evc4kf-0.ketKXK.fqofCs > div > div > div > ul > li:nth-child(1) > button\").click()")
(e/wait 3)
; Download savings file
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-toggle-button-accountsListDropdown > div > div > div\")")
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#dropdown-listbox-accountsListDropdown-1 > div > div > div.AccountCardstyle__StyledAccountTypeContainer-sc-16p3i3w-1.cLXpwB\")")
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"#export-button > div\").click()")
(e/js-execute driver "document.querySelector(\"#bodyContent > miniapp-transactions\").shadowRoot.querySelector(\"div > div > div > div:nth-child(2) > div > div > div > div > div.StyledScreenView-jlloJx.gsnnCS > div > div > div > div.StyledTopRow-cijGbF.StyledExportCol-fYxmhQ.ibndGm.cAjmIO.sc-dkPtRN.Colstyle__StyledCol-sc-1evc4kf-0.ketKXK.fqofCs > div > div > div > ul > li:nth-child(1) > button\").click()")
(e/wait 3)
(e/quit driver))
(defn move-download
[]
(sh "mv" "~/Downloads/Transactions.csv" "./documents/checking.csv")
(sh "mv" "~/Downloads/Transactions\(1\).csv" "./documents/savings.csv"))

346
src/muneem/import.clj Normal file
View File

@ -0,0 +1,346 @@
(ns muneem.import
(:require [clojure.data.csv :as csv]
[clojure.java.io]
[cljc.java-time.local-date :as jt]
[cljc.java-time.format.date-time-formatter :as pt]
[clojure.string]))
(def beancount-file "./documents/test.beancount")
(def transaction-details (partial #(nth % 5)))
(def category (partial #(nth % 7)))
(defn merchant-name [trans]
(if (= (count trans) 9)
(nth trans 8)
""))
(def amount (partial #(nth % 1)))
(def trans-date (partial #(nth % 0)))
(def transaction-type (partial #(nth % 4)))
(defn take-csv
"Takes file name and reads data."
[file]
(-> (csv/read-csv (slurp file))
; We don't care about the header
(rest)))
(defn make-date
"Reads date from the 5th column (Transaction Details)
The 2nd element split by space is DD/MM of the actal transaction
Add 2022 to that and use it as the transaction date"
[trans statement]
; TODO this doesn't work for various transactions and other years
; Parse real date -> check that it is in fact a date
; Use normal date to determine year
; If it's not a date, use regular date
(let [real-date (-> trans
transaction-details
(clojure.string/split #" ")
second)
trans-date-year (-> (trans-date trans)
(clojure.string/split #" ")
(last)
(->> (str "20")))
with-year (-> real-date
(str "/" trans-date-year)
(clojure.string/split #"/")
reverse)
parse-year (-> (trans-date trans)
(jt/parse (pt/of-pattern "dd MMM uu"))
(jt/format (pt/of-pattern "YYYY-MM-dd")))]
(if (and (not (nil? real-date)) (re-matches #"../.." real-date))
(conj statement (clojure.string/join "-" with-year))
(conj statement parse-year))))
(defn add-transaction-flag
"Bean count requires transaction type, we use * which means txn"
[statement]
(conj statement "*"))
(defn add-narration
"Add some comments about the transaction from the 5th column (Transaction Details)"
[trans statement]
(let [narration (-> trans
transaction-details
(clojure.string/split #" ")
; first 2 are payment method and date
rest
rest
(conj "\"")
(cons "\""))]
(conj statement narration)))
(defn sanitize
[string]
(-> string
(clojure.string/capitalize)
(clojure.string/replace #" " "")
(clojure.string/replace #"'" "")
(clojure.string/replace #"\." "")
(clojure.string/replace #"\(.+\)" "")
(clojure.string/replace #"&" "-")))
; These should all be the same method tbh
; Or they should be a map?
; check out how-to-rules for more
(defn add-account-from-and-amount
"We're only handling checking account for now
All amounts are from it's perspsective"
[trans statement]
(let [amount (amount trans)
account "Assets:Checking"
string (str " " account " " amount " AUD")]
(conj statement string)))
(defn add-account-from-and-amount-savings
[trans statement]
(let [amount (amount trans)
account "Assets:Savings"
string (str " " account " " amount " AUD")]
(conj statement string)))
(defn debit-account-to
[trans statement]
(let [category (sanitize (category trans))
merchant-name (sanitize (merchant-name trans))
; Uncategorized strings need to be reconciled
; TODO we'd probably want rules based on the transaction details
; to automatically categorize further ones
account-to (if (= category "Uncategorised")
(if (> (count merchant-name) 0)
(str " ! Expenses:" category ":" merchant-name)
(str " ! Expenses:" category))
(if (> (count merchant-name) 0)
(str " Expenses:" category ":" merchant-name)
(str " Expenses:" category)))]
(conj statement account-to)))
(defn fees-account-to
[trans statement]
(let [category (sanitize (category trans))
merchant-name (sanitize (merchant-name trans))
account-to (str " Expenses:Fees")]
(conj statement account-to)))
(defn atm-debit-account-to
[trans statement]
(let [category (sanitize (category trans))
merchant-name (sanitize (merchant-name trans))
account-to (str " Assets:Cash")]
(conj statement account-to)))
(defn transfer-credit-account-to
[_ statement]
(let [account-to (str " Assets:Savings")]
(conj statement account-to)))
(defn bank-credit-account-to
[_ statement]
(let [account-to (str " ! Income:ThreatDefence")]
(conj statement account-to)))
(defn dividend-savings
[_ statement]
(let [account-to (str " Income:Dividends")]
(conj statement account-to)))
(defn salary-savings
[_ statement]
(let [account-to (str " Income:Cubiko")]
(conj statement account-to)))
(defn interest-account-to
[_ statement]
(let [account-to (str " Expenses:NAB")]
(conj statement account-to)))
(defn interest-savings
[_ statement]
(let [account-to (str " Income:NAB")]
(conj statement account-to)))
(defn automatic-drawing-account-to
[_ statement]
(let [account-to (str " Assets:Spaceship")]
(conj statement account-to)))
(defn misc-credit-account-to
[_ statement]
(let [account-to (str " Income:Beem")]
(conj statement account-to)))
(defn deposit-account-to
"This function is going to change a lot
It currently only works with Expenses
but in the future will need to handle determining account to"
[trans statement]
(let [account-to (str " ! Expenses:OverCharged")]
(conj statement account-to)))
(defn transfer-debit-account-to
"This function is going to change a lot
It currently only works with Expenses
but in the future will need to handle determining account to"
[trans statement]
(let [rent (str " Expenses:Rent")
trans-type (transaction-type trans)
amt (amount trans)
else-account (str " Expenses:" (-> trans-type
(clojure.string/split #" ")
(nth 1)))]
(if (= amt 170)
(conj statement rent)
(conj statement else-account))))
(defn transfer-debit-savings
"This function is going to change a lot
It currently only works with Expenses
but in the future will need to handle determining account to"
[trans statement]
(let [account-to (str " Assets:Checking")]
(conj statement account-to)))
;; RULES
; {:applies-to (fn [{trans-type :transaction-type
; amount :amount
; category :category
; merchant-name :merchant-name)))
(defn determine-category
"We probably want some sort of rule engine
that takes rules and applies the first match or something (see docs)
but this will do for now
Writes out the account-to to the statement"
[trans statement]
(let [trans-type (transaction-type trans)]
(case trans-type
"EFTPOS DEBIT" (debit-account-to trans statement)
"TRANSFER CREDIT" (transfer-credit-account-to trans statement)
"MISCELLANEOUS DEBIT" (debit-account-to trans statement)
"TRANSFER DEBIT" (transfer-debit-account-to trans statement)
"INTEREST PAID" (interest-account-to trans statement)
"AUTOMATIC DRAWING" (automatic-drawing-account-to trans statement)
"FEES" (fees-account-to trans statement)
"REVERSAL CREDIT" (automatic-drawing-account-to trans statement)
"ATM DEBIT" (atm-debit-account-to trans statement)
"DEPOSIT" (deposit-account-to trans statement)
"INTER-BANK CREDIT" (bank-credit-account-to trans statement)
"MISCELLANEOUS CREDIT" (misc-credit-account-to trans statement))))
(defn determine-category
"We probably want some sort of rule engine
that takes rules and applies the first match or something (see docs)
but this will do for now
Writes out the account-to to the statement"
[trans statement]
(let [trans-type (transaction-type trans)]
(case trans-type
"TRANSFER DEBIT" (transfer-debit-savings trans statement)
"INTEREST PAID" (interest-savings trans statement)
"INTER-BANK CREDIT" (bank-credit-account-to trans statement)
"DIVIDEND" (dividend-savings trans statement)
"SALARY" (salary-savings trans statement))))
(defn write-statement
[statement]
(spit beancount-file (clojure.string/join "\n" (conj statement "\n")) :append true))
(defn tap
[s]
(println s)
s)
(defn organize-correctly
[statement]
(-> statement
(flatten)
(into [])
(->> (clojure.string/join " "))
(->> (conj []))))
(def accounts (atom #{}))
(def new-accounts (atom #{}))
(defn create-account
[transaction]
(let [to-account (second (clojure.string/split (second transaction) #" "))
from-account (second (clojure.string/split (nth transaction 2) #" "))
marshalled-from-account (clojure.string/replace from-account #"! " "")]
(if (not (contains? (deref accounts) to-account))
(swap! new-accounts conj to-account))
(if (not (contains? (deref accounts) marshalled-from-account))
(swap! new-accounts conj marshalled-from-account))
transaction))
(defn write-transaction-to-file
[transaction]
(->> []
(make-date transaction)
(add-transaction-flag)
(add-narration transaction)
(organize-correctly)
(add-account-from-and-amount transaction)
(determine-category transaction)
#_(tap)
(create-account)
(write-statement)))
(defn write-transaction-to-file-savings
[transaction]
(->> []
(make-date transaction)
(add-transaction-flag)
(add-narration transaction)
(organize-correctly)
(add-account-from-and-amount-savings transaction)
(determine-category transaction)
#_(tap)
(create-account)
(write-statement)))
(defn index-of [x coll]
(let [idx? (fn [i a] (when (= x a) i))]
(first (keep-indexed idx? coll))))
(defn get-account
"Returns the account from statements where the account is opened"
[account-info]
(let [split (into [] (clojure.string/split account-info #" "))
account (last split)]
account))
(defn read-accounts
[ledger]
(let [file-contents (slurp ledger)
split (into [] (rest (clojure.string/split file-contents #"\n")))
empty-index (index-of "" split)
only-accounts (if (not (nil? empty-index))
(subvec split 0 empty-index)
split)]
(->> (mapv get-account only-accounts)
(set)
(reset! accounts))))
(defn open-account-string
[account]
(str "2016-01-01 open " account))
(defn write-accounts
[ledger]
(let [file-contents (slurp ledger)
split (clojure.string/split file-contents #"\n")
empty-index (index-of "" split)
[before after] (split-at empty-index split)
middle (mapv open-account-string (into [] (deref new-accounts)))
full-file (vec (concat before middle after "\n"))]
(spit ledger (clojure.string/join "\n" full-file))))
(defn main
[csv checking? & _args]
(read-accounts beancount-file)
(if checking?
(mapv write-transaction-to-file (drop-last (take-csv csv)))
(mapv write-transaction-to-file-savings (drop-last (take-csv csv))))
(write-accounts beancount-file))

12
src/muneem/main.clj Normal file
View File

@ -0,0 +1,12 @@
(ns muneem.main
(:require [muneem.import :as import]
[muneem.etaoin :as download]))
(defn -main
[& _args]
(download/download-csv)
(download/move-download)
(import/main "./documents/checking.csv" true)
(import/main "./documents/savings.csv" true))
(-main)