wallabag initial commit
This commit is contained in:
		
							parent
							
								
									8b9381d5e1
								
							
						
					
					
						commit
						58344f0a37
					
				|  | @ -75,7 +75,11 @@ | |||
|         shellHook = '' | ||||
|           [[ -z $SHAARLI_TOKEN ]] && echo "::dhyan:: Necessary Env Var missing: SHAARLI_TOKEN" | ||||
|           [[ -z $MATRIX_PASSWORD ]] && echo "::dhyan:: Necessary Env Var missing: MATRIX_PASSWORD" | ||||
|           [[ -z $folder ]] && echo "::dhyan:: Necessary Env Var missing: folder" | ||||
|           [[ -z $WALLABAG_CLIENT_ID ]] && echo "::dhyan:: Necessary Env Var missing: WALLABAG_CLIENT_ID" | ||||
|           [[ -z $WALLABAG_CLIENT_SECRET ]] && echo "::dhyan:: Necessary Env Var missing: WALLABAG_CLIENT_SECRET" | ||||
|           [[ -z $WALLABAG_USERNAME ]] && echo "::dhyan:: Necessary Env Var missing: WALLABAG_USERNAME" | ||||
|           [[ -z $WALLABAG_PASSWORD ]] && echo "::dhyan:: Necessary Env Var missing: WALLABAG_PASSWORD" | ||||
|           # [[ -z $folder ]] && echo "::dhyan:: Necessary Env Var missing: folder" | ||||
|         ''; | ||||
|       }; | ||||
|     }; | ||||
|  |  | |||
							
								
								
									
										141
									
								
								src/dhyan/connections/wallabag.clj
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										141
									
								
								src/dhyan/connections/wallabag.clj
									
									
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,141 @@ | |||
| (ns dhyan.connections.wallabag | ||||
|   (:require [cheshire.core :as json] | ||||
|             [clojure.walk] | ||||
|             [babashka.curl :as curl])) | ||||
| 
 | ||||
| (defn generate-token | ||||
|   "Generates a token from either inputs or the environment variables" | ||||
|   ([] | ||||
|    (let [client-id (System/getenv "WALLABAG_CLIENT_ID") | ||||
|          client-secret (System/getenv "WALLABAG_CLIENT_SECRET") | ||||
|          username (System/getenv "WALLABAG_USERNAME") | ||||
|          password (System/getenv "WALLABAG_PASSWORD")] | ||||
|      (generate-token client-id client-secret username password))) | ||||
|   ([client-id client-secret username password] | ||||
|    (-> (curl/post (str "http://read.mossnet.lan/oauth/v2/token") | ||||
|                   {:raw-args ["-d" "grant_type=password" | ||||
|                               "-d" (str "client_id=" client-id) | ||||
|                               "-d" (str "client_secret=" client-secret) | ||||
|                               "-d" (str "username=" username) | ||||
|                               "-d" (str "password=" password)]}) | ||||
|        :body | ||||
|        (json/decode) | ||||
|        (clojure.walk/keywordize-keys)))) | ||||
| 
 | ||||
| ; Setup Token as an atom for use later on | ||||
| (def token (atom 1)) | ||||
| 
 | ||||
| (reset! token (generate-token)) | ||||
| 
 | ||||
| (defn make-request | ||||
|   "Given a link and a token make a get request to wallabag at the link" | ||||
|   [link token] | ||||
|   (-> (curl/get link {:headers {"Authorization" (str "Bearer " token)}}) | ||||
|       :body | ||||
|       (json/decode) | ||||
|       (clojure.walk/keywordize-keys))) | ||||
| 
 | ||||
| (defn build-link | ||||
|   "Creates a wallabag link at mossnet.lan/api/entries given certain parameters" | ||||
|   ([] | ||||
|    (build-link 1 20 0 "metadata")) | ||||
|   ([page per-page archive detail] | ||||
|    (let [link (str "http://read.mossnet.lan/api/entries?" | ||||
|                    "&perPage=" per-page | ||||
|                    "&archive=" archive | ||||
|                    "&page=" page | ||||
|                    "&detail=" detail)] | ||||
|      link))) | ||||
| 
 | ||||
| (defn select-useful-keys | ||||
|   "Given a list of *entries* filter out all the garbage keys" | ||||
|   [entries] | ||||
|   (let [useful-keys [:tags :title :url :annotations :id]] | ||||
|     (mapv #(select-keys % useful-keys) entries))) | ||||
| 
 | ||||
| (defn make-wallabag-link | ||||
|   "Given a single entry turn the :id key into a hardcoded mossnet wallabag clickable link" | ||||
|   [entry] | ||||
|   (let [wallabag-link (get entry :id) | ||||
|         full-link (str "https://read.mossnet.lan/view/" wallabag-link)] | ||||
|     (-> entry | ||||
|         (assoc :wallabag-url full-link) | ||||
|         (dissoc :id)))) | ||||
| 
 | ||||
| (defn map-wallabag-link | ||||
|   "Maps the function that turns id into clickable mossnet wallabag link" | ||||
|   [entries] | ||||
|   (mapv make-wallabag-link entries)) | ||||
| 
 | ||||
| (defn get-entries | ||||
|   "Turns a response from wallabag client into entries" | ||||
|   [resp] | ||||
|   (-> resp | ||||
|       (get-in [:_embedded :items]))) | ||||
| 
 | ||||
| (defn make-resp-to-entries | ||||
|   "Turns a response into a filtered list of entries that we can then use" | ||||
|   [resp] | ||||
|   (-> resp | ||||
|       (get-entries) | ||||
|       (select-useful-keys) | ||||
|       (map-wallabag-link))) | ||||
| 
 | ||||
| (defn get-all-entries | ||||
|   "The big pal function | ||||
|   Recursively pages through the wallabag api to get all the entries given a set of parameters | ||||
|   Defaults to unarchived metadata at 20 links per page | ||||
|   Hardcoded to start from page 1 (we want all the entries afterall) | ||||
|   Handles making requests, turning them into filtered entries, and accumalating them across pages" | ||||
|   ([] | ||||
|    (get-all-entries {:per-page 20 :archive 0 :detail "metadata"})) | ||||
|   ([{per-page :per-page archive :archive detail :detail}] | ||||
|    (let [link (build-link 1 per-page archive detail) | ||||
|          token (get @token :access_token) | ||||
|          acc (vector) | ||||
|          resp (make-request link token) | ||||
|          entries (make-resp-to-entries resp) | ||||
|          next-link (-> resp | ||||
|                        (get-in [:_links :next :href])) | ||||
|          acc-new (-> entries | ||||
|                      ((fn foo [items] (into [] (concat acc items)))))] | ||||
|      (get-all-entries next-link acc-new))) | ||||
|   ([link acc] | ||||
|    (let [token (get @token :access_token) | ||||
|          resp (make-request link token) | ||||
|          entries (make-resp-to-entries resp) | ||||
|          next-link (-> resp | ||||
|                        (get-in [:_links :next :href])) | ||||
|          last-link (-> resp | ||||
|                        (get-in [:_links :last :href])) | ||||
|          acc-new (-> entries | ||||
|                      ((fn foo [items] (into [] (concat acc items)))))] | ||||
|      (if (not (empty? next-link)) ; clj-kondo: (seq? str) returns false always | ||||
|        (get-all-entries next-link acc-new) | ||||
|        (if (= link last-link) | ||||
|          acc-new | ||||
|          (get-all-entries last-link acc)))))) | ||||
| 
 | ||||
| (defn get-all-archived-entries | ||||
|   "Returns a filtered list of all archived entries" | ||||
|   [] | ||||
|   (get-all-entries {:per-page 20 :archive 1 :detail "metadata"})) | ||||
| 
 | ||||
| (get-all-archived-entries) | ||||
| 
 | ||||
| (comment | ||||
|   (-> (make-request (build-link) (get @token :access_token)) | ||||
|       (make-resp-to-entries)) | ||||
|   (count (get-all-archived-entries)) | ||||
|   (def next-link (atom "")) | ||||
|   ; Repeatedly get next link until it's not there anymore | ||||
|   (let [link (build-link 1 20 1 "metadata") | ||||
|         token (get @token :access_token) | ||||
|         resp (make-request link token)] | ||||
|     (reset! next-link (get-in resp [:_links :next :href])) | ||||
|     (println @next-link) | ||||
|     (while (not (empty? @next-link)) | ||||
|       (do | ||||
|         (println @next-link) | ||||
|         (reset! next-link (get-in (make-request @next-link token) [:_links :next :href])) | ||||
|         (println @next-link))))) | ||||
		Loading…
	
		Reference in a new issue