From 6264de978ad9d161da7a8fab9b6e7d804285f1cc Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 13 Jan 2019 15:34:13 +0000 Subject: [PATCH] Use emacs capture template syntax --- README.md | 4 +- TODO.org | 6 +- package.json | 2 +- server/ci.sh | 2 + server/grasp_server.py | 127 ++++++++++++++++++---------------- server/org_tools.py | 150 +++++++++++++++++++++++++++-------------- server/setup | 6 +- 7 files changed, 182 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index f32d583..3044985 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,9 @@ possibly selected text, additional comments or tags and send it into your [Org M No third party dependencies! Just `python 3.6`. # Running -1. Install server counterpart as systemd service (to autostart it): `server/setup --path /path/to/your/capture.org [--port ]`. +1. Install server counterpart as systemd service (to autostart it): `server/setup --path /path/to/your/capture.org [--port ] [--template ]`. - Or alternatively, just run it directly if you don't want to autostart it: `server/grasp_server.py --path /path/to/your/capture.org [--port ]`. + Or alternatively, just run it directly if you don't want to autostart it: `server/grasp_server.py --path /path/to/your/capture.org [--port ] [--template ]`. 2. Install chrome extension and configure hotkeys That's it! If you're using custom port make sure it's same as in the extension settings. diff --git a/TODO.org b/TODO.org index 0d68b80..57be934 100644 --- a/TODO.org +++ b/TODO.org @@ -3,10 +3,12 @@ * TODO [#C] update screens and video now that the notification is nicer * TODO [#C] how to test it? try on different pages? + https://www.blazemeter.com/blog/6-easy-steps-testing-your-chrome-extension-selenium + looks tricky... * TODO [#C] some sort of delayed append logic? e.g. don't dump immediately and add as subitems? * TODO [#C] or, add menu to extension? -* TODO [#C] custom capture template? +* TODO [#C] hmm. if template is specified on client site, it's easier to dispatch todo vs non-todo etc?.. * TODO [#D] do not minfy for dev * TODO [#D] make manifest templated so we don't have to hardcode actions @@ -40,6 +42,8 @@ Now, the content of `build` folder will be the extension ready to be submitted t * DONE [#B] docs on using CLOSED: [2019-01-06 Sun 21:39] +* DONE [#C] custom capture template? + CLOSED: [2019-01-13 Sun 14:57] * DONE [#C] make notification configurable or only in case of errors CLOSED: [2019-01-13 Sun 13:01] * DONE [#C] motivation diff --git a/package.json b/package.json index ab23993..2b83389 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grasp-extension", - "version": "0.4.1", + "version": "0.5.0", "description": "A reliable way of capturing web pages and content", "scripts": { "build": "webpack --progress --profile --colors", diff --git a/server/ci.sh b/server/ci.sh index f16e0b5..29d7377 100755 --- a/server/ci.sh +++ b/server/ci.sh @@ -4,3 +4,5 @@ set -eux pylint -E *.py setup mypy --check-untyped-defs *.py setup + +pytest -s org_tools.py diff --git a/server/grasp_server.py b/server/grasp_server.py index b53b296..9f24f5b 100755 --- a/server/grasp_server.py +++ b/server/grasp_server.py @@ -3,90 +3,92 @@ import argparse from http.server import BaseHTTPRequestHandler, HTTPServer import json import os +import logging from pathlib import Path import re -from typing import List +from typing import List, Optional -from org_tools import append_org_entry +from org_tools import as_org, empty, DEFAULT_TEMPLATE CAPTURE_PATH_VAR = 'GRASP_CAPTURE_PATH' +CAPTURE_TEMPLATE_VAR = 'GRASP_CAPTURE_TEMPLATE' -def log(*things): - # TODO proper logging - print(*things) + +def get_logger(): + return logging.getLogger('grasp-server') + +def append_org( + path: Path, + org: str +): + logger = get_logger() + # TODO perhaps should be an error?... + if not path.exists(): + logger.warning("path %s didn't exist!", path) + # https://stackoverflow.com/a/13232181 + if len(org.encode('utf8')) > 4096: + logger.warning("writing out %s might be non-atomic", org) + with path.open('a') as fo: + fo.write(org) def capture( - url, + url: str, title, selection, comment, tag_str, ): - def empty(s) -> bool: - return s is None or len(s.strip()) == 0 - - log("capturing", url, title, selection, comment) - + logger = get_logger() + # protect strings against None + def safe(s: Optional[str]) -> str: + if s is None: + return '' + else: + return s capture_path = Path(os.environ[CAPTURE_PATH_VAR]).expanduser() + org_template = os.environ[CAPTURE_TEMPLATE_VAR] + logger.info('capturing %s to %s', (url, title, selection, comment, tag_str), capture_path) + + url = safe(url) + title = safe(title) + selection = safe(selection) + comment = safe(comment) + tag_str = safe(tag_str) - heading = url - parts = [] tags: List[str] = [] if not empty(tag_str): tags = re.split('[\s,]', tag_str) tags = [t for t in tags if not empty(t)] # just in case - if not empty(title): - heading = title - parts.append(url) - - # TODO not sure, maybe add as org quote? - if not empty(selection): - parts.extend([ - 'Selection:', - selection, # TODO tabulate? - ]) - if not empty(comment): - parts.extend([ - 'Comment:', - comment - ]) - body = None if len(parts) == 0 else '\n'.join(parts) + org = as_org( + url=url, + title=title, + selection=selection, + comment=comment, + tags=tags, + org_template=org_template, + ) + append_org( + path=capture_path, + org=org, + ) response = { 'path': str(capture_path), - } - # try: - append_org_entry( - capture_path, - heading=heading, - body=body, - tags=tags, - todo=False, - ) - response.update({ 'status': 'ok', - }) - # just rely on setting 500 instead - # except Exception as e: - # log(str(e)) - # response.update({ - # 'status': 'error', - # 'error' : str(e), - # }) - + } return json.dumps(response).encode('utf8') - class GraspRequestHandler(BaseHTTPRequestHandler): def handle_POST(self): + logger = get_logger() + content_length = int(self.headers['Content-Length']) post_data = self.rfile.read(content_length) payload = json.loads(post_data.decode('utf8')) - log("incoming request:") - log(payload) + logger.info("incoming request %s", payload) res = capture(**payload) self.send_response(200) self.send_header('Content-Type', 'application/json') @@ -100,30 +102,41 @@ class GraspRequestHandler(BaseHTTPRequestHandler): self.wfile.write(message.encode('utf8')) def do_POST(self): + logger = get_logger() try: self.handle_POST() except Exception as e: - log("Error during processing") - log(str(e)) + logger.error("Error during processing") + logger.exception(e) self.respond_error(message=str(e)) -def run(port: str, capture_path: str): +def run(port: str, capture_path: str, template: str): + logger = get_logger() + if template == repr(DEFAULT_TEMPLATE): + template = DEFAULT_TEMPLATE # hack so argparse renders default template with \n characters... + # not sure if there is a simpler way to communicate with the server... os.environ[CAPTURE_PATH_VAR] = capture_path + os.environ[CAPTURE_TEMPLATE_VAR] = template httpd = HTTPServer(('', int(port)), GraspRequestHandler) - log(f"Starting httpd on port {port}") + logger.info(f"Starting httpd on port {port}") httpd.serve_forever() def setup_parser(p): p.add_argument('--port', type=str, default='12212', help='Port for communicating with extension') p.add_argument('--path', type=str, default='~/capture.org', help='File to capture into') + p.add_argument('--template', type=str, default=repr(DEFAULT_TEMPLATE), help=f""" + {as_org.__doc__} + """) def main(): - p = argparse.ArgumentParser('grasp server setup', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + + p = argparse.ArgumentParser('grasp server', formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=100)) # type: ignore setup_parser(p) args = p.parse_args() - run(args.port, args.path) + run(args.port, args.path, args.template) if __name__ == '__main__': main() diff --git a/server/org_tools.py b/server/org_tools.py index 2024bf1..e62a6ee 100644 --- a/server/org_tools.py +++ b/server/org_tools.py @@ -1,9 +1,9 @@ from datetime import datetime -import logging -from pathlib import Path import re from typing import List, Optional +DEFAULT_TEMPLATE = "* %U %:description %:tags\n%:link\n%:initial\n" + def date2org(t: datetime) -> str: return t.strftime("%Y-%m-%d %a") @@ -13,63 +13,109 @@ def datetime2orgtime(t: datetime) -> str: def datetime2org(t: datetime) -> str: return date2org(t) + " " + datetime2orgtime(t) -# TODO priority maybe?? -# TODO need to sanitize! -def as_org_entry( - heading: Optional[str] = None, - tags: List[str] = [], - body: Optional[str] = None, - created: Optional[datetime]=None, - todo=True, + +def empty(s) -> bool: + return s is None or len(s.strip()) == 0 + + +def as_org( + url: str, + title: str, + selection: str, + comment: str, + tags: List[str], + org_template: str, ): - if heading is None: - if body is None: - raise RuntimeError('Both heading and body are empty!!') - heading = body.splitlines()[0] # TODO ?? + """ +Formats captured results according to org template. Supports all sensible (e.g. non-interactive) template expansions from https://orgmode.org/manual/Template-expansion.html#Template-expansion. - if body is None: - body = '' +Additionally, supports :selection, :comment and :tags expansions. - # TODO FIXME escape everything properly! - heading = re.sub(r'\s', ' ', heading) - # TODO remove newlines from body +You can look at `test_templates` for some specific examples. + """ + py_template = re.sub(r'%[:]?([?\w]+)', r'{\1}', org_template) - NOW = datetime.now() # TODO tz?? - # appended only added if it's different from created - app = [f':APPENDED: [{datetime2org(NOW)}]'] if created is not None else [] - if created is None: - created = NOW + NOW = datetime.now() + org_date = date2org(NOW) + org_datetime = datetime2org(NOW) + tags_s = '' if len(tags) == 0 else (':' + ':'.join(tags) + ':') - todo_s = ' TODO' if todo else '' - tag_s = ':'.join(tags) + # TODO tabulate selection and comments? + # TODO not sure, maybe add as org quote? + parts = [] + if not empty(selection): + parts.extend([ + 'Selection:', + selection, + ]) + if not empty(comment): + parts.extend([ + 'Comment:', + comment + ]) + initial = '\n'.join(parts) - sch = [f' SCHEDULED: <{date2org(NOW)}>'] if todo else [] + res = py_template.format( + t=f'<{org_date}>', + u=f'[{org_date}]', + T=f'<{org_datetime}>', + U=f'[{org_datetime}]', + description=title, + link=url, - if len(tag_s) != 0: - tag_s = f':{tag_s}:' - lines = [ - f"""*{todo_s} {heading} {tag_s}""", - *sch, - ':PROPERTIES:', - *app, - f':CREATED: [{datetime2org(created)}]', - ':END:', - body, - "", - "", + initial=initial, + i=initial, + + tags=tags_s, + selection=selection, + comment=comment, + annotation=f'[[{url}][{title}]]', # https://orgmode.org/manual/capture-protocol.html + **{'?': ''}, # just ignore it + ) + return res + +# TODO escaping? +# just checks it's not crashing for now +def test_templates(): + url = 'https://whatever' + title = 'hello' + selection = """some + selected + text + """ + comment = 'fafewfewf' + tags = ['aba', 'caba'] + + # https://orgmode.org/guide/Capture-templates.html + org_templates = [ + DEFAULT_TEMPLATE, + "* %? [[%:link][%:description]] \nCaptured On: %U", + "* %:annotation", + "* %U %:description :protocol: \n %:link \n%:initial \n\n", + "* %t captured stuff %:tags \n ", + "\n** Selection\n%:selection\n** Comment\n%:comment\n", + """* TODO [#A] %:link +** Selection +%:selection +** Comment +%:comment + """, + """* %T %:link %:description +#+BEGIN_QUOTE +%:selection +#+END_QUOTE + """, ] - return '\n'.join(lines) + for org_template in org_templates: + res = as_org( + url, + title, + selection, + comment, + tags, + org_template=org_template -# TODO should we check if it exists first? -def append_org_entry( - path: Path, - *args, - **kwargs, -): - res = as_org_entry(*args, **kwargs) - # https://stackoverflow.com/a/13232181 - if len(res.encode('utf8')) > 4096: - logging.warning("writing out %s might be non-atomic", res) - with path.open('a') as fo: - fo.write(res) + ) + print() + print(res) diff --git a/server/setup b/server/setup index ebd853c..bdeb59d 100755 --- a/server/setup +++ b/server/setup @@ -31,7 +31,9 @@ def setup(args): print(f"Writing systemd config to {out}:") server_bin = Path(__file__).parent.joinpath('grasp_server.py').absolute() - extra_args = f'--port {args.port} --path {args.path}' + + template = args.template.replace('%', '%%') # escape for systemd... + extra_args = f'--port {args.port} --path {args.path} --template \"{template}\"' with out.open('w') as fo: fo.write(SYSTEMD_CONFIG.format( @@ -50,7 +52,7 @@ def setup(args): raise e def main(): - p = argparse.ArgumentParser('grasp server setup', formatter_class=argparse.ArgumentDefaultsHelpFormatter) + p = argparse.ArgumentParser('grasp server setup', formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=100)) # type: ignore p.add_argument('--unit-name', type=str, default='grasp.service', help='Systemd unit name') setup_parser(p) args = p.parse_args()