From 6ab099a3301c9c4f4c7fbadbd469b4d9038c2745 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 21 Dec 2022 01:06:38 +0000 Subject: [PATCH 01/32] server test: use requests instead of httpie --- server/test_grasp.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/server/test_grasp.py b/server/test_grasp.py index c730e3f..e71f656 100644 --- a/server/test_grasp.py +++ b/server/test_grasp.py @@ -4,16 +4,10 @@ from subprocess import Popen, check_call from time import sleep -# ugh. +import requests + + @contextmanager -def killmepls(*args, **kwargs): - with Popen(*args, **kwargs) as p: - try: - yield p - finally: - p.kill() - - def grasp_test_server(capture_file: Path, port: str, template=None): server = str((Path(__file__).parent / 'grasp_server.py').absolute()) cmdline = [ @@ -21,23 +15,29 @@ def grasp_test_server(capture_file: Path, port: str, template=None): ] if template is not None: cmdline.extend(['--template', template]) - return killmepls(cmdline) + with Popen(cmdline) as p: + try: + yield p + finally: + p.kill() -def test_server(tmp_path: Path): +def test_server(tmp_path: Path) -> None: cfile = tmp_path / 'test-capture.org' PORT = '17890' def send(url: str, title: str): - # TODO eh, not sure if there is anything easier than httpie - check_call([ - 'http', '--ignore-stdin', 'POST', f'http://localhost:{PORT}', - f'url={url}', - f'title={title}', - 'selection=null', - 'comment=null', - 'tag_str=null', - ]) + r = requests.post( + f'http://localhost:{PORT}', + json = { + 'url': url, + 'title': title, + 'selection': None, + 'comment': None, + 'tag_str': None, + }, + ) + assert r.status_code == 200, r with grasp_test_server(capture_file=cfile, port=PORT): From 4375365415c9d5882eeef5dd75883f3c706002ff Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 21 Dec 2022 01:25:27 +0000 Subject: [PATCH 02/32] org_tools: add a proper test comparing default template output --- server/org_tools.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/server/org_tools.py b/server/org_tools.py index 708796e..7dc4756 100644 --- a/server/org_tools.py +++ b/server/org_tools.py @@ -56,6 +56,8 @@ def as_org( tags: List[str], org_template: str, config: Optional[Config] = None, + *, + _now=None, ): """ Formats captured results according to org template. Supports all sensible (e.g. non-interactive) template expansions from https://orgmode.org/manual/Template-expansion.html#Template-expansion. @@ -66,7 +68,7 @@ You can look at `test_templates` for some specific examples. """ py_template = re.sub(r'%[:]?([?\w]+)', r'{\1}', org_template) - NOW = datetime.now() + NOW = datetime.now() if _now is None else _now org_date = date2org(NOW) org_datetime = datetime2org(NOW) @@ -103,9 +105,8 @@ You can look at `test_templates` for some specific examples. ) return res -# TODO escaping? -# just checks it's not crashing for now -def test_templates(): + +def test_templates() -> None: url = 'https://whatever' title = 'hello' selection = """some @@ -135,6 +136,7 @@ def test_templates(): #+END_QUOTE """, ] + # check they aren't crashing for org_template in org_templates: res = as_org( url, @@ -147,3 +149,24 @@ def test_templates(): ) print() print(res) + res = as_org( + url, + title, + selection, + comment, + tags, + org_template=DEFAULT_TEMPLATE, + _now=datetime(1111, 11, 11), + ) + expected = ''' +* [1111-11-11 Sat 00:00] hello :aba:caba: +https://whatever +Selection: +some + selected + text + +Comment: +fafewfewf +'''.lstrip() + assert res == expected From e019265aa124457b991c4cb5096d3295c6b16065 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 21 Dec 2022 01:49:53 +0000 Subject: [PATCH 03/32] org_tools: support escaping percents in templates via \% also add a regression test addresses https://github.com/karlicoss/grasp/issues/35 https://github.com/karlicoss/grasp/pull/43 Thanks @Chickensoupwithrice for spotting and fix attempt --- server/org_tools.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/org_tools.py b/server/org_tools.py index 7dc4756..ac9332d 100644 --- a/server/org_tools.py +++ b/server/org_tools.py @@ -66,7 +66,9 @@ Additionally, supports :selection, :comment and :tags expansions. You can look at `test_templates` for some specific examples. """ - py_template = re.sub(r'%[:]?([?\w]+)', r'{\1}', org_template) + py_template = re.sub(r'([^\\])%[:]?([?\w]+)', r'\1{\2}', org_template) + # replace all escaped % with regular %, safe to do after substitution + py_template = py_template.replace(r'\%', '%') NOW = datetime.now() if _now is None else _now org_date = date2org(NOW) @@ -116,6 +118,7 @@ def test_templates() -> None: comment = 'fafewfewf' tags = ['aba', 'caba'] + vimzettel_template = r'zettel \%title [%:description] testing' # https://orgmode.org/guide/Capture-templates.html org_templates = [ DEFAULT_TEMPLATE, @@ -135,6 +138,7 @@ def test_templates() -> None: %:selection #+END_QUOTE """, + vimzettel_template, ] # check they aren't crashing for org_template in org_templates: @@ -170,3 +174,13 @@ Comment: fafewfewf '''.lstrip() assert res == expected + + res = as_org( + url, + title, + selection, + comment, + tags, + org_template=vimzettel_template, + ) + assert res == 'zettel %title [hello] testing' From 417a02ba6f9838ad5509c7e7e91f01ed0d046ffb Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 21 Dec 2022 02:05:47 +0000 Subject: [PATCH 04/32] ci: bring up to date with pymplate, setup github actions --- .ci/release | 66 ++++++++++++++++++++++++++++++++++++++ .ci/run | 40 +++++++++++++++++++++++ .github/workflows/main.yml | 53 ++++++++++++++++++++++++++++++ build => build_extension | 0 ci/mypy | 8 ----- ci/pylint | 7 ---- ci/pytest | 7 ---- mypy.ini | 10 ++++++ pytest.ini | 13 ++++++++ server/org_tools.py | 4 +-- setup.py | 57 ++++++++++++++++++++++++++++++++ tox.ini | 32 ++++++++++++++++++ 12 files changed, 273 insertions(+), 24 deletions(-) create mode 100755 .ci/release create mode 100755 .ci/run create mode 100644 .github/workflows/main.yml rename build => build_extension (100%) delete mode 100755 ci/mypy delete mode 100755 ci/pylint delete mode 100755 ci/pytest create mode 100644 mypy.ini create mode 100644 pytest.ini create mode 100644 setup.py create mode 100644 tox.ini diff --git a/.ci/release b/.ci/release new file mode 100755 index 0000000..0ec687f --- /dev/null +++ b/.ci/release @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +''' +Run [[file:scripts/release][scripts/release]] to deploy Python package onto [[https://pypi.org][PyPi]] and [[https://test.pypi.org][test PyPi]]. + +The script expects =TWINE_PASSWORD= environment variable to contain the [[https://pypi.org/help/#apitoken][PyPi token]] (not the password!). + +The script can be run manually. +It's also running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. Packages are deployed on: +- every master commit, onto test pypi +- every new tag, onto production pypi + +You'll need to set =TWINE_PASSWORD= and =TWINE_PASSWORD_TEST= in [[https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets][secrets]] +for Github Actions deployment to work. +''' + +import os +import sys +from pathlib import Path +from subprocess import check_call +import shutil + +is_ci = os.environ.get('CI') is not None + +def main(): + import argparse + p = argparse.ArgumentParser() + p.add_argument('--test', action='store_true', help='use test pypi') + args = p.parse_args() + + extra = [] + if args.test: + extra.extend(['--repository-url', 'https://test.pypi.org/legacy/']) + + root = Path(__file__).absolute().parent.parent + os.chdir(root) # just in case + + if is_ci: + # see https://github.com/actions/checkout/issues/217 + check_call('git fetch --prune --unshallow'.split()) + + dist = root / 'dist' + if dist.exists(): + shutil.rmtree(dist) + + check_call('python3 setup.py sdist bdist_wheel', shell=True) + + TP = 'TWINE_PASSWORD' + password = os.environ.get(TP) + if password is None: + print(f"WARNING: no {TP} passed", file=sys.stderr) + import pip_secrets + password = pip_secrets.token_test if args.test else pip_secrets.token # meh + + check_call([ + 'python3', '-m', 'twine', + 'upload', *dist.iterdir(), + *extra, + ], env={ + 'TWINE_USERNAME': '__token__', + TP: password, + **os.environ, + }) + + +if __name__ == '__main__': + main() diff --git a/.ci/run b/.ci/run new file mode 100755 index 0000000..303de20 --- /dev/null +++ b/.ci/run @@ -0,0 +1,40 @@ +#!/bin/bash +set -eu + +cd "$(dirname "$0")" +cd .. # git root + +if ! command -v sudo; then + # CI or Docker sometimes doesn't have it, so useful to have a dummy + function sudo { + "$@" + } +fi + +if [ -n "${CI-}" ]; then + # install OS specific stuff here + case "$OSTYPE" in + darwin*) + # macos + : + ;; + cygwin* | msys* | win*) + # windows + : + ;; + *) + # must be linux? + : + ;; + esac +fi + + +PY_BIN="python3" +# some systems might have python pointing to python3 +if ! command -v python3 &> /dev/null; then + PY_BIN="python" +fi + +"$PY_BIN" -m pip install --user tox +"$PY_BIN" -m tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..6f936ee --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,53 @@ +# see https://github.com/karlicoss/pymplate for up-to-date reference + +name: CI +on: + push: + branches: '*' + tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi + # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug: + # P.S. fuck made up yaml DSLs. + pull_request: # needed to trigger on others' PRs + # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". + workflow_dispatch: # needed to trigger workflows manually + # todo cron? + + +jobs: + build: + strategy: + matrix: + platform: [ubuntu-latest, macos-latest] # , windows-latest] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + # 3.11 on windows has this bug, lxml setup fails + #https://bugs.launchpad.net/lxml/+bug/1977998 + exclude: [{platform: windows-latest, python-version: '3.11'}] + # vvv just an example of excluding stuff from matrix + # exclude: [{platform: macos-latest, python-version: '3.6'}] + + runs-on: ${{ matrix.platform }} + + steps: + # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/checkout@v3 + with: + submodules: recursive + + # uncomment for SSH debugging + # - uses: mxschmitt/action-tmate@v3 + + # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd... + - run: bash .ci/run + + - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms + uses: actions/upload-artifact@v3 + with: + name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} + path: .coverage.mypy/ + diff --git a/build b/build_extension similarity index 100% rename from build rename to build_extension diff --git a/ci/mypy b/ci/mypy deleted file mode 100755 index 2284e95..0000000 --- a/ci/mypy +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -eux - -# TODO ugh. doesn't work from circleci... -# cd "$(git rev-parse --show-toplevel)" - -# TODO use strict? -python3 -m mypy --check-untyped-defs "$@" server/*.py server/setup diff --git a/ci/pylint b/ci/pylint deleted file mode 100755 index a663c48..0000000 --- a/ci/pylint +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -eux - -# TODO ugh. doesn't work from circleci... -# cd "$(git rev-parse --show-toplevel)" - -python3 -m pylint -E "$@" server/*.py server/setup diff --git a/ci/pytest b/ci/pytest deleted file mode 100755 index 7622e71..0000000 --- a/ci/pytest +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -eux - -# cd "$(git rev-parse --show-toplevel)" - -python3 -m pytest "$@" server/*.py -# TODO force it to collect server/setup diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..0f8386f --- /dev/null +++ b/mypy.ini @@ -0,0 +1,10 @@ +[mypy] +pretty = True +show_error_context = True +show_error_codes = True +check_untyped_defs = True +namespace_packages = True + +# an example of suppressing +# [mypy-my.config.repos.pdfannots.pdfannots] +# ignore_errors = True diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..20c3704 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +# discover files that don't follow test_ naming. Useful to keep tests along with the source code +python_files = *.py +addopts = + # -rap to print tests summary even when they are successful + -rap + --verbose + + # otherwise it won't discover doctests + --doctest-modules + + # show all test durations (unless they are too short) + --durations=0 diff --git a/server/org_tools.py b/server/org_tools.py index ac9332d..0417203 100644 --- a/server/org_tools.py +++ b/server/org_tools.py @@ -23,11 +23,11 @@ def empty(s) -> bool: # TODO put template in config?? class Config: @staticmethod - def format_selection(selection: str) -> List[str]: + def format_selection(selection: str) -> List[str]: # type: ignore[empty-body] ... @staticmethod - def format_comment(comment: str) -> List[str]: + def format_comment(comment: str) -> List[str]: # type: ignore[empty-body] ... diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3cc8c3d --- /dev/null +++ b/setup.py @@ -0,0 +1,57 @@ +# see https://github.com/karlicoss/pymplate for up-to-date reference + + +from setuptools import setup, find_namespace_packages # type: ignore + + +def main() -> None: + # works with both ordinary and namespace packages + #pkgs = find_namespace_packages('src') + pkgs = ['server'] + pkg = min(pkgs) # lexicographically smallest is the correct one usually? + setup( + name=pkg, + use_scm_version={ + 'version_scheme': 'python-simplified-semver', + 'local_scheme': 'dirty-tag', + }, + setup_requires=['setuptools_scm'], + + # otherwise mypy won't work + # https://mypy.readthedocs.io/en/stable/installed_packages.html#making-pep-561-compatible-packages + zip_safe=False, + + packages=pkgs, + #package_dir={'': 'src'}, + # necessary so that package works with mypy + package_data={pkg: ['py.typed']}, + + ## ^^^ this should be mostly automatic and not requiring any changes + + install_requires=[ + # vvv example of git repo dependency + # 'repo @ git+https://github.com/karlicoss/repo.git', + + # vvv example of local file dependency. yes, DUMMY is necessary for some reason + # 'repo @ git+file://DUMMY/path/to/repo', + ], + extras_require={ + 'testing': ['pytest', 'requests'], + 'linting': ['pytest', 'mypy', 'lxml'], # lxml for mypy coverage report + }, + + + # this needs to be set if you're planning to upload to pypi + # url='', + # author='', + # author_email='', + # description='', + + # Rest of the stuff -- classifiers, license, etc, I don't think it matters for pypi + # it's just unnecessary duplication + ) + + +if __name__ == '__main__': + main() + diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..0018b97 --- /dev/null +++ b/tox.ini @@ -0,0 +1,32 @@ +[tox] +minversion = 3.5 +# relies on the correct version of Python installed +envlist = tests,mypy +# https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 +# hack to prevent .tox from crapping to the project directory +toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox + +[testenv] +passenv = +# useful for tests to know they are running under ci + CI + CI_* +# respect user's cache dirs to prevent tox from crapping into project dir + MYPY_CACHE_DIR + PYTHONPYCACHEPREFIX + +[testenv:tests] +commands = + pip install -e .[testing] + # posargs allow test filtering, e.g. tox ... -- -k test_name + python -m pytest server --ignore server/test_with_browser.py {posargs} + +[testenv:mypy] +commands = + pip install -e .[linting] + python -m mypy --install-types --non-interactive \ + server \ + # txt report is a bit more convenient to view on CI + --txt-report .coverage.mypy \ + --html-report .coverage.mypy \ + {posargs} From b57c7877746309da004722cddc0af7b2d7319b41 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 21 Dec 2022 16:52:25 +0000 Subject: [PATCH 05/32] macos: support setup for launchd should resolve https://github.com/karlicoss/grasp/issues/32 --- server/setup | 113 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 22 deletions(-) diff --git a/server/setup b/server/setup index 50606e6..0a0ff04 100755 --- a/server/setup +++ b/server/setup @@ -1,12 +1,18 @@ #!/usr/bin/env python3 import argparse -import shlex +import os from pathlib import Path -from subprocess import check_call, run +import platform +import shlex +from subprocess import check_call, run, DEVNULL from grasp_server import setup_parser -SYSTEMD_CONFIG = """ + +SYSTEM = platform.system() + + +SYSTEMD_TEMPLATE = """ [Unit] Description=Grasp extension server conterpart @@ -17,7 +23,29 @@ WantedBy=default.target ExecStart={server} {extra_args} Type=simple Restart=always -""" +""".lstrip() + + +LAUNCHD_TEMPLATE = ''' + + + + + Label + {service_name} + + ProgramArguments + +{arguments} + + + RunAtLoad + + KeepAlive + + + +'''.lstrip() def systemd(*args, method=check_call): @@ -25,31 +53,36 @@ def systemd(*args, method=check_call): 'systemctl', '--user', *args, ]) -def setup(args): + +def get_command(args) -> list[str]: + return [ + '--port', args.port, + '--path', args.path, + '--template', args.template, + *([] if args.config is None else ['--config' , args.config]), + ] + + +def setup_systemd(args) -> None: + args.template = args.template.replace('%', '%%').replace('\n', r'\n') # escape for systemd... + args.unit_name = args.unit_name.removesuffix('.service') # legacy unit_name default had .service + base_path = '~/.config/systemd/user' if not Path(base_path).expanduser().is_dir(): Path(base_path).expanduser().mkdir(parents=True, exist_ok=True) unit_name = args.unit_name - out = Path(f'{base_path}/{unit_name}').expanduser() - print(f"Writing systemd config to {out}:") + out = Path(f'{base_path}/{unit_name}.service').expanduser() + print(f"Writing unit file to {out}") server_bin = Path(__file__).parent.joinpath('grasp_server.py').absolute() + command = get_command(args) + extra_args = ' '.join(map(shlex.quote, command)) - template = args.template.replace('%', '%%').replace('\n', r'\n') # escape for systemd... - args = [ - '--port', args.port, - '--path', args.path, - '--template', template, - *([] if args.config is None else ['--config' , args.config]), - ] - extra_args = ' '.join(map(shlex.quote, args)) - - with out.open('w') as fo: - fo.write(SYSTEMD_CONFIG.format( + out.write_text(SYSTEMD_TEMPLATE.format( server=server_bin, extra_args=extra_args, - )) + )) try: systemd('stop' , unit_name, method=run) # ignore errors here if it wasn't running in the first place systemd('daemon-reload') @@ -60,12 +93,48 @@ def setup(args): print(f"Something has gone wrong... you might want to use 'journalctl --user -u {unit_name}' to debug") raise e -def main(): + +def setup_launchd(args) -> None: + # launchd keep everything as xml and treats whitespace literally + args.template = args.template.replace(r'\n', '\n') + + base_path = Path('~/Library/LaunchAgents') + + unit_name = args.unit_name + out = Path(f'{base_path}/{unit_name}.plist').expanduser() + print(f"Writing unit file to {out}") + + server_bin = Path(__file__).parent.joinpath('grasp_server.py').absolute() + command = get_command(args) + + + command = [str(server_bin)] + command + arguments = '\n'.join(f'{a}' for a in command) + + out.write_text(LAUNCHD_TEMPLATE.format( + service_name=unit_name, + arguments=arguments, + )) + + DOMAIN = f'gui/{os.getuid()}' + # defensive since might not have been running + run(['launchctl', 'bootout', DOMAIN + '/' + unit_name], stdout=DEVNULL, stderr=DEVNULL) + check_call(['launchctl', 'bootstrap', DOMAIN, out]) + + +def main() -> None: + default_unit_name = 'grasp' if SYSTEM == 'Linux' else 'com.github.karlicoss.grasp' + p = argparse.ArgumentParser('grasp server setup', formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=100)) # type: ignore - p.add_argument('--unit-name', type=str, default='grasp.service', help='Systemd unit name') + p.add_argument('--unit-name', type=str, default=default_unit_name, help='systemd/launchd unit name') setup_parser(p) args = p.parse_args() - setup(args) + if SYSTEM == 'Linux': + setup_systemd(args) + else: # assume macos + setup_launchd(args) + + if __name__ == '__main__': main() From 4127e3de647a7dd14468ecf93405277bc2e60dcd Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 23 Dec 2022 20:58:26 +0000 Subject: [PATCH 06/32] extension: move to a separate dir, update build scripts etc borrowed build scritps etc from promnesia --- .circleci/config.yml | 100 ---------- .github/workflows/main.yml | 33 +++ build_extension | 74 ------- .babelrc => extension/.babelrc | 0 extension/.ci/build | 22 ++ .eslintrc.js => extension/.eslintrc.js | 0 .flowconfig => extension/.flowconfig | 0 extension/build | 107 ++++++++++ extension/jest.config.js | 188 ++++++++++++++++++ package.json => extension/package.json | 30 ++- publish => extension/publish | 0 {src => extension/src}/img/unicorn.png | Bin {src => extension/src}/js/background.js | 0 {src => extension/src}/js/common.js | 0 {src => extension/src}/js/options.js | 0 {src => extension/src}/js/options_page.js | 0 {src => extension/src}/js/permissions.js | 0 {src => extension/src}/js/popup.js | 0 {src => extension/src}/manifest.json | 0 {src => extension/src}/options.html | 0 {src => extension/src}/popup.html | 0 .../webpack.config.js | 22 +- 22 files changed, 391 insertions(+), 185 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100755 build_extension rename .babelrc => extension/.babelrc (100%) create mode 100755 extension/.ci/build rename .eslintrc.js => extension/.eslintrc.js (100%) rename .flowconfig => extension/.flowconfig (100%) create mode 100755 extension/build create mode 100644 extension/jest.config.js rename package.json => extension/package.json (53%) rename publish => extension/publish (100%) rename {src => extension/src}/img/unicorn.png (100%) rename {src => extension/src}/js/background.js (100%) rename {src => extension/src}/js/common.js (100%) rename {src => extension/src}/js/options.js (100%) rename {src => extension/src}/js/options_page.js (100%) rename {src => extension/src}/js/permissions.js (100%) rename {src => extension/src}/js/popup.js (100%) rename {src => extension/src}/manifest.json (100%) rename {src => extension/src}/options.html (100%) rename {src => extension/src}/popup.html (100%) rename webpack.config.js => extension/webpack.config.js (87%) diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index a6e45d6..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,100 +0,0 @@ -# TODO should I use yarn as in circleci node example?? -version: 2 - -jobs: - client_build_and_check: - docker: - - image: circleci/node:latest - steps: - - checkout - - run: sudo npm install -g npm web-ext - - - restore_cache: - keys: - - dependency-cache-{{ checksum "package.json" }} - # fallback to using the latest cache if no exact match is found - - dependency-cache- - - run: npm install - - save_cache: - key: dependency-cache-{{ checksum "package.json" }} - paths: - - ./node_modules - - - run: ./build --chrome --lint - - # TODO ugh. is there a way for a job to inherit a cloned repo after npm run build has ran?? I'd like to separate build and check steps - # TODO ok, apparently it's a custom executor? https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs - - run: - name: js flow - command: npm run flow - when: always - - run: - name: eslint - command: npm run eslint - when: always - - - run: - # TODO need to do some kind of build matrix over [firefox, chrome] x [anyhost, no_anyhost] - # it should be somehow achievable via workspaces, but I've spent 10 minutes and couldn't figure it out... - name: pack - command: | - - for browser in 'chrome'; do - zip -r -j "/tmp/grasp.zip" dist/* - done - - # TODO ugh, also can't use env variables in artifacts... - - store_artifacts: - path: "/tmp/grasp.zip" - destination: "grasp.zip" - - - server_build_and_check: - docker: - - image: circleci/python:latest - steps: - - checkout - - # TODO preserve apt cache.. - - run: sudo apt install python3-pip - - run: python3 -m pip install --user mypy pylint pytest - - # test only dependencies - - run: sudo apt install httpie - - run: python3 -m pip install --user selenium pyautogui - # - - - run: - name: pylint - command: ci/pylint - when: always - - - run: - name: mypy - command: ci/mypy - when: always - - - run: - name: pytest - command: ci/pytest - when: always - -workflows: - version: 2 - - build_and_test: - jobs: - - client_build_and_check - - server_build_and_check - - -# # ugh. circleci only got python3.6 on this node package??? and it's unclear how to combine multiple circleci docker configurations... - -# # - store_artifacts: -# # path: test-results.xml -# # prefix: tests -# # - store_artifacts: -# # path: coverage -# # prefix: coverage -# # - store_test_results: -# # path: test-results.xml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f936ee..61a68d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,3 +51,36 @@ jobs: name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy/ + ### + build_extension: + env: + name: 'grasp' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - uses: actions/setup-node@v3 + with: + node-version: '18' + - run: extension/.ci/build --lint # debug version + - run: extension/.ci/build --lint --release + + # TODO ugh. can't share github actions artifacts publicly... + # TODO for fuck's sake... why does it end up named as .zip.zip ???? + - uses: actions/upload-artifact@v2 + with: + name: '${ env.name }}-chrome-debug-latest.zip' + path: 'extension/dist/artifacts/chrome/${{ env.name }}_dev_-*.zip' + - uses: actions/upload-artifact@v2 + with: + .name : '${{ env.name }}-chrome-release-latest.zip' + path: 'extension/dist/artifacts/chrome/${{ env.name }}-*.zip' + - uses: actions/upload-artifact@v2 + with: + name : '${{ env.name }}-firefox-debug-latest.zip' + path: 'extension/dist/artifacts/firefox/${{ env.name }}_dev_-*.zip' + - uses: actions/upload-artifact@v2 + with: + name : '${{ env.name }}-firefox-release-latest.zip' + path: 'extension/dist/artifacts/firefox/${{ env.name }}-*.zip' diff --git a/build_extension b/build_extension deleted file mode 100755 index 91fa6bc..0000000 --- a/build_extension +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import os -from subprocess import check_call -from pathlib import Path - - -# TODO FIXME copypaste from promnesia -# TODO FIXME use it in ci script -TARGETS = ['chrome', 'firefox'] -def main(): - p = argparse.ArgumentParser() - p.add_argument('--release', action='store_true') - p.add_argument('--sign', action='store_true') - p.add_argument('--lint', action='store_true') - - tg = p.add_mutually_exclusive_group(required=True) - tg.add_argument('--target', type=str, choices=TARGETS) - for b in TARGETS: - tg.add_argument('--' + b, action='store_const', const=b, dest='target') - args = p.parse_args() - target = args.target - - assert target is not None - - dist_dir = Path(__file__).absolute().parent / 'dist' - def webext(*args, **kwargs): - src_dir = dist_dir # / target - src_dir = src_dir.resolve() # TODO ugh. webext can't into symlinks??? - check_call([ - 'web-ext', - '--source-dir', str(src_dir), - *args, - ], **kwargs) - - env = { - 'TARGET': target, - 'RELEASE': 'YES' if args.release else 'NO', - **os.environ, - } - check_call([ - 'npm', 'run', 'build', - ], env=env, cwd=str(Path(__file__).absolute().parent)) - - if args.lint: - # TODO --self-hosted - # TODO warnings as errors?? - webext('lint') - - if args.release: - assert args.lint # TODO not sure.. - - if args.sign: - assert args.release - if target not in ['firefox', 'firefox-mobile']: - raise RuntimeError("{target} is not supported for signing yet".format(target=target)) - - dist_dir = Path(__file__).absolute().parent / 'dist' - src_dir = dist_dir / target - - from firefox_dev_secrets import API_KEY, API_SECRET - webext( - '--source-dir' , str(src_dir), - '--artifacts-dir', str(dist_dir), - '--api-key' , API_KEY, - '--api-secret', API_SECRET, - 'sign', - ) - -if __name__ == '__main__': - main() - -# TODO use UUID? https://addons.mozilla.org/en-GB/developers/addon/grasp/edit -# TODO right, wouldn't work because it's listed.. diff --git a/.babelrc b/extension/.babelrc similarity index 100% rename from .babelrc rename to extension/.babelrc diff --git a/extension/.ci/build b/extension/.ci/build new file mode 100755 index 0000000..9a1cbee --- /dev/null +++ b/extension/.ci/build @@ -0,0 +1,22 @@ +#!/bin/bash +set -eux + +cd "$(dirname "$0")" +cd ../.. # git root + +cd extension + +npm install + +FAILED=0 + +npm run flow || FAILED=1 +npm run eslint || FAILED=1 +# TODO later +#npm run test || FAILED=1 + +for browser in 'firefox' 'chrome'; do + ./build --target "$browser" "$@" +done + +exit "$FAILED" diff --git a/.eslintrc.js b/extension/.eslintrc.js similarity index 100% rename from .eslintrc.js rename to extension/.eslintrc.js diff --git a/.flowconfig b/extension/.flowconfig similarity index 100% rename from .flowconfig rename to extension/.flowconfig diff --git a/extension/build b/extension/build new file mode 100755 index 0000000..ea25d1e --- /dev/null +++ b/extension/build @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +import argparse +import os +from subprocess import check_call +from pathlib import Path + +# right, specifying id in manifest doesn't seem to work +# AMO responds with: Server response: Duplicate add-on ID found. (status: 400) +IDS = { + 'firefox' : 'FIXME', + 'chrome' : 'FIXME', +} + + +TARGETS = [ + 'chrome', + 'firefox', +] + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--release', action='store_true', help="Use release flavor of build") + p.add_argument('--lint' , action='store_true') + p.add_argument('--publish', action='store_true', help="Publish on chrome web store/addons.mozilla.org") + + tg = p.add_mutually_exclusive_group(required=True) + tg.add_argument('--target', type=str, choices=TARGETS) + for b in TARGETS: + tg.add_argument('--' + b, action='store_const', const=b, dest='target') + args = p.parse_args() + target = args.target + + assert target is not None + + base_ext_dir = Path(__file__).absolute().parent / 'dist' + ext_dir = (base_ext_dir / target).resolve() # webext can't into symlinks + # sadly no way to specify zip name in the regex.. + artifacts_dir = (base_ext_dir / 'artifacts' / target).resolve() + def webext(*args, **kwargs): + check_call([ + 'npm', 'run', 'web-ext', + '--', + '--source-dir' , ext_dir, + '--artifacts-dir', artifacts_dir, + *args, + ], **kwargs) + + env = { + 'TARGET': target, + 'RELEASE': 'YES' if args.release else 'NO', + 'PUBLISH': 'YES' if args.publish else 'NO', + **os.environ, + } + check_call([ + 'npm', 'run', 'build', + ], env=env, cwd=str(Path(__file__).absolute().parent)) + + if args.lint: + # TODO --self-hosted + # TODO warnings as errors?? + webext('lint') + + # TODO move somewhere more appropriate.. + webext('build', '-o') # -o overwrites existing artifact + + + if args.release: + assert args.lint # TODO not sure.. + + def firefox_release_args(): + from firefox_dev_secrets import API_KEY, API_SECRET + return [ + '--artifacts-dir', str(artifacts_dir), + '--api-key' , API_KEY, + '--api-secret' , API_SECRET, + '--id' , IDS[target], + ] + + if args.publish: + assert args.release + if 'firefox' in target: + check_call([ + 'npm', 'run', 'release:amo', + '--', + '--channel', 'listed', + '--source-dir', str(ext_dir), + *firefox_release_args(), + ]) + elif target == 'chrome': + from chrome_dev_secrets import CLIENT_ID, CLIENT_SECRET, REFRESH_TOKEN + check_call([ + 'npm', 'run', 'release:cws', + '--', + 'upload', + # '--auto-publish', + '--source' , str(ext_dir), + '--client-id' , CLIENT_ID, + '--client-secret' , CLIENT_SECRET, + '--refresh--token', REFRESH_TOKEN, + '--extension-id' , IDS[target], + # TODO trusted testers? + ]) + else: + raise RuntimeError("{target} is not supported for publishing yet".format(target=target)) + +if __name__ == '__main__': + main() diff --git a/extension/jest.config.js b/extension/jest.config.js new file mode 100644 index 0000000..8dbab0a --- /dev/null +++ b/extension/jest.config.js @@ -0,0 +1,188 @@ +// For a detailed explanation regarding each configuration property, visit: +// https://jestjs.io/docs/en/configuration.html + +module.exports = { + // All imported modules in your tests should be mocked automatically + // automock: false, + + // Stop running tests after `n` failures + // bail: 0, + + // Respect "browser" field in package.json when resolving modules + // browser: false, + + // The directory where Jest should store its cached dependency information + // cacheDirectory: "/tmp/jest_rs", + + // Automatically clear mock calls and instances between every test + clearMocks: true, + + // Indicates whether the coverage information should be collected while executing the test + // collectCoverage: false, + + // An array of glob patterns indicating a set of files for which coverage information should be collected + // collectCoverageFrom: null, + + // The directory where Jest should output its coverage files + // coverageDirectory: null, + + // An array of regexp pattern strings used to skip coverage collection + // coveragePathIgnorePatterns: [ + // "/node_modules/" + // ], + + // A list of reporter names that Jest uses when writing coverage reports + // coverageReporters: [ + // "json", + // "text", + // "lcov", + // "clover" + // ], + + // An object that configures minimum threshold enforcement for coverage results + // coverageThreshold: null, + + // A path to a custom dependency extractor + // dependencyExtractor: null, + + // Make calling deprecated APIs throw helpful error messages + // errorOnDeprecated: false, + + // Force coverage collection from ignored files using an array of glob patterns + // forceCoverageMatch: [], + + // A path to a module which exports an async function that is triggered once before all test suites + // globalSetup: null, + + // A path to a module which exports an async function that is triggered once after all test suites + // globalTeardown: null, + + // A set of global variables that need to be available in all test environments + // globals: {}, + + // An array of directory names to be searched recursively up from the requiring module's location + // moduleDirectories: [ + // "node_modules" + // ], + + // An array of file extensions your modules use + // moduleFileExtensions: [ + // "js", + // "json", + // "jsx", + // "ts", + // "tsx", + // "node" + // ], + + // A map from regular expressions to module names that allow to stub out resources with a single module + // moduleNameMapper: {}, + + // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader + // modulePathIgnorePatterns: [], + + // Activates notifications for test results + // notify: false, + + // An enum that specifies notification mode. Requires { notify: true } + // notifyMode: "failure-change", + + // A preset that is used as a base for Jest's configuration + // preset: null, + + // Run tests from one or more projects + // projects: null, + + // Use this configuration option to add custom reporters to Jest + // reporters: undefined, + + // Automatically reset mock state between every test + // resetMocks: false, + + // Reset the module registry before running each individual test + // resetModules: false, + + // A path to a custom resolver + // resolver: null, + + // Automatically restore mock state between every test + // restoreMocks: false, + + // The root directory that Jest should scan for tests and modules within + // rootDir: null, + + // A list of paths to directories that Jest should use to search for files in + // roots: [ + // "" + // ], + + // Allows you to use a custom runner instead of Jest's default test runner + // runner: "jest-runner", + + // The paths to modules that run some code to configure or set up the testing environment before each test + // setupFiles: [], + + // A list of paths to modules that run some code to configure or set up the testing framework before each test + // setupFilesAfterEnv: [], + + // A list of paths to snapshot serializer modules Jest should use for snapshot testing + // snapshotSerializers: [], + + // The test environment that will be used for testing + testEnvironment: "jsdom", + + // Options that will be passed to the testEnvironment + // testEnvironmentOptions: {}, + + // Adds a location field to test results + // testLocationInResults: false, + + // The glob patterns Jest uses to detect test files + // testMatch: [ + // "**/__tests__/**/*.[jt]s?(x)", + // "**/?(*.)+(spec|test).[tj]s?(x)" + // ], + + // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped + // testPathIgnorePatterns: [ + // "/node_modules/" + // ], + + // The regexp pattern or array of patterns that Jest uses to detect test files + // testRegex: [], + + // This option allows the use of a custom results processor + // testResultsProcessor: null, + + // This option allows use of a custom test runner + // testRunner: "jasmine2", + + // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href + // testURL: "http://localhost", + + // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout" + // timers: "real", + + // A map from regular expressions to paths to transformers + // transform: null, + + // ??? https://github.com/facebook/jest/issues/9395#issuecomment-665239070 + // otherwise fails on importing + transformIgnorePatterns: ['node_modules/?!@webext-options-sync/'] + // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation + // transformIgnorePatterns: [ + // "/node_modules/" + // ], + + // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them + // unmockedModulePathPatterns: undefined, + + // Indicates whether each individual test should be reported during the run + // verbose: null, + + // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode + // watchPathIgnorePatterns: [], + + // Whether to use watchman for file crawling + // watchman: true, +}; diff --git a/package.json b/extension/package.json similarity index 53% rename from package.json rename to extension/package.json index 0408443..f2c3e76 100644 --- a/package.json +++ b/extension/package.json @@ -1,15 +1,30 @@ { "name": "grasp", "version": "0.6.6", + "version_name": "released on TODO", "description": "Reliably capture links and webpage content in a plaintext file (org-mode/markdown)", "main": "dist/background.js", "scripts": { - "build": "webpack --progress --profile --colors", - "watch": "webpack --watch --progress --colors", + "test": "jest", + "build": "webpack --progress --profile", + "watch": "webpack --watch --progress", "dev": "npm run watch", "eslint": "eslint src", - "flow": "flow" + "flow": "flow", + "web-ext": "web-ext", + "release:cws": "webstore", + "release:amo": "web-ext-submit" }, + "repository": { + "type": "git", + "url": "git+https://github.com/karlicoss/grasp.git" + }, + "author": "Dmitrii Gerasimov ", + "license": "MIT", + "bugs": { + "url": "https://github.com/karlicoss/grasp/issues" + }, + "homepage": "https://github.com/karlicoss/grasp#readme", "devDependencies": { "@babel/core": "^7.2.2", "@babel/preset-env": "^7.2.3", @@ -24,10 +39,17 @@ "eslint-plugin-flowtype": "^3.2.0", "flow-bin": "^0.89.0", "flow-interfaces-chrome": "^0.5.3", + "jest": "^28.1.0", + "jest-environment-jsdom": "^28.1.0", + "jest-fetch-mock": "^3.0.3", + "node-fetch": "^2.6.1", "style-loader": "^1.0.0", + "web-ext": "^6.8.0", + "web-ext-submit": "^6.8.0", "webpack": "^4.28.3", "webpack-cli": "^3.2.0", "webpack-dev-server": "^3.1.14", "webpack-extension-manifest-plugin": "^0.4.3" - } + }, + "private": "true" } diff --git a/publish b/extension/publish similarity index 100% rename from publish rename to extension/publish diff --git a/src/img/unicorn.png b/extension/src/img/unicorn.png similarity index 100% rename from src/img/unicorn.png rename to extension/src/img/unicorn.png diff --git a/src/js/background.js b/extension/src/js/background.js similarity index 100% rename from src/js/background.js rename to extension/src/js/background.js diff --git a/src/js/common.js b/extension/src/js/common.js similarity index 100% rename from src/js/common.js rename to extension/src/js/common.js diff --git a/src/js/options.js b/extension/src/js/options.js similarity index 100% rename from src/js/options.js rename to extension/src/js/options.js diff --git a/src/js/options_page.js b/extension/src/js/options_page.js similarity index 100% rename from src/js/options_page.js rename to extension/src/js/options_page.js diff --git a/src/js/permissions.js b/extension/src/js/permissions.js similarity index 100% rename from src/js/permissions.js rename to extension/src/js/permissions.js diff --git a/src/js/popup.js b/extension/src/js/popup.js similarity index 100% rename from src/js/popup.js rename to extension/src/js/popup.js diff --git a/src/manifest.json b/extension/src/manifest.json similarity index 100% rename from src/manifest.json rename to extension/src/manifest.json diff --git a/src/options.html b/extension/src/options.html similarity index 100% rename from src/options.html rename to extension/src/options.html diff --git a/src/popup.html b/extension/src/popup.html similarity index 100% rename from src/popup.html rename to extension/src/popup.html diff --git a/webpack.config.js b/extension/webpack.config.js similarity index 87% rename from webpack.config.js rename to extension/webpack.config.js index 8d25036..2446ece 100644 --- a/webpack.config.js +++ b/extension/webpack.config.js @@ -13,12 +13,14 @@ const T = { const env = { TARGET : process.env.TARGET, // TODO assert not null? RELEASE: process.env.RELEASE, + PUBLISH: process.env.PUBLISH, }; const pkg = require('./package.json'); const baseManifest = require('./src/manifest.json'); const release = env.RELEASE == 'YES' ? true : false; +const publish = env.PUBLISH == 'YES' ? true : false; const dev = !release; // meh. maybe make up my mind? const target = env.TARGET; @@ -78,14 +80,14 @@ if (target === T.FIREFOX) { manifestExtra.browser_action = {browser_style: true}; } -const build_path = path.join(__dirname, "dist"); // TODO target?? +const buildPath = path.join(__dirname, 'dist', target); const options = { mode: dev ? 'development' : 'production', // TODO map directly from MODE env var? - optimization: { - // https://webpack.js.org/configuration/optimization - // don't think minimize worth it for such a tiny extension - minimize: false + node: { + // no idea what does it mean... https://github.com/webpack/webpack/issues/5627#issuecomment-394290231 + // but it does get rid of some Function() which webpacka apparently generates + global: false, }, entry: { background : path.join(__dirname, "src", "js", "background"), @@ -93,9 +95,15 @@ const options = { options_page: path.join(__dirname, "src", "js", "options_page"), }, output: { - path: build_path, + publicPath: '', // https://stackoverflow.com/a/64715069 + path: buildPath, filename: "[name].js", }, + optimization: { + // https://webpack.js.org/configuration/optimization + // don't think minimize worth it for such a tiny extension + minimize: false + }, module: { rules: [ { @@ -118,7 +126,7 @@ const options = { ] }, plugins: [ - new CleanWebpackPlugin([build_path + "/*"]), + new CleanWebpackPlugin([buildPath + "/*"]), new CopyWebpackPlugin([ { from: 'src/img/*.png', flatten: true }, { from: 'src/*.html' , flatten: true }, From ddb5fe5c317050b5df119ad2febc26f7145a3044 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 23 Dec 2022 21:27:08 +0000 Subject: [PATCH 07/32] extension: update webpack/babel/eslint/flow mostly borrowed version from promnesia for now --- .github/workflows/main.yml | 2 +- extension/.babelrc | 3 --- extension/.eslintrc.js | 14 +++++++------- extension/babel.config.js | 15 +++++++++++++++ extension/package.json | 34 +++++++++++++++++----------------- extension/src/js/popup.js | 8 ++++---- extension/webpack.config.js | 23 ++++++++++++----------- 7 files changed, 56 insertions(+), 43 deletions(-) delete mode 100644 extension/.babelrc create mode 100644 extension/babel.config.js diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 61a68d4..22e90af 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,7 +70,7 @@ jobs: # TODO for fuck's sake... why does it end up named as .zip.zip ???? - uses: actions/upload-artifact@v2 with: - name: '${ env.name }}-chrome-debug-latest.zip' + name: '${{ env.name }}-chrome-debug-latest.zip' path: 'extension/dist/artifacts/chrome/${{ env.name }}_dev_-*.zip' - uses: actions/upload-artifact@v2 with: diff --git a/extension/.babelrc b/extension/.babelrc deleted file mode 100644 index 844b18f..0000000 --- a/extension/.babelrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "presets": ["@babel/preset-env", "@babel/preset-flow"] -} diff --git a/extension/.eslintrc.js b/extension/.eslintrc.js index 42f2c35..987c768 100644 --- a/extension/.eslintrc.js +++ b/extension/.eslintrc.js @@ -1,5 +1,5 @@ module.exports = { - "parser": "babel-eslint", + "parser": "@babel/eslint-parser", "plugins": [ "flowtype" ], @@ -27,12 +27,12 @@ module.exports = { "indent": "off", "comma-dangle": "off", "no-console": "off", + "no-inner-declarations": "off", + "flowtype/space-before-type-colon": "off", + "no-unused-vars": ["error", { + "argsIgnorePattern": "^_", + "varsIgnorePattern": "^_" + }], } // TODO use flow? }; -//module.exports = { -// // "extends": "google", -// 'extends': [ -// "plugin:react/recommended", -// ], -//}; diff --git a/extension/babel.config.js b/extension/babel.config.js new file mode 100644 index 0000000..a764417 --- /dev/null +++ b/extension/babel.config.js @@ -0,0 +1,15 @@ +const presets = [ + [ + '@babel/preset-env', + // https://caniuse.com/usage-table + {targets: {chrome: 75, firefox: 75}} + ], + '@babel/preset-flow', +] +const plugins = [] + +// if (process.env["ENV"] === "prod") { +// plugins.push(...); +// } + +module.exports = { presets, plugins } diff --git a/extension/package.json b/extension/package.json index f2c3e76..faba5b7 100644 --- a/extension/package.json +++ b/extension/package.json @@ -26,19 +26,19 @@ }, "homepage": "https://github.com/karlicoss/grasp#readme", "devDependencies": { - "@babel/core": "^7.2.2", - "@babel/preset-env": "^7.2.3", - "@babel/preset-flow": "^7.0.0", - "babel-loader": "^8.0.5", - "babel-eslint": "^10.0.1", - "clean-webpack-plugin": "^1.0.0", - "copy-webpack-plugin": "^4.2.0", - "css-loader": "^2.1.0", - "eslint": "^5.12.0", - "eslint-config-google": "^0.11.0", - "eslint-plugin-flowtype": "^3.2.0", - "flow-bin": "^0.89.0", - "flow-interfaces-chrome": "^0.5.3", + "@babel/core": "^7.16.0", + "@babel/eslint-parser": "^7.16.3", + "@babel/preset-env": "^7.16.4", + "@babel/preset-flow": "^7.16.0", + "babel-loader": "^8.2.3", + "chrome-webstore-upload-cli": "^2.1.0", + "clean-webpack-plugin": "^4.0.0", + "copy-webpack-plugin": "^10.0.0", + "css-loader": "^6.5.1", + "eslint": "^8.15.0", + "eslint-plugin-flowtype": "^8.0.3", + "flow-bin": "^0.133.0", + "flow-interfaces-chrome": "^0.6.0", "jest": "^28.1.0", "jest-environment-jsdom": "^28.1.0", "jest-fetch-mock": "^3.0.3", @@ -46,10 +46,10 @@ "style-loader": "^1.0.0", "web-ext": "^6.8.0", "web-ext-submit": "^6.8.0", - "webpack": "^4.28.3", - "webpack-cli": "^3.2.0", - "webpack-dev-server": "^3.1.14", - "webpack-extension-manifest-plugin": "^0.4.3" + "webpack": "^5.75.0", + "webpack-cli": "^5.0.1", + "webpack-dev-server": "^4.11.1", + "webpack-extension-manifest-plugin": "^0.8.0" }, "private": "true" } diff --git a/extension/src/js/popup.js b/extension/src/js/popup.js index fd3466c..bbe4b1f 100644 --- a/extension/src/js/popup.js +++ b/extension/src/js/popup.js @@ -9,10 +9,10 @@ const COMMENT_ID = 'comment_id'; const TAGS_ID = 'tags_id'; -type State = { +type State = {| comment: string, tag_str: string, -}; +|}; function save_state(state: ?State) { @@ -72,8 +72,8 @@ function submitComment () { window.close(); } -// $FlowFixMe -function ctrlEnterSubmit(e) { +// todo not sure if can define const here?? +var ctrlEnterSubmit: KeyboardEventListener = (e) => { if (e.ctrlKey && e.key === 'Enter') { submitComment(); } diff --git a/extension/webpack.config.js b/extension/webpack.config.js index 2446ece..47d0671 100644 --- a/extension/webpack.config.js +++ b/extension/webpack.config.js @@ -1,9 +1,8 @@ const webpack = require('webpack'), path = require('path'), - CleanWebpackPlugin = require('clean-webpack-plugin'), + {CleanWebpackPlugin} = require('clean-webpack-plugin'), CopyWebpackPlugin = require('copy-webpack-plugin'), WebpackExtensionManifestPlugin = require('webpack-extension-manifest-plugin'); -// TODO remove plugins from package.json const T = { CHROME : 'chrome', @@ -114,9 +113,8 @@ const options = { } }, { - test: /\.css$/, - loader: "style-loader!css-loader", // TODO different from promnesia?? - exclude: /node_modules/ + test: /\.css$/i, + use: ['style-loader', 'css-loader'], }, { test: /\.html$/, @@ -126,11 +124,13 @@ const options = { ] }, plugins: [ - new CleanWebpackPlugin([buildPath + "/*"]), - new CopyWebpackPlugin([ - { from: 'src/img/*.png', flatten: true }, - { from: 'src/*.html' , flatten: true }, - ]), + new CleanWebpackPlugin(), // ok, respects symlinks + new CopyWebpackPlugin({ + patterns: [ + { from: 'src/*.html' , to: '[name][ext]'}, + { from: 'src/img/*.png', to: '[name][ext]'}, + ] + }), new WebpackExtensionManifestPlugin({ config: { base: baseManifest, @@ -142,7 +142,8 @@ const options = { // TODO https://webpack.js.org/configuration/devtool if (dev) { - options.devtool = "cheap-module-eval-source-map"; + // ??? don't remember what it was for, but webpack complains about it now + // options.devtool = "cheap-module-eval-source-map"; } module.exports = options; From 83e7de9be99aebf4ebf9738bf22d96da340bfb45 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 23 Dec 2022 22:34:37 +0000 Subject: [PATCH 08/32] extension: simplify webpack config a bit, remove some legacy, update dev packages --- extension/babel.config.js | 1 + extension/package.json | 14 +++++++------- extension/src/js/background.js | 8 +++++--- extension/src/js/common.js | 1 + extension/src/js/options.js | 2 +- extension/src/js/permissions.js | 8 ++++---- extension/src/js/popup.js | 3 ++- extension/src/manifest.json | 4 ++-- extension/webpack.config.js | 19 +++++++++---------- 9 files changed, 32 insertions(+), 28 deletions(-) diff --git a/extension/babel.config.js b/extension/babel.config.js index a764417..5f5580d 100644 --- a/extension/babel.config.js +++ b/extension/babel.config.js @@ -2,6 +2,7 @@ const presets = [ [ '@babel/preset-env', // https://caniuse.com/usage-table + // TODO ideally should use https://github.com/browserslist/browserslist {targets: {chrome: 75, firefox: 75}} ], '@babel/preset-flow', diff --git a/extension/package.json b/extension/package.json index faba5b7..acedaca 100644 --- a/extension/package.json +++ b/extension/package.json @@ -30,22 +30,22 @@ "@babel/eslint-parser": "^7.16.3", "@babel/preset-env": "^7.16.4", "@babel/preset-flow": "^7.16.0", - "babel-loader": "^8.2.3", + "babel-loader": "^9.1.0", "chrome-webstore-upload-cli": "^2.1.0", "clean-webpack-plugin": "^4.0.0", - "copy-webpack-plugin": "^10.0.0", + "copy-webpack-plugin": "^11.0.0", "css-loader": "^6.5.1", "eslint": "^8.15.0", "eslint-plugin-flowtype": "^8.0.3", - "flow-bin": "^0.133.0", - "flow-interfaces-chrome": "^0.6.0", + "flow-bin": "^0.196.3", + "flow-interfaces-chrome": "^0.7.0", "jest": "^28.1.0", "jest-environment-jsdom": "^28.1.0", "jest-fetch-mock": "^3.0.3", "node-fetch": "^2.6.1", - "style-loader": "^1.0.0", - "web-ext": "^6.8.0", - "web-ext-submit": "^6.8.0", + "style-loader": "^3.3.1", + "web-ext": "^7.4.0", + "web-ext-submit": "^7.4.0", "webpack": "^5.75.0", "webpack-cli": "^5.0.1", "webpack-dev-server": "^4.11.1", diff --git a/extension/src/js/background.js b/extension/src/js/background.js index 4555b38..f8c3c0e 100644 --- a/extension/src/js/background.js +++ b/extension/src/js/background.js @@ -1,8 +1,10 @@ /* @flow */ // TODO do I really need to annotate all files with @flow?? -import {COMMAND_CAPTURE_SIMPLE, METHOD_CAPTURE_WITH_EXTRAS, showNotification} from './common'; -import {get_options} from './options'; +import {COMMAND_CAPTURE_SIMPLE, METHOD_CAPTURE_WITH_EXTRAS, showNotification} from './common' +import {get_options} from './options' +import type {Options} from './options' + type Params = { url: string, @@ -15,7 +17,7 @@ type Params = { function makeCaptureRequest( params: Params, - options, + options: Options, ) { if (params.tag_str == null) { params.tag_str = options.default_tags; diff --git a/extension/src/js/common.js b/extension/src/js/common.js index b1010a4..7e578db 100644 --- a/extension/src/js/common.js +++ b/extension/src/js/common.js @@ -16,6 +16,7 @@ export function showNotification(text: string, priority: number=0) { // $FlowFixMe export function awrap(fn, ...args: Array): Promise { return new Promise((resolve, reject) => { + // $FlowFixMe const cbb = (...xxx) => { const err = chrome.runtime.lastError; if (err) { diff --git a/extension/src/js/options.js b/extension/src/js/options.js index 7b53c32..39341f2 100644 --- a/extension/src/js/options.js +++ b/extension/src/js/options.js @@ -1,6 +1,6 @@ /* @flow */ -type Options = { +export type Options = { endpoint: string; default_tags: string; notification: boolean; diff --git a/extension/src/js/permissions.js b/extension/src/js/permissions.js index 10f4a99..a3396e1 100644 --- a/extension/src/js/permissions.js +++ b/extension/src/js/permissions.js @@ -3,11 +3,11 @@ import {showNotification, awrap} from './common'; import {getOptions} from './options'; -export function chromePermissionsRequest(params: any) { +export function chromePermissionsRequest(params: any): Promise { return awrap(chrome.permissions.request, params); } -export function chromePermissionsContains(params: any) { +export function chromePermissionsContains(params: any): Promise { return awrap(chrome.permissions.contains, params); } @@ -23,7 +23,7 @@ function urlForPermissionsCheck(url: string): string { return u.toString(); } -export function hasPermissions(endpoint: string) { +export function hasPermissions(endpoint: string): Promise { const perms = { origins: [ urlForPermissionsCheck(endpoint), @@ -32,7 +32,7 @@ export function hasPermissions(endpoint: string) { return chromePermissionsContains(perms); } -export function ensurePermissions(endpoint: string) { +export function ensurePermissions(endpoint: string): Promise { const perms = { origins: [ urlForPermissionsCheck(endpoint), diff --git a/extension/src/js/popup.js b/extension/src/js/popup.js index bbe4b1f..bc3aa7b 100644 --- a/extension/src/js/popup.js +++ b/extension/src/js/popup.js @@ -81,9 +81,10 @@ var ctrlEnterSubmit: KeyboardEventListener = (e) => { // https://stackoverflow.com/a/6003829/706389 -function moveCaretToEnd(el) { +function moveCaretToEnd(el: HTMLInputElement) { if (typeof el.selectionStart == "number") { el.selectionStart = el.selectionEnd = el.value.length; + // $FlowFixMe https://github.com/facebook/flow/issues/8705 } else if (typeof el.createTextRange != "undefined") { el.focus(); var range = el.createTextRange(); diff --git a/extension/src/manifest.json b/extension/src/manifest.json index 331c235..4592c06 100644 --- a/extension/src/manifest.json +++ b/extension/src/manifest.json @@ -9,7 +9,7 @@ "page": "options.html" }, "browser_action": { - "default_icon": "unicorn.png", + "default_icon": "img/unicorn.png", "default_popup": "popup.html", "default_title": "Capture page, with extra information" }, @@ -22,7 +22,7 @@ } }, "icons": { - "128": "unicorn.png" + "128": "img/unicorn.png" }, "manifest_version": 2 } diff --git a/extension/webpack.config.js b/extension/webpack.config.js index 47d0671..5953d50 100644 --- a/extension/webpack.config.js +++ b/extension/webpack.config.js @@ -94,9 +94,7 @@ const options = { options_page: path.join(__dirname, "src", "js", "options_page"), }, output: { - publicPath: '', // https://stackoverflow.com/a/64715069 path: buildPath, - filename: "[name].js", }, optimization: { // https://webpack.js.org/configuration/optimization @@ -107,28 +105,29 @@ const options = { rules: [ { test: /\.js$/, + loader: 'babel-loader', exclude: /node_modules/, - use: { - loader: 'babel-loader', - } }, { - test: /\.css$/i, - use: ['style-loader', 'css-loader'], + test: /\.css$/, + use: 'style-loader!css-loader', + exclude: /node_modules/, }, { test: /\.html$/, - loader: "html-loader", + loader: 'html-loader', exclude: /node_modules/ } ] }, plugins: [ new CleanWebpackPlugin(), // ok, respects symlinks + + // without copy pluging, webpack only bundles js/json files new CopyWebpackPlugin({ patterns: [ - { from: 'src/*.html' , to: '[name][ext]'}, - { from: 'src/img/*.png', to: '[name][ext]'}, + { context: 'src', from: '**/*.html' }, + { context: 'src', from: '**/*.png' }, ] }), new WebpackExtensionManifestPlugin({ From 20022db1f6d10b03b1ef2afa8ac2b7b7b060dc33 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 23 Dec 2022 23:18:24 +0000 Subject: [PATCH 09/32] mac: allow using Shift-Enter to complete capture --- extension/src/js/common.js | 2 +- extension/src/js/popup.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/extension/src/js/common.js b/extension/src/js/common.js index 7e578db..2defa37 100644 --- a/extension/src/js/common.js +++ b/extension/src/js/common.js @@ -9,7 +9,7 @@ export function showNotification(text: string, priority: number=0) { 'title': "grasp", 'message': text, 'priority': priority, - 'iconUrl': 'unicorn.png', + 'iconUrl': 'img/unicorn.png', }); } diff --git a/extension/src/js/popup.js b/extension/src/js/popup.js index bc3aa7b..8490071 100644 --- a/extension/src/js/popup.js +++ b/extension/src/js/popup.js @@ -74,7 +74,9 @@ function submitComment () { // todo not sure if can define const here?? var ctrlEnterSubmit: KeyboardEventListener = (e) => { - if (e.ctrlKey && e.key === 'Enter') { + // note: there is also e.metaKey which triggers on mac when cmd is pressed + // but it doesn't seem to allow chords like cmd+Enter + if ((e.ctrlKey || e.shiftKey) && e.key === 'Enter') { submitComment(); } } From 5bdd5116e7c524f622a19c8db8e27b69006137fa Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 24 Dec 2022 03:28:54 +0000 Subject: [PATCH 10/32] switch makeCaptureRequest to fetch api in preparation for manifest v3 --- extension/src/js/background.js | 83 ++++++++++++++-------------------- 1 file changed, 34 insertions(+), 49 deletions(-) diff --git a/extension/src/js/background.js b/extension/src/js/background.js index f8c3c0e..e85e9d5 100644 --- a/extension/src/js/background.js +++ b/extension/src/js/background.js @@ -15,65 +15,50 @@ type Params = { } -function makeCaptureRequest( +async function makeCaptureRequest( params: Params, options: Options, ) { if (params.tag_str == null) { - params.tag_str = options.default_tags; + params.tag_str = options.default_tags } - const data = JSON.stringify(params); - console.log(`[background] capturing ${data}`); + const endpoint = options.endpoint - var request = new XMLHttpRequest(); - const curl = options.endpoint; + const data = JSON.stringify(params) + console.log(`capturing ${data} via ${endpoint}`) - request.timeout = 10 * 1000; // TODO should it be configurable? - request.open('POST', curl, true); - request.onreadystatechange = () => { - const XHR_DONE = 4; - if (request.readyState != XHR_DONE) { - return; + + // TODO maybe handle catch first and then again? + + await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: data, + }).catch((err: Error) => { + // fetch only rejects when host is unavailable + console.error(err) + throw new Error(`${endpoint} unavailable, check if server is running`) + }).then(async response => { + if (!response.ok) { // http error + throw new Error(`${endpoint}: HTTP ${response.status} ${response.statusText}`) } - const status = request.status; - const rtext = request.responseText; - var had_error = false; - var error_message = `status ${status}, response ${rtext}`; - console.log(`[background] status: ${status}, response: ${rtext}`); - if (status >= 200 && status < 400) { // success - try { - // TODO handle json parsing defensively here - const response = JSON.parse(rtext); - const path = response.path; - console.log(`[background] success: ${rtext}`); - if (options.notification) { - showNotification(`OK: captured to ${path}`); - } - } catch (err) { - had_error = true; - error_message = error_message.concat(String(err)); - console.error(err); - } - } else { - had_error = true; - if (status == 0) { - error_message = error_message.concat(` ${curl} must be unavailable `); - } + const jres = await response.json() + const path = jres.path + console.log(`success: ${jres}`); + if (options.notification) { + showNotification(`OK: captured to ${path}`); } - - if (had_error) { - console.error(`[background] ERROR: ${error_message}`); - showNotification(`ERROR: ${error_message}`, 1); - // TODO crap, doesn't really seem to respect urgency... - } - }; - request.onerror = () => { - console.error(request); - }; - - request.setRequestHeader("Content-Type", "application/json;charset=UTF-8"); - request.send(data); + }).catch((err: Error) => { + console.error(err) + // todo make sure we catch errors from then clause too? + const error_message = `ERROR: ${err.toString()}` + console.error(error_message); + showNotification(error_message, 1); + // TODO crap, doesn't really seem to respect urgency... + }) } From 11cd3b3f65a26379e934f2288f8e9937b6a5907d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 24 Dec 2022 05:29:44 +0000 Subject: [PATCH 11/32] extension: some 'cosmetic' preparations for manifest v3 tested on chromium/firefox stable --- extension/src/js/background.js | 60 +++++++++++++++--------- extension/src/manifest.json | 13 ------ extension/webpack.config.js | 84 ++++++++++++++++++++++++---------- 3 files changed, 100 insertions(+), 57 deletions(-) diff --git a/extension/src/js/background.js b/extension/src/js/background.js index e85e9d5..32b640b 100644 --- a/extension/src/js/background.js +++ b/extension/src/js/background.js @@ -65,31 +65,49 @@ async function makeCaptureRequest( // TODO FIXME ugh. need defensive error handling on the very top... function capture(comment: ?string = null, tag_str: ?string = null) { chrome.tabs.query({currentWindow: true, active: true }, tabs => { - const tab = tabs[0]; + const tab = tabs[0] if (tab.url == null) { - showNotification('ERROR: trying to capture null'); - return; + showNotification('ERROR: trying to capture null') + return } - const url: string = tab.url; - const title: ?string = tab.title; + const url: string = tab.url + const title: ?string = tab.title + + + const payload = (selection: ?string) => { + return { + url: url, + title: title, + selection: selection, + comment: comment, + tag_str: tag_str, + } + } + + // TODO await properly and handle errors + const has_scripting = 'scripting' in chrome get_options(opts => { - // console.log('action!'); - // ugh.. https://stackoverflow.com/a/19165930/706389 - chrome.tabs.executeScript( { - code: "window.getSelection().toString();" - }, selections => { - const selection = selections == null ? null : selections[0]; - makeCaptureRequest({ - url: url, - title: title, - selection: selection, - comment: comment, - tag_str: tag_str, - }, opts); - }); - }); - }); + if (has_scripting) { + // $FlowFixMe + chrome.scripting.executeScript({ + target: {tabId: tab.id}, + func: () => window.getSelection().toString() + }, results => { + const [res] = results // should only inject in one frame, so just one result + const selection = res.result + makeCaptureRequest(payload(selection), opts) + }) + } else { + chrome.tabs.executeScript( { + code: "window.getSelection().toString();" + }, selections => { + const selection = selections == null ? null : selections[0] + makeCaptureRequest(payload(selection), opts) + }) + } + }) + }) } diff --git a/extension/src/manifest.json b/extension/src/manifest.json index 4592c06..66df0da 100644 --- a/extension/src/manifest.json +++ b/extension/src/manifest.json @@ -8,19 +8,6 @@ "options_ui": { "page": "options.html" }, - "browser_action": { - "default_icon": "img/unicorn.png", - "default_popup": "popup.html", - "default_title": "Capture page, with extra information" - }, - "commands": { - "capture-simple": { - "description": "Quick capture: url, title and selection" - }, - "_execute_browser_action": { - "description": "Capture page, with extra information" - } - }, "icons": { "128": "img/unicorn.png" }, diff --git a/extension/webpack.config.js b/extension/webpack.config.js index 5953d50..b82997f 100644 --- a/extension/webpack.config.js +++ b/extension/webpack.config.js @@ -25,34 +25,74 @@ const target = env.TARGET; const name = "grasp" + (dev ? ' [dev]' : ''); -// Firefox wouldn't let you rebind its default shortcuts most of which use Shift -// On the other hand, Chrome wouldn't let you use Alt -const modifier = target === T.CHROME ? 'Shift' : 'Alt'; // ugh. declarative formats are shit. -const commandsExtra = { + +// Firefox wouldn't let you rebind its default shortcuts most of which use Shift +// On the other hand, Chrome wouldn't let you use Alt +const modifier = target === T.CHROME ? 'Shift' : 'Alt' + +const action_name = 'browser_action' + +const commands = { "capture-simple": { + "description": "Quick capture: url, title and selection", "suggested_key": { "default": `Ctrl+${modifier}+C`, - "mac": `Command+${modifier}+C` - } - }, - "_execute_browser_action": { + "mac": `Command+${modifier}+C`, + }, + } +} + +commands[`_execute_${action_name}`] = { + "description": "Capture page, with extra information", "suggested_key": { "default": `Ctrl+${modifier}+Y`, - "mac": `Command+${modifier}+Y` - } - }, -}; + "mac": `Command+${modifier}+Y`, + }, +} + + +const action = { + "default_icon": "img/unicorn.png", + "default_popup": "popup.html", + "default_title": "Capture page, with extra information", +} +if (target == T.FIREFOX) { + action['browser_style'] = true +} + + +// prepare for manifest v3 +const host_permissions = [ + "http://localhost/capture", + "https://localhost/capture", +] + + +const optional_permissions = [ + "http://*/capture", + "https://*/capture", +] + // TODO make permissions literate const permissions = [ "storage", "notifications", - "activeTab", - "http://localhost/capture", - "https://localhost/capture" -]; + + // need to query active tab and get its url/title + "activeTab", + + ...host_permissions, +] + +if (target === T.FIREFOX) { + // chrome v2 doesn't support scripting api + // code has a fallback just for that + // need to get selected text + permissions.push("scripting") +} const manifestExtra = { @@ -60,12 +100,11 @@ const manifestExtra = { version: pkg.version, description: pkg.description, permissions: permissions, - commands: commandsExtra, - optional_permissions: [ - "http://*/capture", - "https://*/capture", - ], -}; + commands: commands, + optional_permissions: optional_permissions, +} +manifestExtra[action_name] = action + if (dev) { manifestExtra.content_security_policy = "script-src 'self' 'unsafe-eval'; object-src 'self'"; @@ -76,7 +115,6 @@ if (target === T.CHROME) { } if (target === T.FIREFOX) { manifestExtra.options_ui = {browser_style: true}; - manifestExtra.browser_action = {browser_style: true}; } const buildPath = path.join(__dirname, 'dist', target); From 7dcd827704889c95fa409a37896c503bc3c1994b Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 24 Dec 2022 16:15:26 +0000 Subject: [PATCH 12/32] bump extension version --- extension/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/package.json b/extension/package.json index acedaca..1eed9c7 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1,7 +1,7 @@ { "name": "grasp", - "version": "0.6.6", - "version_name": "released on TODO", + "version": "0.6.7", + "version_name": "released on 2022.12.24", "description": "Reliably capture links and webpage content in a plaintext file (org-mode/markdown)", "main": "dist/background.js", "scripts": { From c1acbc114ba805630447d22162efd2b2afef9051 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 24 Dec 2022 21:06:40 +0000 Subject: [PATCH 13/32] add extension ids for firefox and chrome --- extension/build | 4 ++-- extension/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extension/build b/extension/build index ea25d1e..73e93ee 100755 --- a/extension/build +++ b/extension/build @@ -7,8 +7,8 @@ from pathlib import Path # right, specifying id in manifest doesn't seem to work # AMO responds with: Server response: Duplicate add-on ID found. (status: 400) IDS = { - 'firefox' : 'FIXME', - 'chrome' : 'FIXME', + 'firefox' : '{37e42980-a7c9-473c-96d5-13f18e0efc74}', + 'chrome' : 'ohhbcfjmnbmgkajljopdjcaokbpgbgfa', } diff --git a/extension/package.json b/extension/package.json index 1eed9c7..859ae73 100644 --- a/extension/package.json +++ b/extension/package.json @@ -12,7 +12,7 @@ "eslint": "eslint src", "flow": "flow", "web-ext": "web-ext", - "release:cws": "webstore", + "release:cws": "chrome-webstore-upload", "release:amo": "web-ext-submit" }, "repository": { From 709220eb10989e86a06366ef53f5786c1d4ee087 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 26 Dec 2022 13:34:24 +0000 Subject: [PATCH 14/32] build script: add watch mode --- extension/build | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extension/build b/extension/build index 73e93ee..715cd99 100755 --- a/extension/build +++ b/extension/build @@ -20,6 +20,7 @@ TARGETS = [ def main(): p = argparse.ArgumentParser() p.add_argument('--release', action='store_true', help="Use release flavor of build") + p.add_argument('--watch' , action='store_true') p.add_argument('--lint' , action='store_true') p.add_argument('--publish', action='store_true', help="Publish on chrome web store/addons.mozilla.org") @@ -51,6 +52,14 @@ def main(): 'PUBLISH': 'YES' if args.publish else 'NO', **os.environ, } + + if args.watch: + check_call([ + 'npm', 'run', 'watch', + ], env=env, cwd=str(Path(__file__).absolute().parent)) + # TODO exec instead? + return + check_call([ 'npm', 'run', 'build', ], env=env, cwd=str(Path(__file__).absolute().parent)) From a9ef5bd86ab829f4b7e44989ecc3ea13a5fde834 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 27 Dec 2022 04:28:24 +0000 Subject: [PATCH 15/32] extension: fix for deprecated window unload listener, it doesn't always trigger anymore + some minor refactoring/renaming --- extension/src/js/background.js | 3 ++ extension/src/js/popup.js | 88 ++++++++++++++++++++++++---------- 2 files changed, 65 insertions(+), 26 deletions(-) diff --git a/extension/src/js/background.js b/extension/src/js/background.js index 32b640b..f9f0551 100644 --- a/extension/src/js/background.js +++ b/extension/src/js/background.js @@ -118,6 +118,9 @@ chrome.commands.onCommand.addListener(command => { }); chrome.runtime.onMessage.addListener((message: any, sender: chrome$MessageSender, sendResponse) => { // eslint-disable-line no-unused-vars + if (message.method === 'logging') { + console.error("[%s] %o", message.source, message.data) + } if (message.method === METHOD_CAPTURE_WITH_EXTRAS) { const comment = message.comment; const tag_str = message.tag_str; diff --git a/extension/src/js/popup.js b/extension/src/js/popup.js index 8490071..52bc48c 100644 --- a/extension/src/js/popup.js +++ b/extension/src/js/popup.js @@ -9,37 +9,61 @@ const COMMENT_ID = 'comment_id'; const TAGS_ID = 'tags_id'; +/* + * normal popup logging is pretty annoying because chrome closes devtools + */ +/* eslint-disable no-unused-vars */ +function logbg(msg: any) { + console.error('%o', msg) + chrome.runtime.sendMessage({ + method: 'logging', + source: 'popup.js', + data: msg, + }) +} + + type State = {| comment: string, tag_str: string, -|}; +|} -function save_state(state: ?State) { - window.localStorage.setItem('state', JSON.stringify(state)); +function saveState(state: ?State) { + window.localStorage.setItem('state', JSON.stringify(state)) } -function load_state(): ?State { - const sts = window.localStorage.getItem('state') || null; - return JSON.parse(sts); + +function clearState() { + saveState(null) } + +function loadState(): ?State { + const sts = window.localStorage.getItem('state') || null + return JSON.parse(sts) +} + + function getComment(): HTMLInputElement { const comment = ((document.getElementById(COMMENT_ID): any): HTMLInputElement); return comment; } + function getTags(): HTMLInputElement { const tags = ((document.getElementById(TAGS_ID): any): HTMLInputElement); return tags; } + function getButton(): HTMLElement { const button = ((document.getElementById(BUTTON_ID): any): HTMLElement); return button; } -function getState(): State { + +function getUiState(): State { const comment_text = getComment().value; const tag_str = getTags().value; return { @@ -48,7 +72,9 @@ function getState(): State { }; } + function restoreState(state: ?State) { + window.justSubmitted = false if (state == null) { // comment just relies on default get_options(opts => {getTags().value = opts.default_tags;}); @@ -58,26 +84,31 @@ function restoreState(state: ?State) { } } -function submitComment () { - // TODO focus - const state = getState(); + +function submitCapture () { + const state = getUiState() chrome.runtime.sendMessage({ 'method': METHOD_CAPTURE_WITH_EXTRAS, ...state, }, () => { + window.justSubmitted = true + clearState() + + // TODO not sure about this? + window.close() console.log("[popup] captured!"); - }); - window.submitted = true; - window.close(); + }) + } + // todo not sure if can define const here?? var ctrlEnterSubmit: KeyboardEventListener = (e) => { // note: there is also e.metaKey which triggers on mac when cmd is pressed // but it doesn't seem to allow chords like cmd+Enter if ((e.ctrlKey || e.shiftKey) && e.key === 'Enter') { - submitComment(); + submitCapture() } } @@ -95,6 +126,7 @@ function moveCaretToEnd(el: HTMLInputElement) { } } + function setupPage () { const comment = getComment(); comment.focus(); @@ -104,20 +136,24 @@ function setupPage () { tags.addEventListener('keydown', ctrlEnterSubmit); tags.addEventListener('focus', () => moveCaretToEnd(tags)); // to put cursor to the end of tags when tabbed - getButton().addEventListener('click', submitComment); + getButton().addEventListener('click', submitCapture) - window.submitted = false; - - const state = load_state(); - restoreState(state); - save_state(null); // clean + const state = loadState() + restoreState(state) } -document.addEventListener('DOMContentLoaded', setupPage); - -window.addEventListener('unload', () => { - if (!window.submitted) { - save_state(getState()); +// hmm interesting that on chrome we can use visibilitychange for both +// on firefox dev edition however 'visible' doesn't fire, only 'hidden' :shrug: +document.addEventListener('DOMContentLoaded', setupPage) +window.addEventListener('visibilitychange', (_: Event) => { + // event doesn't contain visibility status + // docs recomment to use this instead of unload + const visible = document.visibilityState === 'visible' + if (!visible) { + // if we just sent, no need to save anything + if (!window.justSubmitted) { + saveState(getUiState()) + } } -}, true); +}) From 8dc554fd617f1c80f92ee039a05c4da14d5c3549 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 27 Dec 2022 14:37:21 +0000 Subject: [PATCH 16/32] release extension v0.6.9 --- extension/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extension/package.json b/extension/package.json index 859ae73..7ada904 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1,7 +1,7 @@ { "name": "grasp", - "version": "0.6.7", - "version_name": "released on 2022.12.24", + "version": "0.6.9", + "version_name": "released on 2022.12.27", "description": "Reliably capture links and webpage content in a plaintext file (org-mode/markdown)", "main": "dist/background.js", "scripts": { From f906728bf8bf45bb24a42231af5e805499597cd3 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 27 Dec 2022 23:26:10 +0000 Subject: [PATCH 17/32] user browser polyfill with Promise for options access --- extension/.flowconfig | 1 + extension/package.json | 1 + extension/src/js/background.js | 5 ++- extension/src/js/browser_types.js | 25 +++++++++++ extension/src/js/options.js | 32 ++++++-------- extension/src/js/options_page.js | 69 ++++++++++++++++--------------- extension/src/js/popup.js | 14 ++++--- extension/src/manifest.json | 5 ++- extension/src/options.html | 1 + extension/src/popup.html | 1 + extension/webpack.config.js | 1 + 11 files changed, 93 insertions(+), 62 deletions(-) create mode 100644 extension/src/js/browser_types.js diff --git a/extension/.flowconfig b/extension/.flowconfig index 5aaad51..4995da2 100644 --- a/extension/.flowconfig +++ b/extension/.flowconfig @@ -5,6 +5,7 @@ [libs] node_modules/flow-interfaces-chrome/interfaces/ +src/js/browser_types.js [lints] diff --git a/extension/package.json b/extension/package.json index 7ada904..7f35e4a 100644 --- a/extension/package.json +++ b/extension/package.json @@ -46,6 +46,7 @@ "style-loader": "^3.3.1", "web-ext": "^7.4.0", "web-ext-submit": "^7.4.0", + "webextension-polyfill": "^0.10.0", "webpack": "^5.75.0", "webpack-cli": "^5.0.1", "webpack-dev-server": "^4.11.1", diff --git a/extension/src/js/background.js b/extension/src/js/background.js index f9f0551..ed3c726 100644 --- a/extension/src/js/background.js +++ b/extension/src/js/background.js @@ -1,8 +1,9 @@ /* @flow */ // TODO do I really need to annotate all files with @flow?? + import {COMMAND_CAPTURE_SIMPLE, METHOD_CAPTURE_WITH_EXTRAS, showNotification} from './common' -import {get_options} from './options' +import {getOptions} from './options' import type {Options} from './options' @@ -87,7 +88,7 @@ function capture(comment: ?string = null, tag_str: ?string = null) { // TODO await properly and handle errors const has_scripting = 'scripting' in chrome - get_options(opts => { + getOptions().then(opts => { if (has_scripting) { // $FlowFixMe chrome.scripting.executeScript({ diff --git a/extension/src/js/browser_types.js b/extension/src/js/browser_types.js new file mode 100644 index 0000000..472e1c5 --- /dev/null +++ b/extension/src/js/browser_types.js @@ -0,0 +1,25 @@ +/* @flow */ + +/* + * Based on https://github.com/Shraymonks/flow-interfaces-chrome + * not sure if browser polyfill has flow types?? + * https://github.com/mozilla/webextension-polyfill/issues/421 + */ + + +type browser$StorageArea = { + get: (keys: string | Array | Object | null) => Promise, + set: (items: Object) => Promise, +} + + +type browser$storage = { + local: browser$StorageArea, + managed: browser$StorageArea, + sync: browser$StorageArea, +} + + +declare var browser: { + storage: browser$storage, +} diff --git a/extension/src/js/options.js b/extension/src/js/options.js index 39341f2..caa8b02 100644 --- a/extension/src/js/options.js +++ b/extension/src/js/options.js @@ -6,31 +6,23 @@ export type Options = { notification: boolean; } -function default_options(): Options { + +function defaultOptions(): Options { return { endpoint: "http://localhost:12212/capture", default_tags: "grasp", notification: true, - }; -} - -export function get_options(cb: (Options) => void) { - chrome.storage.local.get(null, res => { - res = {...default_options(), ...res}; - cb(res); - }); -} - -export function set_options(opts: Options, cb: () => void) { - console.log('Saving %s', JSON.stringify(opts)); - chrome.storage.local.set(opts, cb); + } } - -export function getOptions(): Promise { - return new Promise((resolve) => { - // TODO FIXME reject? - get_options(resolve); - }); +export async function getOptions(): Promise { + const res = await browser.storage.local.get(null) + return {...defaultOptions(), ...res} +} + + +export async function setOptions(opts: Options): Promise { + console.log('Saving %o', opts) + await browser.storage.local.set(opts) } diff --git a/extension/src/js/options_page.js b/extension/src/js/options_page.js index 8938ac6..89782ed 100644 --- a/extension/src/js/options_page.js +++ b/extension/src/js/options_page.js @@ -1,15 +1,15 @@ /* @flow */ -import {get_options, set_options} from './options'; -import {ensurePermissions, hasPermissions} from './permissions'; +import {getOptions, setOptions} from './options' +import {ensurePermissions, hasPermissions} from './permissions' + const ENDPOINT_ID = 'endpoint_id'; const HAS_PERMISSION_ID = 'has_permission_id'; const NOTIFICATION_ID = 'notification_id'; const DEFAULT_TAGS_ID = 'default_tags_id'; -// TODO specify capture path here? - const SAVE_ID = 'save_id'; + function getEndpoint(): HTMLInputElement { return ((document.getElementById(ENDPOINT_ID): any): HTMLInputElement); } @@ -31,47 +31,50 @@ function getHasPermission(): HTMLElement { return ((document.getElementById(HAS_PERMISSION_ID): any): HTMLElement); } + // ugh. needs a better name.. -function refreshPermissionValidation(endpoint: string) { - hasPermissions(endpoint).then(has => { - const pstyle = getHasPermission().style; - if (has) { - console.debug('Got permisssions, nothing to worry about.'); - pstyle.display = 'none'; - return; - } - console.debug('Whoops, no permissions to access %s', endpoint); - // TODO maybe just show button? but then it would need to be reactive.. - pstyle.display = 'block'; - }); +async function refreshPermissionValidation(endpoint: string) { + const has = await hasPermissions(endpoint) + const pstyle = getHasPermission().style + if (has) { + console.debug('Got permisssions, nothing to worry about.') + pstyle.display = 'none' + return + } + console.debug('Whoops, no permissions to access %s', endpoint) + // TODO maybe just show button? but then it would need to be reactive.. + pstyle.display = 'block' } -function restoreOptions() { - getSaveButton().addEventListener('click', saveOptions); - get_options(opts => { - const ep = opts.endpoint; - getEndpoint().value = ep; - getDefaultTags().value = opts.default_tags; - getEnableNotification().checked = opts.notification; - refreshPermissionValidation(ep); - }); + +async function restoreOptions() { + getSaveButton().addEventListener('click', saveOptions) + + const opts = await getOptions() + getEndpoint().value = opts.endpoint + getDefaultTags().value = opts.default_tags + getEnableNotification().checked = opts.notification + await refreshPermissionValidation(opts.endpoint) } -function saveOptions() { + +async function saveOptions() { // TODO could also check for permissions and display message? - const endpoint = getEndpoint().value; - ensurePermissions(endpoint).finally(() => { - refreshPermissionValidation(endpoint); - }); + const endpoint = getEndpoint().value + await ensurePermissions(endpoint) + refreshPermissionValidation(endpoint) const opts = { endpoint: endpoint, default_tags: getDefaultTags().value, notification: getEnableNotification().checked, - }; - set_options(opts, () => { alert('Saved!'); }); + } + await setOptions(opts) + + // hmm seems that regular alert() doesn't work in chrome anymore + chrome.extension.getBackgroundPage().alert('Saved!') } -document.addEventListener('DOMContentLoaded', restoreOptions); +document.addEventListener('DOMContentLoaded', restoreOptions) diff --git a/extension/src/js/popup.js b/extension/src/js/popup.js index 52bc48c..615c0ee 100644 --- a/extension/src/js/popup.js +++ b/extension/src/js/popup.js @@ -1,7 +1,8 @@ /* @flow */ -import {METHOD_CAPTURE_WITH_EXTRAS} from './common'; -import {get_options} from './options'; + +import {METHOD_CAPTURE_WITH_EXTRAS} from './common' +import {getOptions} from './options' // TODO template it in html too? const BUTTON_ID = 'button_id'; @@ -73,14 +74,15 @@ function getUiState(): State { } -function restoreState(state: ?State) { +async function restoreState(state: ?State) { window.justSubmitted = false if (state == null) { // comment just relies on default - get_options(opts => {getTags().value = opts.default_tags;}); + const opts = await getOptions() + getTags().value = opts.default_tags } else { - getComment().value = state.comment; - getTags().value = state.tag_str; + getComment().value = state.comment + getTags().value = state.tag_str } } diff --git a/extension/src/manifest.json b/extension/src/manifest.json index 66df0da..22c3921 100644 --- a/extension/src/manifest.json +++ b/extension/src/manifest.json @@ -2,7 +2,10 @@ "name": "", "version": "", "background": { - "scripts": ["background.js"], + "scripts": [ + "browser-polyfill.js", + "background.js" + ], "persistent": false }, "options_ui": { diff --git a/extension/src/options.html b/extension/src/options.html index d669a1a..3c28a6e 100644 --- a/extension/src/options.html +++ b/extension/src/options.html @@ -2,6 +2,7 @@ +