extension: move to a separate dir, update build scripts etc

borrowed build scritps etc from promnesia

.circleci/config.yml

@@ -1,100 +0,0 @@
-# TODO should I use yarn as in circleci node example??
-version: 2
-
-jobs:
-  client_build_and_check:
-    docker:
-      - image: circleci/node:latest
-    steps:
-      - checkout
-      - run: sudo npm install -g npm web-ext
-
-      - restore_cache:
-          keys:
-          - dependency-cache-{{ checksum "package.json" }}
-          # fallback to using the latest cache if no exact match is found
-          - dependency-cache-
-      - run: npm install
-      - save_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
-          paths:
-            - ./node_modules
-
-      - run: ./build --chrome --lint
-
-      # TODO ugh. is there a way for a job to inherit a cloned repo after npm run build has ran?? I'd like to separate build and check steps
-      # TODO ok, apparently it's a custom executor? https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
-      - run:
-          name: js flow
-          command: npm run flow
-          when: always
-      - run:
-          name: eslint
-          command: npm run eslint
-          when: always
-
-      - run:
-          # TODO need to do some kind of build matrix over [firefox, chrome] x [anyhost, no_anyhost]
-          # it should be somehow achievable via workspaces, but I've spent 10 minutes and couldn't figure it out...
-          name: pack
-          command: |
-
-            for browser in 'chrome'; do
-                zip -r -j "/tmp/grasp.zip" dist/*
-            done
-
-      # TODO ugh, also can't use env variables in artifacts...
-      - store_artifacts:
-          path: "/tmp/grasp.zip"
-          destination: "grasp.zip"
-
-
-  server_build_and_check:
-    docker:
-      - image: circleci/python:latest
-    steps:
-      - checkout
-
-      # TODO preserve apt cache..
-      - run: sudo apt install python3-pip
-      - run: python3 -m pip install --user mypy pylint pytest
-
-      # test only dependencies
-      - run: sudo apt install httpie
-      - run: python3 -m pip install --user selenium pyautogui
-      #
-
-      - run:
-          name: pylint
-          command: ci/pylint
-          when: always
-
-      - run:
-          name: mypy
-          command: ci/mypy
-          when: always
-
-      - run:
-          name: pytest
-          command: ci/pytest
-          when: always
-
-workflows:
-  version: 2
-
-  build_and_test:
-    jobs:
-      - client_build_and_check
-      - server_build_and_check
-
-
-#       # ugh. circleci only got python3.6 on this node package??? and it's unclear how to combine multiple circleci docker configurations...
-
-#       # - store_artifacts:
-#       #     path: test-results.xml
-#       #     prefix: tests
-#       # - store_artifacts:
-#       #     path: coverage
-#       #     prefix: coverage
-#       # - store_test_results:
-#       #     path: test-results.xml

.github/workflows/main.yml

@@ -51,3 +51,36 @@ jobs:
         name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy/
 
+  ###
+  build_extension:
+    env:
+      name: 'grasp'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+    - uses: actions/setup-node@v3
+      with:
+        node-version: '18'
+    - run: extension/.ci/build --lint # debug version
+    - run: extension/.ci/build --lint --release
+
+    # TODO ugh. can't share github actions artifacts publicly...
+    # TODO for fuck's sake... why does it end up named as .zip.zip ????
+    - uses: actions/upload-artifact@v2
+      with:
+        name: '${ env.name }}-chrome-debug-latest.zip'
+        path: 'extension/dist/artifacts/chrome/${{ env.name }}_dev_-*.zip'
+    - uses: actions/upload-artifact@v2
+      with:
+        .name : '${{ env.name }}-chrome-release-latest.zip'
+        path: 'extension/dist/artifacts/chrome/${{ env.name }}-*.zip'
+    - uses: actions/upload-artifact@v2
+      with:
+        name : '${{ env.name }}-firefox-debug-latest.zip'
+        path: 'extension/dist/artifacts/firefox/${{ env.name }}_dev_-*.zip'
+    - uses: actions/upload-artifact@v2
+      with:
+        name : '${{ env.name }}-firefox-release-latest.zip'
+        path: 'extension/dist/artifacts/firefox/${{ env.name }}-*.zip'

build_extension

@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import os
-from subprocess import check_call
-from pathlib import Path
-
-
-# TODO FIXME copypaste from promnesia
-# TODO FIXME use it in ci script
-TARGETS = ['chrome', 'firefox']
-def main():
-    p = argparse.ArgumentParser()
-    p.add_argument('--release', action='store_true')
-    p.add_argument('--sign', action='store_true')
-    p.add_argument('--lint', action='store_true')
-
-    tg = p.add_mutually_exclusive_group(required=True)
-    tg.add_argument('--target', type=str, choices=TARGETS)
-    for b in TARGETS:
-        tg.add_argument('--' + b, action='store_const', const=b, dest='target')
-    args = p.parse_args()
-    target = args.target
-
-    assert target is not None
-
-    dist_dir = Path(__file__).absolute().parent / 'dist'
-    def webext(*args, **kwargs):
-        src_dir = dist_dir # / target
-        src_dir = src_dir.resolve() # TODO ugh. webext can't into symlinks???
-        check_call([
-            'web-ext',
-            '--source-dir', str(src_dir),
-            *args,
-        ], **kwargs)
-
-    env = {
-        'TARGET': target,
-        'RELEASE': 'YES' if args.release else 'NO',
-        **os.environ,
-    }
-    check_call([
-        'npm', 'run', 'build',
-    ], env=env, cwd=str(Path(__file__).absolute().parent))
-
-    if args.lint:
-        # TODO --self-hosted
-        # TODO warnings as errors??
-        webext('lint')
-
-    if args.release:
-        assert args.lint # TODO not sure..
-
-    if args.sign:
-        assert args.release
-        if target not in ['firefox', 'firefox-mobile']:
-            raise RuntimeError("{target} is not supported for signing yet".format(target=target))
-
-        dist_dir = Path(__file__).absolute().parent / 'dist'
-        src_dir = dist_dir / target
-
-        from firefox_dev_secrets import API_KEY, API_SECRET
-        webext(
-            '--source-dir'   , str(src_dir),
-            '--artifacts-dir', str(dist_dir),
-            '--api-key'   , API_KEY,
-            '--api-secret', API_SECRET,
-            'sign',
-        )
-
-if __name__ == '__main__':
-    main()
-
-# TODO use UUID? https://addons.mozilla.org/en-GB/developers/addon/grasp/edit
-# TODO right, wouldn't work because it's listed..

extension/.ci/build

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -eux
+
+cd "$(dirname "$0")"
+cd ../.. # git root
+
+cd extension
+
+npm install
+
+FAILED=0
+
+npm run flow   || FAILED=1
+npm run eslint || FAILED=1
+# TODO later
+#npm run test   || FAILED=1
+
+for browser in 'firefox' 'chrome'; do
+    ./build --target "$browser" "$@"
+done
+
+exit "$FAILED"

extension/build

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+import argparse
+import os
+from subprocess import check_call
+from pathlib import Path
+
+# right, specifying id in manifest doesn't seem to work
+# AMO responds with: Server response: Duplicate add-on ID found. (status: 400)
+IDS = {
+    'firefox'       : 'FIXME',
+    'chrome'        : 'FIXME',
+}
+
+
+TARGETS = [
+    'chrome',
+    'firefox',
+]
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument('--release', action='store_true', help="Use release flavor of build")
+    p.add_argument('--lint'   , action='store_true')
+    p.add_argument('--publish', action='store_true', help="Publish on chrome web store/addons.mozilla.org")
+
+    tg = p.add_mutually_exclusive_group(required=True)
+    tg.add_argument('--target', type=str, choices=TARGETS)
+    for b in TARGETS:
+        tg.add_argument('--' + b, action='store_const', const=b, dest='target')
+    args = p.parse_args()
+    target = args.target
+
+    assert target is not None
+
+    base_ext_dir = Path(__file__).absolute().parent / 'dist'
+    ext_dir = (base_ext_dir / target).resolve() # webext can't into symlinks
+    # sadly no way to specify zip name in the regex..
+    artifacts_dir = (base_ext_dir / 'artifacts' / target).resolve()
+    def webext(*args, **kwargs):
+        check_call([
+            'npm', 'run', 'web-ext',
+            '--',
+            '--source-dir'   , ext_dir,
+            '--artifacts-dir', artifacts_dir,
+            *args,
+        ], **kwargs)
+
+    env = {
+        'TARGET': target,
+        'RELEASE': 'YES' if args.release else 'NO',
+        'PUBLISH': 'YES' if args.publish else 'NO',
+        **os.environ,
+    }
+    check_call([
+        'npm', 'run', 'build',
+    ], env=env, cwd=str(Path(__file__).absolute().parent))
+
+    if args.lint:
+        # TODO --self-hosted
+        # TODO warnings as errors??
+        webext('lint')
+
+        # TODO move somewhere more appropriate..
+        webext('build', '-o') # -o overwrites existing artifact
+
+
+    if args.release:
+        assert args.lint # TODO not sure..
+
+    def firefox_release_args():
+        from firefox_dev_secrets import API_KEY, API_SECRET
+        return [
+            '--artifacts-dir', str(artifacts_dir),
+            '--api-key'      , API_KEY,
+            '--api-secret'   , API_SECRET,
+            '--id'           , IDS[target],
+        ]
+
+    if args.publish:
+        assert args.release
+        if 'firefox' in target:
+            check_call([
+                'npm', 'run', 'release:amo',
+                '--',
+                '--channel', 'listed',
+                '--source-dir', str(ext_dir),
+                *firefox_release_args(),
+            ])
+        elif target == 'chrome':
+            from chrome_dev_secrets import CLIENT_ID, CLIENT_SECRET, REFRESH_TOKEN
+            check_call([
+                'npm', 'run', 'release:cws',
+                '--',
+                'upload',
+                # '--auto-publish',
+                '--source'        , str(ext_dir),
+                '--client-id'     , CLIENT_ID,
+                '--client-secret' , CLIENT_SECRET,
+                '--refresh--token', REFRESH_TOKEN,
+                '--extension-id'  , IDS[target],
+                # TODO trusted testers?
+            ])
+        else:
+            raise RuntimeError("{target} is not supported for publishing yet".format(target=target))
+
+if __name__ == '__main__':
+    main()

extension/jest.config.js

@@ -0,0 +1,188 @@
+// For a detailed explanation regarding each configuration property, visit:
+// https://jestjs.io/docs/en/configuration.html
+
+module.exports = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // Respect "browser" field in package.json when resolving modules
+  // browser: false,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/tmp/jest_rs",
+
+  // Automatically clear mock calls and instances between every test
+  clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  // collectCoverage: false,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: null,
+
+  // The directory where Jest should output its coverage files
+  // coverageDirectory: null,
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: null,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: null,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: null,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: null,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "json",
+  //   "jsx",
+  //   "ts",
+  //   "tsx",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names that allow to stub out resources with a single module
+  // moduleNameMapper: {},
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  // preset: null,
+
+  // Run tests from one or more projects
+  // projects: null,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state between every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: null,
+
+  // Automatically restore mock state between every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: null,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  // setupFilesAfterEnv: [],
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  testEnvironment: "jsdom",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  // testMatch: [
+  //   "**/__tests__/**/*.[jt]s?(x)",
+  //   "**/?(*.)+(spec|test).[tj]s?(x)"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: null,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jasmine2",
+
+  // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href
+  // testURL: "http://localhost",
+
+  // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout"
+  // timers: "real",
+
+  // A map from regular expressions to paths to transformers
+  // transform: null,
+
+  // ??? https://github.com/facebook/jest/issues/9395#issuecomment-665239070
+  // otherwise fails on importing
+  transformIgnorePatterns: ['node_modules/?!@webext-options-sync/']
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: null,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};

extension/package.json

@@ -1,15 +1,30 @@
 {
   "name": "grasp",
   "version": "0.6.6",
+  "version_name": "released on TODO",
   "description": "Reliably capture links and webpage content in a plaintext file (org-mode/markdown)",
   "main": "dist/background.js",
   "scripts": {
-    "build": "webpack --progress --profile --colors",
+    "test": "jest",
-    "watch": "webpack --watch --progress --colors",
+    "build": "webpack --progress --profile",
+    "watch": "webpack --watch --progress",
     "dev": "npm run watch",
     "eslint": "eslint src",
-    "flow": "flow"
+    "flow": "flow",
+    "web-ext": "web-ext",
+    "release:cws": "webstore",
+    "release:amo": "web-ext-submit"
   },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/karlicoss/grasp.git"
+  },
+  "author": "Dmitrii Gerasimov <karlicoss@gmail.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/karlicoss/grasp/issues"
+  },
+  "homepage": "https://github.com/karlicoss/grasp#readme",
   "devDependencies": {
     "@babel/core": "^7.2.2",
     "@babel/preset-env": "^7.2.3",
@@ -24,10 +39,17 @@
     "eslint-plugin-flowtype": "^3.2.0",
     "flow-bin": "^0.89.0",
     "flow-interfaces-chrome": "^0.5.3",
+    "jest": "^28.1.0",
+    "jest-environment-jsdom": "^28.1.0",
+    "jest-fetch-mock": "^3.0.3",
+    "node-fetch": "^2.6.1",
     "style-loader": "^1.0.0",
+    "web-ext": "^6.8.0",
+    "web-ext-submit": "^6.8.0",
     "webpack": "^4.28.3",
     "webpack-cli": "^3.2.0",
     "webpack-dev-server": "^3.1.14",
     "webpack-extension-manifest-plugin": "^0.4.3"
-  }
+  },
+  "private": "true"
 }

extension/webpack.config.js

@@ -13,12 +13,14 @@ const T = {
 const env = {
     TARGET : process.env.TARGET, // TODO assert not null?
     RELEASE: process.env.RELEASE,
+    PUBLISH: process.env.PUBLISH,
 };
 
 const pkg = require('./package.json');
 const baseManifest = require('./src/manifest.json');
 
 const release = env.RELEASE == 'YES' ? true : false;
+const publish = env.PUBLISH == 'YES' ? true : false;
 const dev = !release; // meh. maybe make up my mind?
 const target = env.TARGET;
 
@@ -78,14 +80,14 @@ if (target === T.FIREFOX) {
     manifestExtra.browser_action = {browser_style: true};
 }
 
-const build_path = path.join(__dirname, "dist"); // TODO target??
+const buildPath = path.join(__dirname, 'dist', target);
 
 const options = {
   mode: dev ? 'development' : 'production', // TODO map directly from MODE env var?
-  optimization: {
+  node: {
-    // https://webpack.js.org/configuration/optimization
+    // no idea what does it mean... https://github.com/webpack/webpack/issues/5627#issuecomment-394290231
-    // don't think minimize worth it for such a tiny extension
+    // but it does get rid of some Function() which webpacka apparently generates
-    minimize: false
+    global: false,
   },
   entry: {
     background  : path.join(__dirname, "src", "js", "background"),
@@ -93,9 +95,15 @@ const options = {
     options_page: path.join(__dirname, "src", "js", "options_page"),
   },
   output: {
-      path: build_path,
+      publicPath: '', // https://stackoverflow.com/a/64715069
+      path: buildPath,
       filename: "[name].js",
   },
+  optimization: {
+    // https://webpack.js.org/configuration/optimization
+    // don't think minimize worth it for such a tiny extension
+    minimize: false
+  },
   module: {
     rules: [
       {
@@ -118,7 +126,7 @@ const options = {
     ]
   },
   plugins: [
-    new CleanWebpackPlugin([build_path + "/*"]),
+    new CleanWebpackPlugin([buildPath + "/*"]),
     new CopyWebpackPlugin([
         { from: 'src/img/*.png', flatten: true },
         { from: 'src/*.html'   , flatten: true },