From 4127e3de647a7dd14468ecf93405277bc2e60dcd Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 23 Dec 2022 20:58:26 +0000 Subject: [PATCH] extension: move to a separate dir, update build scripts etc borrowed build scritps etc from promnesia --- .circleci/config.yml | 100 ---------- .github/workflows/main.yml | 33 +++ build_extension | 74 ------- .babelrc => extension/.babelrc | 0 extension/.ci/build | 22 ++ .eslintrc.js => extension/.eslintrc.js | 0 .flowconfig => extension/.flowconfig | 0 extension/build | 107 ++++++++++ extension/jest.config.js | 188 ++++++++++++++++++ package.json => extension/package.json | 30 ++- publish => extension/publish | 0 {src => extension/src}/img/unicorn.png | Bin {src => extension/src}/js/background.js | 0 {src => extension/src}/js/common.js | 0 {src => extension/src}/js/options.js | 0 {src => extension/src}/js/options_page.js | 0 {src => extension/src}/js/permissions.js | 0 {src => extension/src}/js/popup.js | 0 {src => extension/src}/manifest.json | 0 {src => extension/src}/options.html | 0 {src => extension/src}/popup.html | 0 .../webpack.config.js | 22 +- 22 files changed, 391 insertions(+), 185 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100755 build_extension rename .babelrc => extension/.babelrc (100%) create mode 100755 extension/.ci/build rename .eslintrc.js => extension/.eslintrc.js (100%) rename .flowconfig => extension/.flowconfig (100%) create mode 100755 extension/build create mode 100644 extension/jest.config.js rename package.json => extension/package.json (53%) rename publish => extension/publish (100%) rename {src => extension/src}/img/unicorn.png (100%) rename {src => extension/src}/js/background.js (100%) rename {src => extension/src}/js/common.js (100%) rename {src => extension/src}/js/options.js (100%) rename {src => extension/src}/js/options_page.js (100%) rename {src => extension/src}/js/permissions.js (100%) rename {src => extension/src}/js/popup.js (100%) rename {src => extension/src}/manifest.json (100%) rename {src => extension/src}/options.html (100%) rename {src => extension/src}/popup.html (100%) rename webpack.config.js => extension/webpack.config.js (87%) diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index a6e45d6..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,100 +0,0 @@ -# TODO should I use yarn as in circleci node example?? -version: 2 - -jobs: - client_build_and_check: - docker: - - image: circleci/node:latest - steps: - - checkout - - run: sudo npm install -g npm web-ext - - - restore_cache: - keys: - - dependency-cache-{{ checksum "package.json" }} - # fallback to using the latest cache if no exact match is found - - dependency-cache- - - run: npm install - - save_cache: - key: dependency-cache-{{ checksum "package.json" }} - paths: - - ./node_modules - - - run: ./build --chrome --lint - - # TODO ugh. is there a way for a job to inherit a cloned repo after npm run build has ran?? I'd like to separate build and check steps - # TODO ok, apparently it's a custom executor? https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs - - run: - name: js flow - command: npm run flow - when: always - - run: - name: eslint - command: npm run eslint - when: always - - - run: - # TODO need to do some kind of build matrix over [firefox, chrome] x [anyhost, no_anyhost] - # it should be somehow achievable via workspaces, but I've spent 10 minutes and couldn't figure it out... - name: pack - command: | - - for browser in 'chrome'; do - zip -r -j "/tmp/grasp.zip" dist/* - done - - # TODO ugh, also can't use env variables in artifacts... - - store_artifacts: - path: "/tmp/grasp.zip" - destination: "grasp.zip" - - - server_build_and_check: - docker: - - image: circleci/python:latest - steps: - - checkout - - # TODO preserve apt cache.. - - run: sudo apt install python3-pip - - run: python3 -m pip install --user mypy pylint pytest - - # test only dependencies - - run: sudo apt install httpie - - run: python3 -m pip install --user selenium pyautogui - # - - - run: - name: pylint - command: ci/pylint - when: always - - - run: - name: mypy - command: ci/mypy - when: always - - - run: - name: pytest - command: ci/pytest - when: always - -workflows: - version: 2 - - build_and_test: - jobs: - - client_build_and_check - - server_build_and_check - - -# # ugh. circleci only got python3.6 on this node package??? and it's unclear how to combine multiple circleci docker configurations... - -# # - store_artifacts: -# # path: test-results.xml -# # prefix: tests -# # - store_artifacts: -# # path: coverage -# # prefix: coverage -# # - store_test_results: -# # path: test-results.xml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f936ee..61a68d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,3 +51,36 @@ jobs: name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy/ + ### + build_extension: + env: + name: 'grasp' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - uses: actions/setup-node@v3 + with: + node-version: '18' + - run: extension/.ci/build --lint # debug version + - run: extension/.ci/build --lint --release + + # TODO ugh. can't share github actions artifacts publicly... + # TODO for fuck's sake... why does it end up named as .zip.zip ???? + - uses: actions/upload-artifact@v2 + with: + name: '${ env.name }}-chrome-debug-latest.zip' + path: 'extension/dist/artifacts/chrome/${{ env.name }}_dev_-*.zip' + - uses: actions/upload-artifact@v2 + with: + .name : '${{ env.name }}-chrome-release-latest.zip' + path: 'extension/dist/artifacts/chrome/${{ env.name }}-*.zip' + - uses: actions/upload-artifact@v2 + with: + name : '${{ env.name }}-firefox-debug-latest.zip' + path: 'extension/dist/artifacts/firefox/${{ env.name }}_dev_-*.zip' + - uses: actions/upload-artifact@v2 + with: + name : '${{ env.name }}-firefox-release-latest.zip' + path: 'extension/dist/artifacts/firefox/${{ env.name }}-*.zip' diff --git a/build_extension b/build_extension deleted file mode 100755 index 91fa6bc..0000000 --- a/build_extension +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import os -from subprocess import check_call -from pathlib import Path - - -# TODO FIXME copypaste from promnesia -# TODO FIXME use it in ci script -TARGETS = ['chrome', 'firefox'] -def main(): - p = argparse.ArgumentParser() - p.add_argument('--release', action='store_true') - p.add_argument('--sign', action='store_true') - p.add_argument('--lint', action='store_true') - - tg = p.add_mutually_exclusive_group(required=True) - tg.add_argument('--target', type=str, choices=TARGETS) - for b in TARGETS: - tg.add_argument('--' + b, action='store_const', const=b, dest='target') - args = p.parse_args() - target = args.target - - assert target is not None - - dist_dir = Path(__file__).absolute().parent / 'dist' - def webext(*args, **kwargs): - src_dir = dist_dir # / target - src_dir = src_dir.resolve() # TODO ugh. webext can't into symlinks??? - check_call([ - 'web-ext', - '--source-dir', str(src_dir), - *args, - ], **kwargs) - - env = { - 'TARGET': target, - 'RELEASE': 'YES' if args.release else 'NO', - **os.environ, - } - check_call([ - 'npm', 'run', 'build', - ], env=env, cwd=str(Path(__file__).absolute().parent)) - - if args.lint: - # TODO --self-hosted - # TODO warnings as errors?? - webext('lint') - - if args.release: - assert args.lint # TODO not sure.. - - if args.sign: - assert args.release - if target not in ['firefox', 'firefox-mobile']: - raise RuntimeError("{target} is not supported for signing yet".format(target=target)) - - dist_dir = Path(__file__).absolute().parent / 'dist' - src_dir = dist_dir / target - - from firefox_dev_secrets import API_KEY, API_SECRET - webext( - '--source-dir' , str(src_dir), - '--artifacts-dir', str(dist_dir), - '--api-key' , API_KEY, - '--api-secret', API_SECRET, - 'sign', - ) - -if __name__ == '__main__': - main() - -# TODO use UUID? https://addons.mozilla.org/en-GB/developers/addon/grasp/edit -# TODO right, wouldn't work because it's listed.. diff --git a/.babelrc b/extension/.babelrc similarity index 100% rename from .babelrc rename to extension/.babelrc diff --git a/extension/.ci/build b/extension/.ci/build new file mode 100755 index 0000000..9a1cbee --- /dev/null +++ b/extension/.ci/build @@ -0,0 +1,22 @@ +#!/bin/bash +set -eux + +cd "$(dirname "$0")" +cd ../.. # git root + +cd extension + +npm install + +FAILED=0 + +npm run flow || FAILED=1 +npm run eslint || FAILED=1 +# TODO later +#npm run test || FAILED=1 + +for browser in 'firefox' 'chrome'; do + ./build --target "$browser" "$@" +done + +exit "$FAILED" diff --git a/.eslintrc.js b/extension/.eslintrc.js similarity index 100% rename from .eslintrc.js rename to extension/.eslintrc.js diff --git a/.flowconfig b/extension/.flowconfig similarity index 100% rename from .flowconfig rename to extension/.flowconfig diff --git a/extension/build b/extension/build new file mode 100755 index 0000000..ea25d1e --- /dev/null +++ b/extension/build @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +import argparse +import os +from subprocess import check_call +from pathlib import Path + +# right, specifying id in manifest doesn't seem to work +# AMO responds with: Server response: Duplicate add-on ID found. (status: 400) +IDS = { + 'firefox' : 'FIXME', + 'chrome' : 'FIXME', +} + + +TARGETS = [ + 'chrome', + 'firefox', +] + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--release', action='store_true', help="Use release flavor of build") + p.add_argument('--lint' , action='store_true') + p.add_argument('--publish', action='store_true', help="Publish on chrome web store/addons.mozilla.org") + + tg = p.add_mutually_exclusive_group(required=True) + tg.add_argument('--target', type=str, choices=TARGETS) + for b in TARGETS: + tg.add_argument('--' + b, action='store_const', const=b, dest='target') + args = p.parse_args() + target = args.target + + assert target is not None + + base_ext_dir = Path(__file__).absolute().parent / 'dist' + ext_dir = (base_ext_dir / target).resolve() # webext can't into symlinks + # sadly no way to specify zip name in the regex.. + artifacts_dir = (base_ext_dir / 'artifacts' / target).resolve() + def webext(*args, **kwargs): + check_call([ + 'npm', 'run', 'web-ext', + '--', + '--source-dir' , ext_dir, + '--artifacts-dir', artifacts_dir, + *args, + ], **kwargs) + + env = { + 'TARGET': target, + 'RELEASE': 'YES' if args.release else 'NO', + 'PUBLISH': 'YES' if args.publish else 'NO', + **os.environ, + } + check_call([ + 'npm', 'run', 'build', + ], env=env, cwd=str(Path(__file__).absolute().parent)) + + if args.lint: + # TODO --self-hosted + # TODO warnings as errors?? + webext('lint') + + # TODO move somewhere more appropriate.. + webext('build', '-o') # -o overwrites existing artifact + + + if args.release: + assert args.lint # TODO not sure.. + + def firefox_release_args(): + from firefox_dev_secrets import API_KEY, API_SECRET + return [ + '--artifacts-dir', str(artifacts_dir), + '--api-key' , API_KEY, + '--api-secret' , API_SECRET, + '--id' , IDS[target], + ] + + if args.publish: + assert args.release + if 'firefox' in target: + check_call([ + 'npm', 'run', 'release:amo', + '--', + '--channel', 'listed', + '--source-dir', str(ext_dir), + *firefox_release_args(), + ]) + elif target == 'chrome': + from chrome_dev_secrets import CLIENT_ID, CLIENT_SECRET, REFRESH_TOKEN + check_call([ + 'npm', 'run', 'release:cws', + '--', + 'upload', + # '--auto-publish', + '--source' , str(ext_dir), + '--client-id' , CLIENT_ID, + '--client-secret' , CLIENT_SECRET, + '--refresh--token', REFRESH_TOKEN, + '--extension-id' , IDS[target], + # TODO trusted testers? + ]) + else: + raise RuntimeError("{target} is not supported for publishing yet".format(target=target)) + +if __name__ == '__main__': + main() diff --git a/extension/jest.config.js b/extension/jest.config.js new file mode 100644 index 0000000..8dbab0a --- /dev/null +++ b/extension/jest.config.js @@ -0,0 +1,188 @@ +// For a detailed explanation regarding each configuration property, visit: +// https://jestjs.io/docs/en/configuration.html + +module.exports = { + // All imported modules in your tests should be mocked automatically + // automock: false, + + // Stop running tests after `n` failures + // bail: 0, + + // Respect "browser" field in package.json when resolving modules + // browser: false, + + // The directory where Jest should store its cached dependency information + // cacheDirectory: "/tmp/jest_rs", + + // Automatically clear mock calls and instances between every test + clearMocks: true, + + // Indicates whether the coverage information should be collected while executing the test + // collectCoverage: false, + + // An array of glob patterns indicating a set of files for which coverage information should be collected + // collectCoverageFrom: null, + + // The directory where Jest should output its coverage files + // coverageDirectory: null, + + // An array of regexp pattern strings used to skip coverage collection + // coveragePathIgnorePatterns: [ + // "/node_modules/" + // ], + + // A list of reporter names that Jest uses when writing coverage reports + // coverageReporters: [ + // "json", + // "text", + // "lcov", + // "clover" + // ], + + // An object that configures minimum threshold enforcement for coverage results + // coverageThreshold: null, + + // A path to a custom dependency extractor + // dependencyExtractor: null, + + // Make calling deprecated APIs throw helpful error messages + // errorOnDeprecated: false, + + // Force coverage collection from ignored files using an array of glob patterns + // forceCoverageMatch: [], + + // A path to a module which exports an async function that is triggered once before all test suites + // globalSetup: null, + + // A path to a module which exports an async function that is triggered once after all test suites + // globalTeardown: null, + + // A set of global variables that need to be available in all test environments + // globals: {}, + + // An array of directory names to be searched recursively up from the requiring module's location + // moduleDirectories: [ + // "node_modules" + // ], + + // An array of file extensions your modules use + // moduleFileExtensions: [ + // "js", + // "json", + // "jsx", + // "ts", + // "tsx", + // "node" + // ], + + // A map from regular expressions to module names that allow to stub out resources with a single module + // moduleNameMapper: {}, + + // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader + // modulePathIgnorePatterns: [], + + // Activates notifications for test results + // notify: false, + + // An enum that specifies notification mode. Requires { notify: true } + // notifyMode: "failure-change", + + // A preset that is used as a base for Jest's configuration + // preset: null, + + // Run tests from one or more projects + // projects: null, + + // Use this configuration option to add custom reporters to Jest + // reporters: undefined, + + // Automatically reset mock state between every test + // resetMocks: false, + + // Reset the module registry before running each individual test + // resetModules: false, + + // A path to a custom resolver + // resolver: null, + + // Automatically restore mock state between every test + // restoreMocks: false, + + // The root directory that Jest should scan for tests and modules within + // rootDir: null, + + // A list of paths to directories that Jest should use to search for files in + // roots: [ + // "" + // ], + + // Allows you to use a custom runner instead of Jest's default test runner + // runner: "jest-runner", + + // The paths to modules that run some code to configure or set up the testing environment before each test + // setupFiles: [], + + // A list of paths to modules that run some code to configure or set up the testing framework before each test + // setupFilesAfterEnv: [], + + // A list of paths to snapshot serializer modules Jest should use for snapshot testing + // snapshotSerializers: [], + + // The test environment that will be used for testing + testEnvironment: "jsdom", + + // Options that will be passed to the testEnvironment + // testEnvironmentOptions: {}, + + // Adds a location field to test results + // testLocationInResults: false, + + // The glob patterns Jest uses to detect test files + // testMatch: [ + // "**/__tests__/**/*.[jt]s?(x)", + // "**/?(*.)+(spec|test).[tj]s?(x)" + // ], + + // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped + // testPathIgnorePatterns: [ + // "/node_modules/" + // ], + + // The regexp pattern or array of patterns that Jest uses to detect test files + // testRegex: [], + + // This option allows the use of a custom results processor + // testResultsProcessor: null, + + // This option allows use of a custom test runner + // testRunner: "jasmine2", + + // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href + // testURL: "http://localhost", + + // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout" + // timers: "real", + + // A map from regular expressions to paths to transformers + // transform: null, + + // ??? https://github.com/facebook/jest/issues/9395#issuecomment-665239070 + // otherwise fails on importing + transformIgnorePatterns: ['node_modules/?!@webext-options-sync/'] + // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation + // transformIgnorePatterns: [ + // "/node_modules/" + // ], + + // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them + // unmockedModulePathPatterns: undefined, + + // Indicates whether each individual test should be reported during the run + // verbose: null, + + // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode + // watchPathIgnorePatterns: [], + + // Whether to use watchman for file crawling + // watchman: true, +}; diff --git a/package.json b/extension/package.json similarity index 53% rename from package.json rename to extension/package.json index 0408443..f2c3e76 100644 --- a/package.json +++ b/extension/package.json @@ -1,15 +1,30 @@ { "name": "grasp", "version": "0.6.6", + "version_name": "released on TODO", "description": "Reliably capture links and webpage content in a plaintext file (org-mode/markdown)", "main": "dist/background.js", "scripts": { - "build": "webpack --progress --profile --colors", - "watch": "webpack --watch --progress --colors", + "test": "jest", + "build": "webpack --progress --profile", + "watch": "webpack --watch --progress", "dev": "npm run watch", "eslint": "eslint src", - "flow": "flow" + "flow": "flow", + "web-ext": "web-ext", + "release:cws": "webstore", + "release:amo": "web-ext-submit" }, + "repository": { + "type": "git", + "url": "git+https://github.com/karlicoss/grasp.git" + }, + "author": "Dmitrii Gerasimov ", + "license": "MIT", + "bugs": { + "url": "https://github.com/karlicoss/grasp/issues" + }, + "homepage": "https://github.com/karlicoss/grasp#readme", "devDependencies": { "@babel/core": "^7.2.2", "@babel/preset-env": "^7.2.3", @@ -24,10 +39,17 @@ "eslint-plugin-flowtype": "^3.2.0", "flow-bin": "^0.89.0", "flow-interfaces-chrome": "^0.5.3", + "jest": "^28.1.0", + "jest-environment-jsdom": "^28.1.0", + "jest-fetch-mock": "^3.0.3", + "node-fetch": "^2.6.1", "style-loader": "^1.0.0", + "web-ext": "^6.8.0", + "web-ext-submit": "^6.8.0", "webpack": "^4.28.3", "webpack-cli": "^3.2.0", "webpack-dev-server": "^3.1.14", "webpack-extension-manifest-plugin": "^0.4.3" - } + }, + "private": "true" } diff --git a/publish b/extension/publish similarity index 100% rename from publish rename to extension/publish diff --git a/src/img/unicorn.png b/extension/src/img/unicorn.png similarity index 100% rename from src/img/unicorn.png rename to extension/src/img/unicorn.png diff --git a/src/js/background.js b/extension/src/js/background.js similarity index 100% rename from src/js/background.js rename to extension/src/js/background.js diff --git a/src/js/common.js b/extension/src/js/common.js similarity index 100% rename from src/js/common.js rename to extension/src/js/common.js diff --git a/src/js/options.js b/extension/src/js/options.js similarity index 100% rename from src/js/options.js rename to extension/src/js/options.js diff --git a/src/js/options_page.js b/extension/src/js/options_page.js similarity index 100% rename from src/js/options_page.js rename to extension/src/js/options_page.js diff --git a/src/js/permissions.js b/extension/src/js/permissions.js similarity index 100% rename from src/js/permissions.js rename to extension/src/js/permissions.js diff --git a/src/js/popup.js b/extension/src/js/popup.js similarity index 100% rename from src/js/popup.js rename to extension/src/js/popup.js diff --git a/src/manifest.json b/extension/src/manifest.json similarity index 100% rename from src/manifest.json rename to extension/src/manifest.json diff --git a/src/options.html b/extension/src/options.html similarity index 100% rename from src/options.html rename to extension/src/options.html diff --git a/src/popup.html b/extension/src/popup.html similarity index 100% rename from src/popup.html rename to extension/src/popup.html diff --git a/webpack.config.js b/extension/webpack.config.js similarity index 87% rename from webpack.config.js rename to extension/webpack.config.js index 8d25036..2446ece 100644 --- a/webpack.config.js +++ b/extension/webpack.config.js @@ -13,12 +13,14 @@ const T = { const env = { TARGET : process.env.TARGET, // TODO assert not null? RELEASE: process.env.RELEASE, + PUBLISH: process.env.PUBLISH, }; const pkg = require('./package.json'); const baseManifest = require('./src/manifest.json'); const release = env.RELEASE == 'YES' ? true : false; +const publish = env.PUBLISH == 'YES' ? true : false; const dev = !release; // meh. maybe make up my mind? const target = env.TARGET; @@ -78,14 +80,14 @@ if (target === T.FIREFOX) { manifestExtra.browser_action = {browser_style: true}; } -const build_path = path.join(__dirname, "dist"); // TODO target?? +const buildPath = path.join(__dirname, 'dist', target); const options = { mode: dev ? 'development' : 'production', // TODO map directly from MODE env var? - optimization: { - // https://webpack.js.org/configuration/optimization - // don't think minimize worth it for such a tiny extension - minimize: false + node: { + // no idea what does it mean... https://github.com/webpack/webpack/issues/5627#issuecomment-394290231 + // but it does get rid of some Function() which webpacka apparently generates + global: false, }, entry: { background : path.join(__dirname, "src", "js", "background"), @@ -93,9 +95,15 @@ const options = { options_page: path.join(__dirname, "src", "js", "options_page"), }, output: { - path: build_path, + publicPath: '', // https://stackoverflow.com/a/64715069 + path: buildPath, filename: "[name].js", }, + optimization: { + // https://webpack.js.org/configuration/optimization + // don't think minimize worth it for such a tiny extension + minimize: false + }, module: { rules: [ { @@ -118,7 +126,7 @@ const options = { ] }, plugins: [ - new CleanWebpackPlugin([build_path + "/*"]), + new CleanWebpackPlugin([buildPath + "/*"]), new CopyWebpackPlugin([ { from: 'src/img/*.png', flatten: true }, { from: 'src/*.html' , flatten: true },