extension: move to a separate dir, update build scripts etc

borrowed build scritps etc from promnesia
This commit is contained in:
Dima Gerasimov 2022-12-23 20:58:26 +00:00 committed by karlicoss
parent b57c787774
commit 4127e3de64
22 changed files with 391 additions and 185 deletions

View file

@ -1,100 +0,0 @@
# TODO should I use yarn as in circleci node example??
version: 2
jobs:
client_build_and_check:
docker:
- image: circleci/node:latest
steps:
- checkout
- run: sudo npm install -g npm web-ext
- restore_cache:
keys:
- dependency-cache-{{ checksum "package.json" }}
# fallback to using the latest cache if no exact match is found
- dependency-cache-
- run: npm install
- save_cache:
key: dependency-cache-{{ checksum "package.json" }}
paths:
- ./node_modules
- run: ./build --chrome --lint
# TODO ugh. is there a way for a job to inherit a cloned repo after npm run build has ran?? I'd like to separate build and check steps
# TODO ok, apparently it's a custom executor? https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
- run:
name: js flow
command: npm run flow
when: always
- run:
name: eslint
command: npm run eslint
when: always
- run:
# TODO need to do some kind of build matrix over [firefox, chrome] x [anyhost, no_anyhost]
# it should be somehow achievable via workspaces, but I've spent 10 minutes and couldn't figure it out...
name: pack
command: |
for browser in 'chrome'; do
zip -r -j "/tmp/grasp.zip" dist/*
done
# TODO ugh, also can't use env variables in artifacts...
- store_artifacts:
path: "/tmp/grasp.zip"
destination: "grasp.zip"
server_build_and_check:
docker:
- image: circleci/python:latest
steps:
- checkout
# TODO preserve apt cache..
- run: sudo apt install python3-pip
- run: python3 -m pip install --user mypy pylint pytest
# test only dependencies
- run: sudo apt install httpie
- run: python3 -m pip install --user selenium pyautogui
#
- run:
name: pylint
command: ci/pylint
when: always
- run:
name: mypy
command: ci/mypy
when: always
- run:
name: pytest
command: ci/pytest
when: always
workflows:
version: 2
build_and_test:
jobs:
- client_build_and_check
- server_build_and_check
# # ugh. circleci only got python3.6 on this node package??? and it's unclear how to combine multiple circleci docker configurations...
# # - store_artifacts:
# # path: test-results.xml
# # prefix: tests
# # - store_artifacts:
# # path: coverage
# # prefix: coverage
# # - store_test_results:
# # path: test-results.xml

View file

@ -51,3 +51,36 @@ jobs:
name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
path: .coverage.mypy/ path: .coverage.mypy/
###
build_extension:
env:
name: 'grasp'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- uses: actions/setup-node@v3
with:
node-version: '18'
- run: extension/.ci/build --lint # debug version
- run: extension/.ci/build --lint --release
# TODO ugh. can't share github actions artifacts publicly...
# TODO for fuck's sake... why does it end up named as .zip.zip ????
- uses: actions/upload-artifact@v2
with:
name: '${ env.name }}-chrome-debug-latest.zip'
path: 'extension/dist/artifacts/chrome/${{ env.name }}_dev_-*.zip'
- uses: actions/upload-artifact@v2
with:
.name : '${{ env.name }}-chrome-release-latest.zip'
path: 'extension/dist/artifacts/chrome/${{ env.name }}-*.zip'
- uses: actions/upload-artifact@v2
with:
name : '${{ env.name }}-firefox-debug-latest.zip'
path: 'extension/dist/artifacts/firefox/${{ env.name }}_dev_-*.zip'
- uses: actions/upload-artifact@v2
with:
name : '${{ env.name }}-firefox-release-latest.zip'
path: 'extension/dist/artifacts/firefox/${{ env.name }}-*.zip'

View file

@ -1,74 +0,0 @@
#!/usr/bin/env python3
import argparse
import os
from subprocess import check_call
from pathlib import Path
# TODO FIXME copypaste from promnesia
# TODO FIXME use it in ci script
TARGETS = ['chrome', 'firefox']
def main():
p = argparse.ArgumentParser()
p.add_argument('--release', action='store_true')
p.add_argument('--sign', action='store_true')
p.add_argument('--lint', action='store_true')
tg = p.add_mutually_exclusive_group(required=True)
tg.add_argument('--target', type=str, choices=TARGETS)
for b in TARGETS:
tg.add_argument('--' + b, action='store_const', const=b, dest='target')
args = p.parse_args()
target = args.target
assert target is not None
dist_dir = Path(__file__).absolute().parent / 'dist'
def webext(*args, **kwargs):
src_dir = dist_dir # / target
src_dir = src_dir.resolve() # TODO ugh. webext can't into symlinks???
check_call([
'web-ext',
'--source-dir', str(src_dir),
*args,
], **kwargs)
env = {
'TARGET': target,
'RELEASE': 'YES' if args.release else 'NO',
**os.environ,
}
check_call([
'npm', 'run', 'build',
], env=env, cwd=str(Path(__file__).absolute().parent))
if args.lint:
# TODO --self-hosted
# TODO warnings as errors??
webext('lint')
if args.release:
assert args.lint # TODO not sure..
if args.sign:
assert args.release
if target not in ['firefox', 'firefox-mobile']:
raise RuntimeError("{target} is not supported for signing yet".format(target=target))
dist_dir = Path(__file__).absolute().parent / 'dist'
src_dir = dist_dir / target
from firefox_dev_secrets import API_KEY, API_SECRET
webext(
'--source-dir' , str(src_dir),
'--artifacts-dir', str(dist_dir),
'--api-key' , API_KEY,
'--api-secret', API_SECRET,
'sign',
)
if __name__ == '__main__':
main()
# TODO use UUID? https://addons.mozilla.org/en-GB/developers/addon/grasp/edit
# TODO right, wouldn't work because it's listed..

22
extension/.ci/build Executable file
View file

@ -0,0 +1,22 @@
#!/bin/bash
set -eux
cd "$(dirname "$0")"
cd ../.. # git root
cd extension
npm install
FAILED=0
npm run flow || FAILED=1
npm run eslint || FAILED=1
# TODO later
#npm run test || FAILED=1
for browser in 'firefox' 'chrome'; do
./build --target "$browser" "$@"
done
exit "$FAILED"

107
extension/build Executable file
View file

@ -0,0 +1,107 @@
#!/usr/bin/env python3
import argparse
import os
from subprocess import check_call
from pathlib import Path
# right, specifying id in manifest doesn't seem to work
# AMO responds with: Server response: Duplicate add-on ID found. (status: 400)
IDS = {
'firefox' : 'FIXME',
'chrome' : 'FIXME',
}
TARGETS = [
'chrome',
'firefox',
]
def main():
p = argparse.ArgumentParser()
p.add_argument('--release', action='store_true', help="Use release flavor of build")
p.add_argument('--lint' , action='store_true')
p.add_argument('--publish', action='store_true', help="Publish on chrome web store/addons.mozilla.org")
tg = p.add_mutually_exclusive_group(required=True)
tg.add_argument('--target', type=str, choices=TARGETS)
for b in TARGETS:
tg.add_argument('--' + b, action='store_const', const=b, dest='target')
args = p.parse_args()
target = args.target
assert target is not None
base_ext_dir = Path(__file__).absolute().parent / 'dist'
ext_dir = (base_ext_dir / target).resolve() # webext can't into symlinks
# sadly no way to specify zip name in the regex..
artifacts_dir = (base_ext_dir / 'artifacts' / target).resolve()
def webext(*args, **kwargs):
check_call([
'npm', 'run', 'web-ext',
'--',
'--source-dir' , ext_dir,
'--artifacts-dir', artifacts_dir,
*args,
], **kwargs)
env = {
'TARGET': target,
'RELEASE': 'YES' if args.release else 'NO',
'PUBLISH': 'YES' if args.publish else 'NO',
**os.environ,
}
check_call([
'npm', 'run', 'build',
], env=env, cwd=str(Path(__file__).absolute().parent))
if args.lint:
# TODO --self-hosted
# TODO warnings as errors??
webext('lint')
# TODO move somewhere more appropriate..
webext('build', '-o') # -o overwrites existing artifact
if args.release:
assert args.lint # TODO not sure..
def firefox_release_args():
from firefox_dev_secrets import API_KEY, API_SECRET
return [
'--artifacts-dir', str(artifacts_dir),
'--api-key' , API_KEY,
'--api-secret' , API_SECRET,
'--id' , IDS[target],
]
if args.publish:
assert args.release
if 'firefox' in target:
check_call([
'npm', 'run', 'release:amo',
'--',
'--channel', 'listed',
'--source-dir', str(ext_dir),
*firefox_release_args(),
])
elif target == 'chrome':
from chrome_dev_secrets import CLIENT_ID, CLIENT_SECRET, REFRESH_TOKEN
check_call([
'npm', 'run', 'release:cws',
'--',
'upload',
# '--auto-publish',
'--source' , str(ext_dir),
'--client-id' , CLIENT_ID,
'--client-secret' , CLIENT_SECRET,
'--refresh--token', REFRESH_TOKEN,
'--extension-id' , IDS[target],
# TODO trusted testers?
])
else:
raise RuntimeError("{target} is not supported for publishing yet".format(target=target))
if __name__ == '__main__':
main()

188
extension/jest.config.js Normal file
View file

@ -0,0 +1,188 @@
// For a detailed explanation regarding each configuration property, visit:
// https://jestjs.io/docs/en/configuration.html
module.exports = {
// All imported modules in your tests should be mocked automatically
// automock: false,
// Stop running tests after `n` failures
// bail: 0,
// Respect "browser" field in package.json when resolving modules
// browser: false,
// The directory where Jest should store its cached dependency information
// cacheDirectory: "/tmp/jest_rs",
// Automatically clear mock calls and instances between every test
clearMocks: true,
// Indicates whether the coverage information should be collected while executing the test
// collectCoverage: false,
// An array of glob patterns indicating a set of files for which coverage information should be collected
// collectCoverageFrom: null,
// The directory where Jest should output its coverage files
// coverageDirectory: null,
// An array of regexp pattern strings used to skip coverage collection
// coveragePathIgnorePatterns: [
// "/node_modules/"
// ],
// A list of reporter names that Jest uses when writing coverage reports
// coverageReporters: [
// "json",
// "text",
// "lcov",
// "clover"
// ],
// An object that configures minimum threshold enforcement for coverage results
// coverageThreshold: null,
// A path to a custom dependency extractor
// dependencyExtractor: null,
// Make calling deprecated APIs throw helpful error messages
// errorOnDeprecated: false,
// Force coverage collection from ignored files using an array of glob patterns
// forceCoverageMatch: [],
// A path to a module which exports an async function that is triggered once before all test suites
// globalSetup: null,
// A path to a module which exports an async function that is triggered once after all test suites
// globalTeardown: null,
// A set of global variables that need to be available in all test environments
// globals: {},
// An array of directory names to be searched recursively up from the requiring module's location
// moduleDirectories: [
// "node_modules"
// ],
// An array of file extensions your modules use
// moduleFileExtensions: [
// "js",
// "json",
// "jsx",
// "ts",
// "tsx",
// "node"
// ],
// A map from regular expressions to module names that allow to stub out resources with a single module
// moduleNameMapper: {},
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
// modulePathIgnorePatterns: [],
// Activates notifications for test results
// notify: false,
// An enum that specifies notification mode. Requires { notify: true }
// notifyMode: "failure-change",
// A preset that is used as a base for Jest's configuration
// preset: null,
// Run tests from one or more projects
// projects: null,
// Use this configuration option to add custom reporters to Jest
// reporters: undefined,
// Automatically reset mock state between every test
// resetMocks: false,
// Reset the module registry before running each individual test
// resetModules: false,
// A path to a custom resolver
// resolver: null,
// Automatically restore mock state between every test
// restoreMocks: false,
// The root directory that Jest should scan for tests and modules within
// rootDir: null,
// A list of paths to directories that Jest should use to search for files in
// roots: [
// "<rootDir>"
// ],
// Allows you to use a custom runner instead of Jest's default test runner
// runner: "jest-runner",
// The paths to modules that run some code to configure or set up the testing environment before each test
// setupFiles: [],
// A list of paths to modules that run some code to configure or set up the testing framework before each test
// setupFilesAfterEnv: [],
// A list of paths to snapshot serializer modules Jest should use for snapshot testing
// snapshotSerializers: [],
// The test environment that will be used for testing
testEnvironment: "jsdom",
// Options that will be passed to the testEnvironment
// testEnvironmentOptions: {},
// Adds a location field to test results
// testLocationInResults: false,
// The glob patterns Jest uses to detect test files
// testMatch: [
// "**/__tests__/**/*.[jt]s?(x)",
// "**/?(*.)+(spec|test).[tj]s?(x)"
// ],
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
// testPathIgnorePatterns: [
// "/node_modules/"
// ],
// The regexp pattern or array of patterns that Jest uses to detect test files
// testRegex: [],
// This option allows the use of a custom results processor
// testResultsProcessor: null,
// This option allows use of a custom test runner
// testRunner: "jasmine2",
// This option sets the URL for the jsdom environment. It is reflected in properties such as location.href
// testURL: "http://localhost",
// Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout"
// timers: "real",
// A map from regular expressions to paths to transformers
// transform: null,
// ??? https://github.com/facebook/jest/issues/9395#issuecomment-665239070
// otherwise fails on importing
transformIgnorePatterns: ['node_modules/?!@webext-options-sync/']
// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
// transformIgnorePatterns: [
// "/node_modules/"
// ],
// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
// unmockedModulePathPatterns: undefined,
// Indicates whether each individual test should be reported during the run
// verbose: null,
// An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
// watchPathIgnorePatterns: [],
// Whether to use watchman for file crawling
// watchman: true,
};

View file

@ -1,15 +1,30 @@
{ {
"name": "grasp", "name": "grasp",
"version": "0.6.6", "version": "0.6.6",
"version_name": "released on TODO",
"description": "Reliably capture links and webpage content in a plaintext file (org-mode/markdown)", "description": "Reliably capture links and webpage content in a plaintext file (org-mode/markdown)",
"main": "dist/background.js", "main": "dist/background.js",
"scripts": { "scripts": {
"build": "webpack --progress --profile --colors", "test": "jest",
"watch": "webpack --watch --progress --colors", "build": "webpack --progress --profile",
"watch": "webpack --watch --progress",
"dev": "npm run watch", "dev": "npm run watch",
"eslint": "eslint src", "eslint": "eslint src",
"flow": "flow" "flow": "flow",
"web-ext": "web-ext",
"release:cws": "webstore",
"release:amo": "web-ext-submit"
}, },
"repository": {
"type": "git",
"url": "git+https://github.com/karlicoss/grasp.git"
},
"author": "Dmitrii Gerasimov <karlicoss@gmail.com>",
"license": "MIT",
"bugs": {
"url": "https://github.com/karlicoss/grasp/issues"
},
"homepage": "https://github.com/karlicoss/grasp#readme",
"devDependencies": { "devDependencies": {
"@babel/core": "^7.2.2", "@babel/core": "^7.2.2",
"@babel/preset-env": "^7.2.3", "@babel/preset-env": "^7.2.3",
@ -24,10 +39,17 @@
"eslint-plugin-flowtype": "^3.2.0", "eslint-plugin-flowtype": "^3.2.0",
"flow-bin": "^0.89.0", "flow-bin": "^0.89.0",
"flow-interfaces-chrome": "^0.5.3", "flow-interfaces-chrome": "^0.5.3",
"jest": "^28.1.0",
"jest-environment-jsdom": "^28.1.0",
"jest-fetch-mock": "^3.0.3",
"node-fetch": "^2.6.1",
"style-loader": "^1.0.0", "style-loader": "^1.0.0",
"web-ext": "^6.8.0",
"web-ext-submit": "^6.8.0",
"webpack": "^4.28.3", "webpack": "^4.28.3",
"webpack-cli": "^3.2.0", "webpack-cli": "^3.2.0",
"webpack-dev-server": "^3.1.14", "webpack-dev-server": "^3.1.14",
"webpack-extension-manifest-plugin": "^0.4.3" "webpack-extension-manifest-plugin": "^0.4.3"
} },
"private": "true"
} }

View file

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 2.8 KiB

View file

@ -13,12 +13,14 @@ const T = {
const env = { const env = {
TARGET : process.env.TARGET, // TODO assert not null? TARGET : process.env.TARGET, // TODO assert not null?
RELEASE: process.env.RELEASE, RELEASE: process.env.RELEASE,
PUBLISH: process.env.PUBLISH,
}; };
const pkg = require('./package.json'); const pkg = require('./package.json');
const baseManifest = require('./src/manifest.json'); const baseManifest = require('./src/manifest.json');
const release = env.RELEASE == 'YES' ? true : false; const release = env.RELEASE == 'YES' ? true : false;
const publish = env.PUBLISH == 'YES' ? true : false;
const dev = !release; // meh. maybe make up my mind? const dev = !release; // meh. maybe make up my mind?
const target = env.TARGET; const target = env.TARGET;
@ -78,14 +80,14 @@ if (target === T.FIREFOX) {
manifestExtra.browser_action = {browser_style: true}; manifestExtra.browser_action = {browser_style: true};
} }
const build_path = path.join(__dirname, "dist"); // TODO target?? const buildPath = path.join(__dirname, 'dist', target);
const options = { const options = {
mode: dev ? 'development' : 'production', // TODO map directly from MODE env var? mode: dev ? 'development' : 'production', // TODO map directly from MODE env var?
optimization: { node: {
// https://webpack.js.org/configuration/optimization // no idea what does it mean... https://github.com/webpack/webpack/issues/5627#issuecomment-394290231
// don't think minimize worth it for such a tiny extension // but it does get rid of some Function() which webpacka apparently generates
minimize: false global: false,
}, },
entry: { entry: {
background : path.join(__dirname, "src", "js", "background"), background : path.join(__dirname, "src", "js", "background"),
@ -93,9 +95,15 @@ const options = {
options_page: path.join(__dirname, "src", "js", "options_page"), options_page: path.join(__dirname, "src", "js", "options_page"),
}, },
output: { output: {
path: build_path, publicPath: '', // https://stackoverflow.com/a/64715069
path: buildPath,
filename: "[name].js", filename: "[name].js",
}, },
optimization: {
// https://webpack.js.org/configuration/optimization
// don't think minimize worth it for such a tiny extension
minimize: false
},
module: { module: {
rules: [ rules: [
{ {
@ -118,7 +126,7 @@ const options = {
] ]
}, },
plugins: [ plugins: [
new CleanWebpackPlugin([build_path + "/*"]), new CleanWebpackPlugin([buildPath + "/*"]),
new CopyWebpackPlugin([ new CopyWebpackPlugin([
{ from: 'src/img/*.png', flatten: true }, { from: 'src/img/*.png', flatten: true },
{ from: 'src/*.html' , flatten: true }, { from: 'src/*.html' , flatten: true },