From 417a02ba6f9838ad5509c7e7e91f01ed0d046ffb Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 21 Dec 2022 02:05:47 +0000 Subject: [PATCH] ci: bring up to date with pymplate, setup github actions --- .ci/release | 66 ++++++++++++++++++++++++++++++++++++++ .ci/run | 40 +++++++++++++++++++++++ .github/workflows/main.yml | 53 ++++++++++++++++++++++++++++++ build => build_extension | 0 ci/mypy | 8 ----- ci/pylint | 7 ---- ci/pytest | 7 ---- mypy.ini | 10 ++++++ pytest.ini | 13 ++++++++ server/org_tools.py | 4 +-- setup.py | 57 ++++++++++++++++++++++++++++++++ tox.ini | 32 ++++++++++++++++++ 12 files changed, 273 insertions(+), 24 deletions(-) create mode 100755 .ci/release create mode 100755 .ci/run create mode 100644 .github/workflows/main.yml rename build => build_extension (100%) delete mode 100755 ci/mypy delete mode 100755 ci/pylint delete mode 100755 ci/pytest create mode 100644 mypy.ini create mode 100644 pytest.ini create mode 100644 setup.py create mode 100644 tox.ini diff --git a/.ci/release b/.ci/release new file mode 100755 index 0000000..0ec687f --- /dev/null +++ b/.ci/release @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +''' +Run [[file:scripts/release][scripts/release]] to deploy Python package onto [[https://pypi.org][PyPi]] and [[https://test.pypi.org][test PyPi]]. + +The script expects =TWINE_PASSWORD= environment variable to contain the [[https://pypi.org/help/#apitoken][PyPi token]] (not the password!). + +The script can be run manually. +It's also running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. Packages are deployed on: +- every master commit, onto test pypi +- every new tag, onto production pypi + +You'll need to set =TWINE_PASSWORD= and =TWINE_PASSWORD_TEST= in [[https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets][secrets]] +for Github Actions deployment to work. +''' + +import os +import sys +from pathlib import Path +from subprocess import check_call +import shutil + +is_ci = os.environ.get('CI') is not None + +def main(): + import argparse + p = argparse.ArgumentParser() + p.add_argument('--test', action='store_true', help='use test pypi') + args = p.parse_args() + + extra = [] + if args.test: + extra.extend(['--repository-url', 'https://test.pypi.org/legacy/']) + + root = Path(__file__).absolute().parent.parent + os.chdir(root) # just in case + + if is_ci: + # see https://github.com/actions/checkout/issues/217 + check_call('git fetch --prune --unshallow'.split()) + + dist = root / 'dist' + if dist.exists(): + shutil.rmtree(dist) + + check_call('python3 setup.py sdist bdist_wheel', shell=True) + + TP = 'TWINE_PASSWORD' + password = os.environ.get(TP) + if password is None: + print(f"WARNING: no {TP} passed", file=sys.stderr) + import pip_secrets + password = pip_secrets.token_test if args.test else pip_secrets.token # meh + + check_call([ + 'python3', '-m', 'twine', + 'upload', *dist.iterdir(), + *extra, + ], env={ + 'TWINE_USERNAME': '__token__', + TP: password, + **os.environ, + }) + + +if __name__ == '__main__': + main() diff --git a/.ci/run b/.ci/run new file mode 100755 index 0000000..303de20 --- /dev/null +++ b/.ci/run @@ -0,0 +1,40 @@ +#!/bin/bash +set -eu + +cd "$(dirname "$0")" +cd .. # git root + +if ! command -v sudo; then + # CI or Docker sometimes doesn't have it, so useful to have a dummy + function sudo { + "$@" + } +fi + +if [ -n "${CI-}" ]; then + # install OS specific stuff here + case "$OSTYPE" in + darwin*) + # macos + : + ;; + cygwin* | msys* | win*) + # windows + : + ;; + *) + # must be linux? + : + ;; + esac +fi + + +PY_BIN="python3" +# some systems might have python pointing to python3 +if ! command -v python3 &> /dev/null; then + PY_BIN="python" +fi + +"$PY_BIN" -m pip install --user tox +"$PY_BIN" -m tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..6f936ee --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,53 @@ +# see https://github.com/karlicoss/pymplate for up-to-date reference + +name: CI +on: + push: + branches: '*' + tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi + # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug: + # P.S. fuck made up yaml DSLs. + pull_request: # needed to trigger on others' PRs + # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". + workflow_dispatch: # needed to trigger workflows manually + # todo cron? + + +jobs: + build: + strategy: + matrix: + platform: [ubuntu-latest, macos-latest] # , windows-latest] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + # 3.11 on windows has this bug, lxml setup fails + #https://bugs.launchpad.net/lxml/+bug/1977998 + exclude: [{platform: windows-latest, python-version: '3.11'}] + # vvv just an example of excluding stuff from matrix + # exclude: [{platform: macos-latest, python-version: '3.6'}] + + runs-on: ${{ matrix.platform }} + + steps: + # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/checkout@v3 + with: + submodules: recursive + + # uncomment for SSH debugging + # - uses: mxschmitt/action-tmate@v3 + + # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd... + - run: bash .ci/run + + - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms + uses: actions/upload-artifact@v3 + with: + name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} + path: .coverage.mypy/ + diff --git a/build b/build_extension similarity index 100% rename from build rename to build_extension diff --git a/ci/mypy b/ci/mypy deleted file mode 100755 index 2284e95..0000000 --- a/ci/mypy +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -eux - -# TODO ugh. doesn't work from circleci... -# cd "$(git rev-parse --show-toplevel)" - -# TODO use strict? -python3 -m mypy --check-untyped-defs "$@" server/*.py server/setup diff --git a/ci/pylint b/ci/pylint deleted file mode 100755 index a663c48..0000000 --- a/ci/pylint +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -eux - -# TODO ugh. doesn't work from circleci... -# cd "$(git rev-parse --show-toplevel)" - -python3 -m pylint -E "$@" server/*.py server/setup diff --git a/ci/pytest b/ci/pytest deleted file mode 100755 index 7622e71..0000000 --- a/ci/pytest +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -eux - -# cd "$(git rev-parse --show-toplevel)" - -python3 -m pytest "$@" server/*.py -# TODO force it to collect server/setup diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..0f8386f --- /dev/null +++ b/mypy.ini @@ -0,0 +1,10 @@ +[mypy] +pretty = True +show_error_context = True +show_error_codes = True +check_untyped_defs = True +namespace_packages = True + +# an example of suppressing +# [mypy-my.config.repos.pdfannots.pdfannots] +# ignore_errors = True diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..20c3704 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +# discover files that don't follow test_ naming. Useful to keep tests along with the source code +python_files = *.py +addopts = + # -rap to print tests summary even when they are successful + -rap + --verbose + + # otherwise it won't discover doctests + --doctest-modules + + # show all test durations (unless they are too short) + --durations=0 diff --git a/server/org_tools.py b/server/org_tools.py index ac9332d..0417203 100644 --- a/server/org_tools.py +++ b/server/org_tools.py @@ -23,11 +23,11 @@ def empty(s) -> bool: # TODO put template in config?? class Config: @staticmethod - def format_selection(selection: str) -> List[str]: + def format_selection(selection: str) -> List[str]: # type: ignore[empty-body] ... @staticmethod - def format_comment(comment: str) -> List[str]: + def format_comment(comment: str) -> List[str]: # type: ignore[empty-body] ... diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3cc8c3d --- /dev/null +++ b/setup.py @@ -0,0 +1,57 @@ +# see https://github.com/karlicoss/pymplate for up-to-date reference + + +from setuptools import setup, find_namespace_packages # type: ignore + + +def main() -> None: + # works with both ordinary and namespace packages + #pkgs = find_namespace_packages('src') + pkgs = ['server'] + pkg = min(pkgs) # lexicographically smallest is the correct one usually? + setup( + name=pkg, + use_scm_version={ + 'version_scheme': 'python-simplified-semver', + 'local_scheme': 'dirty-tag', + }, + setup_requires=['setuptools_scm'], + + # otherwise mypy won't work + # https://mypy.readthedocs.io/en/stable/installed_packages.html#making-pep-561-compatible-packages + zip_safe=False, + + packages=pkgs, + #package_dir={'': 'src'}, + # necessary so that package works with mypy + package_data={pkg: ['py.typed']}, + + ## ^^^ this should be mostly automatic and not requiring any changes + + install_requires=[ + # vvv example of git repo dependency + # 'repo @ git+https://github.com/karlicoss/repo.git', + + # vvv example of local file dependency. yes, DUMMY is necessary for some reason + # 'repo @ git+file://DUMMY/path/to/repo', + ], + extras_require={ + 'testing': ['pytest', 'requests'], + 'linting': ['pytest', 'mypy', 'lxml'], # lxml for mypy coverage report + }, + + + # this needs to be set if you're planning to upload to pypi + # url='', + # author='', + # author_email='', + # description='', + + # Rest of the stuff -- classifiers, license, etc, I don't think it matters for pypi + # it's just unnecessary duplication + ) + + +if __name__ == '__main__': + main() + diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..0018b97 --- /dev/null +++ b/tox.ini @@ -0,0 +1,32 @@ +[tox] +minversion = 3.5 +# relies on the correct version of Python installed +envlist = tests,mypy +# https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 +# hack to prevent .tox from crapping to the project directory +toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox + +[testenv] +passenv = +# useful for tests to know they are running under ci + CI + CI_* +# respect user's cache dirs to prevent tox from crapping into project dir + MYPY_CACHE_DIR + PYTHONPYCACHEPREFIX + +[testenv:tests] +commands = + pip install -e .[testing] + # posargs allow test filtering, e.g. tox ... -- -k test_name + python -m pytest server --ignore server/test_with_browser.py {posargs} + +[testenv:mypy] +commands = + pip install -e .[linting] + python -m mypy --install-types --non-interactive \ + server \ + # txt report is a bit more convenient to view on CI + --txt-report .coverage.mypy \ + --html-report .coverage.mypy \ + {posargs}