ci: bring up to date with pymplate, setup github actions

.ci/release

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+'''
+Run [[file:scripts/release][scripts/release]] to deploy Python package onto [[https://pypi.org][PyPi]] and [[https://test.pypi.org][test PyPi]].
+
+The script expects =TWINE_PASSWORD= environment variable to contain the [[https://pypi.org/help/#apitoken][PyPi token]] (not the password!).
+
+The script can be run manually.
+It's also running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. Packages are deployed on:
+- every master commit, onto test pypi
+- every new tag, onto production pypi
+
+You'll need to set =TWINE_PASSWORD= and =TWINE_PASSWORD_TEST= in [[https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets][secrets]]
+for Github Actions deployment to work.
+'''
+
+import os
+import sys
+from pathlib import Path
+from subprocess import check_call
+import shutil
+
+is_ci = os.environ.get('CI') is not None
+
+def main():
+    import argparse
+    p = argparse.ArgumentParser()
+    p.add_argument('--test', action='store_true', help='use test pypi')
+    args = p.parse_args()
+
+    extra = []
+    if args.test:
+        extra.extend(['--repository-url', 'https://test.pypi.org/legacy/'])
+
+    root = Path(__file__).absolute().parent.parent
+    os.chdir(root) # just in case
+
+    if is_ci:
+        # see https://github.com/actions/checkout/issues/217
+        check_call('git fetch --prune --unshallow'.split())
+
+    dist = root / 'dist'
+    if dist.exists():
+        shutil.rmtree(dist)
+
+    check_call('python3 setup.py sdist bdist_wheel', shell=True)
+
+    TP = 'TWINE_PASSWORD'
+    password = os.environ.get(TP)
+    if password is None:
+        print(f"WARNING: no {TP} passed", file=sys.stderr)
+        import pip_secrets
+        password = pip_secrets.token_test if args.test else pip_secrets.token # meh
+
+    check_call([
+        'python3', '-m', 'twine',
+        'upload', *dist.iterdir(),
+        *extra,
+    ], env={
+        'TWINE_USERNAME': '__token__',
+        TP: password,
+        **os.environ,
+    })
+
+
+if __name__ == '__main__':
+    main()

.ci/run

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -eu
+
+cd "$(dirname "$0")"
+cd .. # git root
+
+if ! command -v sudo; then
+    # CI or Docker sometimes doesn't have it, so useful to have a dummy
+    function sudo {
+        "$@"
+    }
+fi
+
+if [ -n "${CI-}" ]; then
+    # install OS specific stuff here
+    case "$OSTYPE" in
+    darwin*) 
+        # macos
+        :
+        ;;
+    cygwin* | msys* | win*)
+        # windows
+        :
+        ;;
+    *)
+        # must be linux?
+        :
+        ;;
+    esac
+fi
+
+
+PY_BIN="python3"
+# some systems might have python pointing to python3
+if ! command -v python3 &> /dev/null; then
+    PY_BIN="python"
+fi
+
+"$PY_BIN" -m pip install --user tox
+"$PY_BIN" -m tox

.github/workflows/main.yml

@@ -0,0 +1,53 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+
+name: CI
+on:
+  push:
+    branches: '*'
+    tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi
+    # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug:
+    # P.S. fuck made up yaml DSLs.
+  pull_request: # needed to trigger on others' PRs
+  # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
+  workflow_dispatch: # needed to trigger workflows manually
+  # todo cron?
+
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        platform: [ubuntu-latest, macos-latest] # , windows-latest]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        # 3.11 on windows has this bug, lxml setup fails
+        #https://bugs.launchpad.net/lxml/+bug/1977998
+        exclude: [{platform: windows-latest, python-version: '3.11'}]
+        # vvv just an example of excluding stuff from matrix
+        # exclude: [{platform: macos-latest, python-version: '3.6'}]
+
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
+    - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    # uncomment for SSH debugging
+    # - uses: mxschmitt/action-tmate@v3
+
+    # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
+    - run: bash .ci/run
+
+    - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
+      uses: actions/upload-artifact@v3
+      with:
+        name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
+        path: .coverage.mypy/
+

ci/mypy

@@ -1,8 +0,0 @@
-#!/bin/bash
-set -eux
-
-# TODO ugh. doesn't work from circleci...
-# cd "$(git rev-parse --show-toplevel)"
-
-# TODO use strict?
-python3 -m mypy --check-untyped-defs "$@" server/*.py server/setup

ci/pylint

@@ -1,7 +0,0 @@
-#!/bin/bash
-set -eux
-
-# TODO ugh. doesn't work from circleci...
-# cd "$(git rev-parse --show-toplevel)"
-
-python3 -m pylint -E "$@" server/*.py server/setup

ci/pytest

@@ -1,7 +0,0 @@
-#!/bin/bash
-set -eux
-
-# cd "$(git rev-parse --show-toplevel)"
-
-python3 -m pytest "$@" server/*.py
-# TODO force it to collect server/setup

mypy.ini

@@ -0,0 +1,10 @@
+[mypy]
+pretty = True
+show_error_context = True
+show_error_codes = True
+check_untyped_defs = True
+namespace_packages = True
+
+# an example of suppressing
+# [mypy-my.config.repos.pdfannots.pdfannots]
+# ignore_errors = True

pytest.ini

@@ -0,0 +1,13 @@
+[pytest]
+# discover files that don't follow test_ naming. Useful to keep tests along with the source code
+python_files = *.py
+addopts =
+  # -rap to print tests summary even when they are successful
+  -rap
+  --verbose
+
+  # otherwise it won't discover doctests
+  --doctest-modules
+
+  # show all test durations (unless they are too short)
+  --durations=0

server/org_tools.py

@@ -23,11 +23,11 @@ def empty(s) -> bool:
 # TODO put template in config??
 class Config:
     @staticmethod
-    def format_selection(selection: str) -> List[str]:
+    def format_selection(selection: str) -> List[str]:  # type: ignore[empty-body]
         ...
 
     @staticmethod
-    def format_comment(comment: str) -> List[str]:
+    def format_comment(comment: str) -> List[str]:  # type: ignore[empty-body]
         ...
 
 

setup.py

@@ -0,0 +1,57 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+
+
+from setuptools import setup, find_namespace_packages # type: ignore
+
+
+def main() -> None:
+    # works with both ordinary and namespace packages
+    #pkgs = find_namespace_packages('src')
+    pkgs = ['server']
+    pkg = min(pkgs) # lexicographically smallest is the correct one usually?
+    setup(
+        name=pkg,
+        use_scm_version={
+            'version_scheme': 'python-simplified-semver',
+            'local_scheme': 'dirty-tag',
+        },
+        setup_requires=['setuptools_scm'],
+
+        # otherwise mypy won't work
+        # https://mypy.readthedocs.io/en/stable/installed_packages.html#making-pep-561-compatible-packages
+        zip_safe=False,
+
+        packages=pkgs,
+        #package_dir={'': 'src'},
+        # necessary so that package works with mypy
+        package_data={pkg: ['py.typed']},
+
+        ## ^^^ this should be mostly automatic and not requiring any changes
+
+        install_requires=[
+            # vvv example of git repo dependency
+            # 'repo @ git+https://github.com/karlicoss/repo.git',
+
+            # vvv  example of local file dependency. yes, DUMMY is necessary for some reason
+            # 'repo @ git+file://DUMMY/path/to/repo',
+        ],
+        extras_require={
+            'testing': ['pytest', 'requests'],
+            'linting': ['pytest', 'mypy', 'lxml'], # lxml for mypy coverage report
+        },
+
+
+        # this needs to be set if you're planning to upload to pypi
+        # url='',
+        # author='',
+        # author_email='',
+        # description='',
+
+        # Rest of the stuff -- classifiers, license, etc, I don't think it matters for pypi
+        # it's just unnecessary duplication
+    )
+
+
+if __name__ == '__main__':
+    main()
+

tox.ini

@@ -0,0 +1,32 @@
+[tox]
+minversion = 3.5
+# relies on the correct version of Python installed
+envlist = tests,mypy
+# https://github.com/tox-dev/tox/issues/20#issuecomment-247788333
+# hack to prevent .tox from crapping to the project directory
+toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox
+
+[testenv]
+passenv = 
+# useful for tests to know they are running under ci
+  CI
+  CI_*
+# respect user's cache dirs to prevent tox from crapping into project dir
+  MYPY_CACHE_DIR
+  PYTHONPYCACHEPREFIX
+
+[testenv:tests]
+commands =
+    pip install -e .[testing]
+    # posargs allow test filtering, e.g. tox ... -- -k test_name
+    python -m pytest server --ignore server/test_with_browser.py {posargs}
+
+[testenv:mypy]
+commands =
+    pip install -e .[linting]
+    python -m mypy --install-types --non-interactive \
+                   server                            \
+                   # txt report is a bit more convenient to view on CI
+                   --txt-report  .coverage.mypy      \
+                   --html-report .coverage.mypy      \
+                   {posargs}