From 2730f72f0e6b71af4ea5834ea74a1c8b6d5e3d4b Mon Sep 17 00:00:00 2001 From: Ben Jeffery Date: Fri, 1 May 2020 01:07:37 +0100 Subject: [PATCH] Build wheels and deploy to PyPI in github action --- .github/workflows/lint.yml | 4 +- .github/workflows/wheels.yml | 215 ++++++++++++++++++ docker/buildwheel.sh | 22 ++ docker/shared.env | 5 + docs/development.rst | 16 +- python/requirements/CI/tests/requirements.txt | 6 + python/setup.py | 2 +- 7 files changed, 263 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/wheels.yml create mode 100644 docker/buildwheel.sh create mode 100644 docker/shared.env create mode 100644 python/requirements/CI/tests/requirements.txt diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3b32e33..fa1662b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,8 +23,8 @@ jobs: pre-commit: runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 - name: set PY run: echo "::set-env name=PY::$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" - uses: actions/cache@v1 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..dbdcc52 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,215 @@ +name: Build and test wheels + +on: + push: + branches: + - master + - test + tags: + - '*' + release: + types: [published, created] + +jobs: + windows: + runs-on: windows-latest + strategy: + matrix: + python: [3.6, 3.7, 3.8] + wordsize: [64] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install deps + env: + PYTHON: "py -${{ matrix.python }}-${{ matrix.wordsize }}" + shell: bash + run: | + set -ex + ${PYTHON} -m pip install --upgrade pip + ${PYTHON} -m pip install setuptools wheel cython + - name: Build C extension + env: + PYTHON: "py -${{ matrix.python }}-${{ matrix.wordsize }}" + shell: bash + run: | + set -ex + # Fix symlinks on windows + git config core.symlinks true + git reset --hard + cd python + ${PYTHON} -VV + ${PYTHON} setup.py build_ext --inplace + - name: Build Wheel + env: + PYTHON: "py -${{ matrix.python }}-${{ matrix.wordsize }}" + shell: bash + run: | + set -ex + cd python + ${PYTHON} setup.py bdist_wheel + - name: Install wheel and run tests + env: + PYTHON: "py -${{ matrix.python }}-${{ matrix.wordsize }}" + shell: bash + run: | + set -ex + # We install in this odd way to make sure we get both deps and a local kastore + ${PYTHON} -m pip install kastore --only-binary kastore -f python/dist/ + ${PYTHON} -m pip uninstall -y kastore + ${PYTHON} -m pip install -v kastore --only-binary kastore -f python/dist/ --no-index + ${PYTHON} -c "import kastore" + ${PYTHON} -m pip install -r python/requirements/CI/tests/requirements.txt + rm -rf python/kastore python/*.pyd + ${PYTHON} -m nose -v python + - name: Upload Wheels + uses: actions/upload-artifact@v2 + with: + name: win-wheel-${{ matrix.python }}-${{ matrix.wordsize }} + path: python/dist + + OSX: + runs-on: macos-latest + strategy: + matrix: + python: [3.6, 3.7, 3.8] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install deps + run: | + pip install --upgrade pip + pip install setuptools wheel + - name: Build C extension + run: | + cd python + python -VV + python setup.py build_ext --inplace + - name: Build Wheel + run: | + cd python + python setup.py bdist_wheel + - name: Install wheel and run tests + run: | + # We install in this odd way to make sure we get both deps and a local kastore + pip install kastore --only-binary kastore -f python/dist/ + pip uninstall -y kastore + pip install -v kastore --only-binary kastore -f python/dist/ --no-index + python -c "import kastore" + pip install -r python/requirements/CI/tests/requirements.txt + rm -rf python/kastore python/*.so + python -m nose -v python + - name: Upload Wheels + uses: actions/upload-artifact@v2 + with: + name: osx-wheel-${{ matrix.python }} + path: python/dist + + manylinux: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Build sdist + shell: bash + run: | + cd python + python setup.py sdist + + - name: Upload sdist + uses: actions/upload-artifact@v2 + with: + name: sdist + path: python/dist + + - name: Build wheels in docker + shell: bash + run: | + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Run tests (3.8) + run: | + python -VV + # We install in this odd way to make sure we get both deps and a local kastore + pip install kastore --only-binary kastore -f python/dist/wheelhouse + pip uninstall -y kastore + pip install -v kastore --only-binary kastore -f python/dist/wheelhouse --no-index + python -c "import kastore" + pip install -r python/requirements/CI/tests/requirements.txt + python -m nose -v python + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Run tests (3.7) + run: | + python -VV + # We install in this odd way to make sure we get both deps and a local kastore + pip install kastore --only-binary kastore -f python/dist/wheelhouse + pip uninstall -y kastore + pip install -v kastore --only-binary kastore -f python/dist/wheelhouse --no-index + python -c "import kastore" + pip install -r python/requirements/CI/tests/requirements.txt + python -m nose -v python + + - name: Set up Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 + + - name: Run tests (3.6) + run: | + python -VV + # We install in this odd way to make sure we get both deps and a local kastore + pip install kastore --only-binary kastore -f python/dist/wheelhouse + pip uninstall -y kastore + pip install -v kastore --only-binary kastore -f python/dist/wheelhouse --no-index + python -c "import kastore" + pip install -r python/requirements/CI/tests/requirements.txt + python -m nose -v python + + - name: Upload Wheels + uses: actions/upload-artifact@v2 + with: + name: linux-wheels + path: python/dist/wheelhouse + + + PyPI_Upload: + runs-on: ubuntu-latest + needs: ['windows', 'OSX', 'manylinux'] + steps: + - name: Download all + uses: actions/download-artifact@v2 + - name: Move to dist + run: | + mkdir dist + cp */*.{whl,gz} dist/. + - name: Publish distribution to Test PyPI + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && !contains(github.event.ref, 'C_') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.test_pypi_password }} + repository_url: https://test.pypi.org/legacy/ + - name: Publish distribution to PRODUCTION PyPI + if: github.event_name == 'release' && !startsWith(github.event.release.tag_name, 'C_') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.test_pypi_password }} diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh new file mode 100644 index 0000000..6394d5e --- /dev/null +++ b/docker/buildwheel.sh @@ -0,0 +1,22 @@ +#!/bin/bash +DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$DOCKER_DIR/shared.env" + +set -e -x + +ARCH=`uname -p` +echo "arch=$ARCH" +cd python + +for V in "${PYTHON_VERSIONS[@]}"; do + PYBIN=/opt/python/$V/bin + rm -rf build/ # Avoid lib build by narrow Python is used by wide python + $PYBIN/python setup.py build_ext --inplace + $PYBIN/python setup.py bdist_wheel +done + +cd dist +for whl in *.whl; do + auditwheel repair "$whl" + rm "$whl" +done \ No newline at end of file diff --git a/docker/shared.env b/docker/shared.env new file mode 100644 index 0000000..4f64294 --- /dev/null +++ b/docker/shared.env @@ -0,0 +1,5 @@ +PYTHON_VERSIONS=( + cp38-cp38 + cp37-cp37m + cp36-cp36m +) diff --git a/docs/development.rst b/docs/development.rst index c64891e..561277c 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -28,9 +28,17 @@ Python To make a release first prepare a pull request that sets the correct version number in ``kastore/_version.py`` and updates the Python CHANGELOG.rst, ensuring that all significant changes since the last release have been listed. -Once this PR is merged, create a release on GitHub with the pattern -``py_MAJOR.MINOR.PATCH``. Create the distribution artefacts for Python and -upload to PyPI. +Once this PR is merged, push a tag to github following PEP440 format:: + + git tag -a MAJOR.MINOR.PATCH -m "Python version MAJOR.MINOR.PATCH" + git push upstream --tags + +This will trigger a build of the distribution artifacts for Python +on `Github Actions `_. and deploy +them to the `test PyPI `_. Check +the release looks good there, then create a release on Github based on the tag you +pushed. Publishing this release will cause the github action to deploy to the +`production PyPI `_. ----- C API @@ -44,7 +52,7 @@ When modifying the C code this will conform it to the project style:: If the C API has been updated, the ``KAS_VERSION_*`` macros should be set appropriately, ensuring that the Changelog has been updated to record the changes. After the commit including these changes has been merged, tag a -release on GitHub using the pattern ``c_MAJOR.MINOR.PATCH``. +release on GitHub using the pattern ``C_MAJOR.MINOR.PATCH``. ----------------- diff --git a/python/requirements/CI/tests/requirements.txt b/python/requirements/CI/tests/requirements.txt new file mode 100644 index 0000000..c6523c0 --- /dev/null +++ b/python/requirements/CI/tests/requirements.txt @@ -0,0 +1,6 @@ +attrs +hypothesis +humanize +nose +numpy +mock diff --git a/python/setup.py b/python/setup.py index 0ca1457..d85e8e4 100644 --- a/python/setup.py +++ b/python/setup.py @@ -70,7 +70,7 @@ def finalize_options(self): "Bug Reports": "https://github.com/tskit-dev/kastore/issues", "Source": "https://github.com/tskit-dev/kastore", }, - setup_requires=[numpy_ver], + setup_requires=["oldest-supported-numpy"], cmdclass={"build_ext": local_build_ext}, license="MIT", platforms=["POSIX", "Windows", "MacOS X"],