Merge pull request #118 from benjeffery/wheels

Build wheels in github action, upload to pypi on tag
tskit-dev · May 19, 2020 · 62acb3a · 62acb3a
2 parents f95c6ce + 2730f72
commit 62acb3a
Show file tree

Hide file tree

Showing 7 changed files with 263 additions and 7 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -23,8 +23,8 @@ jobs:
   pre-commit:
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v1
-    - uses: actions/setup-python@v1
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
     - name: set PY
       run: echo "::set-env name=PY::$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')"
     - uses: actions/cache@v1

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -0,0 +1,215 @@
+name: Build and test wheels
+
+on:
+  push:
+    branches:
+      - master
+      - test
+    tags:
+      - '*'
+  release:
+    types: [published, created]
+
+jobs:
+  windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python: [3.6, 3.7, 3.8]
+        wordsize: [64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Install deps
+        env:
+          PYTHON: "py -${{ matrix.python }}-${{ matrix.wordsize }}"
+        shell: bash
+        run: |
+          set -ex
+          ${PYTHON} -m pip install --upgrade pip
+          ${PYTHON} -m pip install setuptools wheel cython
+      - name: Build C extension
+        env:
+          PYTHON: "py -${{ matrix.python }}-${{ matrix.wordsize }}"
+        shell: bash
+        run: |
+          set -ex
+          # Fix symlinks on windows
+          git config core.symlinks true
+          git reset --hard
+          cd python
+          ${PYTHON} -VV
+          ${PYTHON} setup.py build_ext --inplace
+      - name: Build Wheel
+        env:
+          PYTHON: "py -${{ matrix.python }}-${{ matrix.wordsize }}"
+        shell: bash
+        run: |
+          set -ex
+          cd python
+          ${PYTHON} setup.py bdist_wheel
+      - name: Install wheel and run tests
+        env:
+          PYTHON: "py -${{ matrix.python }}-${{ matrix.wordsize }}"
+        shell: bash
+        run: |
+          set -ex
+          # We install in this odd way to make sure we get both deps and a local kastore
+          ${PYTHON} -m pip install kastore --only-binary kastore -f python/dist/
+          ${PYTHON} -m pip uninstall -y kastore
+          ${PYTHON} -m pip install -v kastore --only-binary kastore -f python/dist/ --no-index
+          ${PYTHON} -c "import kastore"
+          ${PYTHON} -m pip install -r python/requirements/CI/tests/requirements.txt
+          rm -rf python/kastore python/*.pyd
+          ${PYTHON} -m nose -v python
+      - name: Upload Wheels
+        uses: actions/upload-artifact@v2
+        with:
+          name: win-wheel-${{ matrix.python }}-${{ matrix.wordsize }}
+          path: python/dist
+
+  OSX:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        python: [3.6, 3.7, 3.8]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install deps
+        run: |
+          pip install --upgrade pip
+          pip install setuptools wheel
+      - name: Build C extension
+        run: |
+          cd python
+          python -VV
+          python setup.py build_ext --inplace
+      - name: Build Wheel
+        run: |
+          cd python
+          python setup.py bdist_wheel
+      - name: Install wheel and run tests
+        run: |
+          # We install in this odd way to make sure we get both deps and a local kastore
+          pip install kastore --only-binary kastore -f python/dist/
+          pip uninstall -y kastore
+          pip install -v kastore --only-binary kastore -f python/dist/ --no-index
+          python -c "import kastore"
+          pip install -r python/requirements/CI/tests/requirements.txt
+          rm -rf python/kastore python/*.so
+          python -m nose -v python
+      - name: Upload Wheels
+        uses: actions/upload-artifact@v2
+        with:
+          name: osx-wheel-${{ matrix.python }}
+          path: python/dist
+
+  manylinux:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Build sdist
+        shell: bash
+        run: |
+          cd python
+          python setup.py sdist
+
+      - name: Upload sdist
+        uses: actions/upload-artifact@v2
+        with:
+          name: sdist
+          path: python/dist
+
+      - name: Build wheels in docker
+        shell: bash
+        run: |
+          docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Run tests (3.8)
+        run: |
+          python -VV
+          # We install in this odd way to make sure we get both deps and a local kastore
+          pip install kastore --only-binary kastore -f python/dist/wheelhouse
+          pip uninstall -y kastore
+          pip install -v kastore --only-binary kastore -f python/dist/wheelhouse --no-index
+          python -c "import kastore"
+          pip install -r python/requirements/CI/tests/requirements.txt
+          python -m nose -v python
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Run tests (3.7)
+        run: |
+          python -VV
+          # We install in this odd way to make sure we get both deps and a local kastore
+          pip install kastore --only-binary kastore -f python/dist/wheelhouse
+          pip uninstall -y kastore
+          pip install -v kastore --only-binary kastore -f python/dist/wheelhouse --no-index
+          python -c "import kastore"
+          pip install -r python/requirements/CI/tests/requirements.txt
+          python -m nose -v python
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+
+      - name: Run tests (3.6)
+        run: |
+          python -VV
+          # We install in this odd way to make sure we get both deps and a local kastore
+          pip install kastore --only-binary kastore -f python/dist/wheelhouse
+          pip uninstall -y kastore
+          pip install -v kastore --only-binary kastore -f python/dist/wheelhouse --no-index
+          python -c "import kastore"
+          pip install -r python/requirements/CI/tests/requirements.txt
+          python -m nose -v python
+
+      - name: Upload Wheels
+        uses: actions/upload-artifact@v2
+        with:
+          name: linux-wheels
+          path: python/dist/wheelhouse
+
+
+  PyPI_Upload:
+    runs-on: ubuntu-latest
+    needs: ['windows', 'OSX', 'manylinux']
+    steps:
+      - name: Download all
+        uses: actions/download-artifact@v2
+      - name: Move to dist
+        run: |
+          mkdir dist
+          cp */*.{whl,gz} dist/.
+      - name: Publish distribution to Test PyPI
+        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && !contains(github.event.ref, 'C_')
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.test_pypi_password }}
+          repository_url: https://test.pypi.org/legacy/
+      - name: Publish distribution to PRODUCTION PyPI
+        if: github.event_name == 'release' && !startsWith(github.event.release.tag_name, 'C_')
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.test_pypi_password }}
diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$DOCKER_DIR/shared.env"
+
+set -e -x
+
+ARCH=`uname -p`
+echo "arch=$ARCH"
+cd python
+
+for V in "${PYTHON_VERSIONS[@]}"; do
+    PYBIN=/opt/python/$V/bin
+    rm -rf build/       # Avoid lib build by narrow Python is used by wide python
+    $PYBIN/python setup.py build_ext --inplace
+    $PYBIN/python setup.py bdist_wheel
+done
+
+cd dist
+for whl in *.whl; do
+    auditwheel repair "$whl"
+    rm "$whl"
+done
diff --git a/docker/shared.env b/docker/shared.env
@@ -0,0 +1,5 @@
+PYTHON_VERSIONS=(
+    cp38-cp38
+    cp37-cp37m
+    cp36-cp36m
+)
diff --git a/docs/development.rst b/docs/development.rst
@@ -28,9 +28,17 @@ Python
 To make a release first prepare a pull request that sets the correct version
 number in ``kastore/_version.py`` and updates the Python CHANGELOG.rst,
 ensuring that all significant changes since the last release have been listed.
-Once this PR is merged, create a release on GitHub with the pattern
-``py_MAJOR.MINOR.PATCH``. Create the distribution artefacts for Python and
-upload to PyPI.
+Once this PR is merged, push a tag to github following PEP440 format::
+
+    git tag -a MAJOR.MINOR.PATCH -m "Python version MAJOR.MINOR.PATCH"
+    git push upstream --tags
+
+This will trigger a build of the distribution artifacts for Python
+on `Github Actions <https://github.com/tskit-dev/kastore/actions>`_. and deploy
+them to the `test PyPI <https://test.pypi.org/project/kastore/>`_. Check
+the release looks good there, then create a release on Github based on the tag you
+pushed. Publishing this release will cause the github action to deploy to the
+`production PyPI <https://pypi.org/project/kastore/>`_.
 
 -----
 C API
@@ -44,7 +52,7 @@ When modifying the C code this will conform it to the project style::
 If the C API has been updated, the ``KAS_VERSION_*`` macros should be set
 appropriately, ensuring that the Changelog has been updated to record the
 changes. After the commit including these changes has been merged, tag a
-release on GitHub using the pattern ``c_MAJOR.MINOR.PATCH``.
+release on GitHub using the pattern ``C_MAJOR.MINOR.PATCH``.
 
 
 -----------------

diff --git a/python/requirements/CI/tests/requirements.txt b/python/requirements/CI/tests/requirements.txt
@@ -0,0 +1,6 @@
+attrs
+hypothesis
+humanize
+nose
+numpy
+mock
diff --git a/python/setup.py b/python/setup.py
@@ -70,7 +70,7 @@ def finalize_options(self):
         "Bug Reports": "https://github.com/tskit-dev/kastore/issues",
         "Source": "https://github.com/tskit-dev/kastore",
     },
-    setup_requires=[numpy_ver],
+    setup_requires=["oldest-supported-numpy"],
     cmdclass={"build_ext": local_build_ext},
     license="MIT",
     platforms=["POSIX", "Windows", "MacOS X"],