[wip][CI] Automatically publish to AUR #47
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish Rapidgzip to PyPI | |
| on: | |
| push: | |
| branches: | |
| - 'oldmaster' | |
| jobs: | |
| Deploy-Wheels: | |
| if: false | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [macos-latest, windows-latest] | |
| archs: [auto64] | |
| # manylinux1 is not supported anymore because it won't compile C++17 code | |
| # manylinux2010 runs into this problem https://github.com/pypa/cibuildwheel/issues/1001 | |
| # MacOS and Windows ignore the manylinux-image version because they are not Linux. | |
| include: | |
| - os: ubuntu-latest | |
| manylinux-image: manylinux2014 | |
| archs: auto64 | |
| - os: ubuntu-latest | |
| manylinux-image: manylinux2014 | |
| archs: auto32 | |
| - os: ubuntu-latest | |
| manylinux-image: manylinux_2_28 | |
| archs: auto64 | |
| env: | |
| # By default, both build (https://pypi.org/project/build/) | |
| # and cibuildwheel (https://pypi.org/project/cibuildwheel/) seem to try to compile for macOS 10.9 | |
| # based on the wheel output names in the logs. However, C++17 seems to have introduced some alignment | |
| # for classes, which results in this error for the GzipReader class itself even without "alignas" being used: | |
| # > error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' | |
| # > is only available on macOS 10.14 or newer | |
| # See also this discussion https://bugs.chromium.org/p/chromium/issues/detail?id=1274388 | |
| # | |
| # In the end, 10.9 support should not be necessary anymore in almost 2023. Apple-support for that seems | |
| # to have stopped 2016-12-01 (https://endoflife.date/macos) and even support for 10.14, which was introduced | |
| # in 2019-09-24 has already stopped in 2021-10-25, so more than a year ago. | |
| # | |
| # Release | Release Date | End of Life | |
| # -------------------------+---------------+------------- | |
| # macOS 13 (Ventura) | 2022-10-24 | some time after 2022-11 | |
| # macOS 12 (Monterey) | 2021-10-25 | some time after 2022-11 | |
| # macOS 11 (Big Sur) | 2020-11-12 | some time after 2022-11 | |
| # macOS 10.15 (Catalina) | 2019-10-07 | 2022-09-12 | |
| # macOS 10.14 (Mojave) | 2018-09-24 | 2021-10-25 | |
| # macOS 10.13 (High Sierra)| 2017-09-25 | 2020-12-01 | |
| # macOS 10.12 (Sierra) | 2016-09-20 | 2019-10-01 | |
| # OS X 10.11 (El Capitan) | 2015-09-30 | 2018-12-01 | |
| # OS X 10.10 (Yosemite) | 2014-10-16 | 2017-08-01 | |
| # OS X 10.9 (Mavericks) | 2013-10-22 | 2016-12-01 | |
| MACOSX_DEPLOYMENT_TARGET: "10.14" | |
| # The following parts should be an exact copy of the wheels-rapidgzip.yml workflow: | |
| # ------ Begin Copy-Paste of wheels-rapidgzip.yml ------ | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade-strategy eager --upgrade cython twine cibuildwheel build requests | |
| - name: Install NASM (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| python3 .github/workflows/install-python-on-windows.py | |
| echo "${PWD}/nasm" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| - name: Build Tarball | |
| working-directory: python/rapidgzip | |
| run: python3 -m build --sdist | |
| - name: Build Musllinux Wheels | |
| if: matrix.os == 'ubuntu-latest' | |
| env: | |
| # The Musllinux containers are based on Alpine and therefore | |
| # "apk add" needs to be used instead of "yum install". | |
| # https://cibuildwheel.readthedocs.io/en/stable/options/ | |
| CIBW_ARCHS: ${{ matrix.archs }} | |
| CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux-image }} | |
| CIBW_BEFORE_ALL_LINUX: "uname -a; apk add nasm" | |
| CIBW_BUILD_VERBOSITY: 2 | |
| CIBW_BUILD: "*musllinux*" | |
| run: python -m cibuildwheel --output-dir dist python/rapidgzip | |
| - name: Build CPython Wheels | |
| env: | |
| # https://cibuildwheel.readthedocs.io/en/stable/options/ | |
| CIBW_ARCHS: ${{ matrix.archs }} | |
| CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux-image }} | |
| CIBW_BEFORE_ALL_LINUX: "uname -a; yum install -y nasm" | |
| CIBW_BEFORE_ALL_MACOS: | | |
| brew install nasm; | |
| export PATH="$PATH:/usr/local/bin" | |
| CIBW_BUILD_VERBOSITY: 2 | |
| CIBW_SKIP: "*musllinux*" | |
| run: python -m cibuildwheel --output-dir dist python/rapidgzip | |
| - name: Check Wheels | |
| run: twine check dist/* python/rapidgzip/dist/* | |
| - name: Test Wheel | |
| if: matrix.os == 'ubuntu-latest' && matrix.archs == 'auto64' | |
| run: | | |
| PYTHON_VERSION=$( python3 --version | sed -nr 's|.* (3)[.]([0-9]+)[.].*|\1\2|p' ) | |
| python -m pip install dist/*-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}-manylinux*.whl | |
| # Ensure that it is built with ISA-l | |
| rapidgzip --oss-attributions | grep ISA-l | |
| - name: Test Wheel (Windows) | |
| if: matrix.os == 'windows-latest' | |
| shell: bash | |
| run: | | |
| python -m pip install dist/*-cp310-win_amd64.whl | |
| # Ensure that it is built with ISA-l | |
| rapidgzip --oss-attributions | grep ISA-l | |
| # ------ End Copy-Paste of wheels-rapidgzip.yml ------ | |
| - name: Publish Tarball to PyPI | |
| env: | |
| TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_RAPIDGZIP }} | |
| working-directory: python/rapidgzip | |
| # Only one out of the OS matrix needs to upload the tarball. | |
| if: matrix.manylinux-image == 'manylinux2014' && matrix.os == 'ubuntu-latest' | |
| run: twine upload --skip-existing -u __token__ dist/* | |
| - name: Publish Wheels to PyPI | |
| env: | |
| TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_RAPIDGZIP }} | |
| run: twine upload --skip-existing -u __token__ dist/* | |
| Deploy-Conda: | |
| # Not working yet and indexed_bzip2 only has a single download from conda ... | |
| if: false | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [macos-latest, ubuntu-latest, windows-latest] | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: "10.14" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: true | |
| - name: Fix macOS CommandLineTools Bug | |
| if: matrix.os == 'macos-latest' | |
| shell: bash | |
| run: | | |
| # See test.yml workflow for explanation but in short: conda build only works with the older SDK installed. | |
| wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX10.9.sdk.tar.xz | |
| tar -xf MacOSX10.9.sdk.tar.xz | |
| cat <<EOF >> conda/conda_build_config.yaml | |
| CONDA_BUILD_SYSROOT: | |
| - $PWD/MacOSX10.9.sdk | |
| EOF | |
| - name: Build Conda Packages | |
| # Windows yields: fatal error C1001: An internal error has occurred in the compiler. | |
| # (compiler file 'msc1.cpp', line 1518) | |
| # This happens on the line: static constexpr FixedHuffmanCoding m_fixedHC = createFixedHC(); | |
| # This is understandable because it is the most complex code but the wheels do build just fine though. | |
| if: matrix.os != 'windows-latest' | |
| # Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitely declared as shell: | |
| # bash -l {0} on steps that need to be properly activated | |
| shell: bash -l {0} | |
| env: | |
| ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} | |
| run: | | |
| conda install conda conda-build anaconda-client conda-verify packaging | |
| conda config --set anaconda_upload yes | |
| cd python/rapidgzip/conda | |
| python3 -m pip install --user build | |
| python3 -m build -o . --sdist .. | |
| mv rapidgzip-*.tar* src.tar | |
| conda build --token "$ANACONDA_TOKEN" . | |
| Deploy-AUR: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: archlinux | |
| steps: | |
| - name: Set Up Non-Root User # Because makepkg is an ass and forbids root from using it | |
| run: | | |
| groupadd sudo | |
| useradd -m -G sudo user | |
| echo '%sudo ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers | |
| - name: Install System Dependencies | |
| run: | | |
| pacman -Syu --noconfirm | |
| pacman -Sy --noconfirm base-devel git python python-pip python-setuptools rhash openssh | |
| - name: Clone Repository | |
| env: | |
| RAPIDGZIP_AUR_SSH: ${{ secrets.RAPIDGZIP_AUR_SSH }} | |
| run: | | |
| chmod a+rwx . | |
| chown user . | |
| su user -c 'mkdir -p "$HOME/.ssh/"' | |
| su user -c 'echo "$RAPIDGZIP_AUR_SSH" > "$HOME/.ssh/id_ed25519"' | |
| su user -c 'git clone ssh://[email protected]/python-rapidgzip.git .' | |
| # For testing whether the update script works | |
| su user -c 'git checkout HEAD~' | |
| - name: Update PKGBUILD | |
| run: | | |
| # Annoyingly, --no-deps is not sufficient and building the wheels would still be triggered even | |
| # just for downloading the source tarball! But, --no-build-isolation seems to avoid that: | |
| # https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/7 | |
| pip download --no-binary :all: --no-deps --no-build-isolation rapidgzip | |
| pkgver=$( echo rapidgzip-*.tar.gz | sed -E 's|rapidgzip-([0-9.]*).*[.]tar.gz|\1|' ) | |
| blake2b=$( python3 -c 'import hashlib, sys; h=hashlib.blake2b(digest_size=256 // 8); h.update(open(sys.argv[1], "rb").read()); print(h.digest().hex())' rapidgzip-*.tar.gz ) | |
| # Update PKGBUILD | |
| sed -i "s|pkgver=.*|pkgver=$pkgver|" PKGBUILD | |
| sed -i "s|sha256sums=.*|sha256sums=(\"$( rhash -p"%x{sha-256}" rapidgzip-*.tar.gz )\")|" PKGBUILD | |
| sed -i -E "s|(.*pythonhosted.org/packages)/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]*/(.*)|\1/${blake2b:0:2}/${blake2b:2:2}/${blake2b:4}/\2|" PKGBUILD | |
| su user -c 'makepkg --printsrcinfo > .SRCINFO' | |
| su user -c 'git add -f PKGBUILD .SRCINFO' | |
| su user -c 'git config --local user.email "[email protected]"' | |
| su user -c 'git config --local user.name "Maximilian Knespel"' | |
| su user -c "git commit -m 'Update to $pkgver'" | |
| su user -c 'git status' | |
| su user -c 'git show' | |
| - name: Test PKGBUILD | |
| run: | | |
| su user -c 'makepkg -s --noconfirm' | |
| su user -c 'makepkg --noconfirm --install python-rapidgzip-*.tar.zst' |