-
Notifications
You must be signed in to change notification settings - Fork 2
239 lines (209 loc) · 9.61 KB
/
publish-rapidgzip.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
name: Publish Rapidgzip to PyPI
on:
push:
branches:
- 'oldmaster'
jobs:
Deploy-Wheels:
if: false
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-latest, windows-latest]
archs: [auto64]
# manylinux1 is not supported anymore because it won't compile C++17 code
# manylinux2010 runs into this problem https://github.com/pypa/cibuildwheel/issues/1001
# MacOS and Windows ignore the manylinux-image version because they are not Linux.
include:
- os: ubuntu-latest
manylinux-image: manylinux2014
archs: auto64
- os: ubuntu-latest
manylinux-image: manylinux2014
archs: auto32
- os: ubuntu-latest
manylinux-image: manylinux_2_28
archs: auto64
env:
# By default, both build (https://pypi.org/project/build/)
# and cibuildwheel (https://pypi.org/project/cibuildwheel/) seem to try to compile for macOS 10.9
# based on the wheel output names in the logs. However, C++17 seems to have introduced some alignment
# for classes, which results in this error for the GzipReader class itself even without "alignas" being used:
# > error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)'
# > is only available on macOS 10.14 or newer
# See also this discussion https://bugs.chromium.org/p/chromium/issues/detail?id=1274388
#
# In the end, 10.9 support should not be necessary anymore in almost 2023. Apple-support for that seems
# to have stopped 2016-12-01 (https://endoflife.date/macos) and even support for 10.14, which was introduced
# in 2019-09-24 has already stopped in 2021-10-25, so more than a year ago.
#
# Release | Release Date | End of Life
# -------------------------+---------------+-------------
# macOS 13 (Ventura) | 2022-10-24 | some time after 2022-11
# macOS 12 (Monterey) | 2021-10-25 | some time after 2022-11
# macOS 11 (Big Sur) | 2020-11-12 | some time after 2022-11
# macOS 10.15 (Catalina) | 2019-10-07 | 2022-09-12
# macOS 10.14 (Mojave) | 2018-09-24 | 2021-10-25
# macOS 10.13 (High Sierra)| 2017-09-25 | 2020-12-01
# macOS 10.12 (Sierra) | 2016-09-20 | 2019-10-01
# OS X 10.11 (El Capitan) | 2015-09-30 | 2018-12-01
# OS X 10.10 (Yosemite) | 2014-10-16 | 2017-08-01
# OS X 10.9 (Mavericks) | 2013-10-22 | 2016-12-01
MACOSX_DEPLOYMENT_TARGET: "10.14"
# The following parts should be an exact copy of the wheels-rapidgzip.yml workflow:
# ------ Begin Copy-Paste of wheels-rapidgzip.yml ------
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install Dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install --upgrade-strategy eager --upgrade cython twine cibuildwheel build requests
- name: Install NASM (Windows)
if: matrix.os == 'windows-latest'
run: |
python3 .github/workflows/install-python-on-windows.py
echo "${PWD}/nasm" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Build Tarball
working-directory: python/rapidgzip
run: python3 -m build --sdist
- name: Build Musllinux Wheels
if: matrix.os == 'ubuntu-latest'
env:
# The Musllinux containers are based on Alpine and therefore
# "apk add" needs to be used instead of "yum install".
# https://cibuildwheel.readthedocs.io/en/stable/options/
CIBW_ARCHS: ${{ matrix.archs }}
CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux-image }}
CIBW_BEFORE_ALL_LINUX: "uname -a; apk add nasm"
CIBW_BUILD_VERBOSITY: 2
CIBW_BUILD: "*musllinux*"
run: python -m cibuildwheel --output-dir dist python/rapidgzip
- name: Build CPython Wheels
env:
# https://cibuildwheel.readthedocs.io/en/stable/options/
CIBW_ARCHS: ${{ matrix.archs }}
CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux-image }}
CIBW_BEFORE_ALL_LINUX: "uname -a; yum install -y nasm"
CIBW_BEFORE_ALL_MACOS: |
brew install nasm;
export PATH="$PATH:/usr/local/bin"
CIBW_BUILD_VERBOSITY: 2
CIBW_SKIP: "*musllinux*"
run: python -m cibuildwheel --output-dir dist python/rapidgzip
- name: Check Wheels
run: twine check dist/* python/rapidgzip/dist/*
- name: Test Wheel
if: matrix.os == 'ubuntu-latest' && matrix.archs == 'auto64'
run: |
PYTHON_VERSION=$( python3 --version | sed -nr 's|.* (3)[.]([0-9]+)[.].*|\1\2|p' )
python -m pip install dist/*-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}-manylinux*.whl
# Ensure that it is built with ISA-l
rapidgzip --oss-attributions | grep ISA-l
- name: Test Wheel (Windows)
if: matrix.os == 'windows-latest'
shell: bash
run: |
python -m pip install dist/*-cp310-win_amd64.whl
# Ensure that it is built with ISA-l
rapidgzip --oss-attributions | grep ISA-l
# ------ End Copy-Paste of wheels-rapidgzip.yml ------
- name: Publish Tarball to PyPI
env:
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_RAPIDGZIP }}
working-directory: python/rapidgzip
# Only one out of the OS matrix needs to upload the tarball.
if: matrix.manylinux-image == 'manylinux2014' && matrix.os == 'ubuntu-latest'
run: twine upload --skip-existing -u __token__ dist/*
- name: Publish Wheels to PyPI
env:
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_RAPIDGZIP }}
run: twine upload --skip-existing -u __token__ dist/*
Deploy-Conda:
# Not working yet and indexed_bzip2 only has a single download from conda ...
if: false
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-latest, ubuntu-latest, windows-latest]
env:
MACOSX_DEPLOYMENT_TARGET: "10.14"
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
- name: Fix macOS CommandLineTools Bug
if: matrix.os == 'macos-latest'
shell: bash
run: |
# See test.yml workflow for explanation but in short: conda build only works with the older SDK installed.
wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX10.9.sdk.tar.xz
tar -xf MacOSX10.9.sdk.tar.xz
cat <<EOF >> conda/conda_build_config.yaml
CONDA_BUILD_SYSROOT:
- $PWD/MacOSX10.9.sdk
EOF
- name: Build Conda Packages
# Windows yields: fatal error C1001: An internal error has occurred in the compiler.
# (compiler file 'msc1.cpp', line 1518)
# This happens on the line: static constexpr FixedHuffmanCoding m_fixedHC = createFixedHC();
# This is understandable because it is the most complex code but the wheels do build just fine though.
if: matrix.os != 'windows-latest'
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitely declared as shell:
# bash -l {0} on steps that need to be properly activated
shell: bash -l {0}
env:
ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
run: |
conda install conda conda-build anaconda-client conda-verify packaging
conda config --set anaconda_upload yes
cd python/rapidgzip/conda
python3 -m pip install --user build
python3 -m build -o . --sdist ..
mv rapidgzip-*.tar* src.tar
conda build --token "$ANACONDA_TOKEN" .
Deploy-AUR:
runs-on: ubuntu-latest
container:
image: archlinux
steps:
- name: Set Up Non-Root User # Because makepkg is an ass and forbids root from using it
run: |
groupadd sudo
useradd -m -G sudo user
echo '%sudo ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers
- name: Install System Dependencies
run: |
pacman -Syu --noconfirm
pacman -Sy --noconfirm base-devel git python python-pip python-setuptools rhash
- name: Clone Repository
run: |
chmod a+rwx .
chown user .
su user -c 'git clone https://aur.archlinux.org/python-rapidgzip.git .'
# For testing whether the update script works
su user -c 'git checkout HEAD~'
- name: Update PKGBUILD
run: |
# Annoyingly, --no-deps is not sufficient and building the wheels would still be triggered even
# just for downloading the source tarball! But, --no-build-isolation seems to avoid that:
# https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/7
pip download --no-binary --no-deps --no-build-isolation :all: rapidgzip
pkgver=$( echo rapidgzip-*.tar.gz | sed -E 's|rapidgzip-([0-9.]*).*[.]tar.gz|\1|' )
blake2b=$( python3 -c 'import hashlib, sys; h=hashlib.blake2b(digest_size=256 // 8); h.update(open(sys.argv[1], "rb").read()); print(h.digest().hex())' rapidgzip-*.tar.gz )
# Update PKGBUILD
sed -i "s|pkgver=.*|pkgver=$pkgver|" PKBUILD
sed -i "s|sha256sums=.*|sha256sums=($( rhash --sha256 rapidgzip-*.tar.gz ))|" PKBUILD
sed -i "s|(.*pythonhosted.org/packages)/[0-9][0-9]/[0-9][0-9]/[0-9]*/(.*)|\1/${blake2b:0:2}/${blake2b:2:2}/${blake2b:4}\2" PKGBUILD
- name: Test PKGBUILD
run: |
su user -c 'makepkg -s --noconfirm'
su user -c 'makepkg --noconfirm --install python-rapidgzip-*.tar.zst'