introduce cibuildwheel to build workflow (#295)

* use var for cargo target-triple * add manylinux container actions * update workflow for linux arm64 build (w/o pgo) * enable workflow test * fix gha path * introduce cibuildwheel * remove gha and scripts * fix enable option name * chmod +x to gen-profile script * comment out test trigar
WorksApplications · Jan 10, 2025 · b699ec3 · b699ec3
1 parent 7d44a41
commit b699ec3
Show file tree

Hide file tree

Showing 5 changed files with 148 additions and 246 deletions.
diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml
@@ -6,111 +6,73 @@ on:
     tags: [ "v*" ]
 # following two lines must be commented out before merging, we don't want to run this on PRs
 # but for testing this we run this on PRs
-#  pull_request:
-#    branches: [ develop ]
+# pull_request:
+#   branches: [ develop ]
 
 jobs:
-  build-sdist:
-    name: Build source distribution
-    runs-on: ubuntu-latest
+  build-wheels:
+    name: build wheels using cibuildwheel
+    runs-on: ${{ format('{0}-latest', matrix.os) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+            - os: "ubuntu"
+              platform: "linux"
+              arch: "x86_64"
+            - os: "macos"
+              arch: "universal2"
+            - os: "windows"
+              arch: "AMD64"
+    env:
+      build-sdist: ${{ matrix.os == 'ubuntu' && matrix.arch == 'x86_64' }}
+
     steps:
       - uses: actions/checkout@v4
-
-      - name: Setup python
+      - name: setup python for sdist and cibuildwheel
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade setuptools setuptools-rust build
-
-      - name: Build sdist
-        working-directory: ./python
-        run: bash build-sdist.sh
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: artifact-sdist
-          path: python/dist/*
-
-  build-linux-wheels:
-    name: Build Linux Python Wheels (+PGO)
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Download dictionary
+            python -m pip install --upgrade pip
+            python -m pip install --upgrade setuptools setuptools-rust build cibuildwheel
+      - name: download dictionary for PGO
+        if: ${{ matrix.os == 'ubuntu' }}
         run: bash fetch_dictionary.sh "20220519" "core"
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-      - uses: eiennohito/gha-manylinux-build@master
-        with:
-          script: python/build-wheels-manylinux-pgo.sh
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: artifact-manylinux
-          path: python/dist/*manylinux*
-
-  build-non-linux-wheels:
-    name: Build wheels on ${{ matrix.os }} / ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, macOS-latest]
-        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: Quansight-Labs/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
+      - name: Add aarch64/x86 target for macos
+        if: ${{ matrix.os == 'macos' }}
+        run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
       - uses: actions/cache@v4
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Install dependencies
-        run: |
-          python -m pip install -U pip
-          python -m pip install -U setuptools setuptools_rust build
-
-      - name: Add aarch64/x86 target for Rust
-        run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
-        if: startsWith(matrix.os, 'macOS')
-
-      - name: Build wheel
+            path: |
+                ~/.cargo/bin/
+                ~/.cargo/registry/index/
+                ~/.cargo/registry/cache/
+                ~/.cargo/git/db/
+                target/
+            key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: build sdist
+        if: ${{ env.build-sdist == 'true' }}
         working-directory: ./python
-        run: python -m build --wheel
+        run: bash build-sdist.sh
+      - name: cibuildwheel ${{ matrix.platform || matrix.os }}
         env:
-          ARCHFLAGS: -arch x86_64 -arch arm64
-          MACOSX_DEPLOYMENT_TARGET: 10.12
-
+            # most configuration are in python/pyproject.toml
+            CIBW_PLATFORM: ${{ matrix.platform || matrix.os }}
+            CIBW_ARCHS: ${{ matrix.arch }}
+        run: cibuildwheel python/ --output-dir python/dist/
+
       - uses: actions/upload-artifact@v4
         with:
-          name: artifact-${{ matrix.os }}-${{ matrix.python-version }}
-          path: python/dist/*.whl
+          name: artifact-${{ matrix.os }}
+          path: |
+            ./python/dist/*.whl
+            ./python/dist/*.tar.gz
 
   upload-to-pypi: # run only if all have succeeded
-    needs: [ build-sdist, build-non-linux-wheels, build-linux-wheels ]
+    needs: [ build-wheels ]
     # https://stackoverflow.com/questions/58475748/github-actions-how-to-check-if-current-push-has-new-tag-is-new-release
     if: startsWith(github.ref, 'refs/tags/v') # only for tags
     runs-on: ubuntu-latest
@@ -120,10 +82,8 @@ jobs:
           pattern: artifact-*
           path: dist/
           merge-multiple: true
-
       - name: List files to upload
         run: ls -R dist/
-
       - name: Publish distribution to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

diff --git a/.github/workflows/python-upload-test.yml b/.github/workflows/python-upload-test.yml
@@ -5,137 +5,80 @@ on:
     branches: [ pre/v* ]
 
 jobs:
-  build-sdist:
-    name: Build source distribution
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade setuptools setuptools-rust build packaging
-
-      - name: Modify version for TestPyPI upload
-        run: python ./python/modify_version_for_testpypi.py
-
-      - name: Build sdist
-        working-directory: ./python
-        run: bash build-sdist.sh
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: artifact-sdist
-          path: python/dist/*
+  build-wheels:
+    name: build wheels using cibuildwheel
+    runs-on: ${{ format('{0}-latest', matrix.os) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+            - os: "ubuntu"
+              platform: "linux"
+              arch: "x86_64"
+            - os: "macos"
+              arch: "universal2"
+            - os: "windows"
+              arch: "AMD64"
+    env:
+      build-sdist: ${{ matrix.os == 'ubuntu' && matrix.arch == 'x86_64' }}
 
-  build-linux-wheels:
-    name: Build Linux Python Wheels (+PGO)
-    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      - name: Download dictionary
-        run: bash fetch_dictionary.sh "20220519" "core"
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Setup python
+      - name: setup python for sdist and cibuildwheel
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-
       - name: Install dependencies
         run: |
-          python -m pip install -U pip
-          python -m pip install -U packaging
-
-      - name: Modify version for TestPyPI upload
-        run: python ./python/modify_version_for_testpypi.py
-
-      - uses: eiennohito/gha-manylinux-build@master
-        with:
-          script: python/build-wheels-manylinux-pgo.sh
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: artifact-manylinux
-          path: python/dist/*manylinux*
-
-  build-non-linux-wheels:
-    name: Build wheels on ${{ matrix.os }} / ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, macOS-latest]
-        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: Quansight-Labs/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
+            python -m pip install --upgrade pip
+            python -m pip install --upgrade setuptools setuptools-rust build cibuildwheel packaging
+      - name: download dictionary for PGO
+        if: ${{ matrix.os == 'ubuntu' }}
+        run: bash fetch_dictionary.sh "20220519" "core"
+      - name: Add aarch64/x86 target for macos
+        if: ${{ matrix.os == 'macos' }}
+        run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
       - uses: actions/cache@v4
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Install dependencies
-        run: |
-          python -m pip install -U pip
-          python -m pip install -U setuptools setuptools_rust build packaging
+            path: |
+                ~/.cargo/bin/
+                ~/.cargo/registry/index/
+                ~/.cargo/registry/cache/
+                ~/.cargo/git/db/
+                target/
+            key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
       - name: Modify version for TestPyPI upload
         run: python ./python/modify_version_for_testpypi.py
-
-      - name: Add aarch64/x86 target for Rust
-        run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
-        if: startsWith(matrix.os, 'macOS')
-
-      - name: Build wheel
+      - name: build sdist
+        if: ${{ env.build-sdist == 'true' }}
         working-directory: ./python
-        run: python -m build --wheel
+        run: bash build-sdist.sh
+      - name: cibuildwheel ${{ matrix.platform || matrix.os }}
         env:
-          ARCHFLAGS: -arch x86_64 -arch arm64
-          MACOSX_DEPLOYMENT_TARGET: 10.12
-
+            # most configuration are in python/pyproject.toml
+            CIBW_PLATFORM: ${{ matrix.platform || matrix.os }}
+            CIBW_ARCHS: ${{ matrix.arch }}
+        run: cibuildwheel python/ --output-dir python/dist/
+
       - uses: actions/upload-artifact@v4
         with:
-          name: artifact-${{ matrix.os }}-${{ matrix.python-version }}
-          path: python/dist/*.whl
+          name: artifact-${{ matrix.os }}
+          path: |
+            ./python/dist/*.whl
+            ./python/dist/*.tar.gz
 
   upload-to-testpypi: # run only if all have succeeded
-    needs: [ build-sdist, build-non-linux-wheels, build-linux-wheels ]
+    needs: [ build-wheels ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v4
         with:
           pattern: artifact-*
           path: dist/
           merge-multiple: true
-
       - name: List files to upload
         run: ls -R dist/
-
       - name: Publish distribution to Test PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
@@ -152,34 +95,27 @@ jobs:
         python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ]
       fail-fast: false
     runs-on: ${{ matrix.os }}
-
     steps:
       - uses: actions/checkout@v4
       - uses: Quansight-Labs/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-
       - name: Upgrade pip
         run: python -m pip install --upgrade pip
-
       - name: Show compatible tags
         run: python -m pip debug --verbose
-
       - name: Install our module from TestPyPi
         run: python -m pip -vvvv install --pre -U -i https://test.pypi.org/simple/ SudachiPy
-
       - name: Install dependencies
         run: python -m pip install sudachidict_core tokenizers
 
       - name: Run test
         working-directory: ./python
         run: python -m unittest
-
       - name: Check that binary works (C mode)
         run: |
           sudachipy .github/data/input.txt -o result-c.txt
           git diff --color=always --no-index -- result-c.txt .github/data/expected-c.txt
-
       - name: Check that binary works (A mode)
         run: |
           sudachipy .github/data/input.txt -m A -o result-a.txt