diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 8a5ef6d1..65a5a168 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -6,111 +6,73 @@ on: tags: [ "v*" ] # following two lines must be commented out before merging, we don't want to run this on PRs # but for testing this we run this on PRs -# pull_request: -# branches: [ develop ] +# pull_request: +# branches: [ develop ] jobs: - build-sdist: - name: Build source distribution - runs-on: ubuntu-latest + build-wheels: + name: build wheels using cibuildwheel + runs-on: ${{ format('{0}-latest', matrix.os) }} + strategy: + fail-fast: false + matrix: + include: + - os: "ubuntu" + platform: "linux" + arch: "x86_64" + - os: "macos" + arch: "universal2" + - os: "windows" + arch: "AMD64" + env: + build-sdist: ${{ matrix.os == 'ubuntu' && matrix.arch == 'x86_64' }} + steps: - uses: actions/checkout@v4 - - - name: Setup python + - name: setup python for sdist and cibuildwheel uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools setuptools-rust build - - - name: Build sdist - working-directory: ./python - run: bash build-sdist.sh - - - uses: actions/upload-artifact@v4 - with: - name: artifact-sdist - path: python/dist/* - - build-linux-wheels: - name: Build Linux Python Wheels (+PGO) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Download dictionary + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools setuptools-rust build cibuildwheel + - name: download dictionary for PGO + if: ${{ matrix.os == 'ubuntu' }} run: bash fetch_dictionary.sh "20220519" "core" - - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - uses: eiennohito/gha-manylinux-build@master - with: - script: python/build-wheels-manylinux-pgo.sh - - - uses: actions/upload-artifact@v4 - with: - name: artifact-manylinux - path: python/dist/*manylinux* - - build-non-linux-wheels: - name: Build wheels on ${{ matrix.os }} / ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, macOS-latest] - python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ] - - steps: - - uses: actions/checkout@v4 - - - uses: Quansight-Labs/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - + - name: Add aarch64/x86 target for macos + if: ${{ matrix.os == 'macos' }} + run: rustup target add aarch64-apple-darwin x86_64-apple-darwin - uses: actions/cache@v4 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install dependencies - run: | - python -m pip install -U pip - python -m pip install -U setuptools setuptools_rust build - - - name: Add aarch64/x86 target for Rust - run: rustup target add aarch64-apple-darwin x86_64-apple-darwin - if: startsWith(matrix.os, 'macOS') - - - name: Build wheel + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: build sdist + if: ${{ env.build-sdist == 'true' }} working-directory: ./python - run: python -m build --wheel + run: bash build-sdist.sh + - name: cibuildwheel ${{ matrix.platform || matrix.os }} env: - ARCHFLAGS: -arch x86_64 -arch arm64 - MACOSX_DEPLOYMENT_TARGET: 10.12 - + # most configuration are in python/pyproject.toml + CIBW_PLATFORM: ${{ matrix.platform || matrix.os }} + CIBW_ARCHS: ${{ matrix.arch }} + run: cibuildwheel python/ --output-dir python/dist/ + - uses: actions/upload-artifact@v4 with: - name: artifact-${{ matrix.os }}-${{ matrix.python-version }} - path: python/dist/*.whl + name: artifact-${{ matrix.os }} + path: | + ./python/dist/*.whl + ./python/dist/*.tar.gz upload-to-pypi: # run only if all have succeeded - needs: [ build-sdist, build-non-linux-wheels, build-linux-wheels ] + needs: [ build-wheels ] # https://stackoverflow.com/questions/58475748/github-actions-how-to-check-if-current-push-has-new-tag-is-new-release if: startsWith(github.ref, 'refs/tags/v') # only for tags runs-on: ubuntu-latest @@ -120,10 +82,8 @@ jobs: pattern: artifact-* path: dist/ merge-multiple: true - - name: List files to upload run: ls -R dist/ - - name: Publish distribution to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/python-upload-test.yml b/.github/workflows/python-upload-test.yml index 33baed77..ba59cc53 100644 --- a/.github/workflows/python-upload-test.yml +++ b/.github/workflows/python-upload-test.yml @@ -5,126 +5,71 @@ on: branches: [ pre/v* ] jobs: - build-sdist: - name: Build source distribution - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools setuptools-rust build packaging - - - name: Modify version for TestPyPI upload - run: python ./python/modify_version_for_testpypi.py - - - name: Build sdist - working-directory: ./python - run: bash build-sdist.sh - - - uses: actions/upload-artifact@v4 - with: - name: artifact-sdist - path: python/dist/* + build-wheels: + name: build wheels using cibuildwheel + runs-on: ${{ format('{0}-latest', matrix.os) }} + strategy: + fail-fast: false + matrix: + include: + - os: "ubuntu" + platform: "linux" + arch: "x86_64" + - os: "macos" + arch: "universal2" + - os: "windows" + arch: "AMD64" + env: + build-sdist: ${{ matrix.os == 'ubuntu' && matrix.arch == 'x86_64' }} - build-linux-wheels: - name: Build Linux Python Wheels (+PGO) - runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - - name: Download dictionary - run: bash fetch_dictionary.sh "20220519" "core" - - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Setup python + - name: setup python for sdist and cibuildwheel uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Install dependencies run: | - python -m pip install -U pip - python -m pip install -U packaging - - - name: Modify version for TestPyPI upload - run: python ./python/modify_version_for_testpypi.py - - - uses: eiennohito/gha-manylinux-build@master - with: - script: python/build-wheels-manylinux-pgo.sh - - - uses: actions/upload-artifact@v4 - with: - name: artifact-manylinux - path: python/dist/*manylinux* - - build-non-linux-wheels: - name: Build wheels on ${{ matrix.os }} / ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, macOS-latest] - python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ] - - steps: - - uses: actions/checkout@v4 - - - uses: Quansight-Labs/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools setuptools-rust build cibuildwheel packaging + - name: download dictionary for PGO + if: ${{ matrix.os == 'ubuntu' }} + run: bash fetch_dictionary.sh "20220519" "core" + - name: Add aarch64/x86 target for macos + if: ${{ matrix.os == 'macos' }} + run: rustup target add aarch64-apple-darwin x86_64-apple-darwin - uses: actions/cache@v4 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install dependencies - run: | - python -m pip install -U pip - python -m pip install -U setuptools setuptools_rust build packaging + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Modify version for TestPyPI upload run: python ./python/modify_version_for_testpypi.py - - - name: Add aarch64/x86 target for Rust - run: rustup target add aarch64-apple-darwin x86_64-apple-darwin - if: startsWith(matrix.os, 'macOS') - - - name: Build wheel + - name: build sdist + if: ${{ env.build-sdist == 'true' }} working-directory: ./python - run: python -m build --wheel + run: bash build-sdist.sh + - name: cibuildwheel ${{ matrix.platform || matrix.os }} env: - ARCHFLAGS: -arch x86_64 -arch arm64 - MACOSX_DEPLOYMENT_TARGET: 10.12 - + # most configuration are in python/pyproject.toml + CIBW_PLATFORM: ${{ matrix.platform || matrix.os }} + CIBW_ARCHS: ${{ matrix.arch }} + run: cibuildwheel python/ --output-dir python/dist/ + - uses: actions/upload-artifact@v4 with: - name: artifact-${{ matrix.os }}-${{ matrix.python-version }} - path: python/dist/*.whl + name: artifact-${{ matrix.os }} + path: | + ./python/dist/*.whl + ./python/dist/*.tar.gz upload-to-testpypi: # run only if all have succeeded - needs: [ build-sdist, build-non-linux-wheels, build-linux-wheels ] + needs: [ build-wheels ] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v4 @@ -132,10 +77,8 @@ jobs: pattern: artifact-* path: dist/ merge-multiple: true - - name: List files to upload run: ls -R dist/ - - name: Publish distribution to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: @@ -152,34 +95,27 @@ jobs: python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ] fail-fast: false runs-on: ${{ matrix.os }} - steps: - uses: actions/checkout@v4 - uses: Quansight-Labs/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Upgrade pip run: python -m pip install --upgrade pip - - name: Show compatible tags run: python -m pip debug --verbose - - name: Install our module from TestPyPi run: python -m pip -vvvv install --pre -U -i https://test.pypi.org/simple/ SudachiPy - - name: Install dependencies run: python -m pip install sudachidict_core tokenizers - name: Run test working-directory: ./python run: python -m unittest - - name: Check that binary works (C mode) run: | sudachipy .github/data/input.txt -o result-c.txt git diff --color=always --no-index -- result-c.txt .github/data/expected-c.txt - - name: Check that binary works (A mode) run: | sudachipy .github/data/input.txt -m A -o result-a.txt diff --git a/python/build-wheels-manylinux-pgo.sh b/python/build-wheels-manylinux-pgo.sh deleted file mode 100644 index 7584e51b..00000000 --- a/python/build-wheels-manylinux-pgo.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -ex - -# This script is assumed to be used inside https://github.com/pypa/manylinux. - -DIR=$(dirname "$(readlink -f "$0")") - -curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --no-modify-path --component llvm-tools-preview -export PATH="$HOME/.cargo/bin:$PATH" - -cd "$DIR/.." - -PROFDATA=/tmp/sudachi-profdata - -# Compile Binary that will generate PGO data -RUSTFLAGS="-C profile-generate=$PROFDATA -C opt-level=3" \ - cargo build --release -p sudachi-cli --target=x86_64-unknown-linux-gnu - -# Download Kyoto Leads corpus original texts -curl -L https://github.com/ku-nlp/KWDLC/releases/download/release_1_0/leads.org.txt.gz | gzip -dc > leads.txt - -# Generate Profile -target/x86_64-unknown-linux-gnu/release/sudachi -o /dev/null leads.txt -target/x86_64-unknown-linux-gnu/release/sudachi --wakati --mode=A -o /dev/null leads.txt -target/x86_64-unknown-linux-gnu/release/sudachi --all --mode=B -o /dev/null leads.txt - -# Generate Merged PGO data -"$HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata" \ - merge -o /tmp/sudachi-profdata.merged "$PROFDATA" - -cd "$DIR" - -export RUSTFLAGS='-C profile-use=/tmp/sudachi-profdata.merged -C opt-level=3' -export CARGO_BUILD_TARGET=x86_64-unknown-linux-gnu - -# see following link for the list of cpython bin -# https://github.com/pypa/manylinux?tab=readme-ov-file#image-content -for PYBIN in /opt/python/cp{37,38,39,310,311,312,313}-*/bin; do - "${PYBIN}/pip" install -U setuptools wheel setuptools-rust - find . -iname 'sudachipy*.so' - rm -f build/lib/sudachipy/sudachipy*.so - "${PYBIN}/pip" wheel . --no-build-isolation -vvv --wheel-dir ./dist -done - -for whl in dist/*.whl; do - auditwheel repair "$whl" -w dist/ -done diff --git a/python/generate-manylinux-profile.sh b/python/generate-manylinux-profile.sh new file mode 100755 index 00000000..41e8cefa --- /dev/null +++ b/python/generate-manylinux-profile.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -ex + +# Generate profile for PGO. +# The build process need to use generated profile data. + +DIR=$(dirname "$(readlink -f "$0")") +cd "$DIR/.." + +PROFDATA=/tmp/sudachi-profdata +MERGED_PROFDATA=${1} + +# Install rust +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --no-modify-path --component llvm-tools +## target-triple of the default toolchain +TARGET_TRIPLE=$(rustc -vV | awk '/^host/ {print $2}') + +# Compile Binary that will generate PGO data +RUSTFLAGS="-C profile-generate=$PROFDATA -C opt-level=3" \ + cargo build --release -p sudachi-cli + +# Download Kyoto Leads corpus original texts +curl -L https://github.com/ku-nlp/KWDLC/releases/download/release_1_0/leads.org.txt.gz | gzip -dc > leads.txt + +# Generate Profile +target/release/sudachi -o /dev/null leads.txt +target/release/sudachi --wakati --mode=A -o /dev/null leads.txt +target/release/sudachi --all --mode=B -o /dev/null leads.txt + +# Generate Merged PGO data +"$HOME/.rustup/toolchains/stable-$TARGET_TRIPLE/lib/rustlib/$TARGET_TRIPLE/bin/llvm-profdata" \ + merge -o "$MERGED_PROFDATA" "$PROFDATA" diff --git a/python/pyproject.toml b/python/pyproject.toml index 31ffe048..26f86b92 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,2 +1,23 @@ [build-system] requires = ["setuptools", "wheel", "setuptools-rust"] + +[tool.cibuildwheel] +build = "cp39-* cp310-* cp311-* cp312-* cp313-* cp313t-*" +skip = "*t-win* *-win32 *-musllinux_*" +enable = ["cpython-freethreading"] + +manylinux-x86_64-image = "manylinux2014" +manylinux-aarch64-image = "manylinux2014" + +build-verbosity = 1 +before-build = "rustup show" + +[tool.cibuildwheel.linux] +environment = { PATH = "$HOME/.cargo/bin:$PATH", CARGO_TERM_COLOR = "always", RUSTFLAGS = "-C profile-use=/tmp/sudachi-profdata.merged -C opt-level=3" } +before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y && python/generate-manylinux-profile.sh /tmp/sudachi-profdata.merged" + +[tool.cibuildwheel.macos] +environment = { PATH = "$HOME/.cargo/bin:$PATH", CARGO_TERM_COLOR = "always", MACOSX_DEPLOYMENT_TARGET = 10.12 } + +[tool.cibuildwheel.windows] +environment = { PATH = "$UserProfile\\.cargo\\bin;$PATH" }