From 81e7a38514e66cc8e6cea17590379a278fd3dad5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 25 Jul 2023 12:12:02 -0700 Subject: [PATCH] Switch to new CI wheel building pipeline (#1305) Moves the wheel build and test logic out of the workflow into the repo. This matches conda tests more closely and allows each repo to manage its own wheels more easily. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Divye Gala (https://github.com/divyegala) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/rmm/pull/1305 --- .github/workflows/build.yaml | 8 +++----- .github/workflows/pr.yaml | 12 ++++-------- .github/workflows/test.yaml | 5 ++--- ci/build_wheel.sh | 26 ++++++++++++++++++++++++++ ci/test_wheel.sh | 17 +++++++++++++++++ 5 files changed, 52 insertions(+), 16 deletions(-) create mode 100755 ci/build_wheel.sh create mode 100755 ci/test_wheel.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 27e360bba..67af4dafc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,19 +68,17 @@ jobs: run_script: "ci/build_docs.sh" wheel-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: rmm - package-dir: python - skbuild-configure-options: "-DRMM_BUILD_WHEELS=ON" + script: ci/build_wheel.sh wheel-publish: needs: wheel-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 59cce5312..f5b7e3bf5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -64,18 +64,14 @@ jobs: wheel-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: pull-request - package-dir: python - package-name: rmm - skbuild-configure-options: "-DRMM_BUILD_WHEELS=ON" + script: ci/build_wheel.sh wheel-tests: needs: wheel-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: pull-request - package-name: rmm - test-unittest: "python -m pytest ./python/rmm/tests" - test-smoketest: "python ./ci/wheel_smoke_test.py" + script: ci/test_wheel.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0c22c5daf..544147d98 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,11 +32,10 @@ jobs: sha: ${{ inputs.sha }} wheel-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: rmm - test-unittest: "python -m pytest ./python/rmm/tests" + script: ci/test_wheel.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 000000000..09c1e104e --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-configure-sccache +source rapids-date-string + +# Use gha-tools rapids-pip-wheel-version to generate wheel version then +# update the necessary files +version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" +echo "The package name and/or version was modified in the package source. The git diff is:" +git diff + +cd python + +SKBUILD_CONFIGURE_OPTIONS="-DRMM_BUILD_WHEELS=ON" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100755 index 000000000..4b03022f8 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eou pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/rmm*.whl)[test] + +# Run smoke tests for aarch64 pull requests +if [ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]; then + python ./ci/wheel_smoke_test.py +else + python -m pytest ./python/rmm/tests +fi