From c9791a475f4eb8bbb34fbb53f6d01b0596f484c9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 26 Apr 2024 00:39:54 +0000 Subject: [PATCH 1/3] Enable testing of Python limited API builds --- .github/workflows/build-all-rapids-repos.yml | 14 ++++++++++++- .../opt/rapids-build-utils/manifest.yaml | 21 ++++++++++--------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 1d6a0fae..823edd79 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -43,7 +43,19 @@ jobs: build_command: | sccache -z; sccache --show-adv-stats; - clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules; + # Need Python 3.11 for the buffer protocol in the limited API + export PYTHON_VERSION="3.11" + clone-all -j$(nproc) -v -q --clone-upstream --single-branch --shallow-submodules --no-update-env; + rapids-generate-scripts + . rapids-make-${PYTHON_PACKAGE_MANAGER}-env + # The latest version of this branch includes both my patches (required + # for RAPIDS Cython usage to work) and upstream patches required for + # Cython itself to work under the limited API. + export NO_CYTHON_COMPILE=true + python -m pip install git+https://github.com/cython/cython.git@limited-api-preview + # Need to install my branch of scikit-build-core for the way I'm specifying CMAKE_CXX_FLAGS via CMAKE_ARGS to work + python -m pip install git+https://github.com/vyasr/scikit-build-core.git@fix/cmake_args_quoted_spaces + build-all \ -v \ -j$(nproc --ignore=1) \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index a93ba68f..ea2e2fd3 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -24,7 +24,7 @@ repos: - name: rmm sub_dir: python/rmm depends: [rmm] - args: {install: *rapids_build_backend_args} + args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"} - name: ucxx path: ucxx @@ -82,11 +82,11 @@ repos: - name: pylibcudf sub_dir: python/pylibcudf depends: [cudf] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cudf sub_dir: python/cudf depends: [cudf] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: dask_cudf sub_dir: python/dask_cudf args: {install: *rapids_build_backend_args} @@ -115,11 +115,11 @@ repos: - name: pylibraft sub_dir: python/pylibraft depends: [raft] - args: {cmake: -DFIND_RAFT_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_RAFT_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: raft-dask sub_dir: python/raft-dask depends: [ucxx, raft] - args: {cmake: -DFIND_RAFT_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_RAFT_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cuvs path: cuvs @@ -135,7 +135,7 @@ repos: - name: cuvs sub_dir: python/cuvs depends: [cuvs] - args: {cmake: -DFIND_CUVS_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_CUVS_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cumlprims_mg path: cumlprims_mg @@ -158,7 +158,7 @@ repos: - name: cuml sub_dir: python/cuml depends: [cuml] - args: {cmake: -DFIND_CUML_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_CUML_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cugraph-ops path: cugraph-ops @@ -220,11 +220,11 @@ repos: - name: pylibcugraph sub_dir: python/pylibcugraph depends: [cugraph] - args: {cmake: -DFIND_CUGRAPH_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_CUGRAPH_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: cugraph sub_dir: python/cugraph depends: [cugraph] - args: {cmake: -DFIND_CUGRAPH_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_CUGRAPH_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: nx-cugraph sub_dir: python/nx-cugraph args: {install: *rapids_build_backend_args} @@ -263,6 +263,7 @@ repos: # it doesn't have a build dir of its own. That means we can't list it as # a cpp repo after cuspatial (above), and we have to manually point CMake # to the cuspatial build dir here. + # Limited API compilation of cuproje currently fails due to https://github.com/cython/cython/issues/6405 args: cmake: | -DFIND_CUPROJ_CPP=ON @@ -270,4 +271,4 @@ repos: - name: cuspatial sub_dir: python/cuspatial depends: [cuspatial] - args: {install: *rapids_build_backend_args} + args: {cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} From 1e8b74dc516127afc24da2e10ef5acad193aea9c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Oct 2024 00:57:56 +0000 Subject: [PATCH 2/3] Stop cloning merged and deleted SKBC branch --- .github/workflows/build-all-rapids-repos.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 823edd79..d722afa2 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -53,8 +53,6 @@ jobs: # Cython itself to work under the limited API. export NO_CYTHON_COMPILE=true python -m pip install git+https://github.com/cython/cython.git@limited-api-preview - # Need to install my branch of scikit-build-core for the way I'm specifying CMAKE_CXX_FLAGS via CMAKE_ARGS to work - python -m pip install git+https://github.com/vyasr/scikit-build-core.git@fix/cmake_args_quoted_spaces build-all \ -v \ From 9ffb7c19751d1a3c0a9cabc282ca226b0b2e0295 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 28 Oct 2024 10:29:41 -0700 Subject: [PATCH 3/3] Try building ucxx and kvikio with the limited API too --- .../rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index ea2e2fd3..775990c0 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -41,11 +41,11 @@ repos: - name: ucxx sub_dir: python/ucxx depends: [ucxx] - args: {cmake: -DFIND_UCXX_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_UCXX_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: distributed-ucxx sub_dir: python/distributed-ucxx depends: [ucxx] - args: {cmake: -DFIND_UCXX_CPP=ON, install: *rapids_build_backend_args} + args: {cmake: -DFIND_UCXX_CPP=ON -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'", install: *rapids_build_backend_args} - name: kvikio path: kvikio @@ -62,7 +62,7 @@ repos: - name: kvikio sub_dir: python/kvikio depends: [KvikIO] - args: {install: *rapids_build_backend_args} + args: {install: *rapids_build_backend_args, cmake: -DCMAKE_CXX_FLAGS="'-DPy_LIMITED_API=0x030B0000 -DCYTHON_LIMITED_API=1'"} - name: cudf path: cudf