From c7f52472ff5a6b69f1b7449245917fd64eb7bda0 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Thu, 15 Aug 2024 23:52:49 +0200 Subject: [PATCH 01/10] Make several attempts to clone Distributed in case of failure (#269) CI may eventually fail due to unsuccessful attempts to clone the Distributed repository. This change ensures multiple attempts are made to clone it before failing. Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/ucxx/pull/269 --- ci/test_common.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ci/test_common.sh b/ci/test_common.sh index 1480e17f..48592e36 100755 --- a/ci/test_common.sh +++ b/ci/test_common.sh @@ -173,7 +173,23 @@ install_distributed_dev_mode() { # to run non-public API tests in CI. rapids-logger "Install Distributed in developer mode" - git clone https://github.com/dask/distributed /tmp/distributed -b 2024.1.1 + MAX_ATTEMPTS=5 + for attempt in $(seq 1 $MAX_ATTEMPTS); do + rm -rf /tmp/distributed + + if git clone https://github.com/dask/distributed /tmp/distributed -b 2024.1.1; then + break + else + + if [ $attempt -eq $MAX_ATTEMPTS ]; then + rapids-logger "Maximum number of attempts to clone Distributed failed." + exit 1 + fi + + sleep 1 + fi + done + pip install -e /tmp/distributed # `pip install -e` removes files under `distributed` but not the directory, later # causing failures to import modules. From f8e7a646e22894e3d182fd2460db4a1761ff198e Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 16 Aug 2024 16:26:45 -0400 Subject: [PATCH 02/10] Restore `*.pxd` files in ucxx wheel (#270) `*.pxd` files were mistakenly removed in https://github.com/rapidsai/ucxx/pull/260. Resume installing them in the ucxx wheel. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/ucxx/pull/270 --- python/ucxx/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ucxx/pyproject.toml b/python/ucxx/pyproject.toml index 0c07b5a5..4bf1eab2 100644 --- a/python/ucxx/pyproject.toml +++ b/python/ucxx/pyproject.toml @@ -73,7 +73,7 @@ ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["ucxx"] -wheel.exclude = ["*.pyx", "*.pxd", "CMakeLists.txt"] +wheel.exclude = ["*.pyx", "CMakeLists.txt"] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From 0e319a68d88691e015d29ac7f9c5f04f4b2da937 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 22 Aug 2024 10:18:05 +0200 Subject: [PATCH 03/10] Remove NumPy <2 pin (#271) This PR removes the NumPy<2 pin. This is desirable for RAPIDS projects that do not have a strong numpy/cupy dependency. Not doing any math, they don't run into subtle issues, but the pin was still necessary briefly before NumPy 2 was initially released. (Other RAPIDS projects rely heavily on CuPy/NumPy and need the soon to be released 13.3.0 to function properly, e.g. due to promotion related changes in NumPy.) Authors: - Sebastian Berg (https://github.com/seberg) Approvers: - Mike Sarahan (https://github.com/msarahan) - James Lamb (https://github.com/jameslamb) - https://github.com/jakirkham URL: https://github.com/rapidsai/ucxx/pull/271 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/ucxx/meta.yaml | 2 +- dependencies.yaml | 2 +- python/distributed-ucxx/pyproject.toml | 2 +- python/ucxx/pyproject.toml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 1ab7f662..7138aedc 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -26,7 +26,7 @@ dependencies: - libtool - ninja - numba>=0.57.1 -- numpy>=1.23,<2.0a0 +- numpy>=1.23,<3.0a0 - pip - pkg-config - pre-commit diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 68bf3b2b..ff57ce82 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -26,7 +26,7 @@ dependencies: - libtool - ninja - numba>=0.57.1 -- numpy>=1.23,<2.0a0 +- numpy>=1.23,<3.0a0 - pip - pkg-config - pre-commit diff --git a/conda/recipes/ucxx/meta.yaml b/conda/recipes/ucxx/meta.yaml index dafb4004..8d9e3f13 100644 --- a/conda/recipes/ucxx/meta.yaml +++ b/conda/recipes/ucxx/meta.yaml @@ -239,7 +239,7 @@ outputs: - ucx >=1.15.0,<1.18.0 - {{ pin_subpackage('libucxx', exact=True) }} - {{ pin_compatible('rmm', max_pin='x.x') }} - - numpy>=1.23,<2.0a0 + - numpy>=1.23,<3.0a0 - pynvml >=11.4.1 run_constrained: - cupy >=9.5.0 diff --git a/dependencies.yaml b/dependencies.yaml index d784cc55..1cca1248 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -282,7 +282,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - &numpy numpy>=1.23,<2.0a0 + - &numpy numpy>=1.23,<3.0a0 - pynvml>=11.4.1 run_python_distributed_ucxx: common: diff --git a/python/distributed-ucxx/pyproject.toml b/python/distributed-ucxx/pyproject.toml index c013329b..8d7e23e4 100644 --- a/python/distributed-ucxx/pyproject.toml +++ b/python/distributed-ucxx/pyproject.toml @@ -41,7 +41,7 @@ docs = [ test = [ "cudf==24.10.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", - "numpy>=1.23,<2.0a0", + "numpy>=1.23,<3.0a0", "pytest-rerunfailures", "pytest==7.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/ucxx/pyproject.toml b/python/ucxx/pyproject.toml index 4bf1eab2..e2660586 100644 --- a/python/ucxx/pyproject.toml +++ b/python/ucxx/pyproject.toml @@ -20,7 +20,7 @@ license = { text = "BSD-3-Clause" } requires-python = ">=3.9" dependencies = [ "libucxx==0.40.*,>=0.0.0a0", - "numpy>=1.23,<2.0a0", + "numpy>=1.23,<3.0a0", "pynvml>=11.4.1", "rmm==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From c72e739393ee5a3a639bd5b349e9653960096f8f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 22 Aug 2024 08:15:44 -0500 Subject: [PATCH 04/10] Drop Python 3.9 support (#272) Contributes to https://github.com/rapidsai/build-planning/issues/88 Finishes the work of dropping Python 3.9 support. This project stopped building / testing against Python 3.9 as of https://github.com/rapidsai/shared-workflows/pull/235. This PR updates configuration and docs to reflect that. ## Notes for Reviewers ### How I tested this Checked that there were no remaining uses like this: ```shell git grep -E '3\.9' git grep '39' git grep 'py39' ``` And similar for variations on Python 3.8 (to catch things that were missed the last time this was done). Authors: - James Lamb (https://github.com/jameslamb) Approvers: - https://github.com/jakirkham - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/ucxx/pull/272 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/ucxx/conda_build_config.yaml | 1 - dependencies.yaml | 6 +----- python/distributed-ucxx/pyproject.toml | 3 +-- python/ucxx/pyproject.toml | 2 +- 6 files changed, 5 insertions(+), 11 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 7138aedc..2f285d2f 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -34,7 +34,7 @@ dependencies: - pytest-asyncio - pytest-rerunfailures - pytest==7.* -- python>=3.9,<3.12 +- python>=3.10,<3.12 - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.10.*,>=0.0.0a0 - rmm==24.10.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index ff57ce82..cf9815cc 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -34,7 +34,7 @@ dependencies: - pytest-asyncio - pytest-rerunfailures - pytest==7.* -- python>=3.9,<3.12 +- python>=3.10,<3.12 - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.10.*,>=0.0.0a0 - rmm==24.10.*,>=0.0.0a0 diff --git a/conda/recipes/ucxx/conda_build_config.yaml b/conda/recipes/ucxx/conda_build_config.yaml index 28fc4ed2..36891b09 100644 --- a/conda/recipes/ucxx/conda_build_config.yaml +++ b/conda/recipes/ucxx/conda_build_config.yaml @@ -20,7 +20,6 @@ cmake: - ">=3.26.4" python: - - 3.9 - 3.10 - 3.11 diff --git a/dependencies.yaml b/dependencies.yaml index 1cca1248..639e4112 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -263,10 +263,6 @@ dependencies: specific: - output_types: conda matrices: - - matrix: - py: "3.9" - packages: - - python=3.9 - matrix: py: "3.10" packages: @@ -277,7 +273,7 @@ dependencies: - python=3.11 - matrix: packages: - - python>=3.9,<3.12 + - python>=3.10,<3.12 run_python_ucxx: common: - output_types: [conda, requirements, pyproject] diff --git a/python/distributed-ucxx/pyproject.toml b/python/distributed-ucxx/pyproject.toml index 8d7e23e4..78184b00 100644 --- a/python/distributed-ucxx/pyproject.toml +++ b/python/distributed-ucxx/pyproject.toml @@ -14,7 +14,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "BSD-3-Clause" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "numba>=0.57.1", "rapids-dask-dependency==24.10.*,>=0.0.0a0", @@ -24,7 +24,6 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] diff --git a/python/ucxx/pyproject.toml b/python/ucxx/pyproject.toml index e2660586..2437becc 100644 --- a/python/ucxx/pyproject.toml +++ b/python/ucxx/pyproject.toml @@ -17,7 +17,7 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "BSD-3-Clause" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "libucxx==0.40.*,>=0.0.0a0", "numpy>=1.23,<3.0a0", From 014a8449b6de6550d638cdb837edd72b9cc27ad9 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:54:22 -0500 Subject: [PATCH 05/10] fix sed expression (#273) --- ci/release/update-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index a19cd100..d5ead315 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -55,7 +55,7 @@ for FILE in dependencies.yaml conda/environments/*.yaml; do sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done for DEP in "${UCXX_DEPENDENCIES[@]}"; do - sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/,>=0.0.0a0/g" "${FILE}" + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done done for FILE in python/*/pyproject.toml; do From 0563c2551b06d7928764793b9d4509671c11dd5f Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 27 Aug 2024 14:15:58 -0400 Subject: [PATCH 06/10] Update rapidsai/pre-commit-hooks (#275) This PR updates rapidsai/pre-commit-hooks to the version 0.4.0. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/ucxx/pull/275 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c9b6834..56ee6763 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,7 +42,7 @@ repos: files: \.(h|cpp)$ # exclude: path/to/myfile.h - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.3.1 + rev: v0.4.0 hooks: - id: verify-alpha-spec args: From 18ada55c48153900a2780e0ab65f80a8c6c8ea57 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Thu, 29 Aug 2024 09:03:16 +0200 Subject: [PATCH 07/10] Update docs to suggest Miniforge3 (#274) Due to licensing restrictions, suggest users install Miniforge3 from now on. Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/ucxx/pull/274 --- README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ceba20e..64bdc549 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,27 @@ UCXX is an object-oriented C++ interface for UCX, with native support for Python ### Environment setup -Before starting it is necessary to have the necessary dependencies installed. The simplest way to get started is to installed [Miniconda](https://docs.conda.io/en/latest/miniconda.html) and then to create and activate an environment with the provided development file: +Before starting it is necessary to have the necessary dependencies installed. The simplest way to get started is to install [Miniforge](https://github.com/conda-forge/miniforge) and then to create and activate an environment with the provided development file, for CUDA 11.x: ``` $ conda env create -n ucxx -f conda/environments/all_cuda-118_arch-x86_64.yaml +``` + +Or for CUDA 12.x: + +``` +$ conda env create -n ucxx -f conda/environments/all_cuda-125_arch-x86_64.yaml +``` + +And then activate the newly created environment: + +``` $ conda activate ucxx ``` #### Faster conda dependency resolution -The procedure aforementioned should complete without issues, but it may be slower than necessary. One alternative to speed up dependency resolution is to install [mamba](https://mamba.readthedocs.io/en/latest/) before creating the new environment. After installing Miniconda, mamba can be installed with: +The procedure aforementioned should complete without issues, but it may be slower than necessary. One alternative to speed up dependency resolution is to install [mamba](https://mamba.readthedocs.io/en/latest/) before creating the new environment. After installing Miniforge, mamba can be installed with: ``` $ conda install -c conda-forge mamba From bb30a22f5cf08e7a1da5f4836dee1e974fbf027c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 9 Sep 2024 07:14:20 -0700 Subject: [PATCH 08/10] Add support for Python 3.12 (#276) Contributes to https://github.com/rapidsai/build-planning/issues/40 This PR adds support for Python 3.12. ## Notes for Reviewers This is part of ongoing work to add Python 3.12 support across RAPIDS. It temporarily introduces a build/test matrix including Python 3.12, from https://github.com/rapidsai/shared-workflows/pull/213. A follow-up PR will revert back to pointing at the `branch-24.10` branch of `shared-workflows` once all RAPIDS repos have added Python 3.12 support. ### This will fail until all dependencies have been updates to Python 3.12 CI here is expected to fail until all of this project's upstream dependencies support Python 3.12. This can be merged whenever all CI jobs are passing. Authors: - James Lamb (https://github.com/jameslamb) - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Bradley Dice (https://github.com/bdice) - Peter Andreas Entschev (https://github.com/pentschev) URL: https://github.com/rapidsai/ucxx/pull/276 --- .github/workflows/build.yaml | 18 +++++----- .github/workflows/pr.yaml | 24 ++++++------- .github/workflows/test.yaml | 10 +++--- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-125_arch-x86_64.yaml | 2 +- conda/recipes/ucxx/conda_build_config.yaml | 1 + cpp/python/src/future.cpp | 34 ++----------------- dependencies.yaml | 6 +++- python/distributed-ucxx/pyproject.toml | 1 + 9 files changed, 37 insertions(+), 61 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5fdfcff0..afe76e3b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: conda-cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: if: github.ref_type == 'branch' needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: upload-conda: needs: [conda-cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -59,7 +59,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-libucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: wheel-publish-libucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -80,7 +80,7 @@ jobs: wheel-build-ucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -90,7 +90,7 @@ jobs: wheel-publish-ucxx: needs: wheel-build-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: package-type: python wheel-build-distributed-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -110,7 +110,7 @@ jobs: wheel-publish-distributed-ucxx: needs: [wheel-build-ucxx, wheel-build-distributed-ucxx] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c57d575d..6d771cfd 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,22 +24,22 @@ jobs: - wheel-build-distributed-ucxx - wheel-tests-distributed-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 with: build_type: pull-request docs-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -49,21 +49,21 @@ jobs: conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-distributed-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: pull-request script: "ci/test_python_distributed.sh" @@ -71,21 +71,21 @@ jobs: wheel-build-libucxx: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_libucxx.sh wheel-build-ucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_ucxx.sh wheel-tests-ucxx: needs: wheel-build-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" @@ -93,14 +93,14 @@ jobs: wheel-build-distributed-ucxx: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 with: build_type: pull-request script: ci/build_wheel_distributed_ucxx.sh wheel-tests-distributed-ucxx: needs: [wheel-build-ucxx, wheel-build-distributed-ucxx] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 467a8d1d..ea8b738b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -25,7 +25,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-distributed-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 with: build_type: nightly script: "ci/test_python_distributed.sh" @@ -44,7 +44,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" wheel-tests-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} @@ -54,7 +54,7 @@ jobs: script: ci/test_wheel_ucxx.sh wheel-tests-distributed-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2f285d2f..135a01be 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -34,7 +34,7 @@ dependencies: - pytest-asyncio - pytest-rerunfailures - pytest==7.* -- python>=3.10,<3.12 +- python>=3.10,<3.13 - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.10.*,>=0.0.0a0 - rmm==24.10.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index cf9815cc..b63f81c3 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -34,7 +34,7 @@ dependencies: - pytest-asyncio - pytest-rerunfailures - pytest==7.* -- python>=3.10,<3.12 +- python>=3.10,<3.13 - rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.10.*,>=0.0.0a0 - rmm==24.10.*,>=0.0.0a0 diff --git a/conda/recipes/ucxx/conda_build_config.yaml b/conda/recipes/ucxx/conda_build_config.yaml index 36891b09..1fef0694 100644 --- a/conda/recipes/ucxx/conda_build_config.yaml +++ b/conda/recipes/ucxx/conda_build_config.yaml @@ -22,6 +22,7 @@ cmake: python: - 3.10 - 3.11 + - 3.12 ucx: - "==1.15.*" diff --git a/cpp/python/src/future.cpp b/cpp/python/src/future.cpp index b82daddf..0647e0e3 100644 --- a/cpp/python/src/future.cpp +++ b/cpp/python/src/future.cpp @@ -113,41 +113,13 @@ PyObject* create_python_future() return result; } -static PyCFunction get_future_method(const char* method_name) -{ - PyCFunction result = NULL; - - PyGILState_STATE state = PyGILState_Ensure(); - - PyObject* future_object = get_asyncio_future_object(); - if (PyErr_Occurred()) { - ucxx_trace_req("ucxx::python::%s, error getting asyncio.Future method object", __func__); - PyErr_Print(); - } - PyMethodDef* m = reinterpret_cast(future_object)->tp_methods; - - for (; m != NULL; ++m) { - if (m->ml_name && !strcmp(m->ml_name, method_name)) { - result = m->ml_meth; - break; - } - } - - if (!result) - PyErr_Format(PyExc_RuntimeError, "Unable to load function pointer for `Future.set_result`."); - - PyGILState_Release(state); - return result; -} - PyObject* future_set_result(PyObject* future, PyObject* value) { PyObject* result = NULL; PyGILState_STATE state = PyGILState_Ensure(); - PyCFunction f = get_future_method("set_result"); - result = f(future, value); + result = PyObject_CallMethodOneArg(future, set_result_str, value); if (PyErr_Occurred()) { ucxx_trace_req("ucxx::python::%s, error calling `set_result()` from `asyncio.Future` object", __func__); @@ -165,7 +137,6 @@ PyObject* future_set_exception(PyObject* future, PyObject* exception, const char PyObject* message_object = NULL; PyObject* message_tuple = NULL; PyObject* formed_exception = NULL; - PyCFunction f = NULL; PyGILState_STATE state = PyGILState_Ensure(); @@ -176,9 +147,8 @@ PyObject* future_set_exception(PyObject* future, PyObject* exception, const char formed_exception = PyObject_Call(exception, message_tuple, NULL); if (formed_exception == NULL) goto err; - f = get_future_method("set_exception"); + result = PyObject_CallMethodOneArg(future, set_exception_str, formed_exception); - result = f(future, formed_exception); goto finish; err: diff --git a/dependencies.yaml b/dependencies.yaml index 639e4112..329af493 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -272,8 +272,12 @@ dependencies: packages: - python=3.11 - matrix: + py: "3.12" packages: - - python>=3.10,<3.12 + - python=3.12 + - matrix: + packages: + - python>=3.10,<3.13 run_python_ucxx: common: - output_types: [conda, requirements, pyproject] diff --git a/python/distributed-ucxx/pyproject.toml b/python/distributed-ucxx/pyproject.toml index 78184b00..b1a80b09 100644 --- a/python/distributed-ucxx/pyproject.toml +++ b/python/distributed-ucxx/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] [project.entry-points."distributed.comm.backends"] From 471130dc9bf71147a1e619ef04bb3f9c9a0869f8 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Mon, 16 Sep 2024 15:15:28 +0200 Subject: [PATCH 09/10] Update version error message (#277) Update version error message to match the minimum support UCX version and provide instructions to check where UCX is being loaded from. Ported from https://github.com/rapidsai/ucx-py/pull/1069 Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Mads R. B. Kristensen (https://github.com/madsbk) URL: https://github.com/rapidsai/ucxx/pull/277 --- python/ucxx/ucxx/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/ucxx/ucxx/__init__.py b/python/ucxx/ucxx/__init__.py index 4f42d44a..fbb4c2d0 100644 --- a/python/ucxx/ucxx/__init__.py +++ b/python/ucxx/ucxx/__init__.py @@ -103,10 +103,13 @@ def _is_mig_device(handle): from ._version import __git_commit__, __version__ +__ucx_min_version__ = "1.15.0" __ucx_version__ = "%d.%d.%d" % get_ucx_version() -if get_ucx_version() < (1, 11, 1): +if get_ucx_version() < tuple(int(i) for i in __ucx_min_version__.split(".")): raise ImportError( f"Support for UCX {__ucx_version__} has ended. Please upgrade to " - "1.11.1 or newer." + f"{__ucx_min_version__} or newer. If you believe the wrong version " + "is being loaded, please check the path from where UCX is loaded " + "by rerunning with the environment variable `UCX_LOG_LEVEL=debug`." ) From a07a40fe6d00ccc7c8932f02dbc070d8f5c56fc0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 18 Sep 2024 14:44:03 -0500 Subject: [PATCH 10/10] Use CI workflow branch 'branch-24.10' again [skip ci] (#279) --- .github/workflows/build.yaml | 18 +++++++++--------- .github/workflows/pr.yaml | 24 ++++++++++++------------ .github/workflows/test.yaml | 10 +++++----- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index afe76e3b..5fdfcff0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: conda-cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: if: github.ref_type == 'branch' needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: upload-conda: needs: [conda-cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -59,7 +59,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-libucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: wheel-publish-libucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -80,7 +80,7 @@ jobs: wheel-build-ucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -90,7 +90,7 @@ jobs: wheel-publish-ucxx: needs: wheel-build-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: package-type: python wheel-build-distributed-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -110,7 +110,7 @@ jobs: wheel-publish-distributed-ucxx: needs: [wheel-build-ucxx, wheel-build-distributed-ucxx] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6d771cfd..c57d575d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,22 +24,22 @@ jobs: - wheel-build-distributed-ucxx - wheel-tests-distributed-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request docs-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -49,21 +49,21 @@ jobs: conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-distributed-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: pull-request script: "ci/test_python_distributed.sh" @@ -71,21 +71,21 @@ jobs: wheel-build-libucxx: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_libucxx.sh wheel-build-ucxx: needs: wheel-build-libucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_ucxx.sh wheel-tests-ucxx: needs: wheel-build-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" @@ -93,14 +93,14 @@ jobs: wheel-build-distributed-ucxx: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: build_type: pull-request script: ci/build_wheel_distributed_ucxx.sh wheel-tests-distributed-ucxx: needs: [wheel-build-ucxx, wheel-build-distributed-ucxx] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ea8b738b..467a8d1d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -25,7 +25,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-distributed-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 with: build_type: nightly script: "ci/test_python_distributed.sh" @@ -44,7 +44,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" wheel-tests-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }} @@ -54,7 +54,7 @@ jobs: script: ci/test_wheel_ucxx.sh wheel-tests-distributed-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@python-3.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 with: build_type: nightly branch: ${{ inputs.branch }}