From 4b2a8e2ccdeff36ff58d28d80c8742a474e1d69a Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 21 Sep 2023 23:13:31 -0700 Subject: [PATCH] Upgrade to arrow 13 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 10 +++++----- conda/environments/all_cuda-120_arch-x86_64.yaml | 4 ++-- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- cpp/cmake/thirdparty/get_arrow.cmake | 2 +- dependencies.yaml | 6 +++--- python/cudf/cudf/tests/test_decimal.py | 6 +++--- python/cudf/pyproject.toml | 4 ++-- python/cudf_kafka/pyproject.toml | 2 +- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 692ba78f317..b8e95fa5634 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -25,7 +25,7 @@ dependencies: - cxx-compiler - cython>=3.0.0 - dask-core>=2023.7.1 -- dask-cuda==23.10.* +# - dask-cuda==23.10.* - dask>=2023.7.1 - distributed>=2023.7.1 - dlpack>=0.5,<0.6.0a0 @@ -39,7 +39,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==12.0.1.* +- libarrow==13.0.0.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 @@ -68,9 +68,9 @@ dependencies: - pre-commit - protobuf>=4.21,<5 - ptxcompiler -- pyarrow==12.0.1.* +- pyarrow==13.0.0.* - pydata-sphinx-theme -- pyorc +# - pyorc - pytest - pytest-benchmark - pytest-cases @@ -79,7 +79,7 @@ dependencies: - python-confluent-kafka>=1.9.0,<1.10.0a0 - python-snappy>=0.6.0 - python>=3.9,<3.11 -- pytorch<1.12.0 +- pytorch<1.12.0 # potential - rmm==23.10.* - s3fs>=2022.3.0 - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index cf1bf4b8733..318f4b971f2 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -40,7 +40,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==12.0.1.* +- libarrow==13.0.0.* - libcufile-dev - libcurand-dev - libkvikio==23.10.* @@ -65,7 +65,7 @@ dependencies: - pip - pre-commit - protobuf>=4.21,<5 -- pyarrow==12.0.1.* +- pyarrow==13.0.0.* - pydata-sphinx-theme - pyorc - pytest diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index a909b72c878..7f14eb479b7 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -61,7 +61,7 @@ requirements: - scikit-build >=0.13.1 - setuptools - dlpack >=0.5,<0.6.0a0 - - pyarrow =12 + - pyarrow =13 - libcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 25b3f19de77..7dc81cb5e9c 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -23,7 +23,7 @@ gtest_version: - ">=1.13.0" libarrow_version: - - "=12" + - "=13" dlpack_version: - ">=0.5,<0.6.0a0" diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index 894dc9649e2..4ac893f6fa0 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -403,7 +403,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow) set(CUDF_VERSION_Arrow # This version must be kept in sync with the libarrow version pinned for builds in # dependencies.yaml. - 12.0.1 + 13.0.0 CACHE STRING "The version of Arrow to find (or build)" ) endif() diff --git a/dependencies.yaml b/dependencies.yaml index 398ae193fe6..5512d63f6fd 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -224,7 +224,7 @@ dependencies: - &gmock gmock>=1.13.0 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - libarrow==12.0.1.* + - libarrow==13.0.0.* - librdkafka>=1.9.0,<1.10.0a0 - spdlog>=1.11.0,<1.12 specific: @@ -258,7 +258,7 @@ dependencies: - cython>=3.0.0 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==12.0.1.* + - pyarrow==13.0.0.* - numpy>=1.21 build_python: common: @@ -283,7 +283,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: # Allow runtime version to float up to minor version - - pyarrow==12.* + - pyarrow==13.* cudatoolkit: specific: - output_types: conda diff --git a/python/cudf/cudf/tests/test_decimal.py b/python/cudf/cudf/tests/test_decimal.py index e4b2af90448..da77f6f9c14 100644 --- a/python/cudf/cudf/tests/test_decimal.py +++ b/python/cudf/cudf/tests/test_decimal.py @@ -93,15 +93,15 @@ def test_from_arrow_max_precision_decimal32(): ) def test_typecast_from_float_to_decimal(data, from_dtype, to_dtype): got = data.astype(from_dtype) - + import pdb;pdb.set_trace() pa_arr = got.to_arrow().cast( pa.decimal128(to_dtype.precision, to_dtype.scale) ) expected = cudf.Series(Decimal64Column.from_arrow(pa_arr)) - got = got.astype(to_dtype) + new_got = got.astype(to_dtype) - assert_eq(got, expected) + assert_eq(new_got, expected) @pytest.mark.parametrize( diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 574769f68d1..043f0710231 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "ninja", "numpy>=1.21", "protoc-wheel", - "pyarrow==12.0.1.*", + "pyarrow==13.0.0.*", "rmm==23.10.*", "scikit-build>=0.13.1", "setuptools", @@ -38,7 +38,7 @@ dependencies = [ "pandas>=1.3,<1.6.0dev0", "protobuf>=4.21,<5", "ptxcompiler", - "pyarrow==12.*", + "pyarrow==13.*", "rmm==23.10.*", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index a6ef867451b..4a4508ee260 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "cython>=3.0.0", "numpy>=1.21", - "pyarrow==12.0.1.*", + "pyarrow==13.0.0.*", "setuptools", "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.