From 25575de82373b866ce549bbaa3e25944f1272601 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 22 Jan 2025 11:26:07 -0500 Subject: [PATCH 1/2] Add shellcheck to pre-commit and fix warnings --- .pre-commit-config.yaml | 6 ++++++ ci/build_docs.sh | 11 ++++++---- ci/build_wheel_cudf.sh | 4 ++-- ci/build_wheel_cudf_polars.sh | 4 ++-- ci/build_wheel_dask_cudf.sh | 4 ++-- ci/build_wheel_libcudf.sh | 2 +- ci/build_wheel_pylibcudf.sh | 4 ++-- ci/check_style.sh | 6 +++--- ci/checks/doxygen.sh | 16 ++++++++------- ci/cpp_linters.sh | 4 +--- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 5 ++--- ci/cudf_pandas_scripts/pandas-tests/run.sh | 18 ++++++++--------- ci/cudf_pandas_scripts/run_tests.sh | 13 ++++++------ .../run-library-tests.sh | 20 +++++++++---------- .../third-party-integration/test.sh | 16 ++++++--------- ci/release/update-version.sh | 16 +++++++-------- ci/run_cudf_examples.sh | 6 ++++-- ci/run_cudf_memcheck_ctests.sh | 6 +++--- ci/run_cudf_polars_polars_tests.sh | 7 ++++--- ci/test_cpp.sh | 8 ++++---- ci/test_cpp_memcheck.sh | 3 ++- ci/test_cudf_polars_polars_tests.sh | 13 ++++++------ ci/test_notebooks.sh | 12 ++++++----- ci/test_python_common.sh | 4 ++-- ci/test_wheel_cudf.sh | 10 +++++----- ci/test_wheel_cudf_polars.sh | 11 +++++----- ci/test_wheel_dask_cudf.sh | 10 +++++----- ci/utils/nbtest.sh | 20 +++++++++---------- ci/validate_wheel.sh | 6 +++--- 29 files changed, 137 insertions(+), 128 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 052c6cc2cb9..cb2b42261fa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -177,6 +177,12 @@ repos: hooks: - id: rapids-dependency-file-generator args: ["--clean"] + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + args: ["--severity=warning"] + files: ^ci/ default_language_version: python: python3 diff --git a/ci/build_docs.sh b/ci/build_docs.sh index e8a054842c8..c24a58b0232 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -3,8 +3,10 @@ set -euo pipefail -export RAPIDS_VERSION="$(rapids-version)" -export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +RAPIDS_VERSION="$(rapids-version)" +RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" +export RAPIDS_VERSION +export RAPIDS_VERSION_MAJOR_MINOR rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh @@ -33,7 +35,8 @@ rapids-mamba-retry install \ "cudf=${RAPIDS_VERSION}" \ "dask-cudf=${RAPIDS_VERSION}" -export RAPIDS_DOCS_DIR="$(mktemp -d)" +RAPIDS_DOCS_DIR="$(mktemp -d)" +export RAPIDS_DOCS_DIR EXITCODE=0 trap "EXITCODE=1" ERR @@ -41,7 +44,7 @@ set +e rapids-logger "Build CPP docs" pushd cpp/doxygen -aws s3 cp s3://rapidsai-docs/librmm/html/${RAPIDS_VERSION_MAJOR_MINOR}/rmm.tag . || echo "Failed to download rmm Doxygen tag" +aws s3 cp s3://rapidsai-docs/librmm/html/"${RAPIDS_VERSION_MAJOR_MINOR}"/rmm.tag . || echo "Failed to download rmm Doxygen tag" doxygen Doxyfile mkdir -p "${RAPIDS_DOCS_DIR}/libcudf/html" mv html/* "${RAPIDS_DOCS_DIR}/libcudf/html" diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 32dd5a7fa62..0f373547660 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -1,11 +1,11 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail package_dir="python/cudf" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Downloads libcudf and pylibcudf wheels from this current build, # then ensures 'cudf' wheel builds always use the 'libcudf' and 'pylibcudf' just built in the same CI run. diff --git a/ci/build_wheel_cudf_polars.sh b/ci/build_wheel_cudf_polars.sh index 38048125247..898ef3e3f45 100755 --- a/ci/build_wheel_cudf_polars.sh +++ b/ci/build_wheel_cudf_polars.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail @@ -8,5 +8,5 @@ package_dir="python/cudf_polars" ./ci/build_wheel.sh cudf-polars ${package_dir} ./ci/validate_wheel.sh ${package_dir} dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python ${package_dir}/dist diff --git a/ci/build_wheel_dask_cudf.sh b/ci/build_wheel_dask_cudf.sh index b0ae2f23abc..168625f1205 100755 --- a/ci/build_wheel_dask_cudf.sh +++ b/ci/build_wheel_dask_cudf.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail @@ -8,5 +8,5 @@ package_dir="python/dask_cudf" ./ci/build_wheel.sh dask-cudf ${package_dir} ./ci/validate_wheel.sh ${package_dir} dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python ${package_dir}/dist diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index d80e4fef0d0..11fc5058500 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -6,7 +6,7 @@ set -euo pipefail package_name="libcudf" package_dir="python/libcudf" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" rapids-logger "Generating build requirements" diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 5a8f3397714..9091f59d57b 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -1,11 +1,11 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail package_dir="python/pylibcudf" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Downloads libcudf wheel from this current build, # then ensures 'pylibcudf' wheel builds always use the 'libcudf' just built in the same CI run. diff --git a/ci/check_style.sh b/ci/check_style.sh index 634d8b0d702..70daff8f504 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. set -euo pipefail @@ -20,8 +20,8 @@ RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" FORMAT_FILE_URL="https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/cmake-format-rapids-cmake.json" export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json -mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) -wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} +mkdir -p "$(dirname "${RAPIDS_CMAKE_FORMAT_FILE}")" +wget -O ${RAPIDS_CMAKE_FORMAT_FILE} "${FORMAT_FILE_URL}" # Run pre-commit checks pre-commit run --all-files --show-diff-on-failure diff --git a/ci/checks/doxygen.sh b/ci/checks/doxygen.sh index f4d97f91aa8..6870a1beac5 100755 --- a/ci/checks/doxygen.sh +++ b/ci/checks/doxygen.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. ############################### # cuDF doxygen warnings check # ############################### @@ -14,8 +14,8 @@ fi function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } # doxygen supported version 1.9.1 -DOXYGEN_VERSION=`doxygen --version` -if [ ! $(version "$DOXYGEN_VERSION") -eq $(version "1.9.1") ] ; then +DOXYGEN_VERSION=$(doxygen --version) +if [ ! "$(version "$DOXYGEN_VERSION")" -eq "$(version "1.9.1")" ] ; then echo -e "warning: Unsupported doxygen version $DOXYGEN_VERSION" echo -e "Expecting doxygen version 1.9.1" exit 0 @@ -23,16 +23,18 @@ fi # Set variables for doxygen # We can't use gha-tools' rapids-version and rapids-version-major-minor here because this script can run outside of CI -export RAPIDS_VERSION="$(sed -E -e "s/^([0-9]{2})\.([0-9]{2})\.([0-9]{2}).*$/\1.\2.\3/" VERSION)" -export RAPIDS_VERSION_MAJOR_MINOR="$(sed -E -e "s/^([0-9]{2})\.([0-9]{2})\.([0-9]{2}).*$/\1.\2/" VERSION)" +RAPIDS_VERSION="$(sed -E -e "s/^([0-9]{2})\.([0-9]{2})\.([0-9]{2}).*$/\1.\2.\3/" VERSION)" +RAPIDS_VERSION_MAJOR_MINOR="$(sed -E -e "s/^([0-9]{2})\.([0-9]{2})\.([0-9]{2}).*$/\1.\2/" VERSION)" +export RAPIDS_VERSION +export RAPIDS_VERSION_MAJOR_MINOR # Run doxygen, ignore missing tag files error TAG_ERROR1="error: Tag file '.*.tag' does not exist or is not a file. Skipping it..." TAG_ERROR2="error: cannot open tag file .*.tag for writing" -DOXYGEN_STDERR=`cd cpp/doxygen && { cat Doxyfile ; echo QUIET = YES; echo GENERATE_HTML = NO; } | doxygen - 2>&1 | sed "/\($TAG_ERROR1\|$TAG_ERROR2\)/d"` +DOXYGEN_STDERR=$(cd cpp/doxygen && { cat Doxyfile ; echo QUIET = YES; echo GENERATE_HTML = NO; } | doxygen - 2>&1 | sed "/\($TAG_ERROR1\|$TAG_ERROR2\)/d") RETVAL=$? -if [ "$RETVAL" != "0" ] || [ ! -z "$DOXYGEN_STDERR" ]; then +if [ "$RETVAL" != "0" ] || [ -n "$DOXYGEN_STDERR" ]; then echo -e "$DOXYGEN_STDERR" RETVAL=1 #because return value is not generated by doxygen 1.8.20 fi diff --git a/ci/cpp_linters.sh b/ci/cpp_linters.sh index 9702b055512..95b153061d2 100755 --- a/ci/cpp_linters.sh +++ b/ci/cpp_linters.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. set -euo pipefail @@ -20,8 +20,6 @@ set +u conda activate clang_tidy set -u -RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" - source rapids-configure-sccache # Run the build via CMake, which will run clang-tidy when CUDF_STATIC_LINTERS is enabled. diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index 5dbb4ba991c..bdf695d5eb4 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -13,7 +13,6 @@ rapids-logger "Github job name: ${GH_JOB_NAME}" rapids-logger "Rapids version: ${RAPIDS_FULL_VERSION}" PY_VER="310" -MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION}-results.json rapids-logger "Fetching latest available results from nightly" @@ -22,7 +21,7 @@ COMPARE_ENV=$(tail -n 1 s3_output.txt) rapids-logger "Latest available results from nightly: ${COMPARE_ENV}" aws s3 cp "s3://rapids-downloads/${COMPARE_ENV}" main-results.json -aws s3 cp $PR_ARTIFACT pr-results.json +aws s3 cp "$PR_ARTIFACT" pr-results.json # Compute the diff and prepare job summary: python -m pip install pandas tabulate diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index e5cd4436a3a..15970a4185c 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -10,7 +10,7 @@ RAPIDS_FULL_VERSION=$(<./VERSION) rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids-version $RAPIDS_FULL_VERSION" rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Download the cudf, libcudf, and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist @@ -19,9 +19,9 @@ RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ - "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" \ - "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" + "$(echo ./dist/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,pandas-tests]" \ + "$(echo ./dist/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo ./dist/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ @@ -34,12 +34,12 @@ bash python/cudf/cudf/pandas/scripts/run-pandas-tests.sh \ --max-worker-restart=3 \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-pandas.xml" \ --dist worksteal \ - --report-log=${PANDAS_TESTS_BRANCH}.json 2>&1 + --report-log="${PANDAS_TESTS_BRANCH}.json" 2>&1 SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-${RAPIDS_FULL_VERSION}-results.json # summarize the results and save them to artifacts: -python python/cudf/cudf/pandas/scripts/summarize-test-results.py --output json pandas-testing/${PANDAS_TESTS_BRANCH}.json > pandas-testing/${SUMMARY_FILE_NAME} +python python/cudf/cudf/pandas/scripts/summarize-test-results.py --output json pandas-testing/"${PANDAS_TESTS_BRANCH}.json" > "pandas-testing/${SUMMARY_FILE_NAME}" RAPIDS_ARTIFACTS_DIR=${RAPIDS_ARTIFACTS_DIR:-"${PWD}/artifacts"} mkdir -p "${RAPIDS_ARTIFACTS_DIR}" -mv pandas-testing/${SUMMARY_FILE_NAME} ${RAPIDS_ARTIFACTS_DIR}/ -rapids-upload-to-s3 ${RAPIDS_ARTIFACTS_DIR}/${SUMMARY_FILE_NAME} "${RAPIDS_ARTIFACTS_DIR}" +mv pandas-testing/"${SUMMARY_FILE_NAME}" "${RAPIDS_ARTIFACTS_DIR}"/ +rapids-upload-to-s3 "${RAPIDS_ARTIFACTS_DIR}"/"${SUMMARY_FILE_NAME}" "${RAPIDS_ARTIFACTS_DIR}" diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 61361fffb07..2c8d5969bdc 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -10,7 +10,6 @@ RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" DEPENDENCIES_PATH="dependencies.yaml" -package_name="pandas" # Use grep to find the line containing the package name and version constraint pandas_version_constraint=$(grep -oP "pandas>=\d+\.\d+,\<\d+\.\d+\.\d+dev\d+" $DEPENDENCIES_PATH) @@ -47,7 +46,7 @@ done if [ "$no_cudf" = true ]; then echo "Skipping cudf install" else - RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Download the cudf, libcudf, and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist @@ -60,9 +59,9 @@ else python -m pip install \ -v \ --constraint ./constraints.txt \ - "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" \ - "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" + "$(echo ./dist/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,cudf-pandas-tests]" \ + "$(echo ./dist/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo ./dist/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" fi python -m pip install ipykernel @@ -77,7 +76,7 @@ python -m pytest -p cudf.pandas \ --cov-report=term \ ./python/cudf/cudf_pandas_tests/ -output=$(python ci/cudf_pandas_scripts/fetch_pandas_versions.py $pandas_version_constraint) +output=$(python ci/cudf_pandas_scripts/fetch_pandas_versions.py "$pandas_version_constraint") # Convert the comma-separated list into an array IFS=',' read -r -a versions <<< "$output" diff --git a/ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh b/ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh index d44d25d658c..ce3291b864a 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh @@ -1,10 +1,10 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 cleanup() { - rm ${TEST_DIR}/results-*.pickle + rm "${TEST_DIR}"/results-*.pickle } trap cleanup EXIT @@ -19,21 +19,21 @@ runtest() { fi pytest \ - $plugin \ + "$plugin" \ -v \ --continue-on-collection-errors \ --cache-clear \ - --numprocesses=${NUM_PROCESSES} \ + --numprocesses="${NUM_PROCESSES}" \ --dist=worksteal \ - ${TEST_DIR}/test_${lib}*.py + "${TEST_DIR}"/test_"${lib}"*.py } main() { local lib=$1 # generation phase - runtest ${lib} "gold" - runtest ${lib} "cudf" + runtest "${lib}" "gold" + runtest "${lib}" "cudf" # assertion phase pytest \ @@ -42,9 +42,9 @@ main() { -v \ --continue-on-collection-errors \ --cache-clear \ - --numprocesses=${NUM_PROCESSES} \ + --numprocesses="${NUM_PROCESSES}" \ --dist=worksteal \ - ${TEST_DIR}/test_${lib}*.py + "${TEST_DIR}"/test_"${lib}"*.py } -main $@ +main "$@" diff --git a/ci/cudf_pandas_scripts/third-party-integration/test.sh b/ci/cudf_pandas_scripts/third-party-integration/test.sh index 30e3ffc9a43..43ed3594917 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/test.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/test.sh @@ -1,22 +1,18 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. # Common setup steps shared by Python test jobs set -euo pipefail -write_output() { - local key="$1" - local value="$2" - echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}" -} extract_lib_from_dependencies_yaml() { local file=$1 # Parse all keys in dependencies.yaml under the "files" section, # extract all the keys that start with "test_", and extract the rest - local extracted_libs="$(yq -o json $file | jq -rc '.files | with_entries(select(.key | contains("test_"))) | keys | map(sub("^test_"; ""))')" - echo $extracted_libs + extracted_libs="$(yq -o json "$file" | jq -rc '.files | with_entries(select(.key | contains("test_"))) | keys | map(sub("^test_"; ""))')" + local extracted_libs + echo "$extracted_libs" } main() { @@ -40,7 +36,7 @@ main() { rapids-dependency-file-generator \ --config "$dependencies_yaml" \ --output conda \ - --file-key test_${lib} \ + --file-key "test_${lib}" \ --matrix "cuda=${CUDA_MAJOR};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test @@ -74,7 +70,7 @@ main() { trap "EXITCODE=1" ERR set +e - TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh ${lib} + TEST_DIR=${TEST_DIR} NUM_PROCESSES=${NUM_PROCESSES} ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh "${lib}" set -e rapids-logger "Test script exiting with value: ${EXITCODE}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 95f36653c2c..f4f31dfbb6f 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. ######################## # cuDF Version Updater # ######################## @@ -13,19 +13,17 @@ NEXT_FULL_TAG=$1 # Get current version CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') -CURRENT_MAJOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[1]}') -CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}') -CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}') +CURRENT_MAJOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[1]}') +CURRENT_MINOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[2]}') CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} # Get . for next version -NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') -NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') -NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}') +NEXT_MAJOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[1]}') +NEXT_MINOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[2]}') +NEXT_PATCH=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[3]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Need to distutils-normalize the versions for some use cases -CURRENT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${CURRENT_SHORT_TAG}'))") NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))") PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_PATCH}'))") @@ -33,7 +31,7 @@ echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" # Inplace sed replace; workaround for Linux and Mac function sed_runner() { - sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak + sed -i.bak ''"$1"'' "$2" && rm -f "${2}".bak } # Centralized version file update diff --git a/ci/run_cudf_examples.sh b/ci/run_cudf_examples.sh index 2439af5b644..adacc304139 100755 --- a/ci/run_cudf_examples.sh +++ b/ci/run_cudf_examples.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. set -uo pipefail @@ -7,9 +7,11 @@ EXITCODE=0 trap "EXITCODE=1" ERR # Support customizing the examples' install location -cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/examples/libcudf/"; +cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/examples/libcudf/" || exit # compute-sanitizer not available before CUDA 11.6 +# Using -lt with decimals doesn't work in `bash` _except_ when comparing version strings +# shellcheck disable=SC2072 if [[ "${RAPIDS_CUDA_VERSION%.*}" < "11.6" ]]; then echo "computer-sanitizer unavailable pre 11.6" exit 0 diff --git a/ci/run_cudf_memcheck_ctests.sh b/ci/run_cudf_memcheck_ctests.sh index 391579b6c59..fac95b5ad8d 100755 --- a/ci/run_cudf_memcheck_ctests.sh +++ b/ci/run_cudf_memcheck_ctests.sh @@ -7,17 +7,17 @@ EXITCODE=0 trap "EXITCODE=1" ERR # Support customizing the ctests' install location -cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcudf/"; +cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcudf/" || exit export GTEST_CUDF_RMM_MODE=cuda export GTEST_BRIEF=1 # compute-sanitizer bug 4553815 export LIBCUDF_MEMCHECK_ENABLED=1 for gt in ./*_TEST ; do - test_name=$(basename ${gt}) + test_name=$(basename "${gt}") # Run gtests with compute-sanitizer echo "Running compute-sanitizer on $test_name" - compute-sanitizer --tool memcheck ${gt} "$@" + compute-sanitizer --tool memcheck "${gt}" "$@" done unset GTEST_BRIEF unset GTEST_CUDF_RMM_MODE diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index c851f65d4f6..95b3cae7b4d 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. set -euo pipefail @@ -38,7 +38,8 @@ else fi fi -DESELECTED_TESTS=$(printf -- " --deselect %s" "${DESELECTED_TESTS[@]}") +# Reusing a variable defined as an array as a string +DESELECTED_TEST_STR=$(printf -- " --deselect %s" "${DESELECTED_TESTS[@]}") python -m pytest \ --import-mode=importlib \ --cache-clear \ @@ -46,6 +47,6 @@ python -m pytest \ -p cudf_polars.testing.plugin \ -v \ --tb=native \ - ${DESELECTED_TESTS} \ + "${DESELECTED_TEST_STR}" \ "$@" \ py-polars/tests diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 7865849bb74..8f33a78948b 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Support invoking test_cpp.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ @@ -17,13 +17,13 @@ rapids-logger "Run libcudf gtests" ./ci/run_cudf_ctests.sh -j20 SUITEERROR=$? -if (( ${SUITEERROR} == 0 )); then +if (( SUITEERROR == 0 )); then rapids-logger "Run libcudf examples" ./ci/run_cudf_examples.sh SUITEERROR=$? fi -if (( ${SUITEERROR} == 0 )); then +if (( SUITEERROR == 0 )); then rapids-logger "Run libcudf_kafka gtests" ./ci/run_cudf_kafka_ctests.sh -j20 SUITEERROR=$? @@ -32,7 +32,7 @@ fi # Ensure that benchmarks are runnable rapids-logger "Run tests of libcudf benchmarks" -if (( ${SUITEERROR} == 0 )); then +if (( SUITEERROR == 0 )); then ./ci/run_cudf_benchmark_smoketests.sh SUITEERROR=$? fi diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index fda11c64155..6317e7e2730 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. # Support invoking test_cpp.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ @@ -11,4 +11,5 @@ rapids-logger "Memcheck gtests with rmm_mode=cuda" ./ci/run_cudf_memcheck_ctests.sh && EXITCODE=$? || EXITCODE=$?; rapids-logger "Test script exiting with value: $EXITCODE" +# shellcheck disable=SC2086 exit ${EXITCODE} diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index fefe26984cb..2f6fd6faca7 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -1,11 +1,11 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. set -eou pipefail rapids-logger "Download wheels" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # Download libcudf and pylibcudf built in the previous step @@ -15,19 +15,20 @@ RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels rapids-logger "Install libcudf, pylibcudf and cudf_polars" python -m pip install \ -v \ - "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ - "$(echo ./local-libcudf-dep/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./local-pylibcudf-dep/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" + "$(echo ./dist/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ + "$(echo ./local-libcudf-dep/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo ./local-pylibcudf-dep/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" TAG=$(python -c 'import polars; print(f"py-{polars.__version__}")') rapids-logger "Clone polars to ${TAG}" -git clone https://github.com/pola-rs/polars.git --branch ${TAG} --depth 1 +git clone https://github.com/pola-rs/polars.git --branch "${TAG}" --depth 1 # Install requirements for running polars tests rapids-logger "Install polars test requirements" python -m pip install -r polars/py-polars/requirements-dev.txt -r polars/py-polars/requirements-ci.txt +# shellcheck disable=SC2317 function set_exitcode() { EXITCODE=$? diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 4197dc5617f..329246ef9d7 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. set -euo pipefail @@ -45,11 +45,13 @@ SKIPNBS="performance-comparisons.ipynb" EXITCODE=0 trap "EXITCODE=1" ERR set +e +# Loops over `find` are fragile but this seems to be working +# shellcheck disable=SC2044 for nb in $(find . -name "*.ipynb"); do - nbBasename=$(basename ${nb}) + nbBasename=$(basename "${nb}") # Skip all notebooks that use dask (in the code or even in their name) - if ((echo ${nb} | grep -qi dask) || \ - (grep -q dask ${nb})); then + if (echo "${nb}" | grep -qi dask) || \ + (grep -q dask "${nb}"); then echo "--------------------------------------------------------------------------------" echo "SKIPPING: ${nb} (suspected Dask usage, not currently automatable)" echo "--------------------------------------------------------------------------------" @@ -59,7 +61,7 @@ for nb in $(find . -name "*.ipynb"); do echo "--------------------------------------------------------------------------------" else nvidia-smi - ${NBTEST} ${nbBasename} + ${NBTEST} "${nbBasename}" fi done diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 4327bfff3af..c6c6a3957b9 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Common setup steps shared by Python test jobs @@ -15,7 +15,7 @@ ENV_YAML_DIR="$(mktemp -d)" FILE_KEY=$1 rapids-dependency-file-generator \ --output conda \ - --file-key ${FILE_KEY} \ + --file-key "${FILE_KEY}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}" \ | tee "${ENV_YAML_DIR}/env.yaml" diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index ce12744c9e3..6eaaf7bb657 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -1,9 +1,9 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -eou pipefail -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Download the cudf, libcudf, and pylibcudf built in the previous step RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist @@ -19,9 +19,9 @@ rapids-generate-pip-constraints py_test_cudf ./constraints.txt python -m pip install \ -v \ --constraint ./constraints.txt \ - "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ - "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" + "$(echo ./dist/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ + "$(echo ./dist/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo ./dist/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 3f818867d49..5827077e826 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -1,11 +1,11 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. set -eou pipefail rapids-logger "Download wheels" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # Download libcudf and pylibcudf built in the previous step @@ -21,12 +21,13 @@ rapids-generate-pip-constraints py_test_cudf_polars ./constraints.txt python -m pip install \ -v \ --constraint ./constraints.txt \ - "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,experimental]" \ - "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" + "$(echo ./dist/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental]" \ + "$(echo ./dist/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo ./dist/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" rapids-logger "Run cudf_polars tests" +# shellcheck disable=SC2317 function set_exitcode() { EXITCODE=$? diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 44f430ce98d..41c59ef59d1 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -3,7 +3,7 @@ set -eou pipefail -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # Download the cudf, libcudf, and pylibcudf built in the previous step @@ -20,10 +20,10 @@ rapids-generate-pip-constraints py_test_dask_cudf ./constraints.txt python -m pip install \ -v \ --constraint ./constraints.txt \ - "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ - "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" + "$(echo ./dist/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo ./dist/dask_cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \ + "$(echo ./dist/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo ./dist/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh index 2a94e2d0695..9fc37d25697 100755 --- a/ci/utils/nbtest.sh +++ b/ci/utils/nbtest.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. MAGIC_OVERRIDE_CODE=" def my_run_line_magic(*args, **kwargs): @@ -24,22 +24,22 @@ get_ipython().run_cell_magic=my_run_cell_magic NO_COLORS=--colors=NoColor EXITCODE=0 NBTMPDIR="$WORKSPACE/tmp" -mkdir -p ${NBTMPDIR} +mkdir -p "${NBTMPDIR}" -for nb in $*; do - NBFILENAME=$1 +for nb in "$@"; do + NBFILENAME=$nb NBNAME=${NBFILENAME%.*} NBNAME=${NBNAME##*/} NBTESTSCRIPT=${NBTMPDIR}/${NBNAME}-test.py shift echo -------------------------------------------------------------------------------- - echo STARTING: ${NBNAME} + echo STARTING: "${NBNAME}" echo -------------------------------------------------------------------------------- - jupyter nbconvert --to script ${NBFILENAME} --output ${NBTMPDIR}/${NBNAME}-test - echo "${MAGIC_OVERRIDE_CODE}" > ${NBTMPDIR}/tmpfile - cat ${NBTESTSCRIPT} >> ${NBTMPDIR}/tmpfile - mv ${NBTMPDIR}/tmpfile ${NBTESTSCRIPT} + jupyter nbconvert --to script "${NBFILENAME}" --output "${NBTMPDIR}"/"${NBNAME}"-test + echo "${MAGIC_OVERRIDE_CODE}" > "${NBTMPDIR}"/tmpfile + cat "${NBTESTSCRIPT}" >> "${NBTMPDIR}"/tmpfile + mv "${NBTMPDIR}"/tmpfile "${NBTESTSCRIPT}" echo "Running \"ipython ${NO_COLORS} ${NBTESTSCRIPT}\" on $(date)" echo @@ -47,7 +47,7 @@ for nb in $*; do NBEXITCODE=$? echo EXIT CODE: ${NBEXITCODE} echo - EXITCODE=$((EXITCODE | ${NBEXITCODE})) + EXITCODE=$((EXITCODE | "${NBEXITCODE}")) done exit ${EXITCODE} diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh index 5910a5c59fe..40bb27feeb3 100755 --- a/ci/validate_wheel.sh +++ b/ci/validate_wheel.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. set -euo pipefail @@ -12,10 +12,10 @@ rapids-logger "validate packages with 'pydistcheck'" pydistcheck \ --inspect \ - "$(echo ${wheel_dir_relative_path}/*.whl)" + "$(echo "${wheel_dir_relative_path}"/*.whl)" rapids-logger "validate packages with 'twine'" twine check \ --strict \ - "$(echo ${wheel_dir_relative_path}/*.whl)" + "$(echo "${wheel_dir_relative_path}"/*.whl)" From bae9c3ccae1fa22ec283624dcf971f629ffbaa56 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 27 Jan 2025 11:33:27 -0500 Subject: [PATCH 2/2] fix polars pytest call --- ci/run_cudf_polars_polars_tests.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index 95b3cae7b4d..dfabe6093a9 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -38,8 +38,11 @@ else fi fi -# Reusing a variable defined as an array as a string -DESELECTED_TEST_STR=$(printf -- " --deselect %s" "${DESELECTED_TESTS[@]}") +DESELECTED_TESTS_STR=$(printf -- " --deselect %s" "${DESELECTED_TESTS[@]}") + +# Don't quote the `DESELECTED_...` variable because `pytest` can't handle +# multiple quoted arguments inline +# shellcheck disable=SC2086 python -m pytest \ --import-mode=importlib \ --cache-clear \ @@ -47,6 +50,6 @@ python -m pytest \ -p cudf_polars.testing.plugin \ -v \ --tb=native \ - "${DESELECTED_TEST_STR}" \ + $DESELECTED_TESTS_STR \ "$@" \ py-polars/tests