From bc304a29d244ad502fbdc6a304c5de0e99aeb57c Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 11 Sep 2023 14:28:37 -0500 Subject: [PATCH] Produce a fatal error if cudf is unable to find pyarrow include directory (#13976) Produce a fatal error if cudf python is unable to find pyarrow include directory. Previously the failure only presented itself while trying to compile cython files which failed to include headers from pyarrow. _Previously:_ ``` FAILED: cudf/_lib/CMakeFiles/avro.dir/avro.cxx.o /usr/bin/sccache /usr/bin/g++ -DFMT_HEADER_ONLY=1 -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_INFO -DSPDLOG_FMT_EXTERNAL -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA -DTHRUST_HOST_SYSTEM=THRUST_HOST_SYSTEM_CPP -Davro_EXPORTS -I/usr/include/python3.10 -I/home/coder/.local/share/venvs/rapids/lib/python3.10/site-packages/numpy/core/include -I/home/coder/cudf/python/cudf/cudf/_lib -I/home/coder/cudf/cpp/build/release/_deps/libcudacxx-src/lib/cmake/libcudacxx/../../../include -I/home/coder/cudf/cpp/build/release/_deps/thrust-src -I/home/coder/cudf/cpp/build/release/_deps/thrust-src/dependencies/cub -isystem /home/coder/cudf/cpp/build/release/_deps/dlpack-src/include -isystem /home/coder/cudf/cpp/build/release/_deps/jitify-src -isystem /home/coder/cudf/cpp/include -isystem /home/coder/cudf/cpp/build/release/include -isystem /home/coder/rmm/include -isystem /usr/local/cuda/include -isystem /home/coder/fmt/include -isystem /home/coder/rmm/build/release/_deps/spdlog-src/include -O3 -DNDEBUG -fPIC -MD -MT cudf/_lib/CMakeFiles/avro.dir/avro.cxx.o -MF cudf/_lib/CMakeFiles/avro.dir/avro.cxx.o.d -o cudf/_lib/CMakeFiles/avro.dir/avro.cxx.o -c /home/coder/cudf/python/cudf/_skbuild/linux-x86_64-3.10/cmake-build/cudf/_lib/avro.cxx /home/coder/cudf/python/cudf/_skbuild/linux-x86_64-3.10/cmake-build/cudf/_lib/avro.cxx:1291:10: fatal error: arrow/python/platform.h: No such file or directory 1291 | #include "arrow/python/platform.h" | ^~~~~~~~~~~~~~~~~~~~~~~~~ compilation terminated. [6/23] Building CXX object cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o FAILED: cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o /usr/bin/sccache /usr/bin/g++ -DFMT_HEADER_ONLY=1 -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_INFO -DSPDLOG_FMT_EXTERNAL -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA -DTHRUST_HOST_SYSTEM=THRUST_HOST_SYSTEM_CPP -Dcsv_EXPORTS -I/usr/include/python3.10 -I/home/coder/.local/share/venvs/rapids/lib/python3.10/site-packages/numpy/core/include -I/home/coder/cudf/python/cudf/cudf/_lib -I/home/coder/cudf/cpp/build/release/_deps/libcudacxx-src/lib/cmake/libcudacxx/../../../include -I/home/coder/cudf/cpp/build/release/_deps/thrust-src -I/home/coder/cudf/cpp/build/release/_deps/thrust-src/dependencies/cub -isystem /home/coder/cudf/cpp/build/release/_deps/dlpack-src/include -isystem /home/coder/cudf/cpp/build/release/_deps/jitify-src -isystem /home/coder/cudf/cpp/include -isystem /home/coder/cudf/cpp/build/release/include -isystem /home/coder/rmm/include -isystem /usr/local/cuda/include -isystem /home/coder/fmt/include -isystem /home/coder/rmm/build/release/_deps/spdlog-src/include -O3 -DNDEBUG -fPIC -MD -MT cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o -MF cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o.d -o cudf/_lib/CMakeFiles/csv.dir/csv.cxx.o -c /home/coder/cudf/python/cudf/_skbuild/linux-x86_64-3.10/cmake-build/cudf/_lib/csv.cxx /home/coder/cudf/python/cudf/_skbuild/linux-x86_64-3.10/cmake-build/cudf/_lib/csv.cxx:1292:10: fatal error: arrow/python/platform.h: No such file or directory 1292 | #include "arrow/python/platform.h" | ^~~~~~~~~~~~~~~~~~~~~~~~~ compilation terminated. ... ``` _With these changes:_ ``` CMake Error at cudf/_lib/CMakeLists.txt:107 (message): Error while trying to obtain pyarrow include dir: Traceback (most recent call last): File "", line 1, in File "/home/coder/.local/share/venvs/rapids/lib/python3.10/site-packages/pyarrow/__init__.py", line 65, in import pyarrow.lib as _lib ImportError: libarrow.so.1200: cannot open shared object file: No such file or directory ``` Authors: - Christopher Harris (https://github.com/cwharris) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/13976 --- python/cudf/cudf/_lib/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 06de6cc825f..947659c290a 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -98,9 +98,15 @@ find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) execute_process( COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_include())" OUTPUT_VARIABLE PYARROW_INCLUDE_DIR + ERROR_VARIABLE PYARROW_ERROR + RESULT_VARIABLE PYARROW_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE ) +if(${PYARROW_RESULT}) + message(FATAL_ERROR "Error while trying to obtain pyarrow include directory:\n${PYARROW_ERROR}") +endif() + set(targets_using_arrow_headers interop avro csv orc json parquet) foreach(target IN LISTS targets_using_arrow_headers) target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}")