From cc4cea46fa457a71567644a0d6fca2cd0fbe4e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 12:02:54 +0100 Subject: [PATCH 01/10] GH-44950: [C++] Bump minimum CMake version to 3.25 Try installing CMake 3.25 manually on Ubuntu Install cmake in linux-apt-lint.dockerfile and update install_cmake.sh to not require platform argument Install CMake 3.25.0 on several Ubuntu jobs and Windows jobs that require the existing version to be bumped Several fixes to install_cmake Some more fixes and upgrade minimal examples to use Ubuntu 24.04 Some more fixes for R Update from CMAKE to CMAKE_VERSION Ensure that versions returned are strings, bump cmake download Try forcing MINGW package to pull a newer CMake Revert trying to pull newer CMake Try to completely remove mingw CMake from CRAN /c/cmake/bin/cmake.exe should be present and in path, try with .exe Try with full path for CMake Try removing manual installation of CMake and installing CMake via pacman before setting up RTools Remove manual Installation of CMake from pacman not present More tries Temporarily try using msys2 GH action to install cmake Remove CMake from PKGBUILD CMake should be in the path Debug new CMake location Try with C:\Program Files\CMake\bin\cmake.exe Just testing at this point Bundle AWSSDK under RTools (#90) Check for mingw Check Windows compiler Try modifying patch due to redefinition Try fixing patch AWS is bundled on Windows R now, do not use MSYS libraries Fix license header on patch + add CMake condition for AWS patch Update configure.win to remove specific AWS CPP libs Update r/configure.win Co-authored-by: Bryce Mecum Update r/configure.win Test turning ccache back on Update ci/scripts/r_windows_build.sh Co-authored-by: Neal Richardson Update r/configure.win Co-authored-by: Neal Richardson Review comments Remove MSYS2_ARG_CONV_EXCL Testing with manually adding missing rtools40 Add correct slash for path Add comment on why MSYS2_ARG_CONV_EXCL is required Use MSYS2_ARG_CONV_EXCL on the same CMake command Update tutorial examples to use a newer Ubuntu Add CMAKE_VERSION to cuda jobs Upgrade minimal Ubuntu to use 24.04 instead of 22.04 Fix minimal build for Ubuntu 24.04 ubuntu-cuda-cpp requires cmake argument Add cmake to docker-compose.yml missing images Add missing cmake to some more images Some more fixing mainly for R binary jobs Set CMAKE to newly installed path Revert "Try modifying patch due to redefinition" This reverts commit b772b23878fb29a8752e82a6214b7c29a3b80352. Reapply "Try modifying patch due to redefinition" This reverts commit 1d713753c9e296a58fa101817332d81b250855da. Hardcode enum value I tried to find a proper way to include-guard this but everything I tried didn't work. Remove outdated refs to aws libs Since we are bundling these we don't have these to copy Move _WIN32_WINNT def into {c,cxx}flags Ignore warnings-as-errors in cmake --- .env | 1 + .github/workflows/cpp.yml | 10 + .github/workflows/dev.yml | 5 + .github/workflows/r.yml | 6 + ci/docker/centos-7-cpp.dockerfile | 7 +- ci/docker/linux-r.dockerfile | 4 + ci/docker/python-wheel-manylinux.dockerfile | 2 +- ci/docker/ubuntu-20.04-cpp-minimal.dockerfile | 5 +- ci/docker/ubuntu-20.04-cpp.dockerfile | 5 +- ci/docker/ubuntu-22.04-cpp-minimal.dockerfile | 4 + ci/docker/ubuntu-22.04-cpp.dockerfile | 5 +- ci/rtools/BuildAwsCCommon.patch | 54 +++++ ci/rtools/aws_c_io_ep.patch | 88 ++++++++ ci/rtools/awssdk_ep.patch | 213 ++++++++++++++++++ ci/scripts/PKGBUILD | 12 +- ci/scripts/install_cmake.sh | 46 ++-- ci/scripts/r_windows_build.sh | 13 +- cpp/CMakeLists.txt | 2 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 31 +++ cpp/examples/minimal_build/CMakeLists.txt | 2 +- cpp/examples/minimal_build/minimal.dockerfile | 2 +- .../system_dependency.dockerfile | 2 +- .../parquet/parquet_arrow/CMakeLists.txt | 2 +- cpp/examples/tutorial_examples/CMakeLists.txt | 2 +- .../tutorial_examples/tutorial.dockerfile | 2 +- dev/tasks/python-wheels/github.osx.yml | 2 +- dev/tasks/tasks.yml | 4 +- dev/tasks/vcpkg-tests/github.windows.yml | 2 +- docker-compose.yml | 12 + docs/source/cpp/build_system.rst | 2 +- matlab/CMakeLists.txt | 2 +- python/CMakeLists.txt | 2 +- .../examples/minimal_build/Dockerfile.ubuntu | 5 +- python/examples/minimal_build/README.md | 2 +- r/configure.win | 7 +- r/tools/nixlibs.R | 10 +- 36 files changed, 516 insertions(+), 59 deletions(-) create mode 100644 ci/rtools/BuildAwsCCommon.patch create mode 100644 ci/rtools/aws_c_io_ep.patch create mode 100644 ci/rtools/awssdk_ep.patch diff --git a/.env b/.env index 252f59ccfb2e6..37b791abd1d23 100644 --- a/.env +++ b/.env @@ -54,6 +54,7 @@ UBUNTU=22.04 # Default versions for various dependencies CLANG_TOOLS=14 +CMAKE_VERSION=3.25.0 CUDA=11.2.2 DASK=latest DOTNET=8.0 diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 7dfe987d2eaff..66f5e8c7148e0 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -165,6 +165,11 @@ jobs: with: fetch-depth: 0 submodules: recursive + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ + echo "/tmp/local/bin" >> $GITHUB_PATH - name: Check CMake presets run: | cd cpp @@ -324,6 +329,11 @@ jobs: with: fetch-depth: 0 submodules: recursive + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh 3.25.0 /c/cmake + echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash run: ci/scripts/download_tz_database.sh diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index f9718cbf7bb18..44cbd66b7fed3 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -53,6 +53,11 @@ jobs: uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: 3.12 + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ + echo "/tmp/local/bin" >> $GITHUB_PATH - name: Install pre-commit run: | python -m pip install pre-commit diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index cb000f8b95c1b..0e9fa781d0e42 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -282,6 +282,12 @@ jobs: restore-keys: | r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- + - name: Install CMake via MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: true + install: mingw-w64-ucrt-x86_64-cmake - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ diff --git a/ci/docker/centos-7-cpp.dockerfile b/ci/docker/centos-7-cpp.dockerfile index 1f30eed694e4e..b012a5abed2e0 100644 --- a/ci/docker/centos-7-cpp.dockerfile +++ b/ci/docker/centos-7-cpp.dockerfile @@ -37,7 +37,6 @@ RUN \ -e 's/mirror\.centos\.org/vault.centos.org/' \ /etc/yum.repos.d/CentOS-SCLo-scl*.repo && \ yum install -y \ - cmake3 \ curl \ devtoolset-8 \ diffutils \ @@ -49,9 +48,13 @@ RUN \ wget \ which +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN bash /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin ENV \ ARROW_R_DEV=TRUE \ - CMAKE=/usr/bin/cmake3 + CMAKE=/usr/local/bin/cmake diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index 7b7e989adc0d1..da378eac43028 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -51,6 +51,10 @@ RUN /arrow/ci/scripts/r_docker_configure.sh COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index 0b5645285b6e1..ffcaa8c0a0741 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -53,7 +53,7 @@ ENV PATH=/opt/python/${CPYTHON_VERSION}-${CPYTHON_VERSION}/bin:${PATH} # Install CMake ARG cmake=3.29.2 COPY ci/scripts/install_cmake.sh arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local # Install Ninja ARG ninja=1.10.2 diff --git a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile index 1b342df596c9d..79a0c786aafba 100644 --- a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile @@ -27,7 +27,6 @@ RUN apt-get update -y -q && \ apt-get install -y -q \ build-essential \ ccache \ - cmake \ curl \ gdb \ git \ @@ -68,6 +67,10 @@ RUN latest_system_llvm=10 && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 259c5fb77fa41..db7e03b8bfb77 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -67,7 +67,6 @@ RUN apt-get update -y -q && \ autoconf \ ca-certificates \ ccache \ - cmake \ curl \ g++ \ gcc \ @@ -121,6 +120,10 @@ RUN apt-get update -y -q && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile index ce31c457e909e..2a90a5637d4df 100644 --- a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile @@ -68,6 +68,10 @@ RUN latest_system_llvm=14 && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 721b37dcae842..4a3d311acbf23 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -71,7 +71,6 @@ RUN apt-get update -y -q && \ ceph \ ceph-fuse \ ceph-mds \ - cmake \ curl \ gdb \ git \ @@ -168,6 +167,10 @@ RUN if [ "${gcc}" = "" ]; then \ # make sure zlib is cached in the EMSDK folder RUN source ~/emsdk/emsdk_env.sh && embuilder --pic build zlib +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/rtools/BuildAwsCCommon.patch b/ci/rtools/BuildAwsCCommon.patch new file mode 100644 index 0000000000000..f129b6f12344d --- /dev/null +++ b/ci/rtools/BuildAwsCCommon.patch @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/cmake/AwsCFlags.cmake b/cmake/AwsCFlags.cmake +index b717bca..5aa8ac9 100644 +--- a/cmake/AwsCFlags.cmake ++++ b/cmake/AwsCFlags.cmake +@@ -120,6 +120,10 @@ function(aws_set_common_properties target) + list(APPEND AWS_C_FLAGS -Wno-strict-aliasing) + endif() + ++ if(CMAKE_C_IMPLICIT_LINK_LIBRARIES MATCHES "mingw32") ++ list(APPEND AWS_C_FLAGS -D__USE_MINGW_ANSI_STDIO=1 -Wno-unused-local-typedefs) ++ endif() ++ + # -moutline-atomics generates code for both older load/store exclusive atomics and also + # Arm's Large System Extensions (LSE) which scale substantially better on large core count systems. + # +diff --git a/include/aws/common/byte_order.inl b/include/aws/common/byte_order.inl +index 1204be0..0abd9cb 100644 +--- a/include/aws/common/byte_order.inl ++++ b/include/aws/common/byte_order.inl +@@ -13,7 +13,7 @@ + # include + #else + # include +-#endif /* _MSC_VER */ ++#endif /* _WIN32 */ + + AWS_EXTERN_C_BEGIN + +@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) { + uint64_t v; + __asm__("bswap %q0" : "=r"(v) : "0"(x)); + return v; +-#elif defined(_MSC_VER) ++#elif defined(_WIN32) + return _byteswap_uint64(x); + #else + uint32_t low = x & UINT32_MAX; diff --git a/ci/rtools/aws_c_io_ep.patch b/ci/rtools/aws_c_io_ep.patch new file mode 100644 index 0000000000000..960b7d0d5e4c4 --- /dev/null +++ b/ci/rtools/aws_c_io_ep.patch @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/source/windows/host_resolver.c b/source/windows/host_resolver.c +index 59fbb85..ad4a99e 100644 +--- a/source/windows/host_resolver.c ++++ b/source/windows/host_resolver.c +@@ -4,6 +4,7 @@ + */ + + /* don't move this below the Windows.h include!!!!*/ ++#define _WIN32_WINNT 0x0601 + #include + #include + +diff --git a/source/windows/iocp/iocp_event_loop.c b/source/windows/iocp/iocp_event_loop.c +index 9ccce30..5cbbef7 100644 +--- a/source/windows/iocp/iocp_event_loop.c ++++ b/source/windows/iocp/iocp_event_loop.c +@@ -12,6 +12,7 @@ + + #include + ++#define _WIN32_WINNT 0x0601 + #include + + /* The next set of struct definitions are taken directly from the +diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure_channel_tls_handler.c +index 50caf02..29fe850 100644 +--- a/source/windows/secure_channel_tls_handler.c ++++ b/source/windows/secure_channel_tls_handler.c +@@ -2,6 +2,7 @@ + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ ++#define _WIN32_WINNT 0x0601 + #define SECURITY_WIN32 + + #include +@@ -35,6 +36,25 @@ + # pragma warning(disable : 4306) /* Identifier is type cast to a larger pointer. */ + #endif + ++#ifndef SP_PROT_TLS1_0_SERVER ++#define SP_PROT_TLS1_0_SERVER SP_PROT_TLS1_SERVER ++#endif ++#ifndef SP_PROT_TLS1_0_CLIENT ++#define SP_PROT_TLS1_0_CLIENT SP_PROT_TLS1_CLIENT ++#endif ++#ifndef SP_PROT_TLS1_1_SERVER ++#define SP_PROT_TLS1_1_SERVER 0x00000100 ++#endif ++#ifndef SP_PROT_TLS1_1_CLIENT ++#define SP_PROT_TLS1_1_CLIENT 0x00000200 ++#endif ++#ifndef SCH_USE_STRONG_CRYPTO ++#define SCH_USE_STRONG_CRYPTO 0x00400000 ++#endif ++#ifndef SECBUFFER_ALERT ++#define SECBUFFER_ALERT 0x11 ++#endif ++ + #define KB_1 1024 + #define READ_OUT_SIZE (16 * KB_1) + #define READ_IN_SIZE READ_OUT_SIZE +@@ -456,7 +476,7 @@ static int s_fillin_alpn_data( + + *extension_length += sizeof(uint32_t) + sizeof(uint16_t); + +- *extension_name = SecApplicationProtocolNegotiationExt_ALPN; ++ *extension_name = 2; + /*now add the protocols*/ + for (size_t i = 0; i < protocols_count; ++i) { + struct aws_byte_cursor *protocol_ptr = NULL; diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch new file mode 100644 index 0000000000000..5920ace5a4968 --- /dev/null +++ b/ci/rtools/awssdk_ep.patch @@ -0,0 +1,213 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/aws-cpp-sdk-core/include/aws/core/utils/Array.h b/aws-cpp-sdk-core/include/aws/core/utils/Array.h +index 2b5bbc566..7cb93bdf0 100644 +--- a/aws-cpp-sdk-core/include/aws/core/utils/Array.h ++++ b/aws-cpp-sdk-core/include/aws/core/utils/Array.h +@@ -54,7 +54,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(arrayToCopy, arrayToCopy + arraySize, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(arrayToCopy, arrayToCopy + arraySize, m_data.get()); +@@ -82,7 +82,7 @@ namespace Aws + if(arr->m_size > 0 && arr->m_data) + { + size_t arraySize = arr->m_size; +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, stdext::checked_array_iterator< T * >(m_data.get() + location, m_size)); + #else + std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, m_data.get() + location); +@@ -101,7 +101,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); +@@ -134,7 +134,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); +diff --git a/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h b/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h +index e26e36b60..3e7189b70 100644 +--- a/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h ++++ b/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h +@@ -29,7 +29,14 @@ namespace Aws + { + namespace Crypto + { +- static const char* SecureRandom_BCrypt_Tag = "SecureRandom_BCrypt"; ++#ifdef __MINGW32__ ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wunused-variable" ++#endif ++ static const char* SecureRandom_BCrypt_Tag = "SecureRandom_BCrypt"; ++#ifdef __MINGW32__ ++#pragma GCC diagnostic pop ++#endif + + class SecureRandomBytes_BCrypt : public SecureRandomBytes + { +diff --git a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +index 4dade6489..a0456cf8e 100644 +--- a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp ++++ b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +@@ -22,6 +22,16 @@ + #include + #include + ++#ifndef WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL ++#define WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL 116 ++#endif ++#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 ++#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 0x00000200 ++#endif ++#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 ++#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 0x00000800 ++#endif ++ + using namespace Aws::Client; + using namespace Aws::Http; + using namespace Aws::Http::Standard; +@@ -272,7 +282,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptr(dwSize / sizeof(wchar_t))); + + WinHttpQueryHeaders(hHttpRequest, WINHTTP_QUERY_CONTENT_TYPE, nullptr, &contentTypeStr, &dwSize, 0); +- if (contentTypeStr[0] != NULL) ++ if (contentTypeStr[0]) + { + Aws::String contentStr = StringUtils::FromWString(contentTypeStr); + response->SetContentType(contentStr); +@@ -303,7 +313,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptrSetContentType(contentTypeStr); + AWS_LOGSTREAM_DEBUG(GetLogTag(), "Received content type " << contentTypeStr); +diff --git a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp +index d7513cc3c..e390a8d4e 100644 +--- a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp ++++ b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp +@@ -349,7 +349,7 @@ std::shared_ptr WinSyncHttpClient::MakeRequest(const std::shared_p + } + } + +- if (!success && !IsRequestProcessingEnabled() || !ContinueRequest(*request)) ++ if (!success && (!IsRequestProcessingEnabled() || !ContinueRequest(*request))) + { + response->SetClientErrorType(CoreErrors::USER_CANCELLED); + response->SetClientErrorMessage("Request processing disabled or continuation cancelled by user's continuation handler."); +diff --git a/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp b/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp +index f6e36077e..13e349aef 100644 +--- a/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp ++++ b/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp +@@ -3,6 +3,7 @@ + * SPDX-License-Identifier: Apache-2.0. + */ + ++#define _WIN32_WINNT 0x0601 + #include + #include + #include +diff --git a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp +index 2ea82de6f..bc423441e 100644 +--- a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp ++++ b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp +@@ -11,7 +11,9 @@ + #include + #include + ++#ifdef _MSC_VER + #pragma warning( disable : 4996) ++#endif + + using namespace Aws::Utils; + namespace Aws +@@ -304,6 +306,9 @@ Aws::String CreateTempFilePath() + { + #ifdef _MSC_VER + #pragma warning(disable: 4996) // _CRT_SECURE_NO_WARNINGS ++#elif !defined(L_tmpnam_s) ++ // Definition from the MSVC stdio.h ++ #define L_tmpnam_s (sizeof("\\") + 16) + #endif + char s_tempName[L_tmpnam_s+1]; + +diff --git a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp +index 0180f7fbf..3adbab313 100644 +--- a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp ++++ b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp +@@ -9,7 +9,9 @@ + + #include + ++#ifdef _MSC_VER + #pragma warning(disable: 4996) ++#endif + #include + #include + namespace Aws +diff --git a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp +index 2ee517b48..3b0dce665 100644 +--- a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp ++++ b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp +@@ -939,7 +939,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateSha256HMACIm + return GetSha256HMACFactory()->CreateImplementation(); + } + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning( push ) + #pragma warning( disable : 4702 ) + #endif +@@ -1032,7 +1032,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateAES_KeyWrapImplementa + return GetAES_KeyWrapFactory()->CreateImplementation(key); + } + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning(pop) + #endif diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index efeed954006c1..56a463f6422fb 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -24,8 +24,7 @@ pkgdesc="Apache Arrow is a cross-language development platform for in-memory dat arch=("any") url="https://arrow.apache.org/" license=("Apache-2.0") -depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" - "${MINGW_PACKAGE_PREFIX}-bzip2" +depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" @@ -36,7 +35,6 @@ depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" "${MINGW_PACKAGE_PREFIX}-zstd" "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" - "${MINGW_PACKAGE_PREFIX}-cmake" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") @@ -82,8 +80,13 @@ build() { # CMAKE_UNITY_BUILD is set to OFF as otherwise some compute functionality # segfaults in tests + # MSYS2_ARG_CONV_EXCL is needed to prevent autoconverting CMAKE_INSTALL_PREFIX + # to Windows paths. See https://www.msys2.org/docs/filesystem-paths/#process-arguments + + # We install CMake via the msys2 GitHub action. We have to use the installed + # CMake to build Arrow which is in the Program Files directory. MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - ${MINGW_PREFIX}/bin/cmake.exe \ + "${PROGRAMFILES}\CMake\bin\cmake.exe" \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ @@ -116,6 +119,7 @@ build() { -DARROW_WITH_BZ2=ON \ -DARROW_ZSTD_USE_SHARED=OFF \ -DARROW_CXXFLAGS="${CPPFLAGS}" \ + -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE="release" \ -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ -DCMAKE_UNITY_BUILD=OFF \ diff --git a/ci/scripts/install_cmake.sh b/ci/scripts/install_cmake.sh index 7fdb06d90f02c..e283288f14a9b 100755 --- a/ci/scripts/install_cmake.sh +++ b/ci/scripts/install_cmake.sh @@ -17,29 +17,49 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi declare -A archs -archs=([amd64]=x86_64 - [arch64]=aarch64 +archs=([x86_64]=x86_64 [arm64]=aarch64 - [arm64v8]=aarch64 - [x86_64]=x86_64) + [aarch64]=aarch64) + +arch=$(uname -m) +if [ -z ${archs[$arch]} ]; then + echo "Unsupported architecture: ${arch}" + exit 0 +fi +arch=${archs[$arch]} declare -A platforms platforms=([linux]=linux [macos]=macos [windows]=windows) -if [ "$#" -ne 4 ]; then - echo "Usage: $0 " - exit 1 -fi +version=$1 +prefix=$2 -arch=${archs[$1]} -platform=${platforms[$2]} -version=$3 -prefix=$4 +platform=$(uname) +case ${platform} in + Linux) + platform=linux + ;; + Darwin) + platform=macos + ;; + MSYS_NT*|MINGW64_NT*) + platform=windows + ;; + *) + echo "Unsupported platform: ${platform}" + exit 0 + ;; +esac mkdir -p ${prefix} url="https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${platform}-" diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index c9395eb243f76..af29858bfe83c 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -23,13 +23,8 @@ set -ex # Make sure it is absolute and exported export ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" -# Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN -# curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf -# curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" -# pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz -# pacman --noconfirm -Scc - pacman --noconfirm -Syy + RWINLIB_LIB_DIR="lib" : ${MINGW_ARCH:="mingw32 mingw64 ucrt64"} @@ -71,7 +66,7 @@ if [ -d mingw64/lib/ ]; then # Move the 64-bit versions of libarrow into the expected location mv mingw64/lib/*.a $DST_DIR/lib/x64 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*,nghttp2}.a $DST_DIR/lib/x64 + cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/x64 fi # Same for the 32-bit versions @@ -79,7 +74,7 @@ if [ -d mingw32/lib/ ]; then ls $MSYS_LIB_DIR/mingw32/lib/ mkdir -p $DST_DIR/lib/i386 mv mingw32/lib/*.a $DST_DIR/lib/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*,nghttp2}.a $DST_DIR/lib/i386 + cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/i386 fi # Do the same also for ucrt64 @@ -87,7 +82,7 @@ if [ -d ucrt64/lib/ ]; then ls $MSYS_LIB_DIR/ucrt64/lib/ mkdir -p $DST_DIR/lib/x64-ucrt mv ucrt64/lib/*.a $DST_DIR/lib/x64-ucrt - cp $MSYS_LIB_DIR/ucrt64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*,nghttp2}.a $DST_DIR/lib/x64-ucrt + cp $MSYS_LIB_DIR/ucrt64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/x64-ucrt fi # Create build artifact diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a7d80c2e96c23..f2500b3a72f40 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") # https://www.cmake.org/cmake/help/latest/policy/CMP0025.html diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f9459f4175c83..698ac2828af09 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5042,6 +5042,12 @@ macro(build_awssdk) string(APPEND AWS_C_FLAGS " -Wno-deprecated") string(APPEND AWS_CXX_FLAGS " -Wno-deprecated") endif() + # GH-44950: This is required to build under Rtools40 and we may be able to + # remove it if/when we no longer need to build under Rtools40 + if(WIN32 AND NOT MSVC) + string(APPEND AWS_C_FLAGS " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args") + string(APPEND AWS_CXX_FLAGS " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args") + endif() set(AWSSDK_COMMON_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} @@ -5079,6 +5085,29 @@ macro(build_awssdk) endif() list(APPEND AWSSDK_PATCH_COMMAND ${AWSSDK_UNUSED_DIRECTORIES}) + # WIP GH-44950 + # Patch parts of the AWSSDK EP so it builds cleanly under Rtools40 + if(WIN32 AND NOT MSVC) + find_program(PATCH patch REQUIRED) + # Patch aws_c_common to build under Rtools40 + set(AWS_C_COMMON_PATCH_COMMAND ${PATCH} -p1 -i + ${CMAKE_SOURCE_DIR}/../ci/rtools/BuildAwsCCommon.patch) + message(STATUS "Hello ${AWS_C_COMMON_PATCH_COMMAND}") + # aws_c_io_ep to build under Rtools40 + set(AWS_C_IO_PATCH_COMMAND ${PATCH} -p1 -i + ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_io_ep.patch) + message(STATUS "Hello ${AWS_C_IO_PATCH_COMMAND}") + # awssdk_ep to build under Rtools40 + list(APPEND + AWSSDK_PATCH_COMMAND + && + ${PATCH} + -p1 + -i + ${CMAKE_SOURCE_DIR}/../ci/rtools/awssdk_ep.patch) + message(STATUS "Hello ${AWSSDK_PATCH_COMMAND}") + endif() + if(UNIX) # on Linux and macOS curl seems to be required find_curl() @@ -5173,6 +5202,7 @@ macro(build_awssdk) ${EP_COMMON_OPTIONS} URL ${AWS_C_COMMON_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM}" + PATCH_COMMAND ${AWS_C_COMMON_PATCH_COMMAND} CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY}) add_dependencies(AWS::aws-c-common aws_c_common_ep) @@ -5268,6 +5298,7 @@ macro(build_awssdk) ${EP_COMMON_OPTIONS} URL ${AWS_C_IO_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" + PATCH_COMMAND ${AWS_C_IO_PATCH_COMMAND} CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} DEPENDS ${AWS_C_IO_DEPENDS}) diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 95dad34221add..689dba437e033 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(ArrowMinimalExample) diff --git a/cpp/examples/minimal_build/minimal.dockerfile b/cpp/examples/minimal_build/minimal.dockerfile index 9361fc5e81d4d..8062e9b698437 100644 --- a/cpp/examples/minimal_build/minimal.dockerfile +++ b/cpp/examples/minimal_build/minimal.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/cpp/examples/minimal_build/system_dependency.dockerfile b/cpp/examples/minimal_build/system_dependency.dockerfile index 926fcaf6f4baa..84a16c4902f3a 100644 --- a/cpp/examples/minimal_build/system_dependency.dockerfile +++ b/cpp/examples/minimal_build/system_dependency.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/cpp/examples/parquet/parquet_arrow/CMakeLists.txt b/cpp/examples/parquet/parquet_arrow/CMakeLists.txt index 0480391e3800e..189d17914d678 100644 --- a/cpp/examples/parquet/parquet_arrow/CMakeLists.txt +++ b/cpp/examples/parquet/parquet_arrow/CMakeLists.txt @@ -16,7 +16,7 @@ # under the License. # Require cmake that supports BYPRODUCTS in add_custom_command, ExternalProject_Add [1]. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(parquet_arrow_example) diff --git a/cpp/examples/tutorial_examples/CMakeLists.txt b/cpp/examples/tutorial_examples/CMakeLists.txt index 8788501484c87..a6f8350c41dfe 100644 --- a/cpp/examples/tutorial_examples/CMakeLists.txt +++ b/cpp/examples/tutorial_examples/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(ArrowTutorialExamples) diff --git a/cpp/examples/tutorial_examples/tutorial.dockerfile b/cpp/examples/tutorial_examples/tutorial.dockerfile index 9361fc5e81d4d..8062e9b698437 100644 --- a/cpp/examples/tutorial_examples/tutorial.dockerfile +++ b/cpp/examples/tutorial_examples/tutorial.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index 031bad94227e8..b353d7f06f2f4 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -63,7 +63,7 @@ jobs: - name: Install CMake 3.29.0 shell: bash run: | - arrow/ci/scripts/install_cmake.sh $(arch) macos 3.29.0 ${PWD}/local + arrow/ci/scripts/install_cmake.sh 3.29.0 ${PWD}/local echo "${PWD}/local/bin" >> $GITHUB_PATH - name: Retrieve VCPKG version from arrow/.env diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 14bd83c90e5b4..f5e26db5c0b57 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1374,8 +1374,8 @@ tasks: template: docker-tests/github.cuda.yml params: env: - UBUNTU: 22.04 CUDA: 11.7.1 + UBUNTU: 22.04 image: ubuntu-cuda-cpp test-cuda-python-ubuntu-22.04-cuda-11.7.1: @@ -1383,8 +1383,8 @@ tasks: template: docker-tests/github.cuda.yml params: env: - UBUNTU: 22.04 CUDA: 11.7.1 + UBUNTU: 22.04 image: ubuntu-cuda-python ############################## Fuzz tests ################################# diff --git a/dev/tasks/vcpkg-tests/github.windows.yml b/dev/tasks/vcpkg-tests/github.windows.yml index af12db595286f..0fd4892c0a67c 100644 --- a/dev/tasks/vcpkg-tests/github.windows.yml +++ b/dev/tasks/vcpkg-tests/github.windows.yml @@ -32,7 +32,7 @@ jobs: - name: Install CMake 3.29.0 shell: bash run: | - arrow/ci/scripts/install_cmake.sh amd64 windows 3.29.0 /c/cmake + arrow/ci/scripts/install_cmake.sh 3.29.0 /c/cmake echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash diff --git a/docker-compose.yml b/docker-compose.yml index 87383568aaba4..46101daed67cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -390,6 +390,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size @@ -426,6 +427,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size @@ -455,6 +457,8 @@ services: dockerfile: ci/docker/centos-7-cpp.dockerfile cache_from: - ${REPO}:centos-7-cpp-static + args: + cmake: ${CMAKE_VERSION} shm_size: *shm-size volumes: - .:/arrow:delegated @@ -477,6 +481,7 @@ services: args: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -571,6 +576,7 @@ services: arch: ${ARCH} base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU} clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -621,6 +627,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -700,6 +707,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -1587,6 +1595,7 @@ services: tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} r_custom_ccache: ${R_CUSTOM_CCACHE} + cmake: ${CMAKE_VERSION} shm_size: *shm-size environment: <<: [*common, *sccache] @@ -1616,6 +1625,7 @@ services: - ${REPO}:amd64-ubuntu-20.04-r-sanitizer args: base: wch1/r-debug:latest + cmake: ${CMAKE_VERSION} r_bin: RDsan tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} @@ -1636,6 +1646,7 @@ services: - ${REPO}:r-rhub-clang-devel-latest args: base: rhub/clang-asan + cmake: ${CMAKE_VERSION} r_dev: ${ARROW_R_DEV} r_bin: R tz: ${TZ} @@ -1667,6 +1678,7 @@ services: - ${REPO}:amd64-ubuntu-20.04-r-valgrind args: base: wch1/r-debug:latest + cmake: ${CMAKE_VERSION} r_bin: RDvalgrind tz: ${TZ} environment: diff --git a/docs/source/cpp/build_system.rst b/docs/source/cpp/build_system.rst index e80bca4c949dc..5f89866a16b75 100644 --- a/docs/source/cpp/build_system.rst +++ b/docs/source/cpp/build_system.rst @@ -46,7 +46,7 @@ file into an executable linked with the Arrow C++ shared library: .. code-block:: cmake - cmake_minimum_required(VERSION 3.16) + cmake_minimum_required(VERSION 3.25) project(MyExample) diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index cd6d8768d03ee..07de5a612b399 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) # Build the Arrow C++ libraries using ExternalProject_Add. function(build_arrow) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 0b0275f200d9d..ac497e844e74f 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -18,7 +18,7 @@ # Includes code assembled from BSD/MIT/Apache-licensed code from some 3rd-party # projects, including Kudu, Impala, and libdynd. See python/LICENSE.txt -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(pyarrow) # This is needed for 3.13 free-threading. CMake used to add Python diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index 07cd69c082461..5c6b113168478 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:jammy +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive @@ -33,7 +33,6 @@ RUN apt-get update -y -q && \ python3-pip \ python3-venv \ tzdata \ + tzdata-legacy \ && \ apt-get clean && rm -rf /var/lib/apt/lists* - -RUN pip3 install --no-cache-dir -U pip setuptools diff --git a/python/examples/minimal_build/README.md b/python/examples/minimal_build/README.md index e9d595b603103..2ed7a653e7b48 100644 --- a/python/examples/minimal_build/README.md +++ b/python/examples/minimal_build/README.md @@ -51,7 +51,7 @@ docker compose run --rm minimal-fedora-conda docker compose run --rm minimal-fedora-venv ``` -## Ubuntu 20.04 +## Ubuntu 24.04 First, build the Docker image using: ``` diff --git a/r/configure.win b/r/configure.win index b6ac19faea2d4..a8f8422478bf8 100755 --- a/r/configure.win +++ b/r/configure.win @@ -61,10 +61,9 @@ function configure_binaries() { OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" BROTLI_LIBS="-lbrotlienc -lbrotlidec -lbrotlicommon" # Common goes last since dec and enc depend on it - AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management \ - -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 \ - -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common \ - -luserenv -lversion -lws2_32 -lbcrypt -lwininet -lwinhttp" + # We build aws-sdk-cpp bundled now, so the AWS libs are included in arrow_bundled_dependencies + # but we also need to include these Windows system libraries + AWS_LIBS="-luserenv -lversion -lws2_32 -lbcrypt -lwininet -lwinhttp -lsecur32 -lshlwapi -lncrypt" # pkg-config --libs libcurl GCS_LIBS="-lcurl -lnormaliz -lssh2 -lgdi32 -lssl -lcrypto -lcrypt32 -lwldap32 \ -lz -lws2_32 -lnghttp2 -ldbghelp" diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 6c68133cd893d..d3b9d2ce12e96 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -635,12 +635,12 @@ build_libarrow <- function(src_dir, dst_dir) { invisible(status) } -ensure_cmake <- function(cmake_minimum_required = "3.16") { +ensure_cmake <- function(cmake_minimum_required = "3.25") { cmake <- find_cmake(version_required = cmake_minimum_required) if (is.null(cmake)) { # If not found, download it - CMAKE_VERSION <- Sys.getenv("CMAKE_VERSION", "3.26.4") + CMAKE_VERSION <- Sys.getenv("CMAKE_VERSION", "3.31.2") if (on_macos) { postfix <- "-macos-universal.tar.gz" } else if (tolower(Sys.info()[["machine"]]) %in% c("arm64", "aarch64")) { @@ -698,7 +698,7 @@ find_cmake <- function(paths = c( if (on_macos) "/Applications/CMake.app/Contents/bin/cmake", Sys.which("cmake3") ), - version_required = "3.16") { + version_required) { # Given a list of possible cmake paths, return the first one that exists and is new enough # version_required should be a string or packageVersion; numeric version # can be misleading (e.g. 3.10 is actually 3.1) @@ -715,7 +715,7 @@ find_cmake <- function(paths = c( } else { # Keep trying lg("Not using cmake found at %s", path, .indent = "****") - if (found_version > 0) { + if (found_version > "0") { lg("Version >= %s required; found %s", version_required, found_version, .indent = "*****") } else { # If cmake_version() couldn't determine version, it returns 0 @@ -737,7 +737,7 @@ cmake_version <- function(cmd = "cmake") { package_version(sub(pat, "\\1", raw_version[which_line])) }, error = function(e) { - return(0) + return("0") } ) } From 4b88388cfd672dc67c371166e4c316b76ad68d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 21 Jan 2025 18:15:31 +0100 Subject: [PATCH 02/10] Fix some review comments Rever CMAKE_VERSION back to CMAKE on .env Fix typo on file name --- .env | 2 +- ...AwsCCommon.patch => aws_c_common_ep.patch} | 2 +- ci/scripts/install_cmake.sh | 5 ----- cpp/cmake_modules/ThirdpartyToolchain.cmake | 12 +++++++--- docker-compose.yml | 22 +++++++++---------- 5 files changed, 22 insertions(+), 21 deletions(-) rename ci/rtools/{BuildAwsCCommon.patch => aws_c_common_ep.patch} (96%) diff --git a/.env b/.env index 37b791abd1d23..0651d189c247f 100644 --- a/.env +++ b/.env @@ -54,7 +54,7 @@ UBUNTU=22.04 # Default versions for various dependencies CLANG_TOOLS=14 -CMAKE_VERSION=3.25.0 +CMAKE=3.25.0 CUDA=11.2.2 DASK=latest DOTNET=8.0 diff --git a/ci/rtools/BuildAwsCCommon.patch b/ci/rtools/aws_c_common_ep.patch similarity index 96% rename from ci/rtools/BuildAwsCCommon.patch rename to ci/rtools/aws_c_common_ep.patch index f129b6f12344d..f1357070429ef 100644 --- a/ci/rtools/BuildAwsCCommon.patch +++ b/ci/rtools/aws_c_common_ep.patch @@ -23,7 +23,7 @@ index b717bca..5aa8ac9 100644 list(APPEND AWS_C_FLAGS -Wno-strict-aliasing) endif() -+ if(CMAKE_C_IMPLICIT_LINK_LIBRARIES MATCHES "mingw32") ++ if (CMAKE_C_IMPLICIT_LINK_LIBRARIES MATCHES "mingw32") + list(APPEND AWS_C_FLAGS -D__USE_MINGW_ANSI_STDIO=1 -Wno-unused-local-typedefs) + endif() + diff --git a/ci/scripts/install_cmake.sh b/ci/scripts/install_cmake.sh index e283288f14a9b..d01a7a744dca8 100755 --- a/ci/scripts/install_cmake.sh +++ b/ci/scripts/install_cmake.sh @@ -36,11 +36,6 @@ if [ -z ${archs[$arch]} ]; then fi arch=${archs[$arch]} -declare -A platforms -platforms=([linux]=linux - [macos]=macos - [windows]=windows) - version=$1 prefix=$2 diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 698ac2828af09..05784c5f6a363 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5045,8 +5045,14 @@ macro(build_awssdk) # GH-44950: This is required to build under Rtools40 and we may be able to # remove it if/when we no longer need to build under Rtools40 if(WIN32 AND NOT MSVC) - string(APPEND AWS_C_FLAGS " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args") - string(APPEND AWS_CXX_FLAGS " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args") + string(APPEND + AWS_C_FLAGS + " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args" + ) + string(APPEND + AWS_CXX_FLAGS + " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args" + ) endif() set(AWSSDK_COMMON_CMAKE_ARGS @@ -5091,7 +5097,7 @@ macro(build_awssdk) find_program(PATCH patch REQUIRED) # Patch aws_c_common to build under Rtools40 set(AWS_C_COMMON_PATCH_COMMAND ${PATCH} -p1 -i - ${CMAKE_SOURCE_DIR}/../ci/rtools/BuildAwsCCommon.patch) + ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_common_ep.patch) message(STATUS "Hello ${AWS_C_COMMON_PATCH_COMMAND}") # aws_c_io_ep to build under Rtools40 set(AWS_C_IO_PATCH_COMMAND ${PATCH} -p1 -i diff --git a/docker-compose.yml b/docker-compose.yml index 46101daed67cd..22a8e99f5ef10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -390,7 +390,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size @@ -427,7 +427,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size @@ -458,7 +458,7 @@ services: cache_from: - ${REPO}:centos-7-cpp-static args: - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} shm_size: *shm-size volumes: - .:/arrow:delegated @@ -481,7 +481,7 @@ services: args: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -576,7 +576,7 @@ services: arch: ${ARCH} base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU} clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -627,7 +627,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -707,7 +707,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -1595,7 +1595,7 @@ services: tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} r_custom_ccache: ${R_CUSTOM_CCACHE} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} shm_size: *shm-size environment: <<: [*common, *sccache] @@ -1625,7 +1625,7 @@ services: - ${REPO}:amd64-ubuntu-20.04-r-sanitizer args: base: wch1/r-debug:latest - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} r_bin: RDsan tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} @@ -1646,7 +1646,7 @@ services: - ${REPO}:r-rhub-clang-devel-latest args: base: rhub/clang-asan - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} r_dev: ${ARROW_R_DEV} r_bin: R tz: ${TZ} @@ -1678,7 +1678,7 @@ services: - ${REPO}:amd64-ubuntu-20.04-r-valgrind args: base: wch1/r-debug:latest - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} r_bin: RDvalgrind tz: ${TZ} environment: From 908b0801fa96914ddfb728a5ec62cb510ff09108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:06:28 +0100 Subject: [PATCH 03/10] Update some workflows to use Ubuntu 24.04 in order to remove manual CMake installation --- .github/workflows/cpp.yml | 12 +----------- .github/workflows/dev.yml | 7 +------ 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 66f5e8c7148e0..6504654400912 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -156,7 +156,7 @@ jobs: build-example: name: C++ Minimal Build Example - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 45 steps: @@ -165,11 +165,6 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ - echo "/tmp/local/bin" >> $GITHUB_PATH - name: Check CMake presets run: | cd cpp @@ -329,11 +324,6 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh 3.25.0 /c/cmake - echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash run: ci/scripts/download_tz_database.sh diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 44cbd66b7fed3..04303f2b1bc9f 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -41,7 +41,7 @@ jobs: lint: name: Lint C++, Python, R, Docker, RAT - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 15 steps: @@ -53,11 +53,6 @@ jobs: uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: python-version: 3.12 - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ - echo "/tmp/local/bin" >> $GITHUB_PATH - name: Install pre-commit run: | python -m pip install pre-commit From 48a6d0a13f084261b05664a330a308e2cf3e8a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:13:34 +0100 Subject: [PATCH 04/10] Try to use newer CMake from base Windows-2019 image on Windows R build instead of installing via MSYS2 --- .github/workflows/r.yml | 6 ------ ci/scripts/PKGBUILD | 4 +--- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 0e9fa781d0e42..cb000f8b95c1b 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -282,12 +282,6 @@ jobs: restore-keys: | r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - - name: Install CMake via MSYS2 - uses: msys2/setup-msys2@v2 - with: - msystem: UCRT64 - update: true - install: mingw-w64-ucrt-x86_64-cmake - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 56a463f6422fb..9ecdc2ad2e92a 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -83,10 +83,8 @@ build() { # MSYS2_ARG_CONV_EXCL is needed to prevent autoconverting CMAKE_INSTALL_PREFIX # to Windows paths. See https://www.msys2.org/docs/filesystem-paths/#process-arguments - # We install CMake via the msys2 GitHub action. We have to use the installed - # CMake to build Arrow which is in the Program Files directory. MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - "${PROGRAMFILES}\CMake\bin\cmake.exe" \ + cmake \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From 0d37aca77fa228440f1cc8a291b8131aaf77f60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:26:53 +0100 Subject: [PATCH 05/10] Locate CMake --- .github/workflows/r.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index cb000f8b95c1b..3eba85bd54738 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -290,6 +290,8 @@ jobs: r-version: "4.1" rtools-version: 40 Ncpus: 2 + - name: Locate CMAKE + run: where cmake - name: Build Arrow C++ shell: bash env: From fb44d93b40a66a6a168500ad637819e8e69954bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:27:07 +0100 Subject: [PATCH 06/10] Try installing cmake on Linux packages --- .../linux-packages/apache-arrow/yum/centos-7/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index b10c40937d32a..3b467a8748683 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -50,7 +50,6 @@ RUN \ brotli-devel \ bzip2-devel \ ccache \ - cmake3 \ curl-devel \ flex \ gflags-devel \ @@ -75,6 +74,10 @@ RUN \ zlib-devel && \ yum clean ${quiet} all +ARG cmake=3.25.0 +COPY ../../../../../ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + ENV \ BOOST_INCLUDEDIR=/usr/include/boost169 \ BOOST_LIBRARYDIR=/usr/lib64/boost169 From 089a9810e3436b91cf410ba51d11be0829f794f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:32:49 +0100 Subject: [PATCH 07/10] Locate CMake v2 --- .github/workflows/r.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 3eba85bd54738..df331ef254ef5 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -291,7 +291,10 @@ jobs: rtools-version: 40 Ncpus: 2 - name: Locate CMAKE - run: where cmake + shell: bash + run: | + cmake --version + which cmake - name: Build Arrow C++ shell: bash env: From 9ae7ca161ea8aa316f06fc1ff0d40e0d66de2edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:44:24 +0100 Subject: [PATCH 08/10] Use full path to CMake on Program Files --- ci/scripts/PKGBUILD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 9ecdc2ad2e92a..c09efe3ee1207 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -83,8 +83,10 @@ build() { # MSYS2_ARG_CONV_EXCL is needed to prevent autoconverting CMAKE_INSTALL_PREFIX # to Windows paths. See https://www.msys2.org/docs/filesystem-paths/#process-arguments + # We require the full path to the CMake executable in order to build Arrow + # which is in the Program Files directory: "/c/Program Files/CMake/bin/cmake" MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - cmake \ + "${PROGRAMFILES}\CMake\bin\cmake.exe" \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From 9cdbc70b706308ac03770b79764a68f24c062afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:45:11 +0100 Subject: [PATCH 09/10] Try centos7 path (this is just testing at this point) --- dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 3b467a8748683..3e687519efd9d 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -75,7 +75,7 @@ RUN \ yum clean ${quiet} all ARG cmake=3.25.0 -COPY ../../../../../ci/scripts/install_cmake.sh /arrow/ci/scripts/ +COPY ../../../../../../../ci/scripts/install_cmake.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ ENV \ From 79de38093161af7cc626787d2ba2eb7c25043142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 12:28:35 +0100 Subject: [PATCH 10/10] Try using link for context on Dockerfile --- dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile | 4 ++-- dev/tasks/linux-packages/scripts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 120000 dev/tasks/linux-packages/scripts diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 3e687519efd9d..db43f56a489a3 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -75,8 +75,8 @@ RUN \ yum clean ${quiet} all ARG cmake=3.25.0 -COPY ../../../../../../../ci/scripts/install_cmake.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ +COPY scripts/install_cmake.sh /ci/scripts/ +RUN /ci/scripts/install_cmake.sh ${cmake} /usr/local/ ENV \ BOOST_INCLUDEDIR=/usr/include/boost169 \ diff --git a/dev/tasks/linux-packages/scripts b/dev/tasks/linux-packages/scripts new file mode 120000 index 0000000000000..1762358f1e023 --- /dev/null +++ b/dev/tasks/linux-packages/scripts @@ -0,0 +1 @@ +../../../ci/scripts \ No newline at end of file