From c4d9db2714442a905efe6e15f57c10ef4ca6e0f5 Mon Sep 17 00:00:00 2001 From: deepa <8832194+deepashreeraghu@users.noreply.github.com> Date: Tue, 23 Jul 2024 19:45:13 +0530 Subject: [PATCH] [VL] Fix for centos9 build of Gluten (#6183) * Merge conflicts resolution * New build script updated * Handled review comments * removed BUILD_PROTOBUF from centos9 * Updated based on review comment * Handling review comments * Typo during merge added back --------- Co-authored-by: Deepashree Gandhi --- .../backendsapi/velox/VeloxListenerApi.scala | 4 +- .../utils/SharedLibraryLoaderCentos9.scala | 49 +++++++++++++++++++ dev/build_helper_functions.sh | 1 + dev/package.sh | 10 +++- ep/build-velox/src/get_velox.sh | 40 +++++++++++++++ 5 files changed, 102 insertions(+), 2 deletions(-) create mode 100755 backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index e5c3cb084819..86925fd1d6a8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -66,6 +66,8 @@ class VeloxListenerApi extends ListenerApi { new SharedLibraryLoaderUbuntu2004 } else if (systemName.contains("Ubuntu") && systemVersion.startsWith("22.04")) { new SharedLibraryLoaderUbuntu2204 + } else if (systemName.contains("CentOS") && systemVersion.startsWith("9")) { + new SharedLibraryLoaderCentos9 } else if (systemName.contains("CentOS") && systemVersion.startsWith("8")) { new SharedLibraryLoaderCentos8 } else if (systemName.contains("CentOS") && systemVersion.startsWith("7")) { @@ -83,7 +85,7 @@ class VeloxListenerApi extends ListenerApi { } else if (system.contains("tencentos") && system.contains("3.2")) { new SharedLibraryLoaderCentos8 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("9")) { - new SharedLibraryLoaderCentos8 + new SharedLibraryLoaderCentos9 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("8")) { new SharedLibraryLoaderCentos8 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("7")) { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala new file mode 100755 index 000000000000..2d9ececb366d --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.utils + +import org.apache.gluten.vectorized.JniLibLoader + +class SharedLibraryLoaderCentos9 extends SharedLibraryLoader { + override def loadLib(loader: JniLibLoader): Unit = { + loader + .newTransaction() + .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) + .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) + .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) + .loadAndCreateLink("libicudata.so.67", "libicudata.so", false) + .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) + .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) + .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + .loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) + .loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) + .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + .loadAndCreateLink("libglog.so.1", "libglog.so", false) + .loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false) + .loadAndCreateLink("libidn.so.12", "libidn.so", false) + .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + .loadAndCreateLink("libre2.so.9", "libre2.so", false) + .loadAndCreateLink("libsodium.so.23", "libsodium.so", false) + .commit() + } +} diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh index 8559911c1cd4..92ae5c29a884 100644 --- a/dev/build_helper_functions.sh +++ b/dev/build_helper_functions.sh @@ -135,6 +135,7 @@ function setup_linux { scripts/setup-ubuntu.sh elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then case "$LINUX_VERSION_ID" in + 9) scripts/setup-centos9.sh ;; 8) scripts/setup-centos8.sh ;; 7) scripts/setup-centos7.sh diff --git a/dev/package.sh b/dev/package.sh index 7e7e793bdabd..db9125c2f473 100755 --- a/dev/package.sh +++ b/dev/package.sh @@ -27,6 +27,12 @@ function process_setup_ubuntu_2204 { cp /usr/local/lib/{libhdfs3.so.1,libprotobuf.so.32,libboost_context.so.1.84.0,libboost_regex.so.1.84.0} $THIRDPARTY_LIB/ } +function process_setup_centos_9 { + cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libgsasl.so.7,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libidn.so.12,libntlm.so.0,libsodium.so.23} $THIRDPARTY_LIB/ + cp /usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32} $THIRDPARTY_LIB/ + cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1} $THIRDPARTY_LIB/ +} + function process_setup_centos_8 { cp /usr/lib64/{libre2.so.0,libdouble-conversion.so.3,libevent-2.1.so.6,libdwarf.so.1,libgsasl.so.7,libicudata.so.60,libicui18n.so.60,libicuuc.so.60,libidn.so.11,libntlm.so.0,libsodium.so.23} $THIRDPARTY_LIB/ cp /usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32} $THIRDPARTY_LIB/ @@ -56,7 +62,9 @@ if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then process_setup_ubuntu_2204 fi elif [ "$LINUX_OS" == "centos" ]; then - if [ "$VERSION" == "8" ]; then + if [ "$VERSION" == "9" ]; then + process_setup_centos_9 + elif [ "$VERSION" == "8" ]; then process_setup_centos_8 elif [ "$VERSION" == "7" ]; then process_setup_centos_7 diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index d01d7eb12b26..818bdb99d2e3 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -137,6 +137,45 @@ function process_setup_ubuntu { sed -i '/run_and_time install_arrow/d' scripts/setup-ubuntu.sh } +function process_setup_centos9 { + # Allows other version of git already installed. + if [ -z "$(which git)" ]; then + dnf install -y -q --setopt=install_weak_deps=False git + fi + # make this function Reentrant + git checkout scripts/setup-centos9.sh + # No need to re-install git. + + ensure_pattern_matched 'dnf_install' scripts/setup-centos9.sh + sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-12 git/dnf_install ninja-build cmake curl ccache gcc-toolset-12/' scripts/setup-centos9.sh + sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}' scripts/setup-centos9.sh + sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos9.sh + + ensure_pattern_matched 'install_gflags' scripts/setup-centos9.sh + sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos9.sh + + ensure_pattern_matched 'install_fbthrift' scripts/setup-centos9.sh + sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos9.sh + sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh + + if [ $ENABLE_HDFS == "ON" ]; then + sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n cmake_install\n}\n' scripts/setup-centos9.sh + sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_libhdfs3' scripts/setup-centos9.sh + sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos9.sh + fi + sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh + if [ $ENABLE_S3 == "ON" ]; then + sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos9.sh + fi + if [ $ENABLE_GCS == "ON" ]; then + sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos9.sh + fi + if [ $ENABLE_ABFS == "ON" ]; then + sed -i '/^ run_and_time install_fbthrift/a \ \ export AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-centos9.sh + fi +} + + function process_setup_centos8 { # Allows other version of git already installed. if [ -z "$(which git)" ]; then @@ -287,6 +326,7 @@ function setup_linux { process_setup_ubuntu elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then case "$LINUX_VERSION_ID" in + 9) process_setup_centos9 ;; 8) process_setup_centos8 ;; 7) process_setup_centos7 ;; *)