From e278d5db44eb8dc71b7b574c884e53c20cece4eb Mon Sep 17 00:00:00 2001 From: Deepashree Gandhi Date: Wed, 17 Jul 2024 22:33:49 +0530 Subject: [PATCH 1/7] Merge conflicts resolution --- .../backendsapi/velox/VeloxListenerApi.scala | 2 + .../utils/SharedLibraryLoaderCentos9.scala | 49 +++++++++++++++++++ dev/package.sh | 10 +++- ep/build-velox/src/get_velox.sh | 40 +++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100755 backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index e5c3cb084819..d5aabda46b03 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -66,6 +66,8 @@ class VeloxListenerApi extends ListenerApi { new SharedLibraryLoaderUbuntu2004 } else if (systemName.contains("Ubuntu") && systemVersion.startsWith("22.04")) { new SharedLibraryLoaderUbuntu2204 + } else if (systemName.contains("CentOS") && systemVersion.startsWith("9")) { + new SharedLibraryLoaderCentos9 } else if (systemName.contains("CentOS") && systemVersion.startsWith("8")) { new SharedLibraryLoaderCentos8 } else if (systemName.contains("CentOS") && systemVersion.startsWith("7")) { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala new file mode 100755 index 000000000000..2d9ececb366d --- /dev/null +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.utils + +import org.apache.gluten.vectorized.JniLibLoader + +class SharedLibraryLoaderCentos9 extends SharedLibraryLoader { + override def loadLib(loader: JniLibLoader): Unit = { + loader + .newTransaction() + .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) + .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) + .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) + .loadAndCreateLink("libicudata.so.67", "libicudata.so", false) + .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) + .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) + .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + .loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) + .loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) + .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + .loadAndCreateLink("libglog.so.1", "libglog.so", false) + .loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false) + .loadAndCreateLink("libidn.so.12", "libidn.so", false) + .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + .loadAndCreateLink("libre2.so.9", "libre2.so", false) + .loadAndCreateLink("libsodium.so.23", "libsodium.so", false) + .commit() + } +} diff --git a/dev/package.sh b/dev/package.sh index 7e7e793bdabd..db9125c2f473 100755 --- a/dev/package.sh +++ b/dev/package.sh @@ -27,6 +27,12 @@ function process_setup_ubuntu_2204 { cp /usr/local/lib/{libhdfs3.so.1,libprotobuf.so.32,libboost_context.so.1.84.0,libboost_regex.so.1.84.0} $THIRDPARTY_LIB/ } +function process_setup_centos_9 { + cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libgsasl.so.7,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libidn.so.12,libntlm.so.0,libsodium.so.23} $THIRDPARTY_LIB/ + cp /usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32} $THIRDPARTY_LIB/ + cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1} $THIRDPARTY_LIB/ +} + function process_setup_centos_8 { cp /usr/lib64/{libre2.so.0,libdouble-conversion.so.3,libevent-2.1.so.6,libdwarf.so.1,libgsasl.so.7,libicudata.so.60,libicui18n.so.60,libicuuc.so.60,libidn.so.11,libntlm.so.0,libsodium.so.23} $THIRDPARTY_LIB/ cp /usr/local/lib/{libhdfs3.so.1,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0,libprotobuf.so.32} $THIRDPARTY_LIB/ @@ -56,7 +62,9 @@ if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then process_setup_ubuntu_2204 fi elif [ "$LINUX_OS" == "centos" ]; then - if [ "$VERSION" == "8" ]; then + if [ "$VERSION" == "9" ]; then + process_setup_centos_9 + elif [ "$VERSION" == "8" ]; then process_setup_centos_8 elif [ "$VERSION" == "7" ]; then process_setup_centos_7 diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 51ea70849a8f..754a6efaeb9b 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -115,6 +115,45 @@ function process_setup_ubuntu { sed -i '/run_and_time install_arrow/d' scripts/setup-ubuntu.sh } +function process_setup_centos9 { + # Allows other version of git already installed. + if [ -z "$(which git)" ]; then + dnf install -y -q --setopt=install_weak_deps=False git + fi + # make this function Reentrant + git checkout scripts/setup-centos9.sh + # need set BUILD_SHARED_LIBS flag for thrift + #sed -i "/cd fbthrift/{n;s/cmake_install -Denable_tests=OFF/cmake_install -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF/;}" scripts/setup-centos9.sh + # No need to re-install git. + sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-12 git/dnf_install ninja-build cmake curl ccache gcc-toolset-12/' scripts/setup-centos9.sh + sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}' scripts/setup-centos9.sh + sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos9.sh + sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos9.sh + sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos9.sh + + if [ $ENABLE_HDFS == "ON" ]; then + sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n cmake_install\n}\n' scripts/setup-centos9.sh + sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_libhdfs3' scripts/setup-centos9.sh + sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos9.sh + fi + if [[ $BUILD_PROTOBUF == "ON" ]] || [[ $ENABLE_HDFS == "ON" ]]; then + sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh + fi + sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh + if [ $ENABLE_S3 == "ON" ]; then + sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos9.sh + # Maybe already installed. + #sed -i 's/rpm -i minio-20220526054841.0.0.x86_64.rpm/rpm -i --replacepkgs minio-20220526054841.0.0.x86_64.rpm/g' scripts/setup-adapters.sh + fi + if [ $ENABLE_GCS == "ON" ]; then + sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos9.sh + fi + if [ $ENABLE_ABFS == "ON" ]; then + sed -i '/^ run_and_time install_fbthrift/a \ \ export AZURE_SDK_DISABLE_AUTO_VCPKG=ON \n '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-centos9.sh + fi +} + + function process_setup_centos8 { # Allows other version of git already installed. if [ -z "$(which git)" ]; then @@ -270,6 +309,7 @@ function setup_linux { process_setup_ubuntu elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then case "$LINUX_VERSION_ID" in + 9) process_setup_centos9 ;; 8) process_setup_centos8 ;; 7) process_setup_centos7 ;; *) From 72dbf52df9493867a9205f88a55e3bc91ef40955 Mon Sep 17 00:00:00 2001 From: Deepashree Gandhi Date: Wed, 17 Jul 2024 22:47:48 +0530 Subject: [PATCH 2/7] New build script updated --- dev/build_helper_functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh index 8559911c1cd4..92ae5c29a884 100644 --- a/dev/build_helper_functions.sh +++ b/dev/build_helper_functions.sh @@ -135,6 +135,7 @@ function setup_linux { scripts/setup-ubuntu.sh elif [[ "$LINUX_DISTRIBUTION" == "centos" ]]; then case "$LINUX_VERSION_ID" in + 9) scripts/setup-centos9.sh ;; 8) scripts/setup-centos8.sh ;; 7) scripts/setup-centos7.sh From 4dbf0c4932d2ddec8e0e823083fdaff8a7af69c5 Mon Sep 17 00:00:00 2001 From: Deepashree Gandhi Date: Wed, 17 Jul 2024 22:51:53 +0530 Subject: [PATCH 3/7] Handled review comments --- ep/build-velox/src/get_velox.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 754a6efaeb9b..7ed6e9d99f1b 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -122,8 +122,6 @@ function process_setup_centos9 { fi # make this function Reentrant git checkout scripts/setup-centos9.sh - # need set BUILD_SHARED_LIBS flag for thrift - #sed -i "/cd fbthrift/{n;s/cmake_install -Denable_tests=OFF/cmake_install -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF/;}" scripts/setup-centos9.sh # No need to re-install git. sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-12 git/dnf_install ninja-build cmake curl ccache gcc-toolset-12/' scripts/setup-centos9.sh sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}' scripts/setup-centos9.sh @@ -142,8 +140,6 @@ function process_setup_centos9 { sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos9.sh - # Maybe already installed. - #sed -i 's/rpm -i minio-20220526054841.0.0.x86_64.rpm/rpm -i --replacepkgs minio-20220526054841.0.0.x86_64.rpm/g' scripts/setup-adapters.sh fi if [ $ENABLE_GCS == "ON" ]; then sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos9.sh From a3696e16c5c6e689d1043a2761a8703c05177d70 Mon Sep 17 00:00:00 2001 From: Deepashree Gandhi Date: Thu, 18 Jul 2024 14:21:51 +0530 Subject: [PATCH 4/7] removed BUILD_PROTOBUF from centos9 --- ep/build-velox/src/get_velox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 7ed6e9d99f1b..c3ec240636f8 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -134,7 +134,7 @@ function process_setup_centos9 { sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_libhdfs3' scripts/setup-centos9.sh sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos9.sh fi - if [[ $BUILD_PROTOBUF == "ON" ]] || [[ $ENABLE_HDFS == "ON" ]]; then + if [[ $ENABLE_HDFS == "ON" ]]; then sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh fi sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh From fd8139e0a679db13c0aef707199401749f8d9574 Mon Sep 17 00:00:00 2001 From: Deepashree Gandhi Date: Fri, 19 Jul 2024 20:29:48 +0530 Subject: [PATCH 5/7] Updated based on review comment --- ep/build-velox/src/get_velox.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index c3ec240636f8..f8c46d4a32f3 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -128,15 +128,13 @@ function process_setup_centos9 { sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos9.sh sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos9.sh sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos9.sh + sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh if [ $ENABLE_HDFS == "ON" ]; then sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n cmake_install\n}\n' scripts/setup-centos9.sh sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_libhdfs3' scripts/setup-centos9.sh sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos9.sh fi - if [[ $ENABLE_HDFS == "ON" ]]; then - sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh - fi sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos9.sh From dfb146d88ccda74bdff277642d10e309115d1e70 Mon Sep 17 00:00:00 2001 From: Deepashree Gandhi Date: Mon, 22 Jul 2024 15:53:33 +0530 Subject: [PATCH 6/7] Handling review comments --- ep/build-velox/src/get_velox.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 49330fd9c784..f136efd5edbc 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -145,10 +145,16 @@ function process_setup_centos9 { # make this function Reentrant git checkout scripts/setup-centos9.sh # No need to re-install git. + + ensure_pattern_matched 'dnf_install' scripts/setup-centos9.sh sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-12 git/dnf_install ninja-build cmake curl ccache gcc-toolset-12/' scripts/setup-centos9.sh sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}' scripts/setup-centos9.sh sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos9.sh + + ensure_pattern_matched 'install_gflags' scripts/setup-centos9.sh sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos9.sh + + ensure_pattern_matched 'install_fbthrift' scripts/setup-centos9.sh sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos9.sh sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos9.sh From 52337862e064e1bc992c62b4074c36226fb9f09e Mon Sep 17 00:00:00 2001 From: Deepashree Gandhi Date: Mon, 22 Jul 2024 22:38:42 +0530 Subject: [PATCH 7/7] Typo during merge added back --- .../org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index d5aabda46b03..86925fd1d6a8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -85,7 +85,7 @@ class VeloxListenerApi extends ListenerApi { } else if (system.contains("tencentos") && system.contains("3.2")) { new SharedLibraryLoaderCentos8 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("9")) { - new SharedLibraryLoaderCentos8 + new SharedLibraryLoaderCentos9 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("8")) { new SharedLibraryLoaderCentos8 } else if (systemName.contains("Red Hat") && systemVersion.startsWith("7")) {