From 96378f4314200c3df50a2d3a912bf1338f602de3 Mon Sep 17 00:00:00 2001 From: Jia Ke Date: Thu, 18 Apr 2024 16:55:56 +0800 Subject: [PATCH] Fix the hdfs connect issue when enable kerberos --- .../apache/gluten/utils/SharedLibraryLoaderCentos7.scala | 2 +- .../apache/gluten/utils/SharedLibraryLoaderCentos8.scala | 2 +- .../gluten/utils/SharedLibraryLoaderDebian11.scala | 2 +- .../gluten/utils/SharedLibraryLoaderDebian12.scala | 2 +- .../gluten/utils/SharedLibraryLoaderUbuntu2004.scala | 2 +- .../gluten/utils/SharedLibraryLoaderUbuntu2204.scala | 2 +- cpp/velox/CMakeLists.txt | 9 +++++++-- 7 files changed, 13 insertions(+), 8 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala index 47ed2c47cbb5d..d77bb145e1497 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala @@ -36,7 +36,7 @@ class SharedLibraryLoaderCentos7 extends SharedLibraryLoader { .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false) .loadAndCreateLink("libre2.so.10", "libre2.so", false) .loadAndCreateLink("libzstd.so.1", "libzstd.so", false) .loadAndCreateLink("liblz4.so.1", "liblz4.so", false) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala index c1d3bf2e26cb7..cf7e01d329fd8 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala @@ -41,7 +41,7 @@ class SharedLibraryLoaderCentos8 extends SharedLibraryLoader { .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false) .loadAndCreateLink("libre2.so.0", "libre2.so", false) .loadAndCreateLink("libsodium.so.23", "libsodium.so", false) .commit() diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala index ca7d1d22d9840..514d84ad0b53d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala @@ -46,7 +46,7 @@ class SharedLibraryLoaderDebian11 extends SharedLibraryLoader { .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false) .commit() } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala index 128c8eaa2aef2..abd82f4bdbf89 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala @@ -52,7 +52,7 @@ class SharedLibraryLoaderDebian12 extends SharedLibraryLoader { .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false) .commit() } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala index 18f2e6cfbeb32..e0985e11589ba 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala @@ -59,7 +59,7 @@ class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader { .loadAndCreateLink("libicudata.so.66", "libicudata.so", false) .loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false) .loadAndCreateLink("libxml2.so.2", "libxml2.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false) .loadAndCreateLink("libre2.so.5", "libre2.so", false) .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala index b23105b7dce05..58569f125f393 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala @@ -44,7 +44,7 @@ class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader { .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) .loadAndCreateLink("libxml2.so.2", "libxml2.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false) .loadAndCreateLink("libre2.so.9", "libre2.so", false) .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) .loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false) diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 770ecb6f80fbf..bbed772872947 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -407,8 +407,13 @@ endif() if(ENABLE_HDFS) add_definitions(-DENABLE_HDFS) - find_libhdfs3() - target_link_libraries(velox PUBLIC HDFS::hdfs3) + set(HADOOP_HOME $ENV{HADOOP_HOME}) + find_library( + LIBHDFS + NAMES libhdfs.so + HINTS "${HADOOP_HOME}/lib/native" REQUIRED) + # find_libhdfs3() + target_link_libraries(velox PUBLIC ${LIBHDFS}) endif() if(ENABLE_S3)