Fix the hdfs connect issue when enable kerberos

apache · Apr 22, 2024 · 96378f4 · 96378f4
1 parent a14f081
commit 96378f4
Show file tree

Hide file tree

Showing 7 changed files with 13 additions and 8 deletions.
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
@@ -36,7 +36,7 @@ class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
       .loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
       .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
       .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
-      .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+      .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false)
       .loadAndCreateLink("libre2.so.10", "libre2.so", false)
       .loadAndCreateLink("libzstd.so.1", "libzstd.so", false)
       .loadAndCreateLink("liblz4.so.1", "liblz4.so", false)

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
@@ -41,7 +41,7 @@ class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
       .loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
       .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
       .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
-      .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+      .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false)
       .loadAndCreateLink("libre2.so.0", "libre2.so", false)
       .loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
       .commit()

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
@@ -46,7 +46,7 @@ class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
       .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
       .loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
       .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
-      .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+      .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false)
       .commit()
   }
 }
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
@@ -52,7 +52,7 @@ class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
       .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
       .loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
       .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
-      .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+      .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false)
       .commit()
   }
 }
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
@@ -59,7 +59,7 @@ class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader {
       .loadAndCreateLink("libicudata.so.66", "libicudata.so", false)
       .loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false)
       .loadAndCreateLink("libxml2.so.2", "libxml2.so", false)
-      .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+      .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false)
       .loadAndCreateLink("libre2.so.5", "libre2.so", false)
       .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
       .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false)

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
@@ -44,7 +44,7 @@ class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader {
       .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
       .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
       .loadAndCreateLink("libxml2.so.2", "libxml2.so", false)
-      .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false)
+      .loadAndCreateLink("libhdfs.so.0.0.0", "libhdfs.so", false)
       .loadAndCreateLink("libre2.so.9", "libre2.so", false)
       .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
       .loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false)

diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
@@ -407,8 +407,13 @@ endif()
 
 if(ENABLE_HDFS)
   add_definitions(-DENABLE_HDFS)
-  find_libhdfs3()
-  target_link_libraries(velox PUBLIC HDFS::hdfs3)
+  set(HADOOP_HOME $ENV{HADOOP_HOME})
+  find_library(
+    LIBHDFS
+    NAMES libhdfs.so
+    HINTS "${HADOOP_HOME}/lib/native" REQUIRED)
+  # find_libhdfs3()
+  target_link_libraries(velox PUBLIC ${LIBHDFS})
 endif()
 
 if(ENABLE_S3)