Support jvm libhdfs in velox

apache · Apr 22, 2024 · a14f081 · a14f081
1 parent 9720043
commit a14f081
Show file tree

Hide file tree

Showing 7 changed files with 17 additions and 119 deletions.
diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml
@@ -64,6 +64,16 @@ jobs:
       - name: Build Gluten Velox third party
         if: ${{ steps.cache.outputs.cache-hit != 'true' }}
         run: |
+          yum update -y && yum install -y java-1.8.0-openjdk-devel wget
+          export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+          
+          echo "JAVA_HOME: $JAVA_HOME"
+
+          wget https://archive.apache.org/dist/hadoop/core/hadoop-2.10.1/hadoop-2.10.1.tar.gz
+          tar xf hadoop-2.10.1.tar.gz -C /usr/local/
+          export HADOOP_HOME='/usr/local/hadoop-2.10.1'
+          echo "HADOOP_HOME: $HADOOP_HOME"
+          
           source dev/ci-velox-buildstatic.sh
       - uses: actions/upload-artifact@v2
         with:

diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
@@ -330,9 +330,9 @@ set(VELOX_SRCS
     utils/Common.cc
     )
 
-if (ENABLE_HDFS)
-  list(APPEND VELOX_SRCS utils/HdfsUtils.cc)
-endif ()
+# if (ENABLE_HDFS)
+#   list(APPEND VELOX_SRCS utils/HdfsUtils.cc)
+# endif ()
 
 if(ENABLE_S3)
   find_package(ZLIB)

diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -24,10 +24,6 @@
 
 #include "utils/ConfigExtractor.h"
 
-#ifdef ENABLE_HDFS
-#include "utils/HdfsUtils.h"
-#endif
-
 using namespace facebook;
 
 namespace gluten {
@@ -113,9 +109,9 @@ WholeStageResultIterator::WholeStageResultIterator(
       scanNodeIds_(scanNodeIds),
       scanInfos_(scanInfos),
       streamIds_(streamIds) {
-#ifdef ENABLE_HDFS
-  gluten::updateHdfsTokens(veloxCfg_.get());
-#endif
+  // #ifdef ENABLE_HDFS
+  //   gluten::updateHdfsTokens(veloxCfg_.get());
+  // #endif
   spillStrategy_ = veloxCfg_->get<std::string>(kSpillStrategy, kSpillStrategyDefaultValue);
   getOrderedNodeIds(veloxPlan_, orderedNodeIds_);
 

diff --git a/cpp/velox/utils/HdfsUtils.cc b/cpp/velox/utils/HdfsUtils.cc
diff --git a/cpp/velox/utils/HdfsUtils.h b/cpp/velox/utils/HdfsUtils.h
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
 set -exu
 
 VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_04_19
+VELOX_BRANCH=libhdfs
 VELOX_HOME=""
 
 #Set on run gluten on HDFS

diff --git a/ep/build-velox/src/modify_velox.patch b/ep/build-velox/src/modify_velox.patch
@@ -39,26 +39,6 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt
 index 53aaf4391..90aba6916 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -243,10 +243,15 @@ if(VELOX_ENABLE_ABFS)
- endif()
-
- if(VELOX_ENABLE_HDFS)
--  find_library(
--    LIBHDFS3
--    NAMES libhdfs3.so libhdfs3.dylib
--    HINTS "${CMAKE_SOURCE_DIR}/hawq/depends/libhdfs3/_build/src/" REQUIRED)
-+  find_package(libhdfs3)
-+  if(libhdfs3_FOUND AND TARGET HDFS::hdfs3)
-+    set(LIBHDFS3 HDFS::hdfs3)
-+  else()
-+    find_library(
-+      LIBHDFS3
-+      NAMES libhdfs3.so libhdfs3.dylib
-+      HINTS "${CMAKE_SOURCE_DIR}/hawq/depends/libhdfs3/_build/src/" REQUIRED)
-+  endif()
-   add_definitions(-DVELOX_ENABLE_HDFS3)
- endif()
-
 @@ -386,7 +391,7 @@ resolve_dependency(Boost 1.77.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES})
  # for reference. find_package(range-v3)