[VL] Extract ORC reader compilation from GenericBenchmark (#3414)

apache · Oct 18, 2023 · 5d2ea9b · 5d2ea9b
1 parent 837f51c
commit 5d2ea9b
Show file tree

Hide file tree

Showing 17 changed files with 415 additions and 342 deletions.
diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml
@@ -79,11 +79,11 @@ jobs:
           cd /opt/gluten && \
           mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
           mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest'
-#       Cpp micro benchmarks will use generated files from unit test in backends-velox module.
-#      - name: Run micro benchmarks
-#        run: |
-#          docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/cpp/build/velox/benchmarks && \
-#          ./generic_benchmark --with-shuffle --threads 1 --iterations 1'
+      # Cpp micro benchmarks will use generated files from unit test in backends-velox module.
+      - name: Run micro benchmarks
+        run: |
+          docker exec ubuntu2004-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/cpp/build/velox/benchmarks && \
+          ./generic_benchmark --with-shuffle --threads 1 --iterations 1'
       - name: Exit docker container
         if: ${{ always() }}
         run: |

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -49,6 +49,7 @@ option(ENABLE_QAT "Enable QAT for de/compression" OFF)
 option(ENABLE_IAA "Enable IAA for de/compression" OFF)
 option(ENABLE_S3 "Enable S3" OFF)
 option(ENABLE_HDFS "Enable HDFS" OFF)
+option(ENABLE_ORC "Enable ORC" OFF)
 
 set(root_directory ${PROJECT_BINARY_DIR})
 get_filename_component(GLUTEN_HOME ${CMAKE_SOURCE_DIR} DIRECTORY)
@@ -187,6 +188,10 @@ if(ENABLE_IAA)
   add_definitions(-DGLUTEN_ENABLE_IAA)
 endif()
 
+if(ENABLE_ORC)
+  add_definitions(-DGLUTEN_ENABLE_ORC)
+endif()
+
 #
 # Subdirectories
 #

diff --git a/cpp/velox/benchmarks/BatchIteratorWrapper.h b/cpp/velox/benchmarks/BatchIteratorWrapper.h
diff --git a/cpp/velox/benchmarks/BatchStreamIterator.h b/cpp/velox/benchmarks/BatchStreamIterator.h
diff --git a/cpp/velox/benchmarks/CMakeLists.txt b/cpp/velox/benchmarks/CMakeLists.txt
@@ -15,23 +15,28 @@
 
 find_arrow_lib(${PARQUET_LIB_NAME})
 
+set(VELOX_BENCHMARK_COMMON_SRCS common/FileReaderIterator.cc common/BenchmarkUtils.cc)
+add_library(velox_benchmark_common STATIC ${VELOX_BENCHMARK_COMMON_SRCS})
+target_include_directories(velox_benchmark_common PUBLIC ${CMAKE_SOURCE_DIR}/velox ${CMAKE_SOURCE_DIR}/core)
+target_link_libraries(velox_benchmark_common PUBLIC Arrow::parquet velox benchmark::benchmark gflags simdjson)
+
 function(add_velox_benchmark BM_EXEC BM_FILE)
-  add_executable(${BM_EXEC} ${BM_FILE} BenchmarkUtils.cc)
-  target_include_directories(${BM_EXEC} PRIVATE ${CMAKE_SOURCE_DIR}/velox ${CMAKE_SOURCE_DIR}/src)
-  target_link_libraries(${BM_EXEC} gflags Arrow::parquet velox benchmark::benchmark simdjson)
+  add_executable(${BM_EXEC} ${BM_FILE})
+  target_link_libraries(${BM_EXEC} PRIVATE velox_benchmark_common)
 endfunction()
 
 # Query benchmark
 add_velox_benchmark(query_benchmark QueryBenchmark.cc)
 
 # Generic benchmark
-# TODO: This benchmark generates ORC file with Arrow ORC file writer.
-# We need to compile Arrow with -DARROW_ORC first in Velox.
-# add_velox_benchmark(generic_benchmark GenericBenchmark.cc)
+add_velox_benchmark(generic_benchmark GenericBenchmark.cc)
 
 add_velox_benchmark(columnar_to_row_benchmark ColumnarToRowBenchmark.cc)
 
 add_velox_benchmark(parquet_write_benchmark ParquetWriteBenchmark.cc)
 
 add_velox_benchmark(shuffle_split_benchmark ShuffleSplitBenchmark.cc)
 
+if(ENABLE_ORC)
+  add_velox_benchmark(orc_converter exec/OrcConverter.cc)
+endif()