[GLUTEN-3582][CH] Remove ArrowParquetBlockInputFormat and OptimizedPa…

…rquetBlockInputFormat (#4626) * remove use_local_format * make ut works * fix style
apache · Feb 4, 2024 · 9e1e5d5 · 9e1e5d5
1 parent 70c0d68
commit 9e1e5d5
Show file tree

Hide file tree

Showing 20 changed files with 5 additions and 10,764 deletions.
diff --git a/...house/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHParquetAQESuite.scala b/...house/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHParquetAQESuite.scala
@@ -44,7 +44,7 @@ class GlutenClickHouseTPCHParquetAQESuite
       .set("spark.sql.shuffle.partitions", "5")
       .set("spark.sql.autoBroadcastJoinThreshold", "10MB")
       .set("spark.sql.adaptive.enabled", "true")
-      .set("spark.gluten.sql.columnar.backend.ch.runtime_config.use_local_format", "true")
+      .set("spark.gluten.sql.columnar.backend.ch.runtime_config.use_local_format", "false")
       .set("spark.gluten.sql.columnar.backend.ch.shuffle.hash.algorithm", "sparkMurmurHash3_32")
   }
 

diff --git a/cpp-ch/local-engine/CMakeLists.txt b/cpp-ch/local-engine/CMakeLists.txt
@@ -139,7 +139,7 @@ PRIVATE
         substrait
 )
 
-target_link_libraries(${LOCALENGINE_SHARED_LIB} PUBLIC ch_parquet)
+#TODO: target_link_libraries(${LOCALENGINE_SHARED_LIB} PUBLIC ch_parquet)
 
 if (ENABLE_JEMALLOC)
     target_link_options(${LOCALENGINE_SHARED_LIB} PRIVATE

diff --git a/cpp-ch/local-engine/Storages/ArrowParquetBlockInputFormat.cpp b/cpp-ch/local-engine/Storages/ArrowParquetBlockInputFormat.cpp
diff --git a/cpp-ch/local-engine/Storages/ArrowParquetBlockInputFormat.h b/cpp-ch/local-engine/Storages/ArrowParquetBlockInputFormat.h
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
@@ -29,7 +29,6 @@
 #include <Processors/Formats/Impl/ArrowBufferedStreams.h>
 #include <Processors/Formats/Impl/ArrowColumnToCHColumn.h>
 #include <Processors/Formats/Impl/ParquetBlockInputFormat.h>
-#include <Storages/ArrowParquetBlockInputFormat.h>
 #include <Storages/SubstraitSource/SubstraitFileSourceStep.h>
 #include <parquet/arrow/reader.h>
 #include <parquet/metadata.h>
@@ -80,9 +79,8 @@ FormatFile::InputFormatPtr ParquetFormatFile::createInputFormat(const DB::Block
         row_group_indices.reserve(required_row_groups.size());
         for (const auto & row_group : required_row_groups)
             row_group_indices.emplace_back(row_group.index);
-
-        res->input
-            = std::make_shared<local_engine::ArrowParquetBlockInputFormat>(*(res->read_buffer), header, format_settings, row_group_indices);
+        //TODO: use_local_format
+        assert(false);
     }
     else
     {

diff --git a/cpp-ch/local-engine/Storages/ch_parquet/CMakeLists.txt b/cpp-ch/local-engine/Storages/ch_parquet/CMakeLists.txt
@@ -12,48 +12,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-set(ARROW_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src")
-
-macro(add_headers_and_sources_including_cc prefix common_path)
-    add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h)
-    add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.c ${common_path}/*.cc ${common_path}/*.h)
-endmacro()
-
-add_headers_and_sources(ch_parquet .)
-add_headers_and_sources_including_cc(ch_parquet arrow)
-
-add_library(ch_parquet ${ch_parquet_sources})
-
-target_compile_options(ch_parquet PRIVATE -fPIC
-    -Wno-shorten-64-to-32
-    -Wno-shadow-field-in-constructor
-    -Wno-return-type
-    -Wno-reserved-identifier
-    -Wno-extra-semi-stmt
-    -Wno-extra-semi
-    -Wno-unused-result
-    -Wno-unreachable-code-return
-    -Wno-unused-parameter
-    -Wno-unreachable-code
-    -Wno-pessimizing-move
-    -Wno-unreachable-code-break
-    -Wno-unused-variable
-    -Wno-inconsistent-missing-override
-    -Wno-shadow-uncaptured-local
-    -Wno-suggest-override
-    -Wno-unused-member-function
-    -Wno-deprecated-this-capture
-    -Wno-tautological-unsigned-zero-compare
-)
-
-target_link_libraries(ch_parquet PUBLIC
-    boost::headers_only
-    clickhouse_common_io
-)
-
-target_include_directories(ch_parquet SYSTEM BEFORE PUBLIC 
-    ${ARROW_INCLUDE_DIR} 
-    ${CMAKE_BINARY_DIR}/contrib/arrow-cmake/cpp/src
-    ${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src
-)