[GLUTEN-1632][CH]Daily Update Clickhouse Version (20240918) (apache#7260

) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240918) * Fix build due to ENABLE_ROCKSDB=OFF caused by apache#7239 * Fix UT build due to apache#6723 * Fix UT build due to apache#7193 * Fix Build due to ClickHouse/ClickHouse#69298 --------- Co-authored-by: kyligence-git <[email protected]> Co-authored-by: Chang Chen <[email protected]>
sharkdtu · Oct 14, 2024 · 5cbba95 · 5cbba95
1 parent 4d04df1
commit 5cbba95
Show file tree

Hide file tree

Showing 26 changed files with 62 additions and 63 deletions.
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
 CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20240906
-CH_COMMIT=4513a954b95
+CH_BRANCH=rebase_ch/20240918
+CH_COMMIT=cc6de0f1995
diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp
@@ -21,7 +21,6 @@
 #include <memory>
 #include <optional>
 #include <unistd.h>
-
 #include <AggregateFunctions/Combinators/AggregateFunctionCombinatorFactory.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <Columns/ColumnArray.h>
@@ -53,7 +52,6 @@
 #include <Parser/SubstraitParserUtils.h>
 #include <Planner/PlannerActionsVisitor.h>
 #include <Processors/Chunk.h>
-#include <Processors/Formats/IOutputFormat.h>
 #include <Processors/QueryPlan/ExpressionStep.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>

diff --git a/cpp-ch/local-engine/Disks/registerGlutenDisks.cpp b/cpp-ch/local-engine/Disks/registerGlutenDisks.cpp
@@ -71,7 +71,7 @@ void registerGlutenDisks(bool global_skip_access_check)
 #if USE_ROCKSDB
             metadata_storage = MetadataStorageFromRocksDB::create(name, config, config_prefix, object_storage);
 #else
-            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RocksDB metadata storage is not enabled in the build");
+            throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "RocksDB metadata storage is not enabled in the build");
 #endif
         }
         else
@@ -118,7 +118,7 @@ void registerGlutenDisks(bool global_skip_access_check)
 #if USE_ROCKSDB
             metadata_storage = MetadataStorageFromRocksDB::create(name, config, config_prefix, object_storage);
 #else
-            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RocksDB metadata storage is not enabled in the build");
+            throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "RocksDB metadata storage is not enabled in the build");
 #endif
         }
         else

diff --git a/cpp-ch/local-engine/Parser/LocalExecutor.cpp b/cpp-ch/local-engine/Parser/LocalExecutor.cpp
@@ -18,7 +18,7 @@
 #include "LocalExecutor.h"
 
 #include <Core/Settings.h>
-#include <Parser/SerializedPlanParser.h>
+#include <Interpreters/Context.h>
 #include <Processors/Executors/PipelineExecutor.h>
 #include <QueryPipeline/printPipeline.h>
 #include <Common/QueryContext.h>

diff --git a/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp b/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp
@@ -24,7 +24,6 @@
 #include <Join/BroadCastJoinBuilder.h>
 #include <Join/StorageJoinFromReadBuffer.h>
 #include <Parser/AdvancedParametersParseUtil.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Processors/QueryPlan/ExpressionStep.h>
 #include <Processors/QueryPlan/FilterStep.h>

diff --git a/cpp-ch/local-engine/Parser/RelParsers/ExpandRelParser.cpp b/cpp-ch/local-engine/Parser/RelParsers/ExpandRelParser.cpp
@@ -21,7 +21,6 @@
 #include <Operator/ExpandStep.h>
 #include <Parser/ExpandField.h>
 #include <Parser/RelParsers/RelParser.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Common/logger_useful.h>
 

diff --git a/cpp-ch/local-engine/Parser/RelParsers/FetchRelParser.cpp b/cpp-ch/local-engine/Parser/RelParsers/FetchRelParser.cpp
@@ -17,9 +17,9 @@
 
 #include <memory>
 #include <optional>
-#include <Parser/SerializedPlanParser.h>
+#include <Parser/RelParsers/RelParser.h>
 #include <Processors/QueryPlan/LimitStep.h>
-#include "RelParser.h"
+
 namespace local_engine
 {
 class FetchRelParser : public RelParser

diff --git a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
@@ -30,7 +30,6 @@
 #include <Join/StorageJoinFromReadBuffer.h>
 #include <Operator/EarlyStopStep.h>
 #include <Parser/AdvancedParametersParseUtil.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Processors/QueryPlan/ExpressionStep.h>
 #include <Processors/QueryPlan/FilterStep.h>

diff --git a/cpp-ch/local-engine/Parser/RelParsers/ProjectRelParser.cpp b/cpp-ch/local-engine/Parser/RelParsers/ProjectRelParser.cpp
@@ -15,7 +15,8 @@
  * limitations under the License.
  */
 #include "ProjectRelParser.h"
-#include <Interpreters/ArrayJoinAction.h>
+
+#include <Interpreters/ArrayJoin.h>
 #include <Operator/EmptyProjectStep.h>
 #include <Processors/QueryPlan/ArrayJoinStep.h>
 #include <Processors/QueryPlan/ExpressionStep.h>
@@ -198,9 +199,12 @@ ProjectRelParser::parseGenerate(DB::QueryPlanPtr query_plan, const substrait::Re
         }
 
         /// ARRAY JOIN
-        NameSet array_joined_columns{findArrayJoinNode(splitted_actions_dags.array_join)->result_name};
-        auto array_join_action = std::make_shared<ArrayJoinAction>(array_joined_columns, generate_rel.outer(), getContext());
-        auto array_join_step = std::make_unique<ArrayJoinStep>(query_plan->getCurrentDataStream(), array_join_action);
+        Names array_joined_columns{findArrayJoinNode(splitted_actions_dags.array_join)->result_name};
+        ArrayJoin array_join;
+        array_join.columns = std::move(array_joined_columns);
+        array_join.is_left = generate_rel.outer();
+        auto array_join_step = std::make_unique<ArrayJoinStep>(
+            query_plan->getCurrentDataStream(), std::move(array_join), false, getContext()->getSettingsRef().max_block_size);
         array_join_step->setStepDescription("ARRAY JOIN In Generate");
         steps.emplace_back(array_join_step.get());
         query_plan->addStep(std::move(array_join_step));

diff --git a/cpp-ch/local-engine/Parser/RelParsers/ReadRelParser.cpp b/cpp-ch/local-engine/Parser/RelParsers/ReadRelParser.cpp
@@ -17,6 +17,20 @@
 
 #include "ReadRelParser.h"
 #include <memory>
+#include <Core/Block.h>
+#include <Core/Settings.h>
+#include <Interpreters/Context.h>
+#include <Operator/BlocksBufferPoolTransform.h>
+#include <Parser/RelParsers/MergeTreeRelParser.h>
+#include <Parser/SubstraitParserUtils.h>
+#include <Parser/TypeParser.h>
+#include <Processors/QueryPlan/ReadFromPreparedSource.h>
+#include <Storages/SourceFromJavaIter.h>
+#include <Storages/SubstraitSource/SubstraitFileSource.h>
+#include <Storages/SubstraitSource/SubstraitFileSourceStep.h>
+#include <google/protobuf/wrappers.pb.h>
+#include <Common/BlockTypeUtils.h>
+
 
 namespace DB::ErrorCodes
 {

diff --git a/cpp-ch/local-engine/Parser/RelParsers/ReadRelParser.h b/cpp-ch/local-engine/Parser/RelParsers/ReadRelParser.h
@@ -15,24 +15,12 @@
  * limitations under the License.
  */
 #pragma once
-#include <Core/Block.h>
-#include <Core/Settings.h>
-#include <Interpreters/Context.h>
-#include <Operator/BlocksBufferPoolTransform.h>
-#include <Parser/RelParsers/MergeTreeRelParser.h>
-#include <Parser/SerializedPlanParser.h>
-#include <Parser/SubstraitParserUtils.h>
-#include <Parser/TypeParser.h>
-#include <Processors/QueryPlan/ReadFromPreparedSource.h>
-#include <Storages/SourceFromJavaIter.h>
-#include <Storages/SubstraitSource/SubstraitFileSource.h>
-#include <Storages/SubstraitSource/SubstraitFileSourceStep.h>
-#include <google/protobuf/wrappers.pb.h>
-#include <Common/BlockTypeUtils.h>
+#include <Parser/RelParsers/RelParser.h>
 #include <Common/JNIUtils.h>
 
 namespace local_engine
 {
+
 class ReadRelParser : public RelParser
 {
 public:

diff --git a/cpp-ch/local-engine/Parser/RelParsers/RelParser.cpp b/cpp-ch/local-engine/Parser/RelParsers/RelParser.cpp
@@ -17,13 +17,10 @@
 #include "RelParser.h"
 
 #include <string>
-#include <google/protobuf/wrappers.pb.h>
-
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <DataTypes/DataTypeAggregateFunction.h>
 #include <DataTypes/IDataType.h>
-#include <DataTypes/DataTypeNullable.h>
-#include <Poco/Logger.h>
+#include <google/protobuf/wrappers.pb.h>
 #include <Poco/StringTokenizer.h>
 #include <Common/Exception.h>
 #include <Common/logger_useful.h>

diff --git a/cpp-ch/local-engine/Parser/RelParsers/RelParser.h b/cpp-ch/local-engine/Parser/RelParsers/RelParser.h
@@ -27,6 +27,7 @@
 #include <base/types.h>
 #include <substrait/extensions/extensions.pb.h>
 #include <substrait/plan.pb.h>
+
 namespace local_engine
 {
 /// parse a single substrait relation
@@ -52,34 +53,37 @@ class RelParser
     inline SerializedPlanParser * getPlanParser() const { return plan_parser; }
     inline ContextPtr getContext() const { return plan_parser->context; }
 
-    inline String getUniqueName(const std::string & name) { return plan_parser->getUniqueName(name); }
+    inline String getUniqueName(const std::string & name) const { return plan_parser->getUniqueName(name); }
 
-    inline const std::unordered_map<std::string, std::string> & getFunctionMapping() { return plan_parser->function_mapping; }
+    inline const std::unordered_map<std::string, std::string> & getFunctionMapping() const { return plan_parser->function_mapping; }
 
     // Get function signature name.
     std::optional<String> parseSignatureFunctionName(UInt32 function_ref);
     // Get coresponding function name in ClickHouse.
     std::optional<String> parseFunctionName(UInt32 function_ref, const substrait::Expression_ScalarFunction & function);
 
-    const DB::ActionsDAG::Node * parseArgument(ActionsDAG & action_dag, const substrait::Expression & rel)
+    const DB::ActionsDAG::Node * parseArgument(ActionsDAG & action_dag, const substrait::Expression & rel) const
     {
         return plan_parser->parseExpression(action_dag, rel);
     }
 
-    const DB::ActionsDAG::Node * parseExpression(ActionsDAG & action_dag, const substrait::Expression & rel)
+    const DB::ActionsDAG::Node * parseExpression(ActionsDAG & action_dag, const substrait::Expression & rel) const
     {
         return plan_parser->parseExpression(action_dag, rel);
     }
-    DB::ActionsDAG expressionsToActionsDAG(const std::vector<substrait::Expression> & expressions, const DB::Block & header)
+    DB::ActionsDAG expressionsToActionsDAG(const std::vector<substrait::Expression> & expressions, const DB::Block & header) const
     {
         return plan_parser->expressionsToActionsDAG(expressions, header, header);
     }
-    std::pair<DataTypePtr, Field> parseLiteral(const substrait::Expression_Literal & literal) { return plan_parser->parseLiteral(literal); }
+    std::pair<DataTypePtr, Field> parseLiteral(const substrait::Expression_Literal & literal) const
+    {
+        return plan_parser->parseLiteral(literal);
+    }
     // collect all steps for metrics
     std::vector<IQueryPlanStep *> steps;
 
     const ActionsDAG::Node *
-    buildFunctionNode(ActionsDAG & action_dag, const String & function, const DB::ActionsDAG::NodeRawConstPtrs & args)
+    buildFunctionNode(ActionsDAG & action_dag, const String & function, const DB::ActionsDAG::NodeRawConstPtrs & args) const
     {
         return plan_parser->toFunctionNode(action_dag, function, args);
     }

diff --git a/cpp-ch/local-engine/Parser/RelParsers/WindowGroupLimitRelParser.h b/cpp-ch/local-engine/Parser/RelParsers/WindowGroupLimitRelParser.h
@@ -17,7 +17,6 @@
 #pragma once
 #include <optional>
 #include <Parser/RelParsers/RelParser.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Poco/Logger.h>
 #include <Common/logger_useful.h>

diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
@@ -85,7 +85,6 @@
 #include <Common/JNIUtils.h>
 #include <Common/logger_useful.h>
 #include <Common/typeid_cast.h>
-#include "RelParsers/RelParser.h"
 
 namespace DB
 {

diff --git a/cpp-ch/local-engine/Shuffle/PartitionWriter.h b/cpp-ch/local-engine/Shuffle/PartitionWriter.h
@@ -21,7 +21,6 @@
 #include <Core/Block.h>
 #include <Core/Settings.h>
 #include <Interpreters/TemporaryDataOnDisk.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Shuffle/ShuffleCommon.h>
 #include <jni/CelebornClient.h>
 #include <Common/GlutenConfig.h>
@@ -68,7 +67,7 @@ friend class Spillable;
     PartitionWriter(const SplitOptions& options, LoggerPtr logger_);
     virtual ~PartitionWriter() = default;
 
-    void initialize(SplitResult * split_result_, const Block & output_header_)
+    void initialize(SplitResult * split_result_, const DB::Block & output_header_)
     {
         if (!init)
         {
@@ -107,8 +106,8 @@ friend class Spillable;
 
     /// Only valid in celeborn partition writer
     size_t last_partition_id;
-    SplitResult* split_result = nullptr;
-    Block output_header;
+    SplitResult * split_result = nullptr;
+    DB::Block output_header;
     LoggerPtr logger = nullptr;
     bool init = false;
 };

diff --git a/cpp-ch/local-engine/Shuffle/ShuffleCommon.cpp b/cpp-ch/local-engine/Shuffle/ShuffleCommon.cpp
@@ -16,7 +16,6 @@
  */
 #include <Shuffle/ShuffleCommon.h>
 #include <Storages/IO/AggregateSerializationUtils.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Poco/StringTokenizer.h>
 
 namespace local_engine

diff --git a/cpp-ch/local-engine/tests/gluten_test_util.cpp b/cpp-ch/local-engine/tests/gluten_test_util.cpp
@@ -21,6 +21,7 @@
 #include <IO/ReadBuffer.h>
 #include <IO/ReadBufferFromFile.h>
 #include <Interpreters/ActionsVisitor.h>
+#include <Parser/LocalExecutor.h>
 #include <Parser/SerializedPlanParser.h>
 #include <Parser/SubstraitParserUtils.h>
 #include <Parsers/ExpressionListParsers.h>

diff --git a/cpp-ch/local-engine/tests/gtest_ch_storages.cpp b/cpp-ch/local-engine/tests/gtest_ch_storages.cpp
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 #include <Functions/FunctionFactory.h>
-#include <Parser/MergeTreeRelParser.h>
+#include <Parser/RelParsers/MergeTreeRelParser.h>
 #include <Processors/Executors/PipelineExecutor.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <Storages/MergeTree/SparkMergeTreeMeta.h>

diff --git a/cpp-ch/local-engine/tests/gtest_clickhouse_pr_verify.cpp b/cpp-ch/local-engine/tests/gtest_clickhouse_pr_verify.cpp
@@ -18,6 +18,7 @@
 #include <incbin.h>
 #include <Core/Settings.h>
 #include <Interpreters/Context.h>
+#include <Parser/LocalExecutor.h>
 #include <Parser/SerializedPlanParser.h>
 #include <Parser/SubstraitParserUtils.h>
 #include <gtest/gtest.h>

diff --git a/cpp-ch/local-engine/tests/gtest_local_engine.cpp b/cpp-ch/local-engine/tests/gtest_local_engine.cpp
@@ -24,7 +24,6 @@
 #include <Formats/FormatFactory.h>
 #include <Interpreters/Context.h>
 #include <Parser/CHColumnToSparkRow.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Parser/SparkRowToCHColumn.h>
 #include <Parser/SubstraitParserUtils.h>
 #include <Processors/Executors/PipelineExecutor.h>

diff --git a/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp b/cpp-ch/local-engine/tests/gtest_parquet_columnindex.cpp
@@ -14,26 +14,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-
 #include "config.h"
 #if USE_PARQUET
 #include <charconv>
 #include <ranges>
 #include <string>
 #include <Columns/ColumnString.h>
-#include <DataTypes/DataTypeString.h>
+#include <IO/ReadBufferFromFile.h>
 #include <Interpreters/ActionsVisitor.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/ExpressionActions.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Parsers/ExpressionListParsers.h>
-#include <Processors/Formats/Impl/ArrowBufferedStreams.h>
 #include <Storages/Parquet/ArrowUtils.h>
 #include <Storages/Parquet/ColumnIndexFilter.h>
 #include <Storages/Parquet/ParquetConverter.h>
 #include <Storages/Parquet/RowRanges.h>
 #include <Storages/Parquet/VectorizedParquetRecordReader.h>
+#include <boost/iterator/counting_iterator.hpp>
 #include <gtest/gtest.h>
 #include <parquet/page_index.h>
 #include <parquet/schema.h>
@@ -42,7 +39,6 @@
 #include <Common/BlockTypeUtils.h>
 #include <Common/QueryContext.h>
 
-
 #    define ASSERT_DURATION_LE(secs, stmt) \
         { \
             std::promise<bool> completed; \

diff --git a/cpp-ch/local-engine/tests/gtest_parquet_columnindex_bug.cpp b/cpp-ch/local-engine/tests/gtest_parquet_columnindex_bug.cpp
@@ -18,9 +18,10 @@
 #include <incbin.h>
 #include <Core/Settings.h>
 #include <Interpreters/Context.h>
-#include <Parser/SerializedPlanParser.h>
+#include <Parser/LocalExecutor.h>
 #include <Parser/SubstraitParserUtils.h>
 #include <gtest/gtest.h>
+#include <substrait/plan.pb.h>
 #include <Common/DebugUtils.h>
 #include <Common/GlutenConfig.h>
 #include <Common/QueryContext.h>

diff --git a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
@@ -21,7 +21,6 @@
 
 #include <ranges>
 #include <Core/Range.h>
-#include <Common/BlockTypeUtils.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeDateTime.h>
@@ -32,7 +31,6 @@
 #include <Interpreters/ActionsDAG.h>
 #include <Interpreters/ActionsVisitor.h>
 #include <Interpreters/ExpressionActions.h>
-#include <Parser/SerializedPlanParser.h>
 #include <Processors/Executors/PullingPipelineExecutor.h>
 #include <Processors/Formats/Impl/ArrowColumnToCHColumn.h>
 #include <Processors/Formats/Impl/ParquetBlockInputFormat.h>
@@ -44,6 +42,7 @@
 #include <parquet/arrow/reader.h>
 #include <parquet/level_conversion.h>
 #include <tests/gluten_test_util.h>
+#include <Common/BlockTypeUtils.h>
 
 using namespace DB;