Merge branch 'clickhouse_backend' into 230327_support_s3_and_cache

Kyligence · Mar 29, 2023 · d5905eb · d5905eb
2 parents e5e3e8d + 6aad021
commit d5905eb
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 11 deletions.
diff --git a/utils/local-engine/Common/common.cpp b/utils/local-engine/Common/common.cpp
@@ -211,7 +211,13 @@ void init(const std::string & plan)
             {
                 settings.set(key, config->getString(settings_path + "." + key));
             }
+
+            /// Fixed settings which must be applied
             settings.set("join_use_nulls", true);
+            settings.set("input_format_orc_allow_missing_columns", true);
+            settings.set("input_format_orc_case_insensitive_column_matching", true);
+            settings.set("input_format_parquet_allow_missing_columns", true);
+            settings.set("input_format_parquet_case_insensitive_column_matching", true);
             LOG_INFO(logger, "Init settings.");
 
             /// Initialize global context

diff --git a/utils/local-engine/Operator/ExpandStep.cpp b/utils/local-engine/Operator/ExpandStep.cpp
@@ -69,7 +69,7 @@ DB::Block ExpandStep::buildOutputHeader(
             cols.push_back(old_col);
         else
         {
-            auto null_map = DB::ColumnInt8::create(0, 0);
+            auto null_map = DB::ColumnUInt8::create(0, 0);
             auto null_col = DB::ColumnNullable::create(old_col.column, std::move(null_map));
             auto null_type = std::make_shared<DB::DataTypeNullable>(old_col.type);
             cols.push_back(DB::ColumnWithTypeAndName(null_col, null_type, old_col.name));

diff --git a/utils/local-engine/Operator/ExpandTransform.cpp b/utils/local-engine/Operator/ExpandTransform.cpp
@@ -86,7 +86,7 @@ void ExpandTransform::work()
             // the output columns should all be nullable.
             if (!sets.contains(i))
             {
-                auto null_map = DB::ColumnInt8::create(rows, 1);
+                auto null_map = DB::ColumnUInt8::create(rows, 1);
                 auto col = DB::ColumnNullable::create(original_col, std::move(null_map));
                 cols.push_back(std::move(col));
             }
@@ -96,7 +96,7 @@ void ExpandTransform::work()
                     cols.push_back(original_col);
                 else
                 {
-                    auto null_map = DB::ColumnInt8::create(rows, 0);
+                    auto null_map = DB::ColumnUInt8::create(rows, 0);
                     auto col = DB::ColumnNullable::create(original_col, std::move(null_map));
                     cols.push_back(std::move(col));
                 }

diff --git a/utils/local-engine/Parser/SerializedPlanParser.cpp b/utils/local-engine/Parser/SerializedPlanParser.cpp
@@ -1268,8 +1268,8 @@ SerializedPlanParser::getFunctionName(const std::string & function_signature, co
     }
     else if (function_name == "check_overflow")
     {
-        if (args.size() != 2)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "check_overflow function requires two args.");
+        if (args.size() < 2)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "check_overflow function requires at least two args.");
         ch_function_name = getDecimalFunction(output_type.decimal(), args.at(1).value().literal().boolean());
     }
     else
@@ -1423,8 +1423,11 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG(
 
         if (function_signature.find("check_overflow:", 0) != function_signature.npos)
         {
-            if (scalar_function.arguments().size() != 2)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "check_overflow function requires two args.");
+            if (scalar_function.arguments().size() < 2)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "check_overflow function requires at least two args.");
+
+            ActionsDAG::NodeRawConstPtrs new_args;
+            new_args.reserve(2);
 
             // if toDecimalxxOrNull, first arg need string type
             if (scalar_function.arguments().at(1).value().literal().boolean())
@@ -1437,15 +1440,19 @@ const ActionsDAG::Node * SerializedPlanParser::parseFunctionWithDAG(
                 join(to_string_args, ',', to_string_cast_args_name);
                 result_name = check_overflow_args_trans_function + "(" + to_string_cast_args_name + ")";
                 const auto * to_string_cast_node = &actions_dag->addFunction(to_string_cast, to_string_args, result_name);
-                args[0] = to_string_cast_node;
+                new_args.emplace_back(to_string_cast_node);
+            }
+            else
+            {
+                new_args.emplace_back(args[0]);
             }
 
-            // delete the latest arg
-            args.pop_back();
             auto type = std::make_shared<DataTypeUInt32>();
             UInt32 scale = rel.scalar_function().output_type().decimal().scale();
-            args.emplace_back(
+            new_args.emplace_back(
                 &actions_dag->addColumn(ColumnWithTypeAndName(type->createColumnConst(1, scale), type, getUniqueName(toString(scale)))));
+
+            args = std::move(new_args);
         }
 
         auto function_builder = FunctionFactory::instance().get(function_name, context);