From 5feaa3f878685ba144e703b6de2c21cb1d2064f3 Mon Sep 17 00:00:00 2001 From: kyligence-git Date: Thu, 8 Aug 2024 06:57:13 +0000 Subject: [PATCH 1/5] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240808) --- cpp-ch/clickhouse.version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 46b23e5f0e61..f9f4912abb46 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,4 +1,4 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20240806 -CH_COMMIT=c8a7d6e496d +CH_BRANCH=rebase_ch/20240808 +CH_COMMIT=641bd1900cd From 8d29fad19b06ce168aa057b83c56ab739f69367e Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Thu, 8 Aug 2024 14:49:21 +0800 Subject: [PATCH 2/5] Fix build due to https://github.com/ClickHouse/ClickHouse/pull/66279 --- .../Storages/Parquet/VectorizedParquetRecordReader.h | 2 +- .../Storages/SubstraitSource/SubstraitFileSource.cpp | 2 +- .../local-engine/Storages/SubstraitSource/SubstraitFileSource.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h b/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h index a9c796a7556b..b930df0cac95 100644 --- a/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h +++ b/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h @@ -235,7 +235,7 @@ class VectorizedParquetBlockInputFormat final : public DB::IInputFormat ColumnIndexFilterPtr column_index_filter_; protected: - void onCancel() override { is_stopped = 1; } + void onCancel() noexcept override { is_stopped = 1; } public: VectorizedParquetBlockInputFormat(DB::ReadBuffer & in_, const DB::Block & header_, const DB::FormatSettings & format_settings); diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp index d4e9f1eb8d4b..d8f0ee0e3552 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp +++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp @@ -144,7 +144,7 @@ bool SubstraitFileSource::tryPrepareReader() } -void SubstraitFileSource::onCancel() +void SubstraitFileSource::onCancel() noexcept { if (file_reader) file_reader->cancel(); diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h index 571e4097107a..113538a92922 100644 --- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h +++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h @@ -131,7 +131,7 @@ class SubstraitFileSource : public DB::SourceWithKeyCondition private: bool tryPrepareReader(); - void onCancel() override; + void onCancel() noexcept override; DB::ContextPtr context; DB::Block output_header; /// Sample header may contains partitions keys From 7d810e8675f6b9bb8392a1cdda72e030a68c0d12 Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Thu, 8 Aug 2024 15:53:29 +0800 Subject: [PATCH 3/5] fix version typo --- cpp-ch/clickhouse.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index f9f4912abb46..cede6e7d0398 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,4 +1,4 @@ CH_ORG=Kyligence CH_BRANCH=rebase_ch/20240808 -CH_COMMIT=641bd1900cd +CH_COMMIT=641bd1900cd From 2591a604cbfaa2c447b0171435805dbbaa331270 Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Thu, 8 Aug 2024 15:57:42 +0800 Subject: [PATCH 4/5] Revert "[GLUTEN-6262][CH]Json input format ignore key case #6263" due to https://github.com/ClickHouse/ClickHouse/pull/67484 revert https://github.com/ClickHouse/ClickHouse/pull/61750 This reverts commit 2be6c86bef0a5de834ec4a88276787cfcd453aed. --- .../src/test/resources/text-data/json-settings/data.txt | 1 - cpp-ch/local-engine/Common/CHUtil.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/backends-clickhouse/src/test/resources/text-data/json-settings/data.txt b/backends-clickhouse/src/test/resources/text-data/json-settings/data.txt index 230b46ec3803..0541ce3469a7 100644 --- a/backends-clickhouse/src/test/resources/text-data/json-settings/data.txt +++ b/backends-clickhouse/src/test/resources/text-data/json-settings/data.txt @@ -1,5 +1,4 @@ {"a":1,"b":2,"c":3} -{"a":"a5", "B":"b6", "c":7} {"a":"4"} {"t":{"ta":"cc","tb":100,"tc":1.234}} {"t":{"ta":"cc","tb":100,"td":"ignore"}} diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp index d32eed92340a..35b4f0c97806 100644 --- a/cpp-ch/local-engine/Common/CHUtil.cpp +++ b/cpp-ch/local-engine/Common/CHUtil.cpp @@ -781,7 +781,6 @@ void BackendInitializerUtil::initSettings(std::map & b settings.set("input_format_parquet_import_nested", true); settings.set("input_format_json_read_numbers_as_strings", true); settings.set("input_format_json_read_bools_as_numbers", false); - settings.set("input_format_json_case_insensitive_column_matching", true); settings.set("input_format_csv_trim_whitespaces", false); settings.set("input_format_csv_allow_cr_end_of_line", true); settings.set("output_format_orc_string_as_string", true); From 21b02eb435a703c3397ac29889a7365945e73560 Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Thu, 8 Aug 2024 16:19:31 +0800 Subject: [PATCH 5/5] fix gtest due to https://github.com/apache/incubator-gluten/pull/6722 --- cpp-ch/local-engine/tests/gtest_spark_row.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp-ch/local-engine/tests/gtest_spark_row.cpp b/cpp-ch/local-engine/tests/gtest_spark_row.cpp index 0350bb16c705..963f7736858f 100644 --- a/cpp-ch/local-engine/tests/gtest_spark_row.cpp +++ b/cpp-ch/local-engine/tests/gtest_spark_row.cpp @@ -136,7 +136,7 @@ TEST(SparkRow, GetArrayElementSize) {std::make_shared(), 4}, {std::make_shared(), 4}, {std::make_shared(), 4}, - {std::make_shared(9, 4), 4}, + {std::make_shared(9, 4), 8}, {std::make_shared(), 8}, {std::make_shared(), 8}, {std::make_shared(), 8}, @@ -152,11 +152,11 @@ TEST(SparkRow, GetArrayElementSize) for (const auto & [type, size] : type_to_size) { - EXPECT_TRUE(BackingDataLengthCalculator::getArrayElementSize(type) == size); + EXPECT_EQ(size, BackingDataLengthCalculator::getArrayElementSize(type)); if (type->canBeInsideNullable()) { const auto type_with_nullable = std::make_shared(type); - EXPECT_TRUE(BackingDataLengthCalculator::getArrayElementSize(type_with_nullable) == size); + EXPECT_EQ(size, BackingDataLengthCalculator::getArrayElementSize(type_with_nullable)); } } }