[GLUTEN-1632][CH]Daily Update Clickhouse Version (20240706) (#6359)

* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240706) * Fix build due to ClickHouse/ClickHouse#63636 * Revert "[GLUTEN-1632][CH]Daily Update Clickhouse Version (20240705) (#6338)" This reverts commit 4a674e5. * exclude"shift left", "shift right","shift right unsigned" due to ClickHouse/ClickHouse#65838 --------- Co-authored-by: kyligence-git <[email protected]> Co-authored-by: Chang Chen <[email protected]>
apache · Jul 6, 2024 · 0cb2db3 · 0cb2db3
1 parent e8b770c
commit 0cb2db3
Show file tree

Hide file tree

Showing 11 changed files with 42 additions and 15 deletions.
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
@@ -1,4 +1,4 @@
 CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20240705
-CH_COMMIT=531a87ed802
+CH_BRANCH=rebase_ch/20240706
+CH_COMMIT=25bf31bfbdf
 
diff --git a/cpp-ch/local-engine/Operator/DefaultHashAggregateResult.cpp b/cpp-ch/local-engine/Operator/DefaultHashAggregateResult.cpp
@@ -116,18 +116,18 @@ class DefaultHashAggrgateResultTransform : public DB::IProcessor
             has_input = true;
             output_chunk = DB::Chunk(result_cols, 1);
             auto info = std::make_shared<DB::AggregatedChunkInfo>();
-            output_chunk.getChunkInfos().add(std::move(info));
+            output_chunk.setChunkInfo(info);
             return Status::Ready;
         }
 
         input.setNeeded();
         if (input.hasData())
         {
             output_chunk = input.pull(true);
-            if (output_chunk.getChunkInfos().empty())
+            if (!output_chunk.hasChunkInfo())
             {
                 auto info = std::make_shared<DB::AggregatedChunkInfo>();
-                output_chunk.getChunkInfos().add(std::move(info));
+                output_chunk.setChunkInfo(info);
             }
             has_input = true;
             return Status::Ready;

diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.cpp
@@ -121,11 +121,12 @@ void SparkMergeTreeWriter::write(const DB::Block & block)
         checkAndMerge();
 }
 
-bool SparkMergeTreeWriter::chunkToPart(Chunk && plan_chunk)
+bool SparkMergeTreeWriter::chunkToPart(Chunk && chunk)
 {
-    if (Chunk result_chunk = DB::Squashing::squash(std::move(plan_chunk)))
+    if (chunk.hasChunkInfo())
     {
-        auto result = squashing->getHeader().cloneWithColumns(result_chunk.detachColumns());
+        Chunk squash_chunk = DB::Squashing::squash(std::move(chunk));
+        Block result = header.cloneWithColumns(squash_chunk.getColumns());
         return blockToPart(result);
     }
     return false;

diff --git a/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h b/cpp-ch/local-engine/Storages/Mergetree/SparkMergeTreeWriter.h
@@ -77,7 +77,7 @@ class SparkMergeTreeWriter
     void saveMetadata();
     void commitPartToRemoteStorageIfNeeded();
     void finalizeMerge();
-    bool chunkToPart(Chunk && plan_chunk);
+    bool chunkToPart(Chunk && chunk);
     bool blockToPart(Block & block);
     bool useLocalStorage() const;
 

diff --git a/cpp-ch/local-engine/Storages/SourceFromJavaIter.cpp b/cpp-ch/local-engine/Storages/SourceFromJavaIter.cpp
@@ -109,13 +109,13 @@ DB::Chunk SourceFromJavaIter::generate()
             auto info = std::make_shared<DB::AggregatedChunkInfo>();
             info->is_overflows = data->info.is_overflows;
             info->bucket_num = data->info.bucket_num;
-            result.getChunkInfos().add(std::move(info));
+            result.setChunkInfo(info);
         }
         else
         {
             result = BlockUtil::buildRowCountChunk(rows);
             auto info = std::make_shared<DB::AggregatedChunkInfo>();
-            result.getChunkInfos().add(std::move(info));
+            result.setChunkInfo(info);
         }
     }
     return result;

diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
@@ -23,7 +23,6 @@
 #include <Disks/IO/AsynchronousBoundedReadBuffer.h>
 #include <Disks/IO/ReadBufferFromAzureBlobStorage.h>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
-#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
 #include <IO/BoundedReadBuffer.h>
 #include <IO/ReadBufferFromFile.h>
 #include <IO/ReadBufferFromS3.h>
@@ -52,6 +51,10 @@
 #include <Common/logger_useful.h>
 #include <Common/safe_cast.h>
 
+#if USE_AZURE_BLOB_STORAGE
+#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
+#endif
+
 #if USE_AWS_S3
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/s3/model/CopyObjectRequest.h>
@@ -687,7 +690,19 @@ class AzureBlobReadBuffer : public ReadBufferBuilder
     {
         if (shared_client)
             return shared_client;
-        shared_client = DB::getAzureBlobContainerClient(context->getConfigRef(), "blob");
+
+        const std::string config_prefix = "blob";
+        const Poco::Util::AbstractConfiguration & config = context->getConfigRef();
+        bool is_client_for_disk = false;
+        auto new_settings = DB::AzureBlobStorage::getRequestSettings(config, config_prefix, context);
+        DB::AzureBlobStorage::ConnectionParams params
+        {
+            .endpoint = DB::AzureBlobStorage::processEndpoint(config, config_prefix),
+            .auth_method = DB::AzureBlobStorage::getAuthMethod(config, config_prefix),
+            .client_options = DB::AzureBlobStorage::getClientOptions(*new_settings, is_client_for_disk),
+        };
+
+        shared_client = DB::AzureBlobStorage::getContainerClient(params, true);
         return shared_client;
     }
 };

diff --git a/cpp-ch/local-engine/tests/gtest_parser.cpp b/cpp-ch/local-engine/tests/gtest_parser.cpp
@@ -101,8 +101,7 @@ TEST(LocalExecutor, StorageObjectStorageSink)
 
     /// 2. Create Chunk
     /// 3. comsume
-    Chunk data = testChunk();
-    sink.consume(data);
+    sink.consume(testChunk());
     sink.onFinish();
 }
 

diff --git a/...ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/...ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -850,6 +850,9 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("atan2")
     .exclude("round/bround")
     .exclude("SPARK-37388: width_bucket")
+    .exclude("shift left")
+    .exclude("shift right")
+    .exclude("shift right unsigned")
   enableSuite[GlutenMiscExpressionsSuite]
   enableSuite[GlutenNondeterministicSuite]
     .exclude("MonotonicallyIncreasingID")

diff --git a/...ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/...ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -809,6 +809,9 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("SPARK-35926: Support YearMonthIntervalType in width-bucket function")
     .exclude("SPARK-35925: Support DayTimeIntervalType in width-bucket function")
     .exclude("SPARK-37388: width_bucket")
+    .exclude("shift left")
+    .exclude("shift right")
+    .exclude("shift right unsigned")
   enableSuite[GlutenMiscExpressionsSuite]
   enableSuite[GlutenNondeterministicSuite]
     .exclude("MonotonicallyIncreasingID")

diff --git a/...ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/...ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -711,6 +711,9 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("SPARK-35926: Support YearMonthIntervalType in width-bucket function")
     .exclude("SPARK-35925: Support DayTimeIntervalType in width-bucket function")
     .exclude("SPARK-37388: width_bucket")
+    .exclude("shift left")
+    .exclude("shift right")
+    .exclude("shift right unsigned")
   enableSuite[GlutenMiscExpressionsSuite]
   enableSuite[GlutenNondeterministicSuite]
     .exclude("MonotonicallyIncreasingID")

diff --git a/...ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/...ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -711,6 +711,9 @@ class ClickHouseTestSettings extends BackendTestSettings {
     .exclude("SPARK-35926: Support YearMonthIntervalType in width-bucket function")
     .exclude("SPARK-35925: Support DayTimeIntervalType in width-bucket function")
     .exclude("SPARK-37388: width_bucket")
+    .exclude("shift left")
+    .exclude("shift right")
+    .exclude("shift right unsigned")
   enableSuite[GlutenMiscExpressionsSuite]
   enableSuite[GlutenNondeterministicSuite]
     .exclude("MonotonicallyIncreasingID")