[GLUTEN-1632][CH]Daily Update Clickhouse Version (20240526)

cpp-ch/clickhouse.version

@@ -1,3 +1,3 @@
 CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20240524
-CH_COMMIT=b1a2a14bdd2
+CH_BRANCH=rebase_ch/20240526
+CH_COMMIT=ff17e067fac

cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp

@@ -17,7 +17,7 @@
 
 #include "GlutenHDFSObjectStorage.h"
 #if USE_HDFS
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
 using namespace DB;
 namespace local_engine
 {
@@ -35,8 +35,10 @@ std::unique_ptr<ReadBufferFromFileBase> GlutenHDFSObjectStorage::readObject( ///
 
 DB::ObjectStorageKey local_engine::GlutenHDFSObjectStorage::generateObjectKeyForPath(const std::string & path) const
 {
-    return DB::ObjectStorageKey::createAsAbsolute(hdfs_root_path + path);
+    initializeHDFSFS();
+    /// what ever data_source_description.description value is, consider that key as relative key
+    chassert(data_directory.starts_with("/"));
+    return ObjectStorageKey::createAsRelative(fs::path(url_without_path) / data_directory.substr(1) / path);
 }
 }
 #endif
-

cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h

@@ -33,7 +33,7 @@ class GlutenHDFSObjectStorage final : public DB::HDFSObjectStorage
             const String & hdfs_root_path_,
             SettingsPtr settings_,
             const Poco::Util::AbstractConfiguration & config_)
-        : HDFSObjectStorage(hdfs_root_path_, std::move(settings_), config_), config(config_)
+        : HDFSObjectStorage(hdfs_root_path_, std::move(settings_), config_, /* lazy_initialize */true), config(config_)
     {
     }
     std::unique_ptr<DB::ReadBufferFromFileBase> readObject( /// NOLINT

cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp

@@ -75,7 +75,7 @@ void registerGlutenS3ObjectStorage(ObjectStorageFactory & factory)
             auto uri = getS3URI(config, config_prefix, context);
             auto s3_capabilities = getCapabilitiesFromConfig(config, config_prefix);
             auto settings = getSettings(config, config_prefix, context);
-            auto client = getClient(config, config_prefix, context, *settings);
+            auto client = getClient(config, config_prefix, context, *settings, true);
             auto key_generator = createObjectStorageKeysGeneratorAsIsWithPrefix(uri.key);
 
             auto object_storage = std::make_shared<S3ObjectStorage>(
@@ -110,7 +110,6 @@ void registerGlutenHDFSObjectStorage(ObjectStorageFactory & factory)
 
             std::unique_ptr<HDFSObjectStorageSettings> settings = std::make_unique<HDFSObjectStorageSettings>(
                 config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024),
-                config.getInt(config_prefix + ".objects_chunk_size_to_delete", 1000),
                 context->getSettingsRef().hdfs_replication
             );
             return std::make_unique<GlutenHDFSObjectStorage>(uri, std::move(settings), config);

cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp

@@ -17,13 +17,11 @@
 #include <memory>
 #include <IO/WriteSettings.h>
 #include <Interpreters/Cache/FileCache.h>
-#include <Interpreters/Cache/FileCacheSettings.h>
 #include <Interpreters/Context_fwd.h>
-#include <Storages/HDFS/HDFSCommon.h>
-#include <Storages/HDFS/WriteBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/WriteBufferFromHDFS.h>
 #include <Storages/Output/WriteBufferBuilder.h>
 #include <hdfs/hdfs.h>
-#include <Poco/Logger.h>
 #include <Poco/URI.h>
 #include <Common/CHUtil.h>
 

cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp

@@ -14,23 +14,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "Common/CHUtil.h"
 #include "ExcelTextFormatFile.h"
-
+#include <Common/CHUtil.h>
 
 #include <memory>
 #include <string>
-#include <utility>
 
 #include <Columns/ColumnNullable.h>
 #include <DataTypes/DataTypeDecimalBase.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/Serializations/SerializationNullable.h>
 #include <Formats/FormatSettings.h>
 #include <IO/PeekableReadBuffer.h>
-#include <IO/SeekableReadBuffer.h>
 #include <Processors/Formats/IRowInputFormat.h>
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
 #include <Storages/Serializations/ExcelDecimalSerialization.h>
 #include <Storages/Serializations/ExcelSerialization.h>
 #include <Storages/Serializations/ExcelStringReader.h>

cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp

@@ -35,9 +35,9 @@
 #include <Interpreters/Cache/FileCacheFactory.h>
 #include <Interpreters/Cache/FileCacheSettings.h>
 #include <Interpreters/Context_fwd.h>
-#include <Storages/HDFS/AsynchronousReadBufferFromHDFS.h>
-#include <Storages/HDFS/HDFSCommon.h>
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
 #include <Storages/StorageS3Settings.h>
 #include <Storages/SubstraitSource/ReadBufferBuilder.h>
 #include <Storages/SubstraitSource/SubstraitFileSource.h>