From 9a19204377a68c3aebfad17fe9f1993f244d75ec Mon Sep 17 00:00:00 2001 From: Kyligence Git Date: Thu, 17 Oct 2024 02:19:52 -0500 Subject: [PATCH] [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241017) (#7567) * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241017) * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/68146 * Fix Build due to rebasing failed with https://github.com/ClickHouse/ClickHouse/pull/69179 --------- Co-authored-by: kyligence-git Co-authored-by: Chang Chen --- cpp-ch/clickhouse.version | 4 ++-- .../ObjectStorages/MetadataStorageFromRocksDB.cpp | 14 +++++++------- .../ObjectStorages/MetadataStorageFromRocksDB.h | 6 +++--- ...dataStorageFromRocksDBTransactionOperations.cpp | 8 +++++++- ...tadataStorageFromRocksDBTransactionOperations.h | 1 + .../Storages/MergeTree/MetaDataHelper.cpp | 12 ++++++------ .../Storages/MergeTree/SparkStorageMergeTree.cpp | 2 +- 7 files changed, 27 insertions(+), 20 deletions(-) diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version index 63308f5a51ed..65feb5e57cb4 100644 --- a/cpp-ch/clickhouse.version +++ b/cpp-ch/clickhouse.version @@ -1,3 +1,3 @@ CH_ORG=Kyligence -CH_BRANCH=rebase_ch/20241016 -CH_COMMIT=c3a1c1b8457 +CH_BRANCH=rebase_ch/20241017 +CH_COMMIT=afe3026c6ce \ No newline at end of file diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp index aba42656414f..cebc34853e63 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp @@ -80,21 +80,21 @@ DB::MetadataStorageType MetadataStorageFromRocksDB::getType() const return DB::MetadataStorageType::None; } -bool MetadataStorageFromRocksDB::exists(const std::string & path) const +bool MetadataStorageFromRocksDB::existsFileOrDirectory(const std::string & path) const { return exist(getRocksDB(), path); } -bool MetadataStorageFromRocksDB::isFile(const std::string & path) const +bool MetadataStorageFromRocksDB::existsFile(const std::string & path) const { - auto data = getData(getRocksDB(), path); - return data != RocksDBCreateDirectoryOperation::DIR_DATA; + std::string data; + return tryGetData(getRocksDB(), path, &data) && data != RocksDBCreateDirectoryOperation::DIR_DATA; } -bool MetadataStorageFromRocksDB::isDirectory(const std::string & path) const +bool MetadataStorageFromRocksDB::existsDirectory(const std::string & path) const { - auto data = getData(getRocksDB(), path); - return data == RocksDBCreateDirectoryOperation::DIR_DATA; + std::string data; + return tryGetData(getRocksDB(), path, &data) && data == RocksDBCreateDirectoryOperation::DIR_DATA; } uint64_t MetadataStorageFromRocksDB::getFileSize(const std::string & path) const diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.h b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.h index 1f53a1efee96..66a9ca4999b7 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.h +++ b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.h @@ -39,9 +39,9 @@ class MetadataStorageFromRocksDB final : public DB::IMetadataStorage DB::MetadataTransactionPtr createTransaction() override; const std::string & getPath() const override; DB::MetadataStorageType getType() const override; - bool exists(const std::string & path) const override; - bool isFile(const std::string & path) const override; - bool isDirectory(const std::string & path) const override; + bool existsFile(const std::string & path) const override; + bool existsDirectory(const std::string & path) const override; + bool existsFileOrDirectory(const std::string & path) const override; uint64_t getFileSize(const std::string & path) const override; Poco::Timestamp getLastModified(const std::string & path) const override; bool supportsChmod() const override; diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.cpp b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.cpp index 34214594bb08..0219dd2629bf 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.cpp +++ b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.cpp @@ -17,6 +17,7 @@ #include #if USE_ROCKSDB #include "MetadataStorageFromRocksDBTransactionOperations.h" + #include namespace DB @@ -39,7 +40,12 @@ void throwRockDBErrorNotOk(const rocksdb::Status & status) bool exist(rocksdb::DB & db, const std::string & path) { std::string data; - auto status = db.Get({}, path, &data); + return tryGetData(db, path, &data); +} + +bool tryGetData(rocksdb::DB & db, const std::string & path, std::string * value) +{ + auto status = db.Get({}, path, value); if (status.IsNotFound()) return false; throwRockDBErrorNotOk(status); diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.h b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.h index 4a246cf7b151..a6c344d84285 100644 --- a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.h +++ b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.h @@ -25,6 +25,7 @@ namespace local_engine { void throwRockDBErrorNotOk(const rocksdb::Status & status); bool exist(rocksdb::DB & db, const std::string & path); +bool tryGetData(rocksdb::DB & db, const std::string & path, std::string* value); String getData(rocksdb::DB & db, const std::string & path); std::vector listKeys(rocksdb::DB & db, const std::string & path); diff --git a/cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp b/cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp index 19a07b06f3d9..a904fd087677 100644 --- a/cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp +++ b/cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp @@ -90,7 +90,7 @@ void restoreMetaData(const SparkStorageMergeTreePtr & storage, const Me for (const auto & part : mergeTreeTable.getPartNames()) { auto part_path = table_path / part; - if (!metadata_storage->exists(part_path)) + if (!metadata_storage->existsDirectory(part_path)) not_exists_part.emplace(part); } @@ -104,7 +104,7 @@ void restoreMetaData(const SparkStorageMergeTreePtr & storage, const Me auto s3 = data_disk->getObjectStorage(); auto transaction = metadata_storage->createTransaction(); - if (!metadata_storage->exists(table_path)) + if (!metadata_storage->existsDirectory(table_path)) transaction->createDirectoryRecursive(table_path.generic_string()); for (const auto & part : not_exists_part) @@ -112,7 +112,7 @@ void restoreMetaData(const SparkStorageMergeTreePtr & storage, const Me auto part_path = table_path / part; auto metadata_file_path = part_path / METADATA_FILE_NAME; - if (metadata_storage->exists(part_path)) + if (metadata_storage->existsDirectory(part_path)) return; else transaction->createDirectoryRecursive(part_path); @@ -143,7 +143,7 @@ void restoreMetaData( for (const auto & part : mergeTreeTable.getPartNames()) { auto part_path = table_path / part; - if (!metadata_disk->exists(part_path)) + if (!metadata_disk->existsDirectory(part_path)) not_exists_part.emplace(part); } @@ -166,7 +166,7 @@ void restoreMetaData( not_exists_part.size()); auto s3 = data_disk->getObjectStorage(); - if (!metadata_disk->exists(table_path)) + if (!metadata_disk->existsDirectory(table_path)) metadata_disk->createDirectories(table_path.generic_string()); for (const auto & part : not_exists_part) @@ -176,7 +176,7 @@ void restoreMetaData( auto part_path = table_path / part; auto metadata_file_path = part_path / METADATA_FILE_NAME; - if (metadata_disk->exists(part_path)) + if (metadata_disk->existsDirectory(part_path)) return; else metadata_disk->createDirectories(part_path); diff --git a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp index 6b1738811306..d85efd03b811 100644 --- a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp +++ b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp @@ -165,7 +165,7 @@ void SparkStorageMergeTree::prefetchMetaDataFile(std::unordered_set read_settings.remote_fs_method = RemoteFSReadMethod::read; for (const auto & meta_path : meta_paths) { - if (!disk->exists(meta_path)) + if (!disk->existsDirectory(meta_path)) continue; auto in = disk->readFile(meta_path, read_settings);