From 38d369caf2fc1c199a440bb6e368978d4c97e98a Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Tue, 5 Nov 2024 15:05:47 +0100 Subject: [PATCH] Configure checksum verification on repository level --- CONFIGURE.md | 5 ++++- conf/minifi.properties | 7 +++++-- extensions/rocksdb-repos/DatabaseContentRepository.cpp | 2 +- extensions/rocksdb-repos/FlowFileRepository.cpp | 2 +- extensions/rocksdb-repos/ProvenanceRepository.cpp | 2 +- .../rocksdb-repos/controllers/RocksDbStateStorage.cpp | 2 +- libminifi/include/properties/Configuration.h | 5 ++++- libminifi/src/Configuration.cpp | 5 ++++- 8 files changed, 21 insertions(+), 9 deletions(-) diff --git a/CONFIGURE.md b/CONFIGURE.md index 14f6629432..51e6219ae2 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -538,7 +538,10 @@ RocksDB has an option to set synchronous writes for its database, ensuring that RocksDB has an option to verify checksums for its database reads. This option is set to false by default for better performance. If you prefer to enable checksum verification you can set this option to true. # in minifi.properties - nifi.rocksdb.read.verify.checksums=false + nifi.content.repository.rocksdb.read.verify.checksums=false + nifi.flowfile.repository.rocksdb.read.verify.checksums=false + nifi.provenance.repository.rocksdb.read.verify.checksums=false + nifi.rocksdb.state.storage.read.verify.checksums=false ### Global RocksDB options diff --git a/conf/minifi.properties b/conf/minifi.properties index 46cd211489..1a18fb076d 100644 --- a/conf/minifi.properties +++ b/conf/minifi.properties @@ -38,8 +38,11 @@ nifi.content.repository.class.name=DatabaseContentRepository # Use synchronous writes for the RocksDB content repository. Disable for better write performance, if data loss is acceptable in case of the host crashing. # nifi.content.repository.rocksdb.use.synchronous.writes=true -# Verify checksum of the data read from the RocksDB content repository. Disabled by default for better read performance. -# nifi.rocksdb.read.verify.checksums=false +# Verify checksum of the data read from a RocksDB repository. Disabled by default for better read performance. +# nifi.content.repository.rocksdb.read.verify.checksums=false +# nifi.flowfile.repository.rocksdb.read.verify.checksums=false +# nifi.provenance.repository.rocksdb.read.verify.checksums=false +# nifi.rocksdb.state.storage.read.verify.checksums=false ## Relates to the internal workings of the rocksdb backend # nifi.flowfile.repository.rocksdb.compaction.period=2 min diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.cpp b/extensions/rocksdb-repos/DatabaseContentRepository.cpp index 577fdd66a8..f7d75aa766 100644 --- a/extensions/rocksdb-repos/DatabaseContentRepository.cpp +++ b/extensions/rocksdb-repos/DatabaseContentRepository.cpp @@ -82,7 +82,7 @@ bool DatabaseContentRepository::initialize(const std::shared_ptrget(Configure::nifi_content_repository_rocksdb_use_synchronous_writes).value_or("true") != "false"; - verify_checksums_in_rocksdb_reads_ = configuration->get(Configure::nifi_rocksdb_read_verify_checksums).value_or("false") == "true"; + verify_checksums_in_rocksdb_reads_ = configuration->get(Configure::nifi_content_repository_rocksdb_read_verify_checksums).value_or("false") == "true"; logger_->log_debug("{} checksum verification in DatabaseContentRepository", verify_checksums_in_rocksdb_reads_ ? "Using" : "Not using"); return is_valid_; } diff --git a/extensions/rocksdb-repos/FlowFileRepository.cpp b/extensions/rocksdb-repos/FlowFileRepository.cpp index 5d3f2fefeb..783e747056 100644 --- a/extensions/rocksdb-repos/FlowFileRepository.cpp +++ b/extensions/rocksdb-repos/FlowFileRepository.cpp @@ -212,7 +212,7 @@ bool FlowFileRepository::initialize(const std::shared_ptr &configure) const auto encrypted_env = createEncryptingEnv(utils::crypto::EncryptionManager{configure->getHome()}, DbEncryptionOptions{directory_, ENCRYPTION_KEY_NAME}); logger_->log_info("Using {} FlowFileRepository", encrypted_env ? "encrypted" : "plaintext"); - verify_checksums_in_rocksdb_reads_ = configure->get(Configure::nifi_rocksdb_read_verify_checksums).value_or("false") == "true"; + verify_checksums_in_rocksdb_reads_ = configure->get(Configure::nifi_flowfile_repository_rocksdb_read_verify_checksums).value_or("false") == "true"; logger_->log_debug("{} checksum verification in FlowFileRepository", verify_checksums_in_rocksdb_reads_ ? "Using" : "Not using"); auto db_options = [encrypted_env] (minifi::internal::Writable& options) { diff --git a/extensions/rocksdb-repos/ProvenanceRepository.cpp b/extensions/rocksdb-repos/ProvenanceRepository.cpp index df1a9db80c..90e035506b 100644 --- a/extensions/rocksdb-repos/ProvenanceRepository.cpp +++ b/extensions/rocksdb-repos/ProvenanceRepository.cpp @@ -39,7 +39,7 @@ bool ProvenanceRepository::initialize(const std::shared_ptrlog_debug("MiNiFi Provenance Max Storage Time: [{}]", max_partition_millis_); - verify_checksums_in_rocksdb_reads_ = config->get(Configure::nifi_rocksdb_read_verify_checksums).value_or("false") == "true"; + verify_checksums_in_rocksdb_reads_ = config->get(Configure::nifi_provenance_repository_rocksdb_read_verify_checksums).value_or("false") == "true"; logger_->log_debug("{} checksum verification in ProvenanceRepository", verify_checksums_in_rocksdb_reads_ ? "Using" : "Not using"); auto db_options = [] (minifi::internal::Writable& db_opts) { diff --git a/extensions/rocksdb-repos/controllers/RocksDbStateStorage.cpp b/extensions/rocksdb-repos/controllers/RocksDbStateStorage.cpp index a086c3bca9..c4855b6281 100644 --- a/extensions/rocksdb-repos/controllers/RocksDbStateStorage.cpp +++ b/extensions/rocksdb-repos/controllers/RocksDbStateStorage.cpp @@ -93,7 +93,7 @@ void RocksDbStateStorage::onEnable() { default_write_options.sync = true; } - verify_checksums_in_rocksdb_reads_ = configuration_->get(Configure::nifi_rocksdb_read_verify_checksums).value_or("false") == "true"; + verify_checksums_in_rocksdb_reads_ = configuration_->get(Configure::nifi_rocksdb_state_storage_read_verify_checksums).value_or("false") == "true"; logger_->log_trace("Enabled RocksDbStateStorage"); } diff --git a/libminifi/include/properties/Configuration.h b/libminifi/include/properties/Configuration.h index f8d38bd8e0..f5b7bc41bc 100644 --- a/libminifi/include/properties/Configuration.h +++ b/libminifi/include/properties/Configuration.h @@ -78,7 +78,10 @@ class Configuration : public Properties { static constexpr const char *nifi_dbcontent_repository_rocksdb_compaction_period = "nifi.database.content.repository.rocksdb.compaction.period"; static constexpr const char *nifi_dbcontent_repository_purge_period = "nifi.database.content.repository.purge.period"; static constexpr const char *nifi_content_repository_rocksdb_use_synchronous_writes = "nifi.content.repository.rocksdb.use.synchronous.writes"; - static constexpr const char *nifi_rocksdb_read_verify_checksums = "nifi.rocksdb.read.verify.checksums"; + static constexpr const char *nifi_content_repository_rocksdb_read_verify_checksums = "nifi.content.repository.rocksdb.read.verify.checksums"; + static constexpr const char *nifi_flowfile_repository_rocksdb_read_verify_checksums = "nifi.flowfile.repository.rocksdb.read.verify.checksums"; + static constexpr const char *nifi_provenance_repository_rocksdb_read_verify_checksums = "nifi.provenance.repository.rocksdb.read.verify.checksums"; + static constexpr const char *nifi_rocksdb_state_storage_read_verify_checksums = "nifi.rocksdb.state.storage.read.verify.checksums"; static constexpr const char *nifi_remote_input_secure = "nifi.remote.input.secure"; static constexpr const char *nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth"; diff --git a/libminifi/src/Configuration.cpp b/libminifi/src/Configuration.cpp index 442d4a4175..df50a7d728 100644 --- a/libminifi/src/Configuration.cpp +++ b/libminifi/src/Configuration.cpp @@ -54,7 +54,10 @@ const std::unordered_map