Configure checksum verification on repository level

apache · Nov 5, 2024 · 38d369c · 38d369c
1 parent bb6b8d0
commit 38d369c
Show file tree

Hide file tree

Showing 8 changed files with 21 additions and 9 deletions.
diff --git a/CONFIGURE.md b/CONFIGURE.md
@@ -538,7 +538,10 @@ RocksDB has an option to set synchronous writes for its database, ensuring that
 RocksDB has an option to verify checksums for its database reads. This option is set to false by default for better performance. If you prefer to enable checksum verification you can set this option to true.
 
     # in minifi.properties
-    nifi.rocksdb.read.verify.checksums=false
+    nifi.content.repository.rocksdb.read.verify.checksums=false
+    nifi.flowfile.repository.rocksdb.read.verify.checksums=false
+    nifi.provenance.repository.rocksdb.read.verify.checksums=false
+    nifi.rocksdb.state.storage.read.verify.checksums=false
 
 ### Global RocksDB options
 

diff --git a/conf/minifi.properties b/conf/minifi.properties
@@ -38,8 +38,11 @@ nifi.content.repository.class.name=DatabaseContentRepository
 # Use synchronous writes for the RocksDB content repository. Disable for better write performance, if data loss is acceptable in case of the host crashing.
 # nifi.content.repository.rocksdb.use.synchronous.writes=true
 
-# Verify checksum of the data read from the RocksDB content repository. Disabled by default for better read performance.
-# nifi.rocksdb.read.verify.checksums=false
+# Verify checksum of the data read from a RocksDB repository. Disabled by default for better read performance.
+# nifi.content.repository.rocksdb.read.verify.checksums=false
+# nifi.flowfile.repository.rocksdb.read.verify.checksums=false
+# nifi.provenance.repository.rocksdb.read.verify.checksums=false
+# nifi.rocksdb.state.storage.read.verify.checksums=false
 
 ## Relates to the internal workings of the rocksdb backend
 # nifi.flowfile.repository.rocksdb.compaction.period=2 min

diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.cpp b/extensions/rocksdb-repos/DatabaseContentRepository.cpp
@@ -82,7 +82,7 @@ bool DatabaseContentRepository::initialize(const std::shared_ptr<minifi::Configu
   }
 
   use_synchronous_writes_ = configuration->get(Configure::nifi_content_repository_rocksdb_use_synchronous_writes).value_or("true") != "false";
-  verify_checksums_in_rocksdb_reads_ = configuration->get(Configure::nifi_rocksdb_read_verify_checksums).value_or("false") == "true";
+  verify_checksums_in_rocksdb_reads_ = configuration->get(Configure::nifi_content_repository_rocksdb_read_verify_checksums).value_or("false") == "true";
   logger_->log_debug("{} checksum verification in DatabaseContentRepository", verify_checksums_in_rocksdb_reads_ ? "Using" : "Not using");
   return is_valid_;
 }

diff --git a/extensions/rocksdb-repos/FlowFileRepository.cpp b/extensions/rocksdb-repos/FlowFileRepository.cpp
@@ -212,7 +212,7 @@ bool FlowFileRepository::initialize(const std::shared_ptr<Configure> &configure)
   const auto encrypted_env = createEncryptingEnv(utils::crypto::EncryptionManager{configure->getHome()}, DbEncryptionOptions{directory_, ENCRYPTION_KEY_NAME});
   logger_->log_info("Using {} FlowFileRepository", encrypted_env ? "encrypted" : "plaintext");
 
-  verify_checksums_in_rocksdb_reads_ = configure->get(Configure::nifi_rocksdb_read_verify_checksums).value_or("false") == "true";
+  verify_checksums_in_rocksdb_reads_ = configure->get(Configure::nifi_flowfile_repository_rocksdb_read_verify_checksums).value_or("false") == "true";
   logger_->log_debug("{} checksum verification in FlowFileRepository", verify_checksums_in_rocksdb_reads_ ? "Using" : "Not using");
 
   auto db_options = [encrypted_env] (minifi::internal::Writable<rocksdb::DBOptions>& options) {

diff --git a/extensions/rocksdb-repos/ProvenanceRepository.cpp b/extensions/rocksdb-repos/ProvenanceRepository.cpp
@@ -39,7 +39,7 @@ bool ProvenanceRepository::initialize(const std::shared_ptr<org::apache::nifi::m
   }
   logger_->log_debug("MiNiFi Provenance Max Storage Time: [{}]", max_partition_millis_);
 
-  verify_checksums_in_rocksdb_reads_ = config->get(Configure::nifi_rocksdb_read_verify_checksums).value_or("false") == "true";
+  verify_checksums_in_rocksdb_reads_ = config->get(Configure::nifi_provenance_repository_rocksdb_read_verify_checksums).value_or("false") == "true";
   logger_->log_debug("{} checksum verification in ProvenanceRepository", verify_checksums_in_rocksdb_reads_ ? "Using" : "Not using");
 
   auto db_options = [] (minifi::internal::Writable<rocksdb::DBOptions>& db_opts) {

diff --git a/extensions/rocksdb-repos/controllers/RocksDbStateStorage.cpp b/extensions/rocksdb-repos/controllers/RocksDbStateStorage.cpp
@@ -93,7 +93,7 @@ void RocksDbStateStorage::onEnable() {
     default_write_options.sync = true;
   }
 
-  verify_checksums_in_rocksdb_reads_ = configuration_->get(Configure::nifi_rocksdb_read_verify_checksums).value_or("false") == "true";
+  verify_checksums_in_rocksdb_reads_ = configuration_->get(Configure::nifi_rocksdb_state_storage_read_verify_checksums).value_or("false") == "true";
 
   logger_->log_trace("Enabled RocksDbStateStorage");
 }

diff --git a/libminifi/include/properties/Configuration.h b/libminifi/include/properties/Configuration.h
@@ -78,7 +78,10 @@ class Configuration : public Properties {
   static constexpr const char *nifi_dbcontent_repository_rocksdb_compaction_period = "nifi.database.content.repository.rocksdb.compaction.period";
   static constexpr const char *nifi_dbcontent_repository_purge_period = "nifi.database.content.repository.purge.period";
   static constexpr const char *nifi_content_repository_rocksdb_use_synchronous_writes = "nifi.content.repository.rocksdb.use.synchronous.writes";
-  static constexpr const char *nifi_rocksdb_read_verify_checksums = "nifi.rocksdb.read.verify.checksums";
+  static constexpr const char *nifi_content_repository_rocksdb_read_verify_checksums = "nifi.content.repository.rocksdb.read.verify.checksums";
+  static constexpr const char *nifi_flowfile_repository_rocksdb_read_verify_checksums = "nifi.flowfile.repository.rocksdb.read.verify.checksums";
+  static constexpr const char *nifi_provenance_repository_rocksdb_read_verify_checksums = "nifi.provenance.repository.rocksdb.read.verify.checksums";
+  static constexpr const char *nifi_rocksdb_state_storage_read_verify_checksums = "nifi.rocksdb.state.storage.read.verify.checksums";
 
   static constexpr const char *nifi_remote_input_secure = "nifi.remote.input.secure";
   static constexpr const char *nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth";

diff --git a/libminifi/src/Configuration.cpp b/libminifi/src/Configuration.cpp
@@ -54,7 +54,10 @@ const std::unordered_map<std::string_view, gsl::not_null<const core::PropertyVal
   {Configuration::nifi_flowfile_repository_rocksdb_compaction_period, gsl::make_not_null(&core::StandardPropertyTypes::TIME_PERIOD_TYPE)},
   {Configuration::nifi_dbcontent_repository_rocksdb_compaction_period, gsl::make_not_null(&core::StandardPropertyTypes::TIME_PERIOD_TYPE)},
   {Configuration::nifi_content_repository_rocksdb_use_synchronous_writes, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
-  {Configuration::nifi_rocksdb_read_verify_checksums, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
+  {Configuration::nifi_content_repository_rocksdb_read_verify_checksums, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
+  {Configuration::nifi_flowfile_repository_rocksdb_read_verify_checksums, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
+  {Configuration::nifi_provenance_repository_rocksdb_read_verify_checksums, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
+  {Configuration::nifi_rocksdb_state_storage_read_verify_checksums, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
   {Configuration::nifi_dbcontent_repository_purge_period, gsl::make_not_null(&core::StandardPropertyTypes::TIME_PERIOD_TYPE)},
   {Configuration::nifi_remote_input_secure, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},
   {Configuration::nifi_security_need_ClientAuth, gsl::make_not_null(&core::StandardPropertyTypes::BOOLEAN_TYPE)},