From 6d1ce1fc9da409396d4b0375007132c2a1de2593 Mon Sep 17 00:00:00 2001 From: marta-lokhova Date: Thu, 14 Nov 2024 09:30:26 -0800 Subject: [PATCH] Close ledgers in parallel when flag is enabled --- lib/libmedida | 2 +- src/bucket/BucketList.cpp | 8 +- src/bucket/BucketList.h | 4 +- src/bucket/BucketListSnapshot.cpp | 2 - src/bucket/BucketManager.h | 10 +- src/bucket/BucketManagerImpl.cpp | 83 ++++----- src/bucket/BucketManagerImpl.h | 12 +- src/bucket/BucketSnapshotManager.cpp | 1 - src/bucket/test/BucketListTests.cpp | 3 +- src/bucket/test/BucketTestUtils.cpp | 12 +- src/catchup/ApplyCheckpointWork.cpp | 2 +- src/catchup/CatchupManagerImpl.cpp | 10 +- src/catchup/CatchupWork.cpp | 5 +- src/catchup/DownloadApplyTxsWork.cpp | 6 +- src/catchup/DownloadApplyTxsWork.h | 2 +- src/database/Database.cpp | 34 ++-- src/database/Database.h | 14 +- src/herder/HerderImpl.cpp | 15 +- src/herder/HerderSCPDriver.cpp | 4 +- src/herder/LedgerCloseData.h | 2 + src/herder/TxSetFrame.cpp | 13 +- src/herder/test/HerderTests.cpp | 99 ++++++----- src/herder/test/QuorumTrackerTests.cpp | 2 +- src/herder/test/TestTxSetUtils.cpp | 18 +- src/herder/test/TestTxSetUtils.h | 5 +- src/herder/test/TransactionQueueTests.cpp | 2 +- src/herder/test/TxSetTests.cpp | 75 ++++---- src/herder/test/UpgradesTests.cpp | 20 ++- src/history/HistoryManagerImpl.cpp | 2 +- src/history/test/HistoryTestsUtils.cpp | 1 + src/ledger/LedgerHeaderUtils.cpp | 4 +- src/ledger/LedgerHeaderUtils.h | 2 +- src/ledger/LedgerManager.h | 6 +- src/ledger/LedgerManagerImpl.cpp | 163 ++++++++++-------- src/ledger/LedgerManagerImpl.h | 8 +- src/ledger/LedgerTxn.cpp | 15 +- src/ledger/LedgerTxnImpl.h | 2 +- src/ledger/test/LedgerHeaderTests.cpp | 6 +- src/ledger/test/LedgerTests.cpp | 2 +- src/main/AppConnector.cpp | 13 +- src/main/AppConnector.h | 5 +- src/main/Application.h | 3 + src/main/ApplicationImpl.cpp | 73 ++++++-- src/main/ApplicationImpl.h | 12 +- src/main/ApplicationUtils.cpp | 2 +- src/main/CommandHandler.cpp | 2 +- src/main/Config.cpp | 13 ++ src/main/Config.h | 4 + src/main/ExternalQueue.cpp | 20 ++- src/main/test/ApplicationUtilsTests.cpp | 4 +- src/main/test/ConfigTests.cpp | 6 +- src/overlay/OverlayManagerImpl.cpp | 22 +-- src/overlay/test/FloodTests.cpp | 6 +- src/simulation/LoadGenerator.cpp | 7 + src/simulation/TxGenerator.cpp | 2 +- src/test/FuzzerImpl.cpp | 5 +- src/test/TestUtils.cpp | 8 +- src/test/TxTests.cpp | 18 +- .../ExtendFootprintTTLOpFrame.cpp | 6 +- .../InvokeHostFunctionOpFrame.cpp | 5 +- src/transactions/OperationFrame.cpp | 3 +- src/transactions/RestoreFootprintOpFrame.cpp | 12 +- src/transactions/TransactionFrame.cpp | 11 +- src/transactions/TransactionUtils.cpp | 2 +- .../test/InvokeHostFunctionTests.cpp | 15 +- src/transactions/test/SorobanTxTestUtils.cpp | 2 +- 66 files changed, 562 insertions(+), 395 deletions(-) diff --git a/lib/libmedida b/lib/libmedida index 62630a115d..299c6f6c89 160000 --- a/lib/libmedida +++ b/lib/libmedida @@ -1 +1 @@ -Subproject commit 62630a115d32e2086ee1b1bc19128db2eb935b80 +Subproject commit 299c6f6c897961fa123c455151a437325c291484 diff --git a/src/bucket/BucketList.cpp b/src/bucket/BucketList.cpp index aa3820dd47..3d49b77652 100644 --- a/src/bucket/BucketList.cpp +++ b/src/bucket/BucketList.cpp @@ -11,6 +11,7 @@ #include "crypto/SHA.h" #include "ledger/LedgerManager.h" #include "ledger/LedgerTxn.h" +#include "ledger/NetworkConfig.h" #include "main/Application.h" #include "util/GlobalChecks.h" #include "util/Logging.h" @@ -56,7 +57,6 @@ BucketLevel::getNext() void BucketLevel::setNext(FutureBucket const& fb) { - releaseAssert(threadIsMain()); mNextCurr = fb; } @@ -75,7 +75,6 @@ BucketLevel::getSnap() const void BucketLevel::setCurr(std::shared_ptr b) { - releaseAssert(threadIsMain()); mNextCurr.clear(); mCurr = b; } @@ -776,7 +775,8 @@ void BucketList::scanForEvictionLegacy(Application& app, AbstractLedgerTxn& ltx, uint32_t ledgerSeq, EvictionCounters& counters, - std::shared_ptr stats) + std::shared_ptr stats, + SorobanNetworkConfig& networkConfig) { releaseAssert(stats); @@ -785,8 +785,6 @@ BucketList::scanForEvictionLegacy(Application& app, AbstractLedgerTxn& ltx, return iter.isCurrBucket ? level.getCurr() : level.getSnap(); }; - auto const& networkConfig = - app.getLedgerManager().getSorobanNetworkConfig(); auto const firstScanLevel = networkConfig.stateArchivalSettings().startingEvictionScanLevel; auto evictionIter = networkConfig.evictionIterator(); diff --git a/src/bucket/BucketList.h b/src/bucket/BucketList.h index 5599132f4c..21a3a67b56 100644 --- a/src/bucket/BucketList.h +++ b/src/bucket/BucketList.h @@ -349,6 +349,7 @@ struct BucketEntryCounters; class Config; struct EvictionCounters; struct InflationWinner; +class SorobanNetworkConfig; namespace testutil { @@ -478,7 +479,8 @@ class BucketList void scanForEvictionLegacy(Application& app, AbstractLedgerTxn& ltx, uint32_t ledgerSeq, EvictionCounters& counters, - std::shared_ptr stats); + std::shared_ptr stats, + SorobanNetworkConfig& networkConfig); // Restart any merges that might be running on background worker threads, // merging buckets between levels. This needs to be called after forcing a diff --git a/src/bucket/BucketListSnapshot.cpp b/src/bucket/BucketListSnapshot.cpp index 5d26fd8296..ddc8856398 100644 --- a/src/bucket/BucketListSnapshot.cpp +++ b/src/bucket/BucketListSnapshot.cpp @@ -17,8 +17,6 @@ BucketListSnapshot::BucketListSnapshot(BucketList const& bl, LedgerHeader header) : mHeader(std::move(header)) { - releaseAssert(threadIsMain()); - for (uint32_t i = 0; i < BucketList::kNumLevels; ++i) { auto const& level = bl.getLevel(i); diff --git a/src/bucket/BucketManager.h b/src/bucket/BucketManager.h index a64bd9181f..dea63e685d 100644 --- a/src/bucket/BucketManager.h +++ b/src/bucket/BucketManager.h @@ -35,6 +35,7 @@ struct HistoryArchiveState; struct InflationWinner; struct LedgerHeader; struct MergeKey; +class SorobanNetworkConfig; // A fine-grained merge-operation-counter structure for tracking various // events during merges. These are not medida counters because we do not @@ -288,12 +289,15 @@ class BucketManager : NonMovableOrCopyable // pointed to by EvictionIterator. Scans until `maxEntriesToEvict` entries // have been evicted or maxEvictionScanSize bytes have been scanned. virtual void scanForEvictionLegacy(AbstractLedgerTxn& ltx, - uint32_t ledgerSeq) = 0; + uint32_t ledgerSeq, + SorobanNetworkConfig& networkConfig) = 0; - virtual void startBackgroundEvictionScan(uint32_t ledgerSeq) = 0; + virtual void startBackgroundEvictionScan(uint32_t ledgerSeq, + bool callFromLedgerClose) = 0; virtual void resolveBackgroundEvictionScan(AbstractLedgerTxn& ltx, uint32_t ledgerSeq, - LedgerKeySet const& modifiedKeys) = 0; + LedgerKeySet const& modifiedKeys, + SorobanNetworkConfig& networkConfig) = 0; virtual medida::Meter& getBloomMissMeter() const = 0; virtual medida::Meter& getBloomLookupMeter() const = 0; diff --git a/src/bucket/BucketManagerImpl.cpp b/src/bucket/BucketManagerImpl.cpp index b228f93e83..d5b8583157 100644 --- a/src/bucket/BucketManagerImpl.cpp +++ b/src/bucket/BucketManagerImpl.cpp @@ -91,7 +91,7 @@ void BucketManagerImpl::initialize() { ZoneScoped; - std::string d = mApp.getConfig().BUCKET_DIR_PATH; + std::string d = mConfig.BUCKET_DIR_PATH; if (!fs::exists(d)) { @@ -122,17 +122,17 @@ BucketManagerImpl::initialize() mLockedBucketDir = std::make_unique(d); mTmpDirManager = std::make_unique(d + "/tmp"); - if (mApp.getConfig().MODE_ENABLES_BUCKETLIST) + if (mConfig.MODE_ENABLES_BUCKETLIST) { mBucketList = std::make_unique(); - if (mApp.getConfig().isUsingBucketListDB()) + if (mConfig.isUsingBucketListDB()) { mSnapshotManager = std::make_unique( mApp, std::make_unique(*mBucketList, LedgerHeader()), - mApp.getConfig().QUERY_SNAPSHOT_LEDGERS); + mConfig.QUERY_SNAPSHOT_LEDGERS); } } @@ -141,11 +141,10 @@ BucketManagerImpl::initialize() // BUCKET_DIR_PATH, HISTORY_FILE_TYPE_SCP is persisted to the database // so create the remaining ledger header, transactions and results // directories - createPublishDir(FileType::HISTORY_FILE_TYPE_LEDGER, mApp.getConfig()); - createPublishDir(FileType::HISTORY_FILE_TYPE_TRANSACTIONS, - mApp.getConfig()); - createPublishDir(FileType::HISTORY_FILE_TYPE_RESULTS, mApp.getConfig()); - HistoryManager::createPublishQueueDir(mApp.getConfig()); + createPublishDir(FileType::HISTORY_FILE_TYPE_LEDGER, mConfig); + createPublishDir(FileType::HISTORY_FILE_TYPE_TRANSACTIONS, mConfig); + createPublishDir(FileType::HISTORY_FILE_TYPE_RESULTS, mConfig); + HistoryManager::createPublishQueueDir(mConfig); } void @@ -178,6 +177,7 @@ EvictionCounters::EvictionCounters(Application& app) BucketManagerImpl::BucketManagerImpl(Application& app) : mApp(app) + , mConfig(app.getConfig()) , mBucketList(nullptr) , mSnapshotManager(nullptr) , mTmpDirManager(nullptr) @@ -299,7 +299,7 @@ void BucketManagerImpl::deleteEntireBucketDir() { ZoneScoped; - std::string d = mApp.getConfig().BUCKET_DIR_PATH; + std::string d = mConfig.BUCKET_DIR_PATH; if (fs::exists(d)) { // First clean out the contents of the tmpdir, as usual. @@ -332,7 +332,7 @@ BucketManagerImpl::deleteTmpDirAndUnlockBucketDir() // Then delete the lockfile $BUCKET_DIR_PATH/stellar-core.lock if (mLockedBucketDir) { - std::string d = mApp.getConfig().BUCKET_DIR_PATH; + std::string d = mConfig.BUCKET_DIR_PATH; std::string lock = d + "/" + kLockFilename; releaseAssert(fs::exists(lock)); fs::unlockFile(lock); @@ -343,14 +343,14 @@ BucketManagerImpl::deleteTmpDirAndUnlockBucketDir() BucketList& BucketManagerImpl::getBucketList() { - releaseAssertOrThrow(mApp.getConfig().MODE_ENABLES_BUCKETLIST); + releaseAssertOrThrow(mConfig.MODE_ENABLES_BUCKETLIST); return *mBucketList; } BucketSnapshotManager& BucketManagerImpl::getBucketSnapshotManager() const { - releaseAssertOrThrow(mApp.getConfig().isUsingBucketListDB()); + releaseAssertOrThrow(mConfig.isUsingBucketListDB()); releaseAssert(mSnapshotManager); return *mSnapshotManager; } @@ -476,7 +476,7 @@ BucketManagerImpl::renameBucketDirFile(std::filesystem::path const& src, std::filesystem::path const& dst) { ZoneScoped; - if (mApp.getConfig().DISABLE_XDR_FSYNC) + if (mConfig.DISABLE_XDR_FSYNC) { return rename(src.string().c_str(), dst.string().c_str()) == 0; } @@ -492,7 +492,7 @@ BucketManagerImpl::adoptFileAsBucket(std::string const& filename, std::unique_ptr index) { ZoneScoped; - releaseAssertOrThrow(mApp.getConfig().MODE_ENABLES_BUCKETLIST); + releaseAssertOrThrow(mConfig.MODE_ENABLES_BUCKETLIST); std::lock_guard lock(mBucketMutex); if (mergeKey) @@ -566,7 +566,7 @@ BucketManagerImpl::adoptFileAsBucket(std::string const& filename, void BucketManagerImpl::noteEmptyMergeOutput(MergeKey const& mergeKey) { - releaseAssertOrThrow(mApp.getConfig().MODE_ENABLES_BUCKETLIST); + releaseAssertOrThrow(mConfig.MODE_ENABLES_BUCKETLIST); // We _do_ want to remove the mergeKey from mLiveFutures, both so that that // map does not grow without bound and more importantly so that we drop the @@ -681,7 +681,7 @@ BucketManagerImpl::putMergeFuture( MergeKey const& key, std::shared_future> wp) { ZoneScoped; - releaseAssertOrThrow(mApp.getConfig().MODE_ENABLES_BUCKETLIST); + releaseAssertOrThrow(mConfig.MODE_ENABLES_BUCKETLIST); std::lock_guard lock(mBucketMutex); CLOG_TRACE( Bucket, @@ -704,7 +704,7 @@ BucketManagerImpl::getBucketListReferencedBuckets() const { ZoneScoped; std::set referenced; - if (!mApp.getConfig().MODE_ENABLES_BUCKETLIST) + if (!mConfig.MODE_ENABLES_BUCKETLIST) { return referenced; } @@ -743,7 +743,7 @@ BucketManagerImpl::getAllReferencedBuckets() const { ZoneScoped; auto referenced = getBucketListReferencedBuckets(); - if (!mApp.getConfig().MODE_ENABLES_BUCKETLIST) + if (!mConfig.MODE_ENABLES_BUCKETLIST) { return referenced; } @@ -788,7 +788,7 @@ void BucketManagerImpl::cleanupStaleFiles() { ZoneScoped; - if (mApp.getConfig().DISABLE_BUCKET_GC) + if (mConfig.DISABLE_BUCKET_GC) { return; } @@ -867,7 +867,7 @@ BucketManagerImpl::forgetUnreferencedBuckets() CLOG_TRACE(Bucket, "BucketManager::forgetUnreferencedBuckets dropping {}", filename); - if (!filename.empty() && !mApp.getConfig().DISABLE_BUCKET_GC) + if (!filename.empty() && !mConfig.DISABLE_BUCKET_GC) { CLOG_TRACE(Bucket, "removing bucket file: {}", filename); std::filesystem::remove(filename); @@ -974,7 +974,7 @@ BucketManagerImpl::snapshotLedger(LedgerHeader& currentHeader) { ZoneScoped; Hash hash; - if (mApp.getConfig().MODE_ENABLES_BUCKETLIST) + if (mConfig.MODE_ENABLES_BUCKETLIST) { hash = mBucketList->getHash(); } @@ -1005,25 +1005,29 @@ BucketManagerImpl::maybeSetIndex(std::shared_ptr b, void BucketManagerImpl::scanForEvictionLegacy(AbstractLedgerTxn& ltx, - uint32_t ledgerSeq) + uint32_t ledgerSeq, + SorobanNetworkConfig& networkConfig) { ZoneScoped; releaseAssert(protocolVersionStartsFrom(ltx.getHeader().ledgerVersion, SOROBAN_PROTOCOL_VERSION)); - mBucketList->scanForEvictionLegacy( - mApp, ltx, ledgerSeq, mBucketListEvictionCounters, mEvictionStatistics); + mBucketList->scanForEvictionLegacy(mApp, ltx, ledgerSeq, + mBucketListEvictionCounters, + mEvictionStatistics, networkConfig); } void -BucketManagerImpl::startBackgroundEvictionScan(uint32_t ledgerSeq) +BucketManagerImpl::startBackgroundEvictionScan(uint32_t ledgerSeq, + bool callFromLedgerClose) { - releaseAssert(mApp.getConfig().isUsingBucketListDB()); + releaseAssert(!threadIsMain() || !mConfig.parallelLedgerClose()); + releaseAssert(mConfig.isUsingBucketListDB()); releaseAssert(mSnapshotManager); releaseAssert(!mEvictionFuture.valid()); releaseAssert(mEvictionStatistics); auto searchableBL = mSnapshotManager->copySearchableBucketListSnapshot(); - auto const& cfg = mApp.getLedgerManager().getSorobanNetworkConfig(); + auto cfg = mApp.getLedgerManager().getSorobanNetworkConfig(); auto const& sas = cfg.stateArchivalSettings(); using task_t = std::packaged_task; @@ -1045,28 +1049,25 @@ BucketManagerImpl::startBackgroundEvictionScan(uint32_t ledgerSeq) void BucketManagerImpl::resolveBackgroundEvictionScan( AbstractLedgerTxn& ltx, uint32_t ledgerSeq, - LedgerKeySet const& modifiedKeys) + LedgerKeySet const& modifiedKeys, SorobanNetworkConfig& networkConfig) { ZoneScoped; - releaseAssert(threadIsMain()); + releaseAssert(!threadIsMain() || !mConfig.parallelLedgerClose()); releaseAssert(mEvictionStatistics); if (!mEvictionFuture.valid()) { - startBackgroundEvictionScan(ledgerSeq); + startBackgroundEvictionScan(ledgerSeq, false); } auto evictionCandidates = mEvictionFuture.get(); - auto const& networkConfig = - mApp.getLedgerManager().getSorobanNetworkConfig(); - // If eviction related settings changed during the ledger, we have to // restart the scan if (!evictionCandidates.isValid(ledgerSeq, networkConfig.stateArchivalSettings())) { - startBackgroundEvictionScan(ledgerSeq); + startBackgroundEvictionScan(ledgerSeq, false); evictionCandidates = mEvictionFuture.get(); } @@ -1176,7 +1177,7 @@ BucketManagerImpl::assumeState(HistoryArchiveState const& has, uint32_t maxProtocolVersion, bool restartMerges) { ZoneScoped; - releaseAssertOrThrow(mApp.getConfig().MODE_ENABLES_BUCKETLIST); + releaseAssertOrThrow(mConfig.MODE_ENABLES_BUCKETLIST); for (uint32_t i = 0; i < BucketList::kNumLevels; ++i) { @@ -1203,7 +1204,7 @@ BucketManagerImpl::assumeState(HistoryArchiveState const& has, // Buckets on the BucketList should always be indexed when // BucketListDB enabled - if (mApp.getConfig().isUsingBucketListDB()) + if (mConfig.isUsingBucketListDB()) { releaseAssert(curr->isEmpty() || curr->isIndexed()); releaseAssert(snap->isEmpty() || snap->isIndexed()); @@ -1328,7 +1329,7 @@ BucketManagerImpl::mergeBuckets(HistoryArchiveState const& has) BucketMetadata meta; MergeCounters mc; auto& ctx = mApp.getClock().getIOContext(); - meta.ledgerVersion = mApp.getConfig().LEDGER_PROTOCOL_VERSION; + meta.ledgerVersion = mConfig.LEDGER_PROTOCOL_VERSION; BucketOutputIterator out(getTmpDir(), /*keepDeadEntries=*/false, meta, mc, ctx, /*doFsync=*/true); for (auto const& pair : ledgerMap) @@ -1539,13 +1540,13 @@ BucketManagerImpl::scheduleVerifyReferencedBucketsWork() Config const& BucketManagerImpl::getConfig() const { - return mApp.getConfig(); + return mConfig; } std::shared_ptr BucketManagerImpl::getSearchableBucketListSnapshot() { - releaseAssert(mApp.getConfig().isUsingBucketListDB()); + releaseAssert(mConfig.isUsingBucketListDB()); // Any other threads must maintain their own snapshot releaseAssert(threadIsMain()); if (!mSearchableBucketListSnapshot) @@ -1560,7 +1561,7 @@ BucketManagerImpl::getSearchableBucketListSnapshot() void BucketManagerImpl::reportBucketEntryCountMetrics() { - if (!mApp.getConfig().isUsingBucketListDB()) + if (!mConfig.isUsingBucketListDB()) { return; } diff --git a/src/bucket/BucketManagerImpl.h b/src/bucket/BucketManagerImpl.h index 50b6479ede..a26a46cb4e 100644 --- a/src/bucket/BucketManagerImpl.h +++ b/src/bucket/BucketManagerImpl.h @@ -3,6 +3,7 @@ #include "bucket/BucketList.h" #include "bucket/BucketManager.h" #include "bucket/BucketMergeMap.h" +#include "ledger/NetworkConfig.h" #include "xdr/Stellar-ledger.h" #include @@ -41,6 +42,7 @@ class BucketManagerImpl : public BucketManager static std::string const kLockFilename; Application& mApp; + Config const mConfig; std::unique_ptr mBucketList; std::unique_ptr mSnapshotManager; std::unique_ptr mTmpDirManager; @@ -149,12 +151,14 @@ class BucketManagerImpl : public BucketManager void snapshotLedger(LedgerHeader& currentHeader) override; void maybeSetIndex(std::shared_ptr b, std::unique_ptr&& index) override; - void scanForEvictionLegacy(AbstractLedgerTxn& ltx, - uint32_t ledgerSeq) override; - void startBackgroundEvictionScan(uint32_t ledgerSeq) override; + void scanForEvictionLegacy(AbstractLedgerTxn& ltx, uint32_t ledgerSeq, + SorobanNetworkConfig& networkConfig) override; + void startBackgroundEvictionScan(uint32_t ledgerSeq, + bool callFromLedgerClose) override; void resolveBackgroundEvictionScan(AbstractLedgerTxn& ltx, uint32_t ledgerSeq, - LedgerKeySet const& modifiedKeys) override; + LedgerKeySet const& modifiedKeys, + SorobanNetworkConfig& networkConfig) override; medida::Meter& getBloomMissMeter() const override; medida::Meter& getBloomLookupMeter() const override; diff --git a/src/bucket/BucketSnapshotManager.cpp b/src/bucket/BucketSnapshotManager.cpp index 52f907307b..8d253b1386 100644 --- a/src/bucket/BucketSnapshotManager.cpp +++ b/src/bucket/BucketSnapshotManager.cpp @@ -111,7 +111,6 @@ BucketSnapshotManager::updateCurrentSnapshot( std::unique_ptr&& newSnapshot) { releaseAssert(newSnapshot); - releaseAssert(threadIsMain()); // Updating the BucketSnapshotManager canonical snapshot, must lock // exclusively for write access. diff --git a/src/bucket/test/BucketListTests.cpp b/src/bucket/test/BucketListTests.cpp index edf229d439..a3893937c3 100644 --- a/src/bucket/test/BucketListTests.cpp +++ b/src/bucket/test/BucketListTests.cpp @@ -706,7 +706,8 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]") for_versions_from(20, *app, [&] { LedgerManagerForBucketTests& lm = app->getLedgerManager(); - auto& networkConfig = app->getLedgerManager().getSorobanNetworkConfig(); + auto& networkConfig = + app->getLedgerManager().getMutableSorobanNetworkConfig(); uint32_t windowSize = networkConfig.stateArchivalSettings() .bucketListSizeWindowSampleSize; diff --git a/src/bucket/test/BucketTestUtils.cpp b/src/bucket/test/BucketTestUtils.cpp index ee4b959857..e08defbb67 100644 --- a/src/bucket/test/BucketTestUtils.cpp +++ b/src/bucket/test/BucketTestUtils.cpp @@ -19,7 +19,7 @@ namespace BucketTestUtils uint32_t getAppLedgerVersion(Application& app) { - auto const& lcl = app.getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app.getLedgerManager().getLastClosedLedgerHeader(); return lcl.header.ledgerVersion; } @@ -170,12 +170,16 @@ LedgerManagerForBucketTests::transferLedgerEntriesToBucketList( if (mApp.getConfig().isUsingBackgroundEviction()) { mApp.getBucketManager().resolveBackgroundEvictionScan( - ltxEvictions, lh.ledgerSeq, keys); + ltxEvictions, lh.ledgerSeq, keys, + mApp.getLedgerManager() + .getMutableSorobanNetworkConfig()); } else { - mApp.getBucketManager().scanForEvictionLegacy(ltxEvictions, - lh.ledgerSeq); + mApp.getBucketManager().scanForEvictionLegacy( + ltxEvictions, lh.ledgerSeq, + mApp.getLedgerManager() + .getMutableSorobanNetworkConfig()); } if (ledgerCloseMeta) diff --git a/src/catchup/ApplyCheckpointWork.cpp b/src/catchup/ApplyCheckpointWork.cpp index 39180e9cdb..28ab85d1c3 100644 --- a/src/catchup/ApplyCheckpointWork.cpp +++ b/src/catchup/ApplyCheckpointWork.cpp @@ -154,7 +154,7 @@ ApplyCheckpointWork::getNextLedgerCloseData() auto& lm = mApp.getLedgerManager(); - auto const& lclHeader = lm.getLastClosedLedgerHeader(); + auto lclHeader = lm.getLastClosedLedgerHeader(); // If we are >1 before LCL, skip if (header.ledgerSeq + 1 < lclHeader.header.ledgerSeq) diff --git a/src/catchup/CatchupManagerImpl.cpp b/src/catchup/CatchupManagerImpl.cpp index 700a7bd889..8963368dee 100644 --- a/src/catchup/CatchupManagerImpl.cpp +++ b/src/catchup/CatchupManagerImpl.cpp @@ -476,24 +476,20 @@ CatchupManagerImpl::tryApplySyncingLedgers() break; } - if (mApp.getConfig() - .ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING.count() > 0) + if (mApp.getConfig().parallelLedgerClose()) { // Close a ledger asynchronously, with an added delay // Usefult to test async extrnalize flow - mApp.postOnMainThread( + mApp.postOnLedgerCloseThread( [&app = mApp, lcd]() { if (app.isStopping()) { return; } - std::this_thread::sleep_for( - app.getConfig() - .ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING); app.getLedgerManager().closeLedger(lcd, /* externalize */ true); }, - "closeLedger"); + "closeLedger queue"); } else { diff --git a/src/catchup/CatchupWork.cpp b/src/catchup/CatchupWork.cpp index 2d06425863..e88bb24dc8 100644 --- a/src/catchup/CatchupWork.cpp +++ b/src/catchup/CatchupWork.cpp @@ -138,7 +138,7 @@ CatchupWork::doReset() mTransactionsVerifyApplySeq.reset(); mGetHistoryArchiveStateWork.reset(); mApplyBufferedLedgersWork.reset(); - auto const& lcl = mApp.getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = mApp.getLedgerManager().getLastClosedLedgerHeader(); mLastClosedLedgerHashPair = LedgerNumHashPair( lcl.header.ledgerSeq, std::make_optional(lcl.hash)); mCatchupSeq.reset(); @@ -531,6 +531,7 @@ CatchupWork::runCatchupStep() // persistently available locally so it will return us to the // correct state. auto& ps = mApp.getPersistentState(); + mApp.getDatabase().clearPreparedStatementCache(); for (auto let : xdr::xdr_traits::enum_values()) { ps.clearRebuildForType(static_cast(let)); @@ -548,7 +549,7 @@ CatchupWork::runCatchupStep() // In this case we should actually have been caught-up during // the replay process and, if judged successful, our LCL should // be the one provided as well. - auto& lastClosed = + auto lastClosed = mApp.getLedgerManager().getLastClosedLedgerHeader(); releaseAssert(mLastApplied.hash == lastClosed.hash); releaseAssert(mLastApplied.header == lastClosed.header); diff --git a/src/catchup/DownloadApplyTxsWork.cpp b/src/catchup/DownloadApplyTxsWork.cpp index aa22f7dd86..90538ce3fa 100644 --- a/src/catchup/DownloadApplyTxsWork.cpp +++ b/src/catchup/DownloadApplyTxsWork.cpp @@ -27,7 +27,7 @@ DownloadApplyTxsWork::DownloadApplyTxsWork( : BatchWork(app, "download-apply-ledgers") , mRange(range) , mDownloadDir(downloadDir) - , mLastQueuedToApply(lastApplied) + , mLastApplied(lastApplied) , mCheckpointToQueue( app.getHistoryManager().checkpointContainingLedger(range.mFirst)) , mWaitForPublish(waitForPublish) @@ -172,7 +172,7 @@ DownloadApplyTxsWork::resetIter() mCheckpointToQueue = mApp.getHistoryManager().checkpointContainingLedger(mRange.mFirst); mLastYieldedWork.reset(); - mLastQueuedToApply = mApp.getLedgerManager().getLastClosedLedgerHeader(); + mLastApplied = mApp.getLedgerManager().getLastClosedLedgerHeader(); } bool @@ -190,7 +190,7 @@ DownloadApplyTxsWork::hasNext() const void DownloadApplyTxsWork::onSuccess() { - mLastQueuedToApply = mApp.getLedgerManager().getLastClosedLedgerHeader(); + mLastApplied = mApp.getLedgerManager().getLastClosedLedgerHeader(); } std::string diff --git a/src/catchup/DownloadApplyTxsWork.h b/src/catchup/DownloadApplyTxsWork.h index 3259fdacfe..e5a81d9240 100644 --- a/src/catchup/DownloadApplyTxsWork.h +++ b/src/catchup/DownloadApplyTxsWork.h @@ -25,7 +25,7 @@ class DownloadApplyTxsWork : public BatchWork { LedgerRange const mRange; TmpDir const& mDownloadDir; - LedgerHeaderHistoryEntry& mLastQueuedToApply; + LedgerHeaderHistoryEntry& mLastApplied; uint32_t mCheckpointToQueue; std::shared_ptr mLastYieldedWork; bool const mWaitForPublish; diff --git a/src/database/Database.cpp b/src/database/Database.cpp index da5b363b2d..4e4f520d97 100644 --- a/src/database/Database.cpp +++ b/src/database/Database.cpp @@ -304,7 +304,7 @@ Database::populateMiscDatabase() void Database::applyMiscSchemaUpgrade(unsigned long vers) { - clearPreparedStatementCache(); + clearPreparedStatementCache(mMiscSession); soci::transaction tx(mMiscSession.session()); switch (vers) { @@ -345,7 +345,7 @@ dropMiscTablesFromMain(Application& app) void Database::applySchemaUpgrade(unsigned long vers) { - clearPreparedStatementCache(); + clearPreparedStatementCache(mSession); soci::transaction tx(mSession.session()); switch (vers) @@ -568,15 +568,29 @@ Database::canUseMiscDB() const void Database::clearPreparedStatementCache() +{ + for (auto& c : mCaches) + { + for (auto& st : c.second) + { + st.second->clean_up(true); + } + } + mCaches.clear(); + mStatementsSize.set_count(0); +} + +void +Database::clearPreparedStatementCache(SessionWrapper& session) { // Flush all prepared statements; in sqlite they represent open cursors // and will conflict with any DROP TABLE commands issued below - for (auto st : mStatements) + for (auto st : mCaches[session.getSessionName()]) { st.second->clean_up(true); + mStatementsSize.dec(); } - mStatements.clear(); - mStatementsSize.set_count(mStatements.size()); + mCaches.erase(session.getSessionName()); } void @@ -757,16 +771,16 @@ StatementContext Database::getPreparedStatement(std::string const& query, SessionWrapper& session) { - auto cacheKey = PrepStatementCacheKey(session.getSessionName(), query); - auto i = mStatements.find(cacheKey); + auto& cache = mCaches[session.getSessionName()]; + auto i = cache.find(query); std::shared_ptr p; - if (i == mStatements.end()) + if (i == cache.end()) { p = std::make_shared(session.session()); p->alloc(); p->prepare(query); - mStatements.insert(std::make_pair(cacheKey, p)); - mStatementsSize.set_count(mStatements.size()); + cache.insert(std::make_pair(query, p)); + mStatementsSize.inc(); } else { diff --git a/src/database/Database.h b/src/database/Database.h index 907af57778..31b98c4e41 100644 --- a/src/database/Database.h +++ b/src/database/Database.h @@ -82,6 +82,11 @@ class SessionWrapper : NonCopyable : mSessionName(std::move(sessionName)) { } + SessionWrapper(std::string sessionName, soci::connection_pool& pool) + : mSession(pool), mSessionName(std::move(sessionName)) + { + } + soci::session& session() { @@ -140,10 +145,10 @@ class Database : NonMovableOrCopyable std::unique_ptr mMiscPool; // Cache key -> session name <> query - using PrepStatementCacheKey = std::pair; - std::unordered_map, - PairHash> - mStatements; + using PreparedStatementCache = + std::unordered_map>; + std::unordered_map mCaches; + medida::Counter& mStatementsSize; static bool gDriversRegistered; @@ -182,6 +187,7 @@ class Database : NonMovableOrCopyable // Purge all cached prepared statements, closing their handles with the // database. + void clearPreparedStatementCache(SessionWrapper& session); void clearPreparedStatementCache(); // Return metric-gathering timers for various families of SQL operation. diff --git a/src/herder/HerderImpl.cpp b/src/herder/HerderImpl.cpp index fa29e42358..f82252965b 100644 --- a/src/herder/HerderImpl.cpp +++ b/src/herder/HerderImpl.cpp @@ -516,7 +516,7 @@ HerderImpl::outOfSyncRecovery() CLOG_INFO(Herder, "Purging slots older than {}", purgeSlot); eraseBelow(purgeSlot); } - auto const& lcl = mLedgerManager.getLastClosedLedgerHeader().header; + auto lcl = mLedgerManager.getLastClosedLedgerHeader().header; for (auto const& e : getSCP().getLatestMessagesSend(lcl.ledgerSeq + 1)) { broadcast(e); @@ -649,7 +649,7 @@ HerderImpl::checkCloseTime(SCPEnvelope const& envelope, bool enforceRecent) auto envLedgerIndex = envelope.statement.slotIndex; auto& scpD = getHerderSCPDriver(); - auto const& lcl = mLedgerManager.getLastClosedLedgerHeader().header; + auto lcl = mLedgerManager.getLastClosedLedgerHeader().header; auto lastCloseIndex = lcl.ledgerSeq; auto lastCloseTime = lcl.scpValue.closeTime; @@ -1163,7 +1163,7 @@ HerderImpl::setupTriggerNextLedger() // core emits SCP messages only for slots it can fully validate // (any closed ledger is fully validated) releaseAssert(isTracking()); - auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + auto lcl = mLedgerManager.getLastClosedLedgerHeader(); releaseAssert(trackingConsensusLedgerIndex() == lcl.header.ledgerSeq); releaseAssert(mLedgerManager.isSynced()); @@ -1350,7 +1350,7 @@ HerderImpl::triggerNextLedger(uint32_t ledgerSeqToTrigger, // our first choice for this round's set is all the tx we have collected // during last few ledger closes - auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + auto lcl = mLedgerManager.getLastClosedLedgerHeader(); TxSetPhaseTransactions txPhases; txPhases.emplace_back(mTransactionQueue.getTransactions(lcl.header)); @@ -2104,7 +2104,7 @@ HerderImpl::maybeHandleUpgrade() // no-op on any earlier protocol return; } - auto const& conf = mApp.getLedgerManager().getSorobanNetworkConfig(); + auto conf = mApp.getLedgerManager().getSorobanNetworkConfig(); auto maybeNewMaxTxSize = conf.txMaxSizeBytes() + getFlowControlExtraBuffer(); @@ -2155,8 +2155,7 @@ HerderImpl::start() .header.ledgerVersion; if (protocolVersionStartsFrom(version, SOROBAN_PROTOCOL_VERSION)) { - auto const& conf = - mApp.getLedgerManager().getSorobanNetworkConfig(); + auto conf = mApp.getLedgerManager().getSorobanNetworkConfig(); mMaxTxSize = std::max(mMaxTxSize, conf.txMaxSizeBytes() + getFlowControlExtraBuffer()); } @@ -2185,7 +2184,7 @@ HerderImpl::start() } // setup a sufficient state that we can participate in consensus - auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + auto lcl = mLedgerManager.getLastClosedLedgerHeader(); if (!mApp.getConfig().FORCE_SCP && lcl.header.ledgerSeq == LedgerManager::GENESIS_LEDGER_SEQ) diff --git a/src/herder/HerderSCPDriver.cpp b/src/herder/HerderSCPDriver.cpp index 47f1453d11..688a037aad 100644 --- a/src/herder/HerderSCPDriver.cpp +++ b/src/herder/HerderSCPDriver.cpp @@ -220,7 +220,7 @@ HerderSCPDriver::validateValueHelper(uint64_t slotIndex, StellarValue const& b, } } - auto const& lcl = mLedgerManager.getLastClosedLedgerHeader().header; + auto lcl = mLedgerManager.getLastClosedLedgerHeader().header; // when checking close time, start with what we have locally lastCloseTime = lcl.scpValue.closeTime; @@ -612,7 +612,7 @@ HerderSCPDriver::combineCandidates(uint64_t slotIndex, std::set aggSet; - auto const& lcl = mLedgerManager.getLastClosedLedgerHeader(); + auto lcl = mLedgerManager.getLastClosedLedgerHeader(); Hash candidatesHash; diff --git a/src/herder/LedgerCloseData.h b/src/herder/LedgerCloseData.h index 31f9192277..575e79233d 100644 --- a/src/herder/LedgerCloseData.h +++ b/src/herder/LedgerCloseData.h @@ -79,6 +79,8 @@ class LedgerCloseData private: uint32_t mLedgerSeq; + // TODO: confirm TxSet pointers don't get purged if ledger close is holding + // a strong reference to it TxSetXDRFrameConstPtr mTxSet; StellarValue mValue; std::optional mExpectedLedgerHash; diff --git a/src/herder/TxSetFrame.cpp b/src/herder/TxSetFrame.cpp index e4f951f458..d22bcc66f2 100644 --- a/src/herder/TxSetFrame.cpp +++ b/src/herder/TxSetFrame.cpp @@ -366,7 +366,7 @@ makeTxSetFromTransactions(TxSetPhaseTransactions const& txPhases, #endif } - auto const& lclHeader = app.getLedgerManager().getLastClosedLedgerHeader(); + auto lclHeader = app.getLedgerManager().getLastClosedLedgerHeader(); // Preliminary applicable frame - we don't know the contents hash yet, but // we also don't return this. std::unique_ptr preliminaryApplicableTxSet( @@ -775,6 +775,12 @@ ApplicableTxSetFrame::ApplicableTxSetFrame(Application& app, bool isGeneralized, , mPhaseInclusionFeeMap(mTxPhases.size()) , mContentsHash(contentsHash) { + // print hashes + CLOG_INFO(Herder, "Creating ApplicableTxSetFrame from txs: {}", + hexAbbrev(previousLedgerHash)); + CLOG_INFO( + Herder, "ApplicableTxSetFrame contents hash: {}", + hexAbbrev(app.getLedgerManager().getLastClosedLedgerHeader().hash)); releaseAssert(previousLedgerHash == app.getLedgerManager().getLastClosedLedgerHeader().hash); } @@ -875,7 +881,7 @@ ApplicableTxSetFrame::checkValid(Application& app, uint64_t upperBoundCloseTimeOffset) const { ZoneScoped; - auto& lcl = app.getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app.getLedgerManager().getLastClosedLedgerHeader(); // Start by checking previousLedgerHash if (lcl.hash != mPreviousLedgerHash) @@ -1388,8 +1394,7 @@ ApplicableTxSetFrame::applySurgePricing(Application& app) ZoneScoped; releaseAssert(mTxPhases.size() <= static_cast(TxSetPhase::PHASE_COUNT)); - auto const& lclHeader = - app.getLedgerManager().getLastClosedLedgerHeader().header; + auto lclHeader = app.getLedgerManager().getLastClosedLedgerHeader().header; for (int i = 0; i < mTxPhases.size(); i++) { TxSetPhase phaseType = static_cast(i); diff --git a/src/herder/test/HerderTests.cpp b/src/herder/test/HerderTests.cpp index 1ec955200a..8acafa41cb 100644 --- a/src/herder/test/HerderTests.cpp +++ b/src/herder/test/HerderTests.cpp @@ -1042,7 +1042,7 @@ TEST_CASE("tx set hits overlay byte limit during construction", cfg.mLedgerMaxInstructions = max; }); - auto const& conf = app->getLedgerManager().getSorobanNetworkConfig(); + auto conf = app->getLedgerManager().getSorobanNetworkConfig(); uint32_t maxContractSize = 0; maxContractSize = conf.maxContractSizeBytes(); @@ -1413,11 +1413,9 @@ TEST_CASE("surge pricing", "[herder][txset][soroban]") SECTION("tx sets over limits are invalid") { TxSetTransactions txs = generateTxs(accounts, conf); - auto txSet = - testtxset::makeNonValidatedGeneralizedTxSet( - {{}, {std::make_pair(500, txs)}}, *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash) - .second; + auto txSet = testtxset::makeNonValidatedGeneralizedTxSet( + {{}, {std::make_pair(500, txs)}}, *app) + .second; REQUIRE(!txSet->checkValid(*app, 0, 0)); } @@ -1825,7 +1823,7 @@ TEST_CASE("generalized tx set applied to ledger", "[herder][txset][soroban]") 1000, std::vector{addTx(3, 3500), addTx(2, 5000)})}, {}}, - *app, app->getLedgerManager().getLastClosedLedgerHeader().hash); + *app); checkFees(txSet, {3000, 2000}); } SECTION("single non-discounted component") @@ -1835,7 +1833,7 @@ TEST_CASE("generalized tx set applied to ledger", "[herder][txset][soroban]") std::vector{ addTx(3, 3500), addTx(2, 5000)})}, {}}, - *app, app->getLedgerManager().getLastClosedLedgerHeader().hash); + *app); checkFees(txSet, {3500, 5000}); } SECTION("multiple components") @@ -1856,9 +1854,8 @@ TEST_CASE("generalized tx set applied to ledger", "[herder][txset][soroban]") std::make_pair(std::nullopt, std::vector{ addTx(5, 35000), addTx(1, 10000)})}; - auto txSet = testtxset::makeNonValidatedGeneralizedTxSet( - {components, {}}, *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash); + auto txSet = + testtxset::makeNonValidatedGeneralizedTxSet({components, {}}, *app); checkFees(txSet, {3000, 2000, 500, 2500, 8000, 35000, 10000}); } SECTION("soroban") @@ -1872,7 +1869,7 @@ TEST_CASE("generalized tx set applied to ledger", "[herder][txset][soroban]") std::vector{ addSorobanTx(5000), addSorobanTx(10000)})}, }, - *app, app->getLedgerManager().getLastClosedLedgerHeader().hash); + *app); SECTION("with validation") { checkFees(txSet, @@ -1905,7 +1902,7 @@ testSCPDriver(uint32 protocolVersion, uint32_t maxTxSetSize, size_t expectedOps) Application::pointer app = createTestApplication(clock, cfg); - auto const& lcl = app->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); auto root = TestAccount::createRoot(*app); std::vector accounts; @@ -2198,8 +2195,7 @@ testSCPDriver(uint32 protocolVersion, uint32_t maxTxSetSize, size_t expectedOps) tx->addSignature(root.getSecretKey()); auto [txSet, applicableTxSet] = testtxset::makeNonValidatedTxSetBasedOnLedgerVersion( - protocolVersion, {tx}, *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash); + protocolVersion, {tx}, *app); // Build a StellarValue containing the transaction set we just // built and the given next closeTime. @@ -2588,10 +2584,10 @@ TEST_CASE("SCP State", "[herder]") REQUIRE(sim->getNode(nodeIDs[0]) ->getLedgerManager() - .getLastClosedLedgerNum() == expectedLedger); + .getLastClosedLedgerNum() >= expectedLedger); REQUIRE(sim->getNode(nodeIDs[1]) ->getLedgerManager() - .getLastClosedLedgerNum() == expectedLedger); + .getLastClosedLedgerNum() >= expectedLedger); lcl = sim->getNode(nodeIDs[0]) ->getLedgerManager() @@ -2689,7 +2685,7 @@ TEST_CASE("SCP State", "[herder]") // then let the nodes run a bit more, they should all externalize the // next ledger sim->crankUntil( - [&]() { return sim->haveAllExternalized(expectedLedger + 1, 5); }, + [&]() { return sim->haveAllExternalized(expectedLedger + 2, 6); }, 2 * numLedgers * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); // nodes are at least on ledger 7 (some may be on 8) @@ -2698,14 +2694,6 @@ TEST_CASE("SCP State", "[herder]") // All nodes are in sync REQUIRE(sim->getNode(nodeIDs[i])->getState() == Application::State::APP_SYNCED_STATE); - auto const& actual = sim->getNode(nodeIDs[i]) - ->getLedgerManager() - .getLastClosedLedgerHeader() - .header; - if (actual.ledgerSeq == expectedLedger + 1) - { - REQUIRE(actual.previousLedgerHash == lcl.hash); - } } } @@ -2732,10 +2720,10 @@ TEST_CASE("SCP State", "[herder]") for (int i = 0; i <= 2; i++) { - auto const& actual = sim->getNode(nodeIDs[i]) - ->getLedgerManager() - .getLastClosedLedgerHeader() - .header; + auto actual = sim->getNode(nodeIDs[i]) + ->getLedgerManager() + .getLastClosedLedgerHeader() + .header; REQUIRE(actual == lcl.header); } @@ -3126,10 +3114,10 @@ TEST_CASE("soroban txs each parameter surge priced", "[soroban][herder]") bool hadSorobanSurgePricing = false; simulation->crankUntil( [&]() { - auto& lclHeader = nodes[0] - ->getLedgerManager() - .getLastClosedLedgerHeader() - .header; + auto lclHeader = nodes[0] + ->getLedgerManager() + .getLastClosedLedgerHeader() + .header; auto txSet = nodes[0]->getHerder().getTxSet( lclHeader.scpValue.txSetHash); GeneralizedTransactionSet xdrTxSet; @@ -3343,14 +3331,34 @@ TEST_CASE("overlay parallel processing") { auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE); - // Set threshold to 1 so all have to vote - auto simulation = - Topologies::core(4, 1, Simulation::OVER_TCP, networkID, [](int i) { - auto cfg = getTestConfig(i); - cfg.TESTING_UPGRADE_MAX_TX_SET_SIZE = 100; - cfg.EXPERIMENTAL_BACKGROUND_OVERLAY_PROCESSING = true; - return cfg; - }); + std::shared_ptr simulation; + + SECTION("background traffic processing") + { + // Set threshold to 1 so all have to vote + simulation = + Topologies::core(4, 1, Simulation::OVER_TCP, networkID, [](int i) { + auto cfg = getTestConfig(i); + cfg.TESTING_UPGRADE_MAX_TX_SET_SIZE = 100; + cfg.EXPERIMENTAL_BACKGROUND_OVERLAY_PROCESSING = true; + return cfg; + }); + } + SECTION("background ledger close") + { + // Set threshold to 1 so all have to vote + simulation = + Topologies::core(4, 1, Simulation::OVER_TCP, networkID, [](int i) { + auto cfg = + getTestConfig(i, Config::TESTDB_BUCKET_DB_PERSISTENT); + cfg.TESTING_UPGRADE_MAX_TX_SET_SIZE = 100; + cfg.EXPERIMENTAL_PARALLEL_LEDGER_CLOSE = true; + cfg.ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING = + std::chrono::seconds(3); + return cfg; + }); + } + simulation->startAllNodes(); auto nodes = simulation->getNodes(); uint32_t desiredTxRate = 1; @@ -3851,6 +3859,7 @@ herderExternalizesValuesWithProtocol(uint32_t version) Herder::ENVELOPE_STATUS_READY); REQUIRE(herder.recvSCPEnvelope(newMsgB.first, qset, newMsgB.second) == Herder::ENVELOPE_STATUS_READY); + simulation->crankForAtLeast(std::chrono::seconds(1), false); }; auto testOutOfOrder = [&](bool partial) { @@ -4325,7 +4334,7 @@ TEST_CASE("In quorum filtering", "[quorum][herder][acceptance]") auto c = sim->getNode(k); HerderImpl& herder = *static_cast(&c->getHerder()); - auto const& lcl = c->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = c->getLedgerManager().getLastClosedLedgerHeader(); herder.getSCP().processCurrentState(lcl.header.ledgerSeq, proc, true); } @@ -4389,7 +4398,7 @@ static void externalize(SecretKey const& sk, LedgerManager& lm, HerderImpl& herder, std::vector const& txs, Application& app) { - auto const& lcl = lm.getLastClosedLedgerHeader(); + auto lcl = lm.getLastClosedLedgerHeader(); auto ledgerSeq = lcl.header.ledgerSeq + 1; auto classicTxs = txs; @@ -4664,8 +4673,6 @@ TEST_CASE("do not flood too many transactions", "[herder][transactionqueue]") cfg.FORCE_SCP = false; cfg.FLOOD_TX_PERIOD_MS = 100; cfg.FLOOD_OP_RATE_PER_LEDGER = 2.0; - cfg.ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING = - std::chrono::seconds(0); return cfg; }); diff --git a/src/herder/test/QuorumTrackerTests.cpp b/src/herder/test/QuorumTrackerTests.cpp index 5dc8f3ad36..5f688531d2 100644 --- a/src/herder/test/QuorumTrackerTests.cpp +++ b/src/herder/test/QuorumTrackerTests.cpp @@ -108,7 +108,7 @@ testQuorumTracker() recvEnvelope(envelope, slotID, k, qSet, pp); }; auto makeValue = [&](int i) { - auto const& lcl = app->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); auto txSet = TxSetXDRFrame::makeEmpty(lcl); StellarValue sv = herder->makeStellarValue( txSet->getContentsHash(), lcl.header.scpValue.closeTime + i, diff --git a/src/herder/test/TestTxSetUtils.cpp b/src/herder/test/TestTxSetUtils.cpp index e27c43c3d5..a92e688a04 100644 --- a/src/herder/test/TestTxSetUtils.cpp +++ b/src/herder/test/TestTxSetUtils.cpp @@ -3,6 +3,7 @@ // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 #include "herder/test/TestTxSetUtils.h" +#include "ledger/LedgerManager.h" #include "ledger/LedgerTxn.h" #include "main/Application.h" #include "util/ProtocolVersion.h" @@ -66,9 +67,10 @@ makeGeneralizedTxSetXDR(std::vector const& txsPerBaseFeePhases, std::pair makeNonValidatedTxSet(std::vector const& txs, - Application& app, Hash const& previousLedgerHash) + Application& app) { - auto xdrTxSet = makeTxSetXDR(txs, previousLedgerHash); + auto xdrTxSet = makeTxSetXDR( + txs, app.getLedgerManager().getLastClosedLedgerHeader().hash); auto txSet = TxSetXDRFrame::makeFromWire(xdrTxSet); auto applicableTxSet = txSet->prepareForApply(app); return std::make_pair(txSet, std::move(applicableTxSet)); @@ -77,10 +79,10 @@ makeNonValidatedTxSet(std::vector const& txs, std::pair makeNonValidatedGeneralizedTxSet( - std::vector const& txsPerBaseFee, Application& app, - Hash const& previousLedgerHash) + std::vector const& txsPerBaseFee, Application& app) { - auto xdrTxSet = makeGeneralizedTxSetXDR(txsPerBaseFee, previousLedgerHash); + auto xdrTxSet = makeGeneralizedTxSetXDR( + txsPerBaseFee, app.getLedgerManager().getLastClosedLedgerHeader().hash); auto txSet = TxSetXDRFrame::makeFromWire(xdrTxSet); return std::make_pair(txSet, txSet->prepareForApply(app)); } @@ -88,16 +90,16 @@ makeNonValidatedGeneralizedTxSet( std::pair makeNonValidatedTxSetBasedOnLedgerVersion( uint32_t ledgerVersion, std::vector const& txs, - Application& app, Hash const& previousLedgerHash) + Application& app) { if (protocolVersionStartsFrom(ledgerVersion, SOROBAN_PROTOCOL_VERSION)) { return makeNonValidatedGeneralizedTxSet( - {{std::make_pair(100LL, txs)}, {}}, app, previousLedgerHash); + {{std::make_pair(100LL, txs)}, {}}, app); } else { - return makeNonValidatedTxSet(txs, app, previousLedgerHash); + return makeNonValidatedTxSet(txs, app); } } diff --git a/src/herder/test/TestTxSetUtils.h b/src/herder/test/TestTxSetUtils.h index be9b7eac1e..a81553889b 100644 --- a/src/herder/test/TestTxSetUtils.h +++ b/src/herder/test/TestTxSetUtils.h @@ -16,12 +16,11 @@ using ComponentPhases = std::vector< std::pair, std::vector>>; std::pair makeNonValidatedGeneralizedTxSet( - std::vector const& txsPerBaseFee, Application& app, - Hash const& previousLedgerHash); + std::vector const& txsPerBaseFee, Application& app); std::pair makeNonValidatedTxSetBasedOnLedgerVersion( uint32_t ledgerVersion, std::vector const& txs, - Application& app, Hash const& previousLedgerHash); + Application& app); } // namespace testtxset } // namespace stellar diff --git a/src/herder/test/TransactionQueueTests.cpp b/src/herder/test/TransactionQueueTests.cpp index 53760d1e8b..3417d3eaed 100644 --- a/src/herder/test/TransactionQueueTests.cpp +++ b/src/herder/test/TransactionQueueTests.cpp @@ -2725,7 +2725,7 @@ TEST_CASE("remove applied", "[herder][transactionqueue]") herder.recvTransaction(tx3, false); { - auto const& lcl = lm.getLastClosedLedgerHeader(); + auto lcl = lm.getLastClosedLedgerHeader(); auto ledgerSeq = lcl.header.ledgerSeq + 1; root.loadSequenceNumber(); diff --git a/src/herder/test/TxSetTests.cpp b/src/herder/test/TxSetTests.cpp index 0f90c62921..438be73082 100644 --- a/src/herder/test/TxSetTests.cpp +++ b/src/herder/test/TxSetTests.cpp @@ -530,12 +530,11 @@ TEST_CASE("generalized tx set XDR conversion", "[txset]") REQUIRE(newXdr == txSetXdr); }; + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); SECTION("empty set") { auto [_, ApplicableTxSetFrame] = - testtxset::makeNonValidatedGeneralizedTxSet( - {{}, {}}, *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash); + testtxset::makeNonValidatedGeneralizedTxSet({{}, {}}, *app); GeneralizedTransactionSet txSetXdr; ApplicableTxSetFrame->toWireTxSetFrame()->toXDR(txSetXdr); @@ -546,8 +545,7 @@ TEST_CASE("generalized tx set XDR conversion", "[txset]") { auto [_, ApplicableTxSetFrame] = testtxset::makeNonValidatedGeneralizedTxSet( - {{std::make_pair(1234LL, createTxs(5, 1234))}, {}}, *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash); + {{std::make_pair(1234LL, createTxs(5, 1234))}, {}}, *app); GeneralizedTransactionSet txSetXdr; ApplicableTxSetFrame->toWireTxSetFrame()->toXDR(txSetXdr); @@ -568,8 +566,7 @@ TEST_CASE("generalized tx set XDR conversion", "[txset]") { auto [_, ApplicableTxSetFrame] = testtxset::makeNonValidatedGeneralizedTxSet( - {{std::make_pair(std::nullopt, createTxs(5, 4321))}, {}}, *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash); + {{std::make_pair(std::nullopt, createTxs(5, 4321))}, {}}, *app); GeneralizedTransactionSet txSetXdr; ApplicableTxSetFrame->toWireTxSetFrame()->toXDR(txSetXdr); @@ -595,7 +592,7 @@ TEST_CASE("generalized tx set XDR conversion", "[txset]") std::make_pair(1234LL, createTxs(2, 1234)), std::make_pair(std::nullopt, createTxs(4, 4321))}, {}}, - *app, app->getLedgerManager().getLastClosedLedgerHeader().hash); + *app); GeneralizedTransactionSet txSetXdr; ApplicableTxSetFrame->toWireTxSetFrame()->toXDR(txSetXdr); @@ -613,10 +610,8 @@ TEST_CASE("generalized tx set XDR conversion", "[txset]") } SECTION("built from transactions") { - auto const& lclHeader = - app->getLedgerManager().getLastClosedLedgerHeader(); std::vector txs = - createTxs(5, lclHeader.header.baseFee, /* isSoroban */ false); + createTxs(5, lcl.header.baseFee, /* isSoroban */ false); std::vector sorobanTxs = createTxs(5, 10'000'000, /* isSoroban */ true); @@ -631,7 +626,7 @@ TEST_CASE("generalized tx set XDR conversion", "[txset]") .phases[0] .v0Components()[0] .txsMaybeDiscountedFee() - .baseFee == lclHeader.header.baseFee); + .baseFee == lcl.header.baseFee); REQUIRE(txSetXdr.v1TxSet() .phases[0] .v0Components()[0] @@ -658,7 +653,7 @@ TEST_CASE("generalized tx set XDR conversion", "[txset]") REQUIRE(phase.v0Components().size() == 1); REQUIRE(*phase.v0Components()[0] .txsMaybeDiscountedFee() - .baseFee == lclHeader.header.baseFee); + .baseFee == lcl.header.baseFee); REQUIRE(phase.v0Components()[0] .txsMaybeDiscountedFee() .txs.size() == 5); @@ -684,7 +679,7 @@ TEST_CASE("generalized tx set XDR conversion", "[txset]") { auto const& phase = txSetXdr.v1TxSet().phases[i]; auto expectedBaseFee = - i == 0 ? lclHeader.header.baseFee + i == 0 ? lcl.header.baseFee : higherFeeSorobanTxs[0]->getInclusionFee(); REQUIRE(phase.v0Components().size() == 1); REQUIRE(*phase.v0Components()[0] @@ -757,29 +752,27 @@ TEST_CASE("generalized tx set with multiple txs per source account", SECTION("invalid") { - auto txSet = - testtxset::makeNonValidatedGeneralizedTxSet( - {{std::make_pair( - 500, - std::vector{ - createTx(1, 1000, false), createTx(3, 1500, false)})}, - {}}, - *app, app->getLedgerManager().getLastClosedLedgerHeader().hash) - .second; + auto txSet = testtxset::makeNonValidatedGeneralizedTxSet( + {{std::make_pair(500, + std::vector{ + createTx(1, 1000, false), + createTx(3, 1500, false)})}, + {}}, + *app) + .second; REQUIRE(!txSet->checkValid(*app, 0, 0)); } SECTION("valid") { - auto txSet = - testtxset::makeNonValidatedGeneralizedTxSet( - {{std::make_pair( - 500, - std::vector{ - createTx(1, 1000, true), createTx(3, 1500, true)})}, - {}}, - *app, app->getLedgerManager().getLastClosedLedgerHeader().hash) - .second; + auto txSet = testtxset::makeNonValidatedGeneralizedTxSet( + {{std::make_pair(500, + std::vector{ + createTx(1, 1000, true), + createTx(3, 1500, true)})}, + {}}, + *app) + .second; REQUIRE(txSet->checkValid(*app, 0, 0)); } @@ -804,7 +797,7 @@ TEST_CASE("generalized tx set with multiple txs per source account", createTx(1, 1000, false), createTx(3, 1500, false)})}, {std::make_pair( 500, std::vector{sorobanTx})}}, - *app, app->getLedgerManager().getLastClosedLedgerHeader().hash) + *app) .second; REQUIRE(!txSet->checkValid(*app, 0, 0)); @@ -894,7 +887,7 @@ TEST_CASE("generalized tx set fees", "[txset][soroban]") std::vector{ createTx(1, 5000, /* isSoroban */ true), createTx(1, 20000, /* isSoroban */ true)})}}, - *app, app->getLedgerManager().getLastClosedLedgerHeader().hash) + *app) .second; REQUIRE(txSet->checkValid(*app, 0, 0)); @@ -925,8 +918,7 @@ TEST_CASE("generalized tx set fees", "[txset][soroban]") std::vector{ createTx(2, 999)})}, {}}, - *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash) + *app) .second; REQUIRE(!txSet->checkValid(*app, 0, 0)); @@ -939,8 +931,7 @@ TEST_CASE("generalized tx set fees", "[txset][soroban]") {std::make_pair( 500, std::vector{createTx( 1, 499, /* isSoroban */ true)})}}, - *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash) + *app) .second; REQUIRE(!txSet->checkValid(*app, 0, 0)); @@ -957,8 +948,7 @@ TEST_CASE("generalized tx set fees", "[txset][soroban]") std::vector{ createTx(2, 199)})}, {}}, - *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash) + *app) .second; REQUIRE(!txSet->checkValid(*app, 0, 0)); @@ -971,8 +961,7 @@ TEST_CASE("generalized tx set fees", "[txset][soroban]") {std::make_pair(std::nullopt, std::vector{ createTx(1, 99, true, false)})}}, - *app, - app->getLedgerManager().getLastClosedLedgerHeader().hash) + *app) .second; REQUIRE(!txSet->checkValid(*app, 0, 0)); @@ -1086,7 +1075,7 @@ TEST_CASE("txset nomination", "[txset]") txToLedgerRatioPercentDistr(rng); }); - auto const& sorobanConfig = + auto sorobanConfig = app->getLedgerManager().getSorobanNetworkConfig(); stellar::uniform_int_distribution<> txReadEntriesDistr( 1, sorobanConfig.txMaxReadLedgerEntries()); diff --git a/src/herder/test/UpgradesTests.cpp b/src/herder/test/UpgradesTests.cpp index c1f86fb7ce..fe14c5a845 100644 --- a/src/herder/test/UpgradesTests.cpp +++ b/src/herder/test/UpgradesTests.cpp @@ -253,7 +253,7 @@ makeBucketListSizeWindowSampleSizeTestUpgrade(Application& app, { // Modify window size auto sas = app.getLedgerManager() - .getSorobanNetworkConfig() + .getMutableSorobanNetworkConfig() .stateArchivalSettings(); sas.bucketListSizeWindowSampleSize = newWindowSize; @@ -580,7 +580,7 @@ TEST_CASE("Ledger Manager applies upgrades properly", "[upgrades]") cfg.USE_CONFIG_FOR_GENESIS = false; auto app = createTestApplication(clock, cfg); - auto const& lcl = app->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); REQUIRE(lcl.header.ledgerVersion == LedgerManager::GENESIS_LEDGER_VERSION); REQUIRE(lcl.header.baseFee == LedgerManager::GENESIS_LEDGER_BASE_FEE); @@ -839,7 +839,7 @@ TEST_CASE("config upgrades applied to ledger", "[soroban][upgrades]") executeUpgrade(*app, makeProtocolVersionUpgrade( static_cast(SOROBAN_PROTOCOL_VERSION))); auto const& sorobanConfig = - app->getLedgerManager().getSorobanNetworkConfig(); + app->getLedgerManager().getMutableSorobanNetworkConfig(); SECTION("unknown config upgrade set is ignored") { auto contractID = autocheck::generator()(5); @@ -907,7 +907,7 @@ TEST_CASE("config upgrades applied to ledger", "[soroban][upgrades]") auto const newSize = 20; populateValuesAndUpgradeSize(newSize); auto const& cfg2 = - app->getLedgerManager().getSorobanNetworkConfig(); + app->getLedgerManager().getMutableSorobanNetworkConfig(); // Verify that we popped the 10 oldest values auto sum = 0; @@ -929,7 +929,7 @@ TEST_CASE("config upgrades applied to ledger", "[soroban][upgrades]") auto const newSize = 40; populateValuesAndUpgradeSize(newSize); auto const& cfg2 = - app->getLedgerManager().getSorobanNetworkConfig(); + app->getLedgerManager().getMutableSorobanNetworkConfig(); // Verify that we backfill 10 copies of the oldest value auto sum = 0; @@ -959,7 +959,7 @@ TEST_CASE("config upgrades applied to ledger", "[soroban][upgrades]") LedgerTxn ltx2(app->getLedgerTxnRoot()); auto const& cfg = - app->getLedgerManager().getSorobanNetworkConfig(); + app->getLedgerManager().getMutableSorobanNetworkConfig(); initialSize = cfg.mStateArchivalSettings.bucketListSizeWindowSampleSize; initialWindow = cfg.mBucketListSizeSnapshots; @@ -974,7 +974,8 @@ TEST_CASE("config upgrades applied to ledger", "[soroban][upgrades]") REQUIRE(configUpgradeSet); executeUpgrade(*app, makeConfigUpgrade(*configUpgradeSet)); - auto const& cfg = app->getLedgerManager().getSorobanNetworkConfig(); + auto const& cfg = + app->getLedgerManager().getMutableSorobanNetworkConfig(); REQUIRE(cfg.mStateArchivalSettings.bucketListSizeWindowSampleSize == initialSize); REQUIRE(cfg.mBucketListSizeSnapshots == initialWindow); @@ -1088,7 +1089,7 @@ TEST_CASE("Soroban max tx set size upgrade applied to ledger", static_cast(SOROBAN_PROTOCOL_VERSION))); auto const& sorobanConfig = - app->getLedgerManager().getSorobanNetworkConfig(); + app->getLedgerManager().getMutableSorobanNetworkConfig(); executeUpgrade(*app, makeMaxSorobanTxSizeUpgrade(123)); REQUIRE(sorobanConfig.ledgerMaxTxCount() == 123); @@ -2246,7 +2247,8 @@ TEST_CASE("configuration initialized in version upgrade", "[upgrades]") InitialSorobanNetworkConfig::MAX_CONTRACT_SIZE); // Check that BucketList size window initialized with current BL size - auto& networkConfig = app->getLedgerManager().getSorobanNetworkConfig(); + auto networkConfig = + app->getLedgerManager().getMutableSorobanNetworkConfig(); REQUIRE(networkConfig.getAverageBucketListSize() == blSize); // Check in memory window diff --git a/src/history/HistoryManagerImpl.cpp b/src/history/HistoryManagerImpl.cpp index 8f4851e357..24a3879729 100644 --- a/src/history/HistoryManagerImpl.cpp +++ b/src/history/HistoryManagerImpl.cpp @@ -195,7 +195,7 @@ HistoryManagerImpl::dropSQLBasedPublish() firstLedgerInCheckpointContaining(lcl), freq, mCheckpointBuilder); } - db.clearPreparedStatementCache(); + db.clearPreparedStatementCache(sess); // Now it's safe to drop obsolete SQL tables sess.session() << "DROP TABLE IF EXISTS publishqueue;"; diff --git a/src/history/test/HistoryTestsUtils.cpp b/src/history/test/HistoryTestsUtils.cpp index 4cd221efd5..e2ead4426b 100644 --- a/src/history/test/HistoryTestsUtils.cpp +++ b/src/history/test/HistoryTestsUtils.cpp @@ -532,6 +532,7 @@ CatchupSimulation::generateRandomLedger(uint32_t version) auto lastSucceeded = txsSucceeded.count(); lm.closeLedger(mLedgerCloseDatas.back()); + testutil::crankFor(getApp().getClock(), std::chrono::milliseconds(10)); if (check) { diff --git a/src/ledger/LedgerHeaderUtils.cpp b/src/ledger/LedgerHeaderUtils.cpp index 41eca7352e..952b0c34d2 100644 --- a/src/ledger/LedgerHeaderUtils.cpp +++ b/src/ledger/LedgerHeaderUtils.cpp @@ -190,10 +190,10 @@ loadBySequence(Database& db, soci::session& sess, uint32_t seq) } void -deleteOldEntries(Database& db, uint32_t ledgerSeq, uint32_t count) +deleteOldEntries(soci::session& sess, uint32_t ledgerSeq, uint32_t count) { ZoneScoped; - DatabaseUtils::deleteOldEntriesHelper(db.getRawSession(), ledgerSeq, count, + DatabaseUtils::deleteOldEntriesHelper(sess, ledgerSeq, count, "ledgerheaders", "ledgerseq"); } diff --git a/src/ledger/LedgerHeaderUtils.h b/src/ledger/LedgerHeaderUtils.h index d67bed277d..ae50c082d7 100644 --- a/src/ledger/LedgerHeaderUtils.h +++ b/src/ledger/LedgerHeaderUtils.h @@ -30,7 +30,7 @@ std::shared_ptr loadBySequence(Database& db, soci::session& sess, uint32_t loadMaxLedgerSeq(Database& db); -void deleteOldEntries(Database& db, uint32_t ledgerSeq, uint32_t count); +void deleteOldEntries(soci::session& sess, uint32_t ledgerSeq, uint32_t count); size_t copyToStream(Database& db, soci::session& sess, uint32_t ledgerSeq, uint32_t ledgerCount, CheckpointBuilder& checkpointBuilder); diff --git a/src/ledger/LedgerManager.h b/src/ledger/LedgerManager.h index 3a9d4b7cc1..37e2e73b3d 100644 --- a/src/ledger/LedgerManager.h +++ b/src/ledger/LedgerManager.h @@ -94,8 +94,7 @@ class LedgerManager bool isLatestSlot) = 0; // Return the LCL header and (complete, immutable) hash. - virtual LedgerHeaderHistoryEntry const& - getLastClosedLedgerHeader() const = 0; + virtual LedgerHeaderHistoryEntry getLastClosedLedgerHeader() const = 0; // return the HAS that corresponds to the last closed ledger as persisted in // the database @@ -129,7 +128,8 @@ class LedgerManager // The config is automatically refreshed on protocol upgrades. // Ledger txn here is needed for the sake of lazy load; it won't be // used most of the time. - virtual SorobanNetworkConfig const& getSorobanNetworkConfig() = 0; + // Return a copy for thread-safety + virtual SorobanNetworkConfig getSorobanNetworkConfig() = 0; virtual bool hasSorobanNetworkConfig() const = 0; #ifdef BUILD_TESTS diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index eece56d3a8..517291a95e 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -342,6 +342,7 @@ LedgerManagerImpl::loadLastKnownLedger(bool restoreBucketlist, { // In no-history mode, this method should only be called when // the LCL is genesis. + std::lock_guard lock(mLedgerStateMutex); releaseAssertOrThrow(mLastClosedLedger.hash == lastLedgerHash); releaseAssertOrThrow(mLastClosedLedger.header.ledgerSeq == GENESIS_LEDGER_SEQ); @@ -458,12 +459,15 @@ LedgerManagerImpl::getDatabase() uint32_t LedgerManagerImpl::getLastMaxTxSetSize() const { + std::lock_guard guard(mLedgerStateMutex); return mLastClosedLedger.header.maxTxSetSize; } uint32_t LedgerManagerImpl::getLastMaxTxSetSizeOps() const { + std::lock_guard guard(mLedgerStateMutex); + auto n = mLastClosedLedger.header.maxTxSetSize; return protocolVersionStartsFrom(mLastClosedLedger.header.ledgerVersion, ProtocolVersion::V_11) @@ -500,7 +504,7 @@ LedgerManagerImpl::maxSorobanTransactionResources() { ZoneScoped; - auto const& conf = mApp.getLedgerManager().getSorobanNetworkConfig(); + auto conf = mApp.getLedgerManager().getSorobanNetworkConfig(); int64_t const opCount = 1; std::vector limits = {opCount, conf.txMaxInstructions(), @@ -515,6 +519,8 @@ LedgerManagerImpl::maxSorobanTransactionResources() int64_t LedgerManagerImpl::getLastMinBalance(uint32_t ownerCount) const { + std::lock_guard guard(mLedgerStateMutex); + auto const& lh = mLastClosedLedger.header; if (protocolVersionIsBefore(lh.ledgerVersion, ProtocolVersion::V_9)) return (2 + ownerCount) * lh.baseReserve; @@ -525,18 +531,23 @@ LedgerManagerImpl::getLastMinBalance(uint32_t ownerCount) const uint32_t LedgerManagerImpl::getLastReserve() const { + std::lock_guard guard(mLedgerStateMutex); + return mLastClosedLedger.header.baseReserve; } uint32_t LedgerManagerImpl::getLastTxFee() const { + std::lock_guard guard(mLedgerStateMutex); + return mLastClosedLedger.header.baseFee; } -LedgerHeaderHistoryEntry const& +LedgerHeaderHistoryEntry LedgerManagerImpl::getLastClosedLedgerHeader() const { + std::lock_guard guard(mLedgerStateMutex); return mLastClosedLedger; } @@ -555,18 +566,22 @@ LedgerManagerImpl::getLastClosedLedgerHAS() uint32_t LedgerManagerImpl::getLastClosedLedgerNum() const { + std::lock_guard guard(mLedgerStateMutex); + return mLastClosedLedger.header.ledgerSeq; } -SorobanNetworkConfig const& +SorobanNetworkConfig LedgerManagerImpl::getSorobanNetworkConfig() { + std::lock_guard guard(mLedgerStateMutex); return *mSorobanNetworkConfig; } bool LedgerManagerImpl::hasSorobanNetworkConfig() const { + std::lock_guard guard(mLedgerStateMutex); return mSorobanNetworkConfig.has_value(); } @@ -592,6 +607,8 @@ LedgerManagerImpl::getSorobanMetrics() void LedgerManagerImpl::publishSorobanMetrics() { + std::lock_guard guard(mLedgerStateMutex); + releaseAssert(mSorobanNetworkConfig); // first publish the network config limits mSorobanMetrics.mConfigContractDataKeySizeBytes.set_count( @@ -674,7 +691,7 @@ LedgerManagerImpl::valueExternalized(LedgerCloseData const& ledgerData, { CLOG_INFO(Ledger, "Can't close ledger: {} in LM because catchup is running", - ledgerAbbrev(mLastClosedLedger)); + ledgerAbbrev(getLastClosedLedgerHeader())); return; } @@ -684,9 +701,10 @@ LedgerManagerImpl::valueExternalized(LedgerCloseData const& ledgerData, if (mState != LM_CATCHING_UP_STATE) { // Out of sync, buffer what we just heard and start catchup. - CLOG_INFO( - Ledger, "Lost sync, local LCL is {}, network closed ledger {}", - mLastClosedLedger.header.ledgerSeq, ledgerData.getLedgerSeq()); + CLOG_INFO(Ledger, + "Lost sync, local LCL is {}, network closed ledger {}", + getLastClosedLedgerHeader().header.ledgerSeq, + ledgerData.getLedgerSeq()); } setState(LM_CATCHING_UP_STATE); @@ -781,6 +799,14 @@ maybeSimulateSleep(Config const& cfg, size_t opSize, } } +asio::io_context& +getMetaIOContext(Application& app) +{ + return app.getConfig().parallelLedgerClose() + ? app.getLedgerCloseIOContext() + : app.getClock().getIOContext(); +} + void ledgerCloseComplete(Application& app, uint32_t lcl, bool externalize, LedgerCloseData const& ledgerData) @@ -815,6 +841,12 @@ void LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData, bool externalize) { + // Don't close new ledgers if app is shutting down + if (mApp.isStopping()) + { + return; + } + #ifdef BUILD_TESTS mLastLedgerTxMeta.clear(); #endif @@ -828,7 +860,7 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData, auto header = ltx.loadHeader(); auto initialLedgerVers = header.current().ledgerVersion; ++header.current().ledgerSeq; - header.current().previousLedgerHash = mLastClosedLedger.hash; + header.current().previousLedgerHash = getLastClosedLedgerHeader().hash; CLOG_DEBUG(Ledger, "starting closeLedger() on ledgerSeq={}", header.current().ledgerSeq); @@ -836,6 +868,7 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData, auto now = mApp.getClock().now(); mLedgerAgeClosed.Update(now - mLastClose); + // mLastClose is only accessed by a single thread mLastClose = now; mLedgerAge.set_count(0); @@ -1003,9 +1036,10 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData, } ledgerClosed(ltx, ledgerCloseMeta, initialLedgerVers); + auto lcl = getLastClosedLedgerHeader(); if (ledgerData.getExpectedHash() && - *ledgerData.getExpectedHash() != mLastClosedLedger.hash) + *ledgerData.getExpectedHash() != lcl.hash) { throw std::runtime_error("Local node's ledger corrupted during close"); } @@ -1013,7 +1047,7 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData, if (mMetaStream || mMetaDebugStream) { releaseAssert(ledgerCloseMeta); - ledgerCloseMeta->ledgerHeader() = mLastClosedLedger; + ledgerCloseMeta->ledgerHeader() = lcl; // At this point we've got a complete meta and we can store it to the // member variable: if we throw while committing below, we will at worst @@ -1029,82 +1063,72 @@ LedgerManagerImpl::closeLedger(LedgerCloseData const& ledgerData, } } - // The next 5 steps happen in a relatively non-obvious, subtle order. - // This is unfortunate and it would be nice if we could make it not - // be so subtle, but for the time being this is where we are. - // - // 1. Queue any history-checkpoint to the database, _within_ the current - // transaction. This way if there's a crash after commit and before - // we've published successfully, we'll re-publish on restart. - // - // 2. Commit the current transaction. - // - // 3. Finalize any new checkpoint files _after_ the commit. If a crash - // occurs - // between commit and this step, core will attempt finalizing files again - // on restart. - // - // 4. Start any queued checkpoint publishing, _after_ the commit so that - // it takes its snapshot of history-rows from the committed state, but - // _before_ we GC any buckets (because this is the step where the - // bucket refcounts are incremented for the duration of the publish). - // - // 5. Start background eviction scan for the next ledger, _after_ the commit - // so that it takes its snapshot of network setting from the - // committed state. - // - // 6. GC unreferenced buckets. Only do this once publishes are in progress. + // TODO: add documentation of the new flow - // step 1 auto& hm = mApp.getHistoryManager(); hm.maybeQueueHistoryCheckpoint(); - // step 2 ltx.commit(); + std::chrono::duration ledgerTimeSeconds = ledgerTime.Stop(); + CLOG_DEBUG(Perf, "Applied ledger {} in {} seconds", ledgerSeq, + ledgerTimeSeconds.count()); + #ifdef BUILD_TESTS mLatestTxResultSet = txResultSet; #endif - // step 3 - hm.maybeCheckpointComplete(); - - // step 4 - hm.publishQueuedHistory(); - hm.logAndUpdatePublishStatus(); - - // step 5 if (protocolVersionStartsFrom(initialLedgerVers, SOROBAN_PROTOCOL_VERSION) && mApp.getConfig().isUsingBackgroundEviction()) { - mApp.getBucketManager().startBackgroundEvictionScan(ledgerSeq + 1); + mApp.getBucketManager().startBackgroundEvictionScan(ledgerSeq + 1, + true); } - // step 6 - mApp.getBucketManager().forgetUnreferencedBuckets(); + auto completionHandler = [this, initialLedgerVers, txs, ledgerSeq, + externalize, ledgerData, + clt = std::move(closeLedgerTime)]() mutable { + auto& hm = mApp.getHistoryManager(); + hm.maybeCheckpointComplete(); - maybeSimulateSleep(mApp.getConfig(), txs.size(), closeLedgerTime); + hm.publishQueuedHistory(); + hm.logAndUpdatePublishStatus(); - std::chrono::duration ledgerTimeSeconds = ledgerTime.Stop(); - CLOG_DEBUG(Perf, "Applied ledger in {} seconds", ledgerTimeSeconds.count()); + mApp.getBucketManager().forgetUnreferencedBuckets(); - ledgerCloseComplete(mApp, ledgerSeq, externalize, ledgerData); + maybeSimulateSleep(mApp.getConfig(), txs.size(), clt); + + ledgerCloseComplete(mApp, ledgerSeq, externalize, ledgerData); + CLOG_INFO(Ledger, "Closed ledger: {}", ledgerSeq); + }; + if (threadIsMain()) + { + completionHandler(); + } + else + { + mApp.postOnMainThread(completionHandler, "ledgerCloseComplete"); + } FrameMark; } - void LedgerManagerImpl::deleteOldEntries(Database& db, uint32_t ledgerSeq, uint32_t count) { ZoneScoped; - soci::transaction txscope(db.getRawSession()); - db.clearPreparedStatementCache(); - LedgerHeaderUtils::deleteOldEntries(db, ledgerSeq, count); - HerderPersistence::deleteOldEntries(db, ledgerSeq, count); - db.clearPreparedStatementCache(); - txscope.commit(); + if (mApp.getConfig().parallelLedgerClose()) + { + auto session = + std::make_unique(mApp.getDatabase().getPool()); + LedgerHeaderUtils::deleteOldEntries(*session, ledgerSeq, count); + } + else + { + LedgerHeaderUtils::deleteOldEntries(db.getRawSession(), ledgerSeq, + count); + } } void @@ -1155,9 +1179,9 @@ LedgerManagerImpl::setupLedgerCloseMetaStream() { // We can't be sure we're writing to a stream that supports fsync; // pipes typically error when you try. So we don't do it. - mMetaStream = std::make_unique( - mApp.getClock().getIOContext(), - /*fsyncOnClose=*/false); + mMetaStream = + std::make_unique(getMetaIOContext(mApp), + /*fsyncOnClose=*/false); std::regex fdrx("^fd:([0-9]+)$"); std::smatch sm; if (std::regex_match(cfg.METADATA_OUTPUT_STREAM, sm, fdrx)) @@ -1222,9 +1246,9 @@ LedgerManagerImpl::maybeResetLedgerCloseMetaDebugStream(uint32_t ledgerSeq) // such stream or a replacement for the one we just handed off to // flush-and-rotate. Either way, we should not have an existing one! releaseAssert(!mMetaDebugStream); - auto tmpStream = std::make_unique( - mApp.getClock().getIOContext(), - /*fsyncOnClose=*/true); + auto tmpStream = + std::make_unique(getMetaIOContext(mApp), + /*fsyncOnClose=*/true); auto metaDebugPath = metautils::getMetaDebugFilePath( mApp.getBucketManager().getBucketDir(), ledgerSeq); @@ -1275,6 +1299,7 @@ void LedgerManagerImpl::advanceLedgerPointers(LedgerHeader const& header, bool debugLog) { + std::lock_guard guard(mLedgerStateMutex); auto ledgerHash = xdrSha256(header); if (debugLog) @@ -1302,6 +1327,7 @@ void LedgerManagerImpl::updateNetworkConfig(AbstractLedgerTxn& rootLtx) { ZoneScoped; + std::lock_guard guard(mLedgerStateMutex); uint32_t ledgerVersion = rootLtx.loadHeader().current().ledgerVersion; @@ -1662,6 +1688,7 @@ LedgerManagerImpl::transferLedgerEntriesToBucketList( if (blEnabled && protocolVersionStartsFrom(initialLedgerVers, SOROBAN_PROTOCOL_VERSION)) { + std::lock_guard guard(mLedgerStateMutex); { auto keys = ltx.getAllTTLKeysWithoutSealing(); LedgerTxn ltxEvictions(ltx); @@ -1669,12 +1696,12 @@ LedgerManagerImpl::transferLedgerEntriesToBucketList( if (mApp.getConfig().isUsingBackgroundEviction()) { mApp.getBucketManager().resolveBackgroundEvictionScan( - ltxEvictions, lh.ledgerSeq, keys); + ltxEvictions, lh.ledgerSeq, keys, *mSorobanNetworkConfig); } else { - mApp.getBucketManager().scanForEvictionLegacy(ltxEvictions, - lh.ledgerSeq); + mApp.getBucketManager().scanForEvictionLegacy( + ltxEvictions, lh.ledgerSeq, *mSorobanNetworkConfig); } if (ledgerCloseMeta) diff --git a/src/ledger/LedgerManagerImpl.h b/src/ledger/LedgerManagerImpl.h index 1905a4741c..2037fff971 100644 --- a/src/ledger/LedgerManagerImpl.h +++ b/src/ledger/LedgerManagerImpl.h @@ -69,6 +69,9 @@ class LedgerManagerImpl : public LedgerManager VirtualClock::time_point mLastClose; bool mRebuildInMemoryState{false}; + // Use mutex to guard read access to LCL and Soroban network config + mutable std::recursive_mutex mLedgerStateMutex; + std::unique_ptr mStartCatchup; medida::Timer& mCatchupDuration; @@ -160,7 +163,7 @@ class LedgerManagerImpl : public LedgerManager uint32_t getLastReserve() const override; uint32_t getLastTxFee() const override; uint32_t getLastClosedLedgerNum() const override; - SorobanNetworkConfig const& getSorobanNetworkConfig() override; + SorobanNetworkConfig getSorobanNetworkConfig() override; bool hasSorobanNetworkConfig() const override; #ifdef BUILD_TESTS @@ -180,7 +183,8 @@ class LedgerManagerImpl : public LedgerManager virtual bool rebuildingInMemoryState() override; virtual void setupInMemoryStateRebuild() override; - LedgerHeaderHistoryEntry const& getLastClosedLedgerHeader() const override; + // TODO: fix if this shows up in profilers (switch to thread-safe copy) + LedgerHeaderHistoryEntry getLastClosedLedgerHeader() const override; HistoryArchiveState getLastClosedLedgerHAS() override; diff --git a/src/ledger/LedgerTxn.cpp b/src/ledger/LedgerTxn.cpp index 90d51a42cb..5bb025191a 100644 --- a/src/ledger/LedgerTxn.cpp +++ b/src/ledger/LedgerTxn.cpp @@ -2554,9 +2554,10 @@ LedgerTxnRoot::Impl::~Impl() SessionWrapper& LedgerTxnRoot::Impl::getSession() const { - // For now, return main app-wide session; - // When application is done in parallel, mSession will be set to a session - // established from the connection pool. + if (mSession) + { + return *mSession; + } return mApp.getDatabase().getSession(); } @@ -2597,10 +2598,10 @@ LedgerTxnRoot::Impl::addChild(AbstractLedgerTxn& child, TransactionMode mode) if (mode == TransactionMode::READ_WRITE_WITH_SQL_TXN) { - if (mApp.getConfig().EXPERIMENTAL_PARALLEL_LEDGER_CLOSE) + if (mApp.getConfig().parallelLedgerClose()) { - mSession = - std::make_unique(mApp.getDatabase().getPool()); + mSession = std::make_unique( + "ledgerClose", mApp.getDatabase().getPool()); } mTransaction = std::make_unique(getSession().session()); @@ -2872,7 +2873,7 @@ LedgerTxnRoot::Impl::commitChild(EntryIterator iter, // committing; on postgres this doesn't matter but on SQLite the passive // WAL-auto-checkpointing-at-commit behaviour will starve if there are // still prepared statements open at commit time. - mApp.getDatabase().clearPreparedStatementCache(); + mApp.getDatabase().clearPreparedStatementCache(getSession()); ZoneNamedN(commitZone, "SOCI commit", true); mTransaction->commit(); } diff --git a/src/ledger/LedgerTxnImpl.h b/src/ledger/LedgerTxnImpl.h index 72ee43f24a..6106f6a716 100644 --- a/src/ledger/LedgerTxnImpl.h +++ b/src/ledger/LedgerTxnImpl.h @@ -732,7 +732,7 @@ class LedgerTxnRoot::Impl size_t const mMaxBestOffersBatchSize; Application& mApp; - std::shared_ptr mSession; + std::unique_ptr mSession; std::unique_ptr mHeader; mutable EntryCache mEntryCache; diff --git a/src/ledger/test/LedgerHeaderTests.cpp b/src/ledger/test/LedgerHeaderTests.cpp index 6152668829..b3f9c5431e 100644 --- a/src/ledger/test/LedgerHeaderTests.cpp +++ b/src/ledger/test/LedgerHeaderTests.cpp @@ -31,7 +31,7 @@ TEST_CASE("genesisledger", "[ledger]") auto app = Application::create(clock, cfg); app->start(); - auto const& lcl = app->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); auto const& header = lcl.header; REQUIRE(header.ledgerVersion == 0); REQUIRE(header.previousLedgerHash == Hash{}); @@ -68,7 +68,7 @@ TEST_CASE("ledgerheader", "[ledger]") Application::pointer app = Application::create(clock, cfg); app->start(); - auto const& lcl = app->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); auto txSet = TxSetXDRFrame::makeEmpty(lcl); // close this ledger @@ -101,7 +101,7 @@ TEST_CASE_VERSIONS("base reserve", "[ledger]") VirtualClock clock; auto app = createTestApplication(clock, cfg); - auto const& lcl = app->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); REQUIRE(lcl.header.baseReserve == 100000000); const uint32 n = 20000; int64 expectedReserve = 2000200000000ll; diff --git a/src/ledger/test/LedgerTests.cpp b/src/ledger/test/LedgerTests.cpp index 495540d97d..431dd05546 100644 --- a/src/ledger/test/LedgerTests.cpp +++ b/src/ledger/test/LedgerTests.cpp @@ -19,7 +19,7 @@ TEST_CASE("cannot close ledger with unsupported ledger version", "[ledger]") app->start(); auto applyEmptyLedger = [&]() { - auto const& lcl = app->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); auto txSet = TxSetXDRFrame::makeEmpty(lcl); StellarValue sv = app->getHerder().makeStellarValue( txSet->getContentsHash(), 1, emptyUpgradeSteps, diff --git a/src/main/AppConnector.cpp b/src/main/AppConnector.cpp index a5953fc6ea..5284d7ad5c 100644 --- a/src/main/AppConnector.cpp +++ b/src/main/AppConnector.cpp @@ -45,24 +45,21 @@ AppConnector::getBanManager() return mApp.getBanManager(); } -SorobanNetworkConfig const& +SorobanNetworkConfig AppConnector::getSorobanNetworkConfig() const { - releaseAssert(threadIsMain()); return mApp.getLedgerManager().getSorobanNetworkConfig(); } medida::MetricsRegistry& AppConnector::getMetrics() const { - releaseAssert(threadIsMain()); return mApp.getMetrics(); } SorobanMetrics& AppConnector::getSorobanMetrics() const { - releaseAssert(threadIsMain()); return mApp.getLedgerManager().getSorobanMetrics(); } @@ -71,7 +68,6 @@ AppConnector::checkOnOperationApply(Operation const& operation, OperationResult const& opres, LedgerTxnDelta const& ltxDelta) { - releaseAssert(threadIsMain()); mApp.getInvariantManager().checkOnOperationApply(operation, opres, ltxDelta); } @@ -79,7 +75,6 @@ AppConnector::checkOnOperationApply(Operation const& operation, Hash const& AppConnector::getNetworkID() const { - releaseAssert(threadIsMain()); return mApp.getNetworkID(); } @@ -129,4 +124,10 @@ AppConnector::getOverlayMetrics() return mApp.getOverlayManager().getOverlayMetrics(); } +LedgerHeaderHistoryEntry +AppConnector::getLastClosedLedgerHeader() const +{ + return mApp.getLedgerManager().getLastClosedLedgerHeader(); +} + } \ No newline at end of file diff --git a/src/main/AppConnector.h b/src/main/AppConnector.h index 6e6d19ce52..130603005e 100644 --- a/src/main/AppConnector.h +++ b/src/main/AppConnector.h @@ -33,8 +33,6 @@ class AppConnector OverlayManager& getOverlayManager(); BanManager& getBanManager(); bool shouldYield() const; - SorobanNetworkConfig const& getSorobanNetworkConfig() const; - medida::MetricsRegistry& getMetrics() const; SorobanMetrics& getSorobanMetrics() const; void checkOnOperationApply(Operation const& operation, OperationResult const& opres, @@ -51,5 +49,8 @@ class AppConnector Config const& getConfig() const; bool overlayShuttingDown() const; OverlayMetrics& getOverlayMetrics(); + SorobanNetworkConfig getSorobanNetworkConfig() const; + medida::MetricsRegistry& getMetrics() const; + LedgerHeaderHistoryEntry getLastClosedLedgerHeader() const; }; } \ No newline at end of file diff --git a/src/main/Application.h b/src/main/Application.h index 0e5bac078f..467b07a6d2 100644 --- a/src/main/Application.h +++ b/src/main/Application.h @@ -229,6 +229,7 @@ class Application virtual asio::io_context& getWorkerIOContext() = 0; virtual asio::io_context& getEvictionIOContext() = 0; virtual asio::io_context& getOverlayIOContext() = 0; + virtual asio::io_context& getLedgerCloseIOContext() = 0; virtual void postOnMainThread( std::function&& f, std::string&& name, @@ -242,6 +243,8 @@ class Application std::string jobName) = 0; virtual void postOnOverlayThread(std::function&& f, std::string jobName) = 0; + virtual void postOnLedgerCloseThread(std::function&& f, + std::string jobName) = 0; // Perform actions necessary to transition from BOOTING_STATE to other // states. In particular: either reload or reinitialize the database, and diff --git a/src/main/ApplicationImpl.cpp b/src/main/ApplicationImpl.cpp index 18c750b182..be5308ea16 100644 --- a/src/main/ApplicationImpl.cpp +++ b/src/main/ApplicationImpl.cpp @@ -102,6 +102,13 @@ ApplicationImpl::ApplicationImpl(VirtualClock& clock, Config const& cfg) , mOverlayWork(mOverlayIOContext ? std::make_unique( *mOverlayIOContext) : nullptr) + , mLedgerCloseIOContext(mConfig.parallelLedgerClose() + ? std::make_unique(1) + : nullptr) + , mLedgerCloseWork( + mLedgerCloseIOContext + ? std::make_unique(*mLedgerCloseIOContext) + : nullptr) , mWorkerThreads() , mEvictionThread() , mStopSignals(clock.getIOContext(), SIGINT) @@ -117,6 +124,8 @@ ApplicationImpl::ApplicationImpl(VirtualClock& clock, Config const& cfg) mMetrics->NewTimer({"app", "post-on-background-thread", "delay"})) , mPostOnOverlayThreadDelay( mMetrics->NewTimer({"app", "post-on-overlay-thread", "delay"})) + , mPostOnLedgerCloseThreadDelay( + mMetrics->NewTimer({"app", "post-on-ledger-close-thread", "delay"})) , mStartedOn(clock.system_now()) { #ifdef SIGQUIT @@ -186,6 +195,12 @@ ApplicationImpl::ApplicationImpl(VirtualClock& clock, Config const& cfg) // Keep priority unchanged as overlay processes time-sensitive tasks mOverlayThread = std::thread{[this]() { mOverlayIOContext->run(); }}; } + + if (mConfig.parallelLedgerClose()) + { + mLedgerCloseThread = + std::thread{[this]() { mLedgerCloseIOContext->run(); }}; + } } static void @@ -296,6 +311,7 @@ maybeRebuildLedger(Application& app, bool applyBuckets) } } + app.getDatabase().clearPreparedStatementCache(); for (auto let : toRebuild) { ps.clearRebuildForType(let); @@ -507,7 +523,7 @@ ApplicationImpl::getJsonInfo(bool verbose) info["protocol_version"] = getConfig().LEDGER_PROTOCOL_VERSION; info["state"] = getStateHuman(); info["startedOn"] = VirtualClock::systemPointToISOString(mStartedOn); - auto const& lcl = lm.getLastClosedLedgerHeader(); + auto lcl = lm.getLastClosedLedgerHeader(); info["ledger"]["num"] = (int)lcl.header.ledgerSeq; info["ledger"]["hash"] = binToHex(lcl.hash); info["ledger"]["closeTime"] = (Json::UInt64)lcl.header.scpValue.closeTime; @@ -669,8 +685,11 @@ ApplicationImpl::getNetworkID() const ApplicationImpl::~ApplicationImpl() { LOG_INFO(DEFAULT_LOG, "Application destructing"); + mStopping = true; try { + // First, shutdown ledger close queue + shutdownLedgerCloseThread(); shutdownWorkScheduler(); if (mProcessManager) { @@ -969,6 +988,7 @@ ApplicationImpl::gracefulStop() return; } mStopping = true; + shutdownLedgerCloseThread(); if (mOverlayManager) { mOverlayManager->shutdown(); @@ -1015,6 +1035,21 @@ ApplicationImpl::shutdownWorkScheduler() } } +void +ApplicationImpl::shutdownLedgerCloseThread() +{ + if (mLedgerCloseThread && !mLedgerCloseThreadStopped) + { + if (mLedgerCloseWork) + { + mLedgerCloseWork.reset(); + } + LOG_INFO(DEFAULT_LOG, "Joining the ledger close thread"); + mLedgerCloseThread->join(); + mLedgerCloseThreadStopped = true; + } +} + void ApplicationImpl::joinAllThreads() { @@ -1029,6 +1064,10 @@ ApplicationImpl::joinAllThreads() { mOverlayWork.reset(); } + if (mEvictionWork) + { + mEvictionWork.reset(); + } LOG_INFO(DEFAULT_LOG, "Joining {} worker threads", mWorkerThreads.size()); for (auto& w : mWorkerThreads) @@ -1036,9 +1075,10 @@ ApplicationImpl::joinAllThreads() w.join(); } - if (mEvictionWork) + if (mOverlayThread) { - mEvictionWork.reset(); + LOG_INFO(DEFAULT_LOG, "Joining the overlay thread"); + mOverlayThread->join(); } if (mEvictionThread) @@ -1047,12 +1087,6 @@ ApplicationImpl::joinAllThreads() mEvictionThread->join(); } - if (mOverlayThread) - { - LOG_INFO(DEFAULT_LOG, "Joining the overlay thread"); - mOverlayThread->join(); - } - LOG_INFO(DEFAULT_LOG, "Joined all {} threads", (mWorkerThreads.size() + 1)); } @@ -1541,6 +1575,13 @@ ApplicationImpl::getOverlayIOContext() return *mOverlayIOContext; } +asio::io_context& +ApplicationImpl::getLedgerCloseIOContext() +{ + releaseAssert(mLedgerCloseIOContext); + return *mLedgerCloseIOContext; +} + void ApplicationImpl::postOnMainThread(std::function&& f, std::string&& name, Scheduler::ActionType type) @@ -1598,6 +1639,19 @@ ApplicationImpl::postOnOverlayThread(std::function&& f, }); } +void +ApplicationImpl::postOnLedgerCloseThread(std::function&& f, + std::string jobName) +{ + releaseAssert(mLedgerCloseIOContext); + LogSlowExecution isSlow{std::move(jobName), LogSlowExecution::Mode::MANUAL, + "executed after"}; + asio::post(*mLedgerCloseIOContext, [this, f = std::move(f), isSlow]() { + mPostOnLedgerCloseThreadDelay.Update(isSlow.checkElapsedTime()); + f(); + }); +} + void ApplicationImpl::enableInvariantsFromConfig() { @@ -1640,7 +1694,6 @@ ApplicationImpl::createDatabase() AbstractLedgerTxnParent& ApplicationImpl::getLedgerTxnRoot() { - releaseAssert(threadIsMain()); return mConfig.MODE_USES_IN_MEMORY_LEDGER ? *mNeverCommittingLedgerTxn : *mLedgerTxnRoot; } diff --git a/src/main/ApplicationImpl.h b/src/main/ApplicationImpl.h index a7553214f9..f5360b69f9 100644 --- a/src/main/ApplicationImpl.h +++ b/src/main/ApplicationImpl.h @@ -82,6 +82,7 @@ class ApplicationImpl : public Application virtual asio::io_context& getWorkerIOContext() override; virtual asio::io_context& getEvictionIOContext() override; virtual asio::io_context& getOverlayIOContext() override; + virtual asio::io_context& getLedgerCloseIOContext() override; virtual void postOnMainThread(std::function&& f, std::string&& name, Scheduler::ActionType type) override; @@ -92,6 +93,8 @@ class ApplicationImpl : public Application virtual void postOnOverlayThread(std::function&& f, std::string jobName) override; + virtual void postOnLedgerCloseThread(std::function&& f, + std::string jobName) override; virtual void start() override; void startServices(); @@ -160,6 +163,9 @@ class ApplicationImpl : public Application std::unique_ptr mOverlayIOContext; std::unique_ptr mOverlayWork; + std::unique_ptr mLedgerCloseIOContext; + std::unique_ptr mLedgerCloseWork; + std::unique_ptr mBucketManager; std::unique_ptr mDatabase; std::unique_ptr mOverlayManager; @@ -205,6 +211,7 @@ class ApplicationImpl : public Application std::vector mWorkerThreads; std::optional mOverlayThread; + std::optional mLedgerCloseThread; // Unlike mWorkerThreads (which are low priority), eviction scans require a // medium priority thread. In the future, this may become a more general @@ -215,7 +222,8 @@ class ApplicationImpl : public Application asio::signal_set mStopSignals; bool mStarted; - bool mStopping; + std::atomic mStopping; + bool mLedgerCloseThreadStopped{false}; VirtualTimer mStoppingTimer; VirtualTimer mSelfCheckTimer; @@ -224,6 +232,7 @@ class ApplicationImpl : public Application medida::Timer& mPostOnMainThreadDelay; medida::Timer& mPostOnBackgroundThreadDelay; medida::Timer& mPostOnOverlayThreadDelay; + medida::Timer& mPostOnLedgerCloseThreadDelay; VirtualClock::system_time_point mStartedOn; @@ -257,5 +266,6 @@ class ApplicationImpl : public Application void upgradeToCurrentSchemaAndMaybeRebuildLedger(bool applyBuckets, bool forceRebuild); + void shutdownLedgerCloseThread(); }; } diff --git a/src/main/ApplicationUtils.cpp b/src/main/ApplicationUtils.cpp index 376012804f..a222dc89e4 100644 --- a/src/main/ApplicationUtils.cpp +++ b/src/main/ApplicationUtils.cpp @@ -430,7 +430,7 @@ setAuthenticatedLedgerHashPair(Application::pointer app, if (lm.isSynced()) { - auto const& lhe = lm.getLastClosedLedgerHeader(); + auto lhe = lm.getLastClosedLedgerHeader(); tryCheckpoint(lhe.header.ledgerSeq, lhe.hash); } else diff --git a/src/main/CommandHandler.cpp b/src/main/CommandHandler.cpp index fd8d9e3034..d923028a8a 100644 --- a/src/main/CommandHandler.cpp +++ b/src/main/CommandHandler.cpp @@ -777,7 +777,7 @@ CommandHandler::sorobanInfo(std::string const& params, std::string& retStr) if (format == "basic") { Json::Value res; - auto const& conf = lm.getSorobanNetworkConfig(); + auto conf = lm.getSorobanNetworkConfig(); // Contract size res["max_contract_size"] = conf.maxContractSizeBytes(); diff --git a/src/main/Config.cpp b/src/main/Config.cpp index fb72c1215b..190d39c4d3 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -1638,6 +1638,12 @@ Config::processConfig(std::shared_ptr t) } } + if (EXPERIMENTAL_PARALLEL_LEDGER_CLOSE && DEPRECATED_SQL_LEDGER_STATE) + { + throw std::invalid_argument( + "EXPERIMENTAL_PARALLEL_LEDGER_CLOSE requires BucketListDB"); + } + if (!OP_APPLY_SLEEP_TIME_DURATION_FOR_TESTING.empty() || !OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING.empty()) { @@ -2334,6 +2340,13 @@ Config::modeStoresAnyHistory() const return MODE_STORES_HISTORY_LEDGERHEADERS || MODE_STORES_HISTORY_MISC; } +bool +Config::parallelLedgerClose() const +{ + return isUsingBucketListDB() && EXPERIMENTAL_PARALLEL_LEDGER_CLOSE && + !(DATABASE.value == ("sqlite3://:memory:")); +} + void Config::setNoListen() { diff --git a/src/main/Config.h b/src/main/Config.h index c19d145036..a3f3c3ef54 100644 --- a/src/main/Config.h +++ b/src/main/Config.h @@ -262,6 +262,9 @@ class Config : public std::enable_shared_from_this // This config should only be enabled when testing. std::chrono::microseconds ARTIFICIALLY_SLEEP_MAIN_THREAD_FOR_TESTING; + // A config parameter that forces stellar-core to sleep every time it closes + // a ledger if order to simulate slow application. This config should only + // be enabled when testing. std::chrono::milliseconds ARTIFICIALLY_DELAY_LEDGER_CLOSE_FOR_TESTING; // Timeout before publishing externalized values to archive @@ -744,6 +747,7 @@ class Config : public std::enable_shared_from_this bool isPersistingBucketListDBIndexes() const; bool modeStoresAllHistory() const; bool modeStoresAnyHistory() const; + bool parallelLedgerClose() const; void logBasicInfo(); void setNoListen(); void setNoPublish(); diff --git a/src/main/ExternalQueue.cpp b/src/main/ExternalQueue.cpp index 511b38f95a..78429ff030 100644 --- a/src/main/ExternalQueue.cpp +++ b/src/main/ExternalQueue.cpp @@ -6,6 +6,7 @@ #include "Application.h" #include "database/Database.h" +#include "herder/HerderPersistence.h" #include "ledger/LedgerManager.h" #include "util/GlobalChecks.h" #include "util/Logging.h" @@ -208,7 +209,24 @@ ExternalQueue::deleteOldEntries(uint32 count) "Trimming history <= ledger {} (rmin={}, qmin={}, lmin={})", cmin, rmin, qmin, lmin); - mApp.getLedgerManager().deleteOldEntries(mApp.getDatabase(), cmin, count); + // Run on main + mApp.getDatabase().clearPreparedStatementCache( + mApp.getDatabase().getMiscSession()); + HerderPersistence::deleteOldEntries(mApp.getDatabase(), cmin, count); + + // Run in the background + if (mApp.getConfig().parallelLedgerClose()) + { + mApp.postOnBackgroundThread( + [&db = mApp.getDatabase(), &lm = mApp.getLedgerManager(), cmin, + count]() { lm.deleteOldEntries(db, cmin, count); }, + "deleteOldEntries"); + } + else + { + mApp.getLedgerManager().deleteOldEntries(mApp.getDatabase(), cmin, + count); + } } void diff --git a/src/main/test/ApplicationUtilsTests.cpp b/src/main/test/ApplicationUtilsTests.cpp index bdbf6942ea..47c676ecc0 100644 --- a/src/main/test/ApplicationUtilsTests.cpp +++ b/src/main/test/ApplicationUtilsTests.cpp @@ -285,7 +285,7 @@ class SimulationHelper return std::make_pair(selectedLedger, selectedHash); } - LedgerHeaderHistoryEntry const& + LedgerHeaderHistoryEntry getMainNodeLCL() { return mSimulation->getNode(mMainNodeID) @@ -293,7 +293,7 @@ class SimulationHelper .getLastClosedLedgerHeader(); } - LedgerHeaderHistoryEntry const& + LedgerHeaderHistoryEntry getTestNodeLCL() { return mSimulation->getNode(mTestNodeID) diff --git a/src/main/test/ConfigTests.cpp b/src/main/test/ConfigTests.cpp index 2b1dcc559f..ed857a1b4e 100644 --- a/src/main/test/ConfigTests.cpp +++ b/src/main/test/ConfigTests.cpp @@ -286,7 +286,7 @@ TEST_CASE("bad validators configs", "[config]") NODE_SEED="SA7FGJMMUIHNE3ZPI2UO5I632A7O5FBAZTXFAIEVFA4DSSGLHXACLAIT a3" {NODE_HOME_DOMAIN} NODE_IS_VALIDATOR=true -DEPRECATED_SQL_LEDGER_STATE=true +DEPRECATED_SQL_LEDGER_STATE=false ############################ # list of HOME_DOMAINS @@ -474,7 +474,7 @@ TEST_CASE("nesting level", "[config]") return secretKey.getStrKeyPublic(); }; std::string configNesting = - "DEPRECATED_SQL_LEDGER_STATE=true\n" // Required for all configs + "DEPRECATED_SQL_LEDGER_STATE=false\n" // Required for all configs "UNSAFE_QUORUM=true"; std::string quorumSetNumber = ""; std::string quorumSetTemplate = R"( @@ -536,7 +536,7 @@ TEST_CASE("operation filter configuration", "[config]") }; std::stringstream ss; - ss << "DEPRECATED_SQL_LEDGER_STATE=true\n"; // required for all configs + ss << "DEPRECATED_SQL_LEDGER_STATE=false\n"; // required for all configs ss << "UNSAFE_QUORUM=true\n"; toConfigStr(vals, ss); ss << "\n[QUORUM_SET]\n"; diff --git a/src/overlay/OverlayManagerImpl.cpp b/src/overlay/OverlayManagerImpl.cpp index 5bbc181901..a777cee998 100644 --- a/src/overlay/OverlayManagerImpl.cpp +++ b/src/overlay/OverlayManagerImpl.cpp @@ -497,17 +497,17 @@ OverlayManagerImpl::triggerPeerResolution() // Trigger DNS resolution on the background thread using task_t = std::packaged_task; - std::shared_ptr task = std::make_shared([this]() { - if (!this->mShuttingDown) - { - auto known = resolvePeers(this->mApp.getConfig().KNOWN_PEERS); - auto preferred = - resolvePeers(this->mApp.getConfig().PREFERRED_PEERS); - return ResolvedPeers{known.first, preferred.first, - known.second || preferred.second}; - } - return ResolvedPeers{{}, {}, false}; - }); + std::shared_ptr task = + std::make_shared([this, cfg = mApp.getConfig()]() { + if (!this->mShuttingDown) + { + auto known = resolvePeers(cfg.KNOWN_PEERS); + auto preferred = resolvePeers(cfg.PREFERRED_PEERS); + return ResolvedPeers{known.first, preferred.first, + known.second || preferred.second}; + } + return ResolvedPeers{{}, {}, false}; + }); mResolvedPeers = task->get_future(); mApp.postOnBackgroundThread(bind(&task_t::operator(), task), diff --git a/src/overlay/test/FloodTests.cpp b/src/overlay/test/FloodTests.cpp index e224530931..1ba68da13c 100644 --- a/src/overlay/test/FloodTests.cpp +++ b/src/overlay/test/FloodTests.cpp @@ -455,8 +455,7 @@ TEST_CASE("Flooding", "[flood][overlay][acceptance]") qset.validators.emplace_back(sources[i]); Hash qSetHash = sha256(xdr::xdr_to_opaque(qset)); - auto const& lcl = - inApp->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = inApp->getLedgerManager().getLastClosedLedgerHeader(); // build an SCP message for the next ledger auto ct = std::max( lcl.header.scpValue.closeTime + 1, @@ -486,8 +485,7 @@ TEST_CASE("Flooding", "[flood][overlay][acceptance]") auto ackedSCP = [&](std::shared_ptr app) { // checks if an app received and processed all SCP messages size_t okCount = 0; - auto const& lcl = - app->getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = app->getLedgerManager().getLastClosedLedgerHeader(); HerderImpl& herder = *static_cast(&app->getHerder()); herder.getSCP().processCurrentState( diff --git a/src/simulation/LoadGenerator.cpp b/src/simulation/LoadGenerator.cpp index fd124f321d..dd4e60809d 100644 --- a/src/simulation/LoadGenerator.cpp +++ b/src/simulation/LoadGenerator.cpp @@ -1188,6 +1188,13 @@ LoadGenerator::checkAccountSynced(Application& app, bool isCreate) account->getAccountId()); result.push_back(account); } + else if (app.getHerder().sourceAccountPending( + account->getPublicKey())) + { + CLOG_TRACE(LoadGen, "Account {} is pending!", + account->getAccountId()); + result.push_back(account); + } } else if (!reloadRes) { diff --git a/src/simulation/TxGenerator.cpp b/src/simulation/TxGenerator.cpp index b5e938baa3..728c7ff825 100644 --- a/src/simulation/TxGenerator.cpp +++ b/src/simulation/TxGenerator.cpp @@ -862,7 +862,7 @@ TxGenerator::sorobanRandomUploadResources() // Estimate VM instantiation cost, with some additional buffer to increase // the chance that this instruction count is sufficient. - ContractCostParamEntry const& vmInstantiationCosts = + ContractCostParamEntry vmInstantiationCosts = mApp.getLedgerManager() .getSorobanNetworkConfig() .cpuCostParams()[VmInstantiation]; diff --git a/src/test/FuzzerImpl.cpp b/src/test/FuzzerImpl.cpp index 23e4c400df..eb3d543de1 100644 --- a/src/test/FuzzerImpl.cpp +++ b/src/test/FuzzerImpl.cpp @@ -888,7 +888,10 @@ resetTxInternalState(Application& app) app.getLedgerTxnRoot().resetForFuzzer(); app.getInvariantManager().resetForFuzzer(); #endif // BUILD_TESTS - app.getDatabase().clearPreparedStatementCache(); + app.getDatabase().clearPreparedStatementCache( + app.getDatabase().getSession()); + app.getDatabase().clearPreparedStatementCache( + app.getDatabase().getMiscSession()); } // FuzzTransactionFrame is a specialized TransactionFrame that includes diff --git a/src/test/TestUtils.cpp b/src/test/TestUtils.cpp index 873bf61e3d..908a90a6d5 100644 --- a/src/test/TestUtils.cpp +++ b/src/test/TestUtils.cpp @@ -244,7 +244,7 @@ upgradeSorobanNetworkConfig(std::function modifyFn, lg.generateLoad(createAccountsLoadConfig); simulation->crankUntil( [&]() { return complete.count() == completeCount + 1; }, - 300 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); + 3 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); } // Create upload wasm transaction. @@ -254,7 +254,7 @@ upgradeSorobanNetworkConfig(std::function modifyFn, completeCount = complete.count(); simulation->crankUntil( [&]() { return complete.count() == completeCount + 1; }, - 300 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); + 3 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); // Create upgrade transaction. auto createUpgradeLoadGenConfig = GeneratedLoadConfig::txLoad( @@ -270,7 +270,7 @@ upgradeSorobanNetworkConfig(std::function modifyFn, completeCount = complete.count(); simulation->crankUntil( [&]() { return complete.count() == completeCount + 1; }, - 4 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); + 3 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); // Arm for upgrade. for (auto app : nodes) @@ -289,7 +289,7 @@ upgradeSorobanNetworkConfig(std::function modifyFn, auto netCfg = app.getLedgerManager().getSorobanNetworkConfig(); return netCfg == cfg; }, - 2 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); + 3 * Herder::EXP_LEDGER_TIMESPAN_SECONDS, false); } void diff --git a/src/test/TxTests.cpp b/src/test/TxTests.cpp index 68be2f5e59..f88f9a1cb7 100644 --- a/src/test/TxTests.cpp +++ b/src/test/TxTests.cpp @@ -639,15 +639,15 @@ loadAccount(AbstractLedgerTxn& ltx, PublicKey const& k, bool mustExist) bool doesAccountExist(Application& app, PublicKey const& k) { - LedgerTxn ltx(app.getLedgerTxnRoot()); - return (bool)stellar::loadAccountWithoutRecord(ltx, k); + LedgerSnapshot lss(app); + return (bool)lss.getAccount(k); } xdr::xvector getAccountSigners(PublicKey const& k, Application& app) { - LedgerTxn ltx(app.getLedgerTxnRoot()); - auto account = stellar::loadAccount(ltx, k); + LedgerSnapshot lss(app); + auto account = lss.getAccount(k); return account.current().data.account().signers; } @@ -709,8 +709,9 @@ transactionFromOperations(Application& app, SecretKey const& from, { uint32_t ledgerVersion; { - LedgerTxn ltx(app.getLedgerTxnRoot()); - ledgerVersion = ltx.loadHeader().current().ledgerVersion; + ledgerVersion = app.getLedgerManager() + .getLastClosedLedgerHeader() + .header.ledgerVersion; } if (protocolVersionIsBefore(ledgerVersion, ProtocolVersion::V_13)) { @@ -1965,8 +1966,9 @@ getBalance(Application& app, AccountID const& accountID, Asset const& asset) uint32_t getLclProtocolVersion(Application& app) { - auto const& lcl = app.getLedgerManager().getLastClosedLedgerHeader(); - return lcl.header.ledgerVersion; + return app.getLedgerManager() + .getLastClosedLedgerHeader() + .header.ledgerVersion; } bool diff --git a/src/transactions/ExtendFootprintTTLOpFrame.cpp b/src/transactions/ExtendFootprintTTLOpFrame.cpp index fcf23d1c8a..7786b3e553 100644 --- a/src/transactions/ExtendFootprintTTLOpFrame.cpp +++ b/src/transactions/ExtendFootprintTTLOpFrame.cpp @@ -57,14 +57,12 @@ ExtendFootprintTTLOpFrame::doApply( releaseAssertOrThrow(sorobanData); ZoneNamedN(applyZone, "ExtendFootprintTTLOpFrame apply", true); - ExtendFootprintTTLMetrics metrics( - app.getLedgerManager().getSorobanMetrics()); + ExtendFootprintTTLMetrics metrics(app.getSorobanMetrics()); auto timeScope = metrics.getExecTimer(); auto const& resources = mParentTx.sorobanResources(); auto const& footprint = resources.footprint; - auto const& sorobanConfig = - app.getLedgerManager().getSorobanNetworkConfig(); + auto sorobanConfig = app.getSorobanNetworkConfig(); rust::Vec rustEntryRentChanges; rustEntryRentChanges.reserve(footprint.readOnly.size()); diff --git a/src/transactions/InvokeHostFunctionOpFrame.cpp b/src/transactions/InvokeHostFunctionOpFrame.cpp index 28a6ec1f83..7c89bb3097 100644 --- a/src/transactions/InvokeHostFunctionOpFrame.cpp +++ b/src/transactions/InvokeHostFunctionOpFrame.cpp @@ -327,10 +327,9 @@ InvokeHostFunctionOpFrame::doApply( ZoneNamedN(applyZone, "InvokeHostFunctionOpFrame apply", true); Config const& appConfig = app.getConfig(); - HostFunctionMetrics metrics(app.getLedgerManager().getSorobanMetrics()); + HostFunctionMetrics metrics(app.getSorobanMetrics()); auto timeScope = metrics.getExecTimer(); - auto const& sorobanConfig = - app.getLedgerManager().getSorobanNetworkConfig(); + auto sorobanConfig = app.getSorobanNetworkConfig(); // Get the entries for the footprint rust::Vec ledgerEntryCxxBufs; diff --git a/src/transactions/OperationFrame.cpp b/src/transactions/OperationFrame.cpp index 69c2660c60..04a7190b85 100644 --- a/src/transactions/OperationFrame.cpp +++ b/src/transactions/OperationFrame.cpp @@ -262,8 +262,7 @@ OperationFrame::checkValid(AppConnector& app, isSoroban()) { releaseAssertOrThrow(sorobanData); - auto const& sorobanConfig = - app.getLedgerManager().getSorobanNetworkConfig(); + auto sorobanConfig = app.getSorobanNetworkConfig(); validationResult = doCheckValidForSoroban(sorobanConfig, app.getConfig(), diff --git a/src/transactions/RestoreFootprintOpFrame.cpp b/src/transactions/RestoreFootprintOpFrame.cpp index 36d5a20953..47dd5fd1e0 100644 --- a/src/transactions/RestoreFootprintOpFrame.cpp +++ b/src/transactions/RestoreFootprintOpFrame.cpp @@ -57,14 +57,13 @@ RestoreFootprintOpFrame::doApply( { ZoneNamedN(applyZone, "RestoreFootprintOpFrame apply", true); - RestoreFootprintMetrics metrics(app.getLedgerManager().getSorobanMetrics()); + RestoreFootprintMetrics metrics(app.getSorobanMetrics()); auto timeScope = metrics.getExecTimer(); auto const& resources = mParentTx.sorobanResources(); auto const& footprint = resources.footprint; auto ledgerSeq = ltx.loadHeader().current().ledgerSeq; - auto const& sorobanConfig = - app.getLedgerManager().getSorobanNetworkConfig(); + auto sorobanConfig = app.getSorobanNetworkConfig(); auto const& appConfig = app.getConfig(); auto const& archivalSettings = sorobanConfig.stateArchivalSettings(); @@ -148,14 +147,11 @@ RestoreFootprintOpFrame::doApply( int64_t rentFee = rust_bridge::compute_rent_fee( app.getConfig().CURRENT_LEDGER_PROTOCOL_VERSION, ledgerVersion, rustEntryRentChanges, - app.getLedgerManager() - .getSorobanNetworkConfig() - .rustBridgeRentFeeConfiguration(), + app.getSorobanNetworkConfig().rustBridgeRentFeeConfiguration(), ledgerSeq); if (!sorobanData->consumeRefundableSorobanResources( 0, rentFee, ltx.loadHeader().current().ledgerVersion, - app.getLedgerManager().getSorobanNetworkConfig(), app.getConfig(), - mParentTx)) + app.getSorobanNetworkConfig(), app.getConfig(), mParentTx)) { innerResult(res).code(RESTORE_FOOTPRINT_INSUFFICIENT_REFUNDABLE_FEE); return false; diff --git a/src/transactions/TransactionFrame.cpp b/src/transactions/TransactionFrame.cpp index 1eb017d768..ccd4e3017f 100644 --- a/src/transactions/TransactionFrame.cpp +++ b/src/transactions/TransactionFrame.cpp @@ -1445,7 +1445,7 @@ TransactionFrame::checkValidWithOptionallyChargedFee( { sorobanResourceFee = computePreApplySorobanResourceFee( ls.getLedgerHeader().current().ledgerVersion, - app.getLedgerManager().getSorobanNetworkConfig(), app.getConfig()); + app.getSorobanNetworkConfig(), app.getConfig()); } bool res = commonValid(app, signatureChecker, ls, current, false, chargeFee, lowerBoundCloseTimeOffset, upperBoundCloseTimeOffset, @@ -1504,8 +1504,7 @@ TransactionFrame::checkSorobanResourceAndSetError( AppConnector& app, uint32_t ledgerVersion, MutableTxResultPtr txResult) const { - auto const& sorobanConfig = - app.getLedgerManager().getSorobanNetworkConfig(); + auto sorobanConfig = app.getSorobanNetworkConfig(); if (!validateSorobanResources(sorobanConfig, app.getConfig(), ledgerVersion, *txResult->getSorobanData())) { @@ -1694,8 +1693,7 @@ TransactionFrame::applyOperations(SignatureChecker& signatureChecker, // If transaction fails, we don't charge for any // refundable resources. auto preApplyFee = computePreApplySorobanResourceFee( - ledgerVersion, - app.getLedgerManager().getSorobanNetworkConfig(), + ledgerVersion, app.getSorobanNetworkConfig(), app.getConfig()); txResult.getSorobanData()->setSorobanFeeRefund( @@ -1797,8 +1795,7 @@ TransactionFrame::apply(AppConnector& app, AbstractLedgerTxn& ltx, isSoroban()) { sorobanResourceFee = computePreApplySorobanResourceFee( - ledgerVersion, app.getLedgerManager().getSorobanNetworkConfig(), - app.getConfig()); + ledgerVersion, app.getSorobanNetworkConfig(), app.getConfig()); auto& sorobanData = *txResult->getSorobanData(); sorobanData.setSorobanConsumedNonRefundableFee( diff --git a/src/transactions/TransactionUtils.cpp b/src/transactions/TransactionUtils.cpp index bcdb02c257..d1c9d8b69b 100644 --- a/src/transactions/TransactionUtils.cpp +++ b/src/transactions/TransactionUtils.cpp @@ -1988,7 +1988,7 @@ isTransactionXDRValidForCurrentProtocol(AppConnector& app, { uint32_t maxProtocol = app.getConfig().CURRENT_LEDGER_PROTOCOL_VERSION; uint32_t currProtocol = - app.getLedgerManager().getLastClosedLedgerHeader().header.ledgerVersion; + app.getLastClosedLedgerHeader().header.ledgerVersion; // If we could parse the XDR when ledger is using the maximum supported // protocol version, then XDR has to be valid. // This check also is pointless before protocol 21 as Soroban environment diff --git a/src/transactions/test/InvokeHostFunctionTests.cpp b/src/transactions/test/InvokeHostFunctionTests.cpp index 441c22fba1..b4b05171cd 100644 --- a/src/transactions/test/InvokeHostFunctionTests.cpp +++ b/src/transactions/test/InvokeHostFunctionTests.cpp @@ -1631,8 +1631,7 @@ TEST_CASE("settings upgrade", "[tx][soroban][upgrades]") auto ledgerUpgrade = LedgerUpgrade{LEDGER_UPGRADE_CONFIG}; ledgerUpgrade.newConfig() = key; - auto const& lcl = - test.getApp().getLedgerManager().getLastClosedLedgerHeader(); + auto lcl = test.getApp().getLedgerManager().getLastClosedLedgerHeader(); auto txSet = TxSetXDRFrame::makeEmpty(lcl); auto lastCloseTime = lcl.header.scpValue.closeTime; @@ -3121,9 +3120,9 @@ TEST_CASE("settings upgrade command line utils", "[tx][soroban][upgrades]") // Update bucketListTargetSizeBytes so bucketListWriteFeeGrowthFactor comes // into play - auto const& blSize = app->getLedgerManager() - .getSorobanNetworkConfig() - .getAverageBucketListSize(); + auto blSize = app->getLedgerManager() + .getSorobanNetworkConfig() + .getAverageBucketListSize(); { LedgerTxn ltx(app->getLedgerTxnRoot()); auto costKey = configSettingKey( @@ -3189,7 +3188,7 @@ TEST_CASE("settings upgrade command line utils", "[tx][soroban][upgrades]") auto ledgerUpgrade = LedgerUpgrade{LEDGER_UPGRADE_CONFIG}; ledgerUpgrade.newConfig() = upgradeSetKey; - auto const& lcl = lm.getLastClosedLedgerHeader(); + auto lcl = lm.getLastClosedLedgerHeader(); auto txSet = TxSetXDRFrame::makeEmpty(lcl); auto lastCloseTime = lcl.header.scpValue.closeTime; @@ -3237,7 +3236,7 @@ TEST_CASE("settings upgrade command line utils", "[tx][soroban][upgrades]") auto ledgerUpgrade = LedgerUpgrade{LEDGER_UPGRADE_CONFIG}; ledgerUpgrade.newConfig() = upgradeSetKey2; - auto const& lcl = lm.getLastClosedLedgerHeader(); + auto lcl = lm.getLastClosedLedgerHeader(); auto txSet = TxSetXDRFrame::makeEmpty(lcl); auto lastCloseTime = lcl.header.scpValue.closeTime; @@ -3270,7 +3269,7 @@ TEST_CASE("settings upgrade command line utils", "[tx][soroban][upgrades]") closeLedger(*app); } - auto const& lcl = lm.getLastClosedLedgerHeader(); + auto lcl = lm.getLastClosedLedgerHeader(); // The only readWrite key in the invoke op is the one that writes the // ConfigUpgradeSet xdr diff --git a/src/transactions/test/SorobanTxTestUtils.cpp b/src/transactions/test/SorobanTxTestUtils.cpp index 0ae49359f6..f583a09b09 100644 --- a/src/transactions/test/SorobanTxTestUtils.cpp +++ b/src/transactions/test/SorobanTxTestUtils.cpp @@ -1055,7 +1055,7 @@ SorobanTest::getDummyAccount() SorobanNetworkConfig const& SorobanTest::getNetworkCfg() { - return getApp().getLedgerManager().getSorobanNetworkConfig(); + return getApp().getLedgerManager().getMutableSorobanNetworkConfig(); } uint32_t