From f5fa23a43ea8fc0f96bc61d043de0b20fc008e81 Mon Sep 17 00:00:00 2001 From: Stan Kladko <13399135+kladkogex@users.noreply.github.com> Date: Mon, 20 May 2024 12:18:47 +0100 Subject: [PATCH 001/314] #806 enable patches --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c5d0defbd..14966537a 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ The SKALE network supports an unlimited number of independent blockchains with z ## Forklessness -Skaled is forkless, meaning that blockchain a linear chain (and not a tree of forks as with ETH 1.0). Every block is provably finalized within finite time. +Skaled is forkless, meaning that blockchain a linear chain (and not a tree of forks as with ETH 1.0). Every block is provably finalized immediately after creation. +Therefore, finalization time for skaled is equal to block time, which is +much faster than 13 minutes for Eth main net. ## Asynchronous block production From 48f898e2126b366819f44097f1cf1a4337a9f7a3 Mon Sep 17 00:00:00 2001 From: Stan Kladko <13399135+kladkogex@users.noreply.github.com> Date: Mon, 20 May 2024 18:52:18 +0100 Subject: [PATCH 002/314] 1545 Adding LevelDB snap --- libdevcore/LevelDB.cpp | 79 ++++++++++++++++++++++++++++++++++ libdevcore/LevelDB.h | 31 ++++++++++++++ libdevcore/LevelDBSnap.cpp | 88 ++++++++++++++++++++++++++++++++++++++ libdevcore/LevelDBSnap.h | 72 +++++++++++++++++++++++++++++++ 4 files changed, 270 insertions(+) create mode 100644 libdevcore/LevelDBSnap.cpp create mode 100644 libdevcore/LevelDBSnap.h diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp index ba019a426..260630594 100644 --- a/libdevcore/LevelDB.cpp +++ b/libdevcore/LevelDB.cpp @@ -19,6 +19,7 @@ #include "LevelDB.h" +#include "LevelDBSnap.h" #include "Assertions.h" #include "Log.h" #include @@ -30,6 +31,8 @@ namespace dev::db { unsigned c_maxOpenLeveldbFiles = 25; const size_t LevelDB::BATCH_CHUNK_SIZE = 10000; +const size_t LevelDB::MAX_OLD_SNAPS_LIFETIME_MS = 10000; +const size_t LevelDB::FORCE_CLOSE_TIME_MS = 3000; namespace { inline leveldb::Slice toLDBSlice( Slice _slice ) { @@ -144,6 +147,7 @@ void LevelDB::openDBInstanceUnsafe() { m_db.reset( db ); m_lastDBOpenTimeMs = getCurrentTimeMs(); + m_dbIdentifier++; cnote << "LEVELDB_OPENED:TIME_MS:" << m_lastDBOpenTimeMs - startTimeMs; } uint64_t LevelDB::getCurrentTimeMs() { @@ -248,6 +252,29 @@ void LevelDB::reopenDataBaseIfNeeded() { if ( currentTimeMs - m_lastDBOpenTimeMs >= ( uint64_t ) m_reopenPeriodMs ) { ExclusiveDBGuard lock( *this ); + + + // we need to close all snaps created for this database + // we wait for FORCE_TIME_MS, hoping that all eth_calls complete nicely + // after that we close snapshots forcefully. This means that + // eth_calls that take more than FORCE_TIME_MS will return exception + // note that in the previous release eth_calls would fail immediately, so + // the new behavior is much nicer + + auto startTimeMs = getCurrentTimeMs(); + + while (getCurrentTimeMs() >= startTimeMs + FORCE_CLOSE_TIME_MS) { + cleanOldSnapsUnsafe(FORCE_CLOSE_TIME_MS); + } + + if (oldSnaps.empty()) { + // there are still open snaps. Close all of them not waiting for + // eth_calls to complete by passing 0 as wait time + cleanOldSnapsUnsafe(0); + } + + LDB_CHECK(oldSnaps.empty()); + // releasing unique pointer will cause database destructor to be called that will close db m_db.reset(); // now open db while holding the exclusive lock @@ -383,6 +410,50 @@ void LevelDB::doCompaction() const { m_db->CompactRange( nullptr, nullptr ); } + +void LevelDB::createBlockSnap( uint64_t _blockId ) { + SharedDBGuard lock( *this ); + + auto newSnapHandle = m_db->GetSnapshot(); + LDB_CHECK(newSnapHandle); + auto newSnap = std::make_shared( + _blockId, newSnapHandle, + m_dbIdentifier); + LDB_CHECK(newSnap); + + { + std::unique_lock< std::shared_mutex > snapLock( m_snapMutex ); + auto oldSnap = m_lastBlockSnap; + m_lastBlockSnap = newSnap; + oldSnaps.emplace( oldSnap->getObjectId(), oldSnap ); + } + + // we clean unneeded old snaps that no-one used or that exist for more that max + // lifetime we give for eth_calls to complete + cleanOldSnapsUnsafe( MAX_OLD_SNAPS_LIFETIME_MS ); +} + + +// this function should be called while holding database reopen lock +void LevelDB::cleanOldSnapsUnsafe( uint64_t _maxSnapLifetimeMs ) { + std::unique_lock< std::shared_mutex > snapLock( m_snapMutex ); + + //now we iterate over oldSnaps closing the ones that are not more in use + auto currentTimeMs = getCurrentTimeMs(); + // Iterate and delete entries with even values + for (auto it = oldSnaps.begin(); it != oldSnaps.end(); ) { + if ( it->second.use_count() == 1 || // no one using this snap anymore except this map + it->second->getCreationTimeMs() + _maxSnapLifetimeMs <= currentTimeMs ) // old + { + it->second->close( m_db, m_dbIdentifier ); + it = oldSnaps.erase(it); // Erase returns the iterator to the next element + } else { + ++it; // Only increment if not erasing + } + } +} + + std::atomic< uint64_t > LevelDB::g_keysToBeDeletedStats = 0; std::atomic< uint64_t > LevelDB::g_keyDeletesStats = 0; @@ -390,4 +461,12 @@ uint64_t LevelDB::getKeyDeletesStats() { return g_keyDeletesStats; } + + + + + + } // namespace dev::db + + diff --git a/libdevcore/LevelDB.h b/libdevcore/LevelDB.h index 5aa1f7df0..9e4ebf50b 100644 --- a/libdevcore/LevelDB.h +++ b/libdevcore/LevelDB.h @@ -29,7 +29,17 @@ #include #include +#define LDB_CHECK( _EXPRESSION_ ) \ + if ( !( _EXPRESSION_ ) ) { \ + auto __msg__ = std::string( "State check failed::" ) + #_EXPRESSION_ + " " + \ + std::string( __FILE__ ) + ":" + std::to_string( __LINE__ ); \ + BOOST_THROW_EXCEPTION(DatabaseError() << errinfo_comment( __msg__ )); \ + } + namespace dev::db { + +class LevelDBSnap; + class LevelDB : public DatabaseFace { public: static leveldb::ReadOptions defaultReadOptions(); @@ -65,6 +75,13 @@ class LevelDB : public DatabaseFace { void doCompaction() const; + void createBlockSnap(uint64_t _blockId); + + std::shared_ptr getLastBlockSnap(uint64_t _blockId); + + void closeSnap(LevelDBSnap& _snap); + + // Return the total count of key deletes since the start static uint64_t getKeyDeletesStats(); // count of the keys that were deleted since the start of skaled @@ -75,6 +92,9 @@ class LevelDB : public DatabaseFace { private: std::unique_ptr< leveldb::DB > m_db; + // this identify is guaranteed to be unique for each m_db reference + // so it can be used to compare to references + std::atomic m_dbIdentifier = 0; leveldb::ReadOptions const m_readOptions; leveldb::WriteOptions const m_writeOptions; leveldb::Options m_options; @@ -85,8 +105,18 @@ class LevelDB : public DatabaseFace { uint64_t m_lastDBOpenTimeMs; mutable std::shared_mutex m_dbMutex; + // old snaps contains snap objects for older blocks + // these objects are alive untils the + // corresponding eth_calls complete + std::map> oldSnaps; + std::shared_ptr m_lastBlockSnap; + // mutex to protect oldSnaps and m_lastBlockSnap; + std::shared_mutex m_snapMutex; + static const size_t BATCH_CHUNK_SIZE; + static const size_t MAX_OLD_SNAPS_LIFETIME_MS; + static const size_t FORCE_CLOSE_TIME_MS; class SharedDBGuard { const LevelDB& m_levedlDB; @@ -125,6 +155,7 @@ class LevelDB : public DatabaseFace { }; void openDBInstanceUnsafe(); void reopenDataBaseIfNeeded(); + void cleanOldSnapsUnsafe( uint64_t _maxSnapLifetimeMs ); }; } // namespace dev::db diff --git a/libdevcore/LevelDBSnap.cpp b/libdevcore/LevelDBSnap.cpp new file mode 100644 index 000000000..0f1e791c8 --- /dev/null +++ b/libdevcore/LevelDBSnap.cpp @@ -0,0 +1,88 @@ +/* + Modifications Copyright (C) 2024- SKALE Labs + + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ + + +#include "LevelDBSnap.h" +#include "Assertions.h" +#include "LevelDB.h" + + +using std::string, std::runtime_error; + +namespace dev::db { + + +LevelDBSnap::LevelDBSnap( + uint64_t _blockId, const leveldb::Snapshot* _snap, uint64_t _parentLevelDBIdentifier ) + : m_blockId( _blockId ), m_snap( _snap ), m_parentLevelDBId( _parentLevelDBIdentifier ) { + LDB_CHECK( m_snap ) + m_creationTimeMs = LevelDB::getCurrentTimeMs(); + m_objectId = objectCounter.fetch_add(1); +} + +uint64_t LevelDBSnap::getParentLevelDBId() const { + return m_parentLevelDBId; +} + +// close LevelDB snap. This will happen when all eth_calls using this snap +// complete, so it is not needed anymore +// reopen the DB +void LevelDBSnap::close( std::unique_ptr< leveldb::DB >& _parentDB, uint64_t _parentDBIdentifier ) { + auto isClosed = m_isClosed.exchange( true ); + if ( isClosed ) { + // this should never happen + cwarn << "Close called twice on a snap"; + return; + } + + LDB_CHECK( _parentDB ); + LDB_CHECK( m_snap ); + // sanity check. We should use the same DB referencr that was used to open this snap + if ( _parentDBIdentifier != m_parentLevelDBId ) { + // this should never happen, since it means that we are attempting to close snapshot + // after its parent database is closed due to reopen + // normally we close all snapshots before reopenining the databasew + cwarn << "Closing the snapshot after the database is closed"; + return; + } + + _parentDB->ReleaseSnapshot( this->m_snap ); + m_snap = nullptr; +} +LevelDBSnap::~LevelDBSnap() { + // LevelDB should be closed before releasing it, otherwise + // we use cerr here since destructor may be called during late stages of + // skaled exit where logging is not available + if ( !m_isClosed ) { + std::cerr << "LevelDB warning: destroying active snap" << std::endl; + } +} + + +uint64_t LevelDBSnap::getObjectId() const { + return m_objectId; +} + +std::atomic LevelDBSnap::objectCounter = 0; +uint64_t LevelDBSnap::getCreationTimeMs() const { + return m_creationTimeMs; +} + + +} // namespace dev::db diff --git a/libdevcore/LevelDBSnap.h b/libdevcore/LevelDBSnap.h new file mode 100644 index 000000000..b582d1342 --- /dev/null +++ b/libdevcore/LevelDBSnap.h @@ -0,0 +1,72 @@ +/* + Modifications Copyright (C) 2024- SKALE Labs + + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ + +#pragma once + +#include "db.h" + +#include +#include +#include +#include + +#include +#include + +namespace dev::db { + +class LevelDB; + +// internal class of LevelDB that represents the +// this class represents a LevelDB snap corresponding to the point immediately +// after processing of a particular block id. +class LevelDBSnap { +public: + LevelDBSnap( uint64_t _blockId, const leveldb::Snapshot* _snap, + uint64_t _parentLevelDBId ); + uint64_t getParentLevelDBId() const; + + // close this snapshot. Use after close will cause an exception + void close(std::unique_ptr< leveldb::DB >& _parentDB, uint64_t _parentDBIdentifier); + + virtual ~LevelDBSnap(); + + +private: + // block id immediately after which this snap is taken + std::atomic m_isClosed = false; + const uint64_t m_blockId; + const leveldb::Snapshot* m_snap = nullptr; + uint64_t m_creationTimeMs; + // LevelDB identifier for which this snapShot has been // created + // this is used to match snaps to leveldb handles + uint64_t m_parentLevelDBId; + +public: + uint64_t getCreationTimeMs() const; + +private: + uint64_t m_objectId; + static std::atomic objectCounter; + +public: + uint64_t getObjectId() const; +}; + +} // namespace dev::db From 049115963774a83fe4252a610fe46edb7825a1ee Mon Sep 17 00:00:00 2001 From: Stan Kladko <13399135+kladkogex@users.noreply.github.com> Date: Mon, 20 May 2024 18:55:21 +0100 Subject: [PATCH 003/314] 1545 Fix comparison --- libdevcore/LevelDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp index 260630594..677e4a4a1 100644 --- a/libdevcore/LevelDB.cpp +++ b/libdevcore/LevelDB.cpp @@ -263,7 +263,7 @@ void LevelDB::reopenDataBaseIfNeeded() { auto startTimeMs = getCurrentTimeMs(); - while (getCurrentTimeMs() >= startTimeMs + FORCE_CLOSE_TIME_MS) { + while (getCurrentTimeMs() <= startTimeMs + FORCE_CLOSE_TIME_MS) { cleanOldSnapsUnsafe(FORCE_CLOSE_TIME_MS); } From 687571b41722b65a5167d41bdeb5bf553ad4bef6 Mon Sep 17 00:00:00 2001 From: Stan Kladko <13399135+kladkogex@users.noreply.github.com> Date: Fri, 24 May 2024 16:40:57 +0100 Subject: [PATCH 004/314] 1545 fix config --- libethereum/ChainParams.cpp | 13 +++++++++++++ test/historicstate/configs/basic_config.json | 1 + 2 files changed, 14 insertions(+) diff --git a/libethereum/ChainParams.cpp b/libethereum/ChainParams.cpp index 4b2be0789..be322b7f6 100644 --- a/libethereum/ChainParams.cpp +++ b/libethereum/ChainParams.cpp @@ -229,6 +229,19 @@ void ChainParams::processSkaleConfigItems( ChainParams& cp, json_spirit::mObject if ( !testSignatures ) { ecdsaKeyName = infoObj.at( "ecdsaKeyName" ).get_str(); + + if (infoObj.count("wallets") == 0) { + BOOST_THROW_EXCEPTION(std::runtime_error("No wallets section in config " + "and testSignature is not set")); + } + + auto infoObj2 = infoObj.at("ima").get_obj(); + + if (infoObj.count("ima") == 0) { + BOOST_THROW_EXCEPTION(std::runtime_error("No wallets/ima section in config")); + } + + js::mObject ima = infoObj.at( "wallets" ).get_obj().at( "ima" ).get_obj(); commonBLSPublicKeys[0] = ima["commonBLSPublicKey0"].get_str(); diff --git a/test/historicstate/configs/basic_config.json b/test/historicstate/configs/basic_config.json index a108a100f..389afd333 100644 --- a/test/historicstate/configs/basic_config.json +++ b/test/historicstate/configs/basic_config.json @@ -317,6 +317,7 @@ "collectionQueueSize": 2, "collectionDuration": 10, "transactionQueueSize": 100, + "testSignatures": true, "maxOpenLeveldbFiles": 25 }, "sChain": From 4294a1f11ebebe6302a38c9f60af45690d8f8a0b Mon Sep 17 00:00:00 2001 From: Stan Kladko <13399135+kladkogex@users.noreply.github.com> Date: Fri, 24 May 2024 19:03:43 +0100 Subject: [PATCH 005/314] 1545 fix config --- libskale/State.cpp | 1082 +++++++++++++++++++++++--------------------- libskale/State.h | 35 +- 2 files changed, 590 insertions(+), 527 deletions(-) diff --git a/libskale/State.cpp b/libskale/State.cpp index 405eb2c69..bd9b59875 100644 --- a/libskale/State.cpp +++ b/libskale/State.cpp @@ -64,34 +64,44 @@ using dev::eth::TransactionReceipt; #define ETH_VMTRACE 0 #endif -State::State( dev::u256 const& _accountStartNonce, boost::filesystem::path const& _dbPath, - dev::h256 const& _genesis, BaseState _bs, dev::u256 _initialFunds, - dev::s256 _contractStorageLimit ) - : x_db_ptr( make_shared< boost::shared_mutex >() ), - m_storedVersion( make_shared< size_t >( 0 ) ), - m_currentVersion( *m_storedVersion ), - m_accountStartNonce( _accountStartNonce ), - m_initial_funds( _initialFunds ), - contractStorageLimit_( _contractStorageLimit ) + +#define STATE_CHECK(_EXPRESSION_) \ + if ( !( _EXPRESSION_ ) ) { \ + auto __msg__ = std::string( "State check failed::" ) + #_EXPRESSION_ + " " + \ + std::string( __FILE__ ) + ":" + std::to_string( __LINE__ ); \ + BOOST_THROW_EXCEPTION(std::runtime_error(__msg__)); \ + } + + +State::State(dev::u256 const &_accountStartNonce, boost::filesystem::path const &_dbPath, + dev::h256 const &_genesis, BaseState _bs, dev::u256 _initialFunds, + dev::s256 _contractStorageLimit) + : x_db_ptr(make_shared()), + m_storedVersion(make_shared(0)), + m_currentVersion(*m_storedVersion), + m_accountStartNonce(_accountStartNonce), + m_initial_funds(_initialFunds), + contractStorageLimit_(_contractStorageLimit) #ifdef HISTORIC_STATE - , - m_historicState( _accountStartNonce, - dev::eth::HistoricState::openDB( - boost::filesystem::path( std::string( _dbPath.string() ) - .append( "/" ) - .append( dev::eth::HISTORIC_STATE_DIR ) ), - _genesis, - _bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ), - dev::eth::HistoricState::openDB( - boost::filesystem::path( std::string( _dbPath.string() ) - .append( "/" ) - .append( dev::eth::HISTORIC_ROOTS_DIR ) ), - _genesis, - _bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ) ) + , + m_historicState(_accountStartNonce, + dev::eth::HistoricState::openDB( + boost::filesystem::path(std::string(_dbPath.string()) + .append("/") + .append(dev::eth::HISTORIC_STATE_DIR)), + _genesis, + _bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill), + dev::eth::HistoricState::openDB( + boost::filesystem::path(std::string(_dbPath.string()) + .append("/") + .append(dev::eth::HISTORIC_ROOTS_DIR)), + _genesis, + _bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill)) #endif { - m_db_ptr = make_shared< OverlayDB >( openDB( _dbPath, _genesis, - _bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ) ); + m_db_ptr = make_shared(openDB(_dbPath, _genesis, + _bs == BaseState::PreExisting ? dev::WithExisting::Trust + : dev::WithExisting::Kill)); auto state = createStateReadOnlyCopy(); totalStorageUsed_ = state.storageUsedTotal(); @@ -99,32 +109,32 @@ State::State( dev::u256 const& _accountStartNonce, boost::filesystem::path const m_historicState.setRootFromDB(); #endif m_fs_ptr = state.fs(); - if ( _bs == BaseState::PreExisting ) { - clog( VerbosityDebug, "statedb" ) << cc::debug( "Using existing database" ); - } else if ( _bs == BaseState::Empty ) { + if (_bs == BaseState::PreExisting) { + clog(VerbosityDebug, "statedb") << cc::debug("Using existing database"); + } else if (_bs == BaseState::Empty) { // Initialise to the state entailed by the genesis block; this guarantees the trie is built // correctly. m_db_ptr->clearDB(); } else { - throw std::logic_error( "Not implemented" ); + throw std::logic_error("Not implemented"); } } -State::State( u256 const& _accountStartNonce, OverlayDB const& _db, +State::State(u256 const &_accountStartNonce, OverlayDB const &_db, #ifdef HISTORIC_STATE - dev::OverlayDB const& _historicDb, dev::OverlayDB const& _historicBlockToStateRootDb, + dev::OverlayDB const &_historicDb, dev::OverlayDB const &_historicBlockToStateRootDb, #endif - skale::BaseState _bs, u256 _initialFunds, s256 _contractStorageLimit ) - : x_db_ptr( make_shared< boost::shared_mutex >() ), - m_db_ptr( make_shared< OverlayDB >( _db ) ), - m_storedVersion( make_shared< size_t >( 0 ) ), - m_currentVersion( *m_storedVersion ), - m_accountStartNonce( _accountStartNonce ), - m_initial_funds( _initialFunds ), - contractStorageLimit_( _contractStorageLimit ) + skale::BaseState _bs, u256 _initialFunds, s256 _contractStorageLimit) + : x_db_ptr(make_shared()), + m_db_ptr(make_shared(_db)), + m_storedVersion(make_shared(0)), + m_currentVersion(*m_storedVersion), + m_accountStartNonce(_accountStartNonce), + m_initial_funds(_initialFunds), + contractStorageLimit_(_contractStorageLimit) #ifdef HISTORIC_STATE - , - m_historicState( _accountStartNonce, _historicDb, _historicBlockToStateRootDb, _bs ) + , + m_historicState(_accountStartNonce, _historicDb, _historicBlockToStateRootDb, _bs) #endif { auto state = createStateReadOnlyCopy(); @@ -133,14 +143,14 @@ State::State( u256 const& _accountStartNonce, OverlayDB const& _db, m_historicState.setRootFromDB(); #endif m_fs_ptr = state.fs(); - if ( _bs == BaseState::PreExisting ) { - clog( VerbosityDebug, "statedb" ) << cc::debug( "Using existing database" ); - } else if ( _bs == BaseState::Empty ) { + if (_bs == BaseState::PreExisting) { + clog(VerbosityDebug, "statedb") << cc::debug("Using existing database"); + } else if (_bs == BaseState::Empty) { // Initialise to the state entailed by the genesis block; this guarantees the trie is built // correctly. m_db_ptr->clearDB(); } else { - throw std::logic_error( "Not implemented" ); + throw std::logic_error("Not implemented"); } } @@ -150,6 +160,7 @@ State::State( u256 const& _accountStartNonce, OverlayDB const& _db, const uint64_t STATE_IMPORT_BATCH_COUNT = 16; void State::populateHistoricStateFromSkaleState() { + STATE_CHECK(!m_levelDBSnap) auto allAccountAddresses = this->addresses(); cout << "Number of addresses in statedb:" << allAccountAddresses.size() << endl; @@ -158,8 +169,8 @@ void State::populateHistoricStateFromSkaleState() { // this is done to save memory, otherwise OverlayDB will frow - for ( uint64_t i = 0; i < STATE_IMPORT_BATCH_COUNT; i++ ) { - populateHistoricStateBatchFromSkaleState( allAccountAddresses, i ); + for (uint64_t i = 0; i < STATE_IMPORT_BATCH_COUNT; i++) { + populateHistoricStateBatchFromSkaleState(allAccountAddresses, i); } cout << "Completed state import" << endl; @@ -167,94 +178,97 @@ void State::populateHistoricStateFromSkaleState() { dev::eth::AccountMap State::getBatchOfAccounts( - std::unordered_map< Address, u256 >& _allAccountAddresses, uint64_t _batchNumber ) { + std::unordered_map &_allAccountAddresses, uint64_t _batchNumber) { + STATE_CHECK(!m_levelDBSnap) dev::eth::AccountMap accountBatch; - for ( auto&& item : _allAccountAddresses ) { + for (auto &&item: _allAccountAddresses) { auto address = item.first; // we split addresses into 16 batches - uint64_t first8BytesOfAddress = *( uint64_t* ) address.asArray().data(); - if ( first8BytesOfAddress % STATE_IMPORT_BATCH_COUNT != _batchNumber ) { + uint64_t first8BytesOfAddress = *(uint64_t *) address.asArray().data(); + if (first8BytesOfAddress % STATE_IMPORT_BATCH_COUNT != _batchNumber) { continue; } - Account account = *this->account( address ); + Account account = *this->account(address); - if ( addressHasCode( address ) ) { + if (addressHasCode(address)) { account.resetCode(); - account.setCode( bytes( code( address ) ), account.version() ); + account.setCode(bytes(code(address)), account.version()); } // mark the account changed so it will be written into the database account.changed(); - accountBatch.emplace( address, account ); + accountBatch.emplace(address, account); } return accountBatch; } void State::populateHistoricStateBatchFromSkaleState( - std::unordered_map< Address, u256 >& _allAccountAddresses, uint64_t _batchNumber ) { + std::unordered_map &_allAccountAddresses, uint64_t _batchNumber) { + STATE_CHECK(!m_levelDBSnap) cout << "Now running batch " << _batchNumber << " out of " << STATE_IMPORT_BATCH_COUNT << endl; - dev::eth::AccountMap accountMap = getBatchOfAccounts( _allAccountAddresses, _batchNumber ); + dev::eth::AccountMap accountMap = getBatchOfAccounts(_allAccountAddresses, _batchNumber); - m_db_ptr->copyStorageIntoAccountMap( accountMap ); + m_db_ptr->copyStorageIntoAccountMap(accountMap); - m_historicState.commitExternalChanges( accountMap ); + m_historicState.commitExternalChanges(accountMap); } + #endif skale::OverlayDB State::openDB( - fs::path const& _basePath, h256 const& _genesisHash, WithExisting _we ) { + fs::path const &_basePath, h256 const &_genesisHash, WithExisting _we) { fs::path path = _basePath.empty() ? eth::Defaults::dbPath() : _basePath; - if ( _we == WithExisting::Kill ) { - clog( VerbosityDebug, "statedb" ) << "Killing state database (WithExisting::Kill)."; - fs::remove_all( path / fs::path( "state" ) ); + if (_we == WithExisting::Kill) { + clog(VerbosityDebug, "statedb") << "Killing state database (WithExisting::Kill)."; + fs::remove_all(path / fs::path("state")); } - path /= fs::path( toHex( _genesisHash.ref().cropped( 0, 4 ) ) ) / - fs::path( toString( eth::c_databaseVersion ) ); - fs::create_directories( path ); - DEV_IGNORE_EXCEPTIONS( fs::permissions( path, fs::owner_all ) ); + path /= fs::path(toHex(_genesisHash.ref().cropped(0, 4))) / + fs::path(toString(eth::c_databaseVersion)); + fs::create_directories(path); + DEV_IGNORE_EXCEPTIONS(fs::permissions(path, fs::owner_all)); - fs::path state_path = path / fs::path( "state" ); + fs::path state_path = path / fs::path("state"); try { - m_orig_db.reset( new db::DBImpl( state_path ) ); - std::unique_ptr< batched_io::batched_db > bdb = make_unique< batched_io::batched_db >(); - bdb->open( m_orig_db ); - assert( bdb->is_open() ); - clog( VerbosityDebug, "statedb" ) << cc::success( "Opened state DB." ); - return OverlayDB( std::move( bdb ) ); - } catch ( boost::exception const& ex ) { - cwarn << boost::diagnostic_information( ex ) << '\n'; - if ( fs::space( path / fs::path( "state" ) ).available < 1024 ) { + m_orig_db.reset(new db::DBImpl(state_path)); + std::unique_ptr bdb = make_unique(); + bdb->open(m_orig_db); + STATE_CHECK(bdb->is_open()); + clog(VerbosityDebug, "statedb") << cc::success("Opened state DB."); + return OverlayDB(std::move(bdb)); + } catch (boost::exception const &ex) { + cwarn << boost::diagnostic_information(ex) << '\n'; + if (fs::space(path / fs::path("state")).available < 1024) { cwarn << "Not enough available space found on hard drive. Please free some up and " "then " "re-run. Bailing."; - BOOST_THROW_EXCEPTION( eth::NotEnoughAvailableSpace() ); + BOOST_THROW_EXCEPTION(eth::NotEnoughAvailableSpace()); } else { - cwarn << "Database " << ( path / fs::path( "state" ) ) + cwarn << "Database " << (path / fs::path("state")) << "already open. You appear to have another instance of ethereum running. " "Bailing."; - BOOST_THROW_EXCEPTION( eth::DatabaseAlreadyOpen() ); + BOOST_THROW_EXCEPTION(eth::DatabaseAlreadyOpen()); } } } -State::State( const State& _s ) +State::State(const State &_s) #ifdef HISTORIC_STATE - : m_historicState( _s.m_historicState ) + : m_historicState(_s.m_historicState) #endif { x_db_ptr = _s.x_db_ptr; - if ( _s.m_db_read_lock ) { - m_db_read_lock.emplace( *x_db_ptr ); + if (_s.m_db_read_lock) { + m_db_read_lock.emplace(*x_db_ptr); } - if ( _s.m_db_write_lock ) { - std::logic_error( "Can't copy locked for writing state object" ); + if (_s.m_db_write_lock) { + std::logic_error("Can't copy locked for writing state object"); } m_db_ptr = _s.m_db_ptr; m_orig_db = _s.m_orig_db; @@ -266,17 +280,18 @@ State::State( const State& _s ) m_accountStartNonce = _s.m_accountStartNonce; m_changeLog = _s.m_changeLog; m_initial_funds = _s.m_initial_funds; + m_levelDBSnap = _s.m_levelDBSnap; contractStorageLimit_ = _s.contractStorageLimit_; totalStorageUsed_ = _s.storageUsedTotal(); } -State& State::operator=( const State& _s ) { +State &State::operator=(const State &_s) { x_db_ptr = _s.x_db_ptr; - if ( _s.m_db_read_lock ) { - m_db_read_lock.emplace( *x_db_ptr ); + if (_s.m_db_read_lock) { + m_db_read_lock.emplace(*x_db_ptr); } - if ( _s.m_db_write_lock ) { - std::logic_error( "Can't copy locked for writing state object" ); + if (_s.m_db_write_lock) { + std::logic_error("Can't copy locked for writing state object"); } m_db_ptr = _s.m_db_ptr; m_orig_db = _s.m_orig_db; @@ -294,254 +309,255 @@ State& State::operator=( const State& _s ) { m_historicState = _s.m_historicState; #endif m_fs_ptr = _s.m_fs_ptr; + m_levelDBSnap = _s.m_levelDBSnap; return *this; } dev::h256 State::safeLastExecutedTransactionHash() { + STATE_CHECK(!m_levelDBSnap) dev::h256 shaLastTx; - if ( m_db_ptr ) + if (m_db_ptr) shaLastTx = m_db_ptr->getLastExecutedTransactionHash(); return shaLastTx; } dev::eth::TransactionReceipts State::safePartialTransactionReceipts() { + STATE_CHECK(!m_levelDBSnap) dev::eth::TransactionReceipts partialTransactionReceipts; - if ( m_db_ptr ) { + if (m_db_ptr) { dev::bytes rawTransactionReceipts = m_db_ptr->getPartialTransactionReceipts(); - if ( !rawTransactionReceipts.empty() ) { - dev::RLP rlp( rawTransactionReceipts ); - dev::eth::BlockReceipts blockReceipts( rlp ); - partialTransactionReceipts.insert( partialTransactionReceipts.end(), - blockReceipts.receipts.begin(), blockReceipts.receipts.end() ); + if (!rawTransactionReceipts.empty()) { + dev::RLP rlp(rawTransactionReceipts); + dev::eth::BlockReceipts blockReceipts(rlp); + partialTransactionReceipts.insert(partialTransactionReceipts.end(), + blockReceipts.receipts.begin(), blockReceipts.receipts.end()); } } return partialTransactionReceipts; } void State::clearPartialTransactionReceipts() { + STATE_CHECK(!m_levelDBSnap) dev::eth::BlockReceipts blockReceipts; - m_db_ptr->setPartialTransactionReceipts( blockReceipts.rlp() ); + m_db_ptr->setPartialTransactionReceipts(blockReceipts.rlp()); } -void State::populateFrom( eth::AccountMap const& _map ) { - for ( auto const& addressAccountPair : _map ) { - const Address& address = addressAccountPair.first; - const eth::Account& account = addressAccountPair.second; +void State::populateFrom(eth::AccountMap const &_map) { + STATE_CHECK(!m_levelDBSnap) + for (auto const &addressAccountPair: _map) { + const Address &address = addressAccountPair.first; + const eth::Account &account = addressAccountPair.second; - if ( account.isDirty() ) { - if ( !account.isAlive() ) { - throw logic_error( "Removing of accounts is not implemented" ); + if (account.isDirty()) { + if (!account.isAlive()) { + throw logic_error("Removing of accounts is not implemented"); } else { - setNonce( address, account.nonce() ); - setBalance( address, account.balance() ); - for ( auto const& storageAddressValuePair : account.storageOverlay() ) { - const u256& storageAddress = storageAddressValuePair.first; - const u256& value = storageAddressValuePair.second; - setStorage( address, storageAddress, value ); + setNonce(address, account.nonce()); + setBalance(address, account.balance()); + for (auto const &storageAddressValuePair: account.storageOverlay()) { + const u256 &storageAddress = storageAddressValuePair.first; + const u256 &value = storageAddressValuePair.second; + setStorage(address, storageAddress, value); } - if ( account.hasNewCode() ) { - setCode( address, account.code(), account.version() ); + if (account.hasNewCode()) { + setCode(address, account.code(), account.version()); } totalStorageUsed_ += currentStorageUsed_; updateStorageUsage(); } } } - commit( dev::eth::CommitBehaviour::KeepEmptyAccounts ); + commit(dev::eth::CommitBehaviour::KeepEmptyAccounts); } -std::unordered_map< Address, u256 > State::addresses() const { - boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr ); - if ( !checkVersion() ) { - cerror << "Current state version is " << m_currentVersion << " but stored version is " - << *m_storedVersion; - BOOST_THROW_EXCEPTION( AttemptToReadFromStateInThePast() ); - } +std::unordered_map State::addresses() const { + SharedDBGuard lock(*this); - std::unordered_map< Address, u256 > addresses; - for ( auto const& h160StringPair : m_db_ptr->accounts() ) { - Address const& address = h160StringPair.first; - string const& rlpString = h160StringPair.second; - RLP account( rlpString ); - u256 balance = account[1].toInt< u256 >(); + std::unordered_map addresses; + for (auto const &h160StringPair: m_db_ptr->accounts()) { + Address const &address = h160StringPair.first; + string const &rlpString = h160StringPair.second; + RLP account(rlpString); + u256 balance = account[1].toInt(); addresses[address] = balance; } - for ( auto const& addressAccountPair : m_cache ) { + for (auto const &addressAccountPair: m_cache) { addresses[addressAccountPair.first] = addressAccountPair.second.balance(); } return addresses; } -std::pair< State::AddressMap, h256 > State::addresses( - const h256& _begin, size_t _maxResults ) const { +std::pair State::addresses( + const h256 &_begin, size_t _maxResults) const { // TODO: do not read all values from database; - unordered_map< Address, u256 > balances = addresses(); + unordered_map balances = addresses(); AddressMap addresses; h256 next; - for ( auto const& pair : balances ) { - Address const& address = pair.first; - auto cache_ptr = m_cache.find( address ); - if ( cache_ptr != m_cache.end() ) { - if ( !cache_ptr->second.isAlive() ) { + for (auto const &pair: balances) { + Address const &address = pair.first; + auto cache_ptr = m_cache.find(address); + if (cache_ptr != m_cache.end()) { + if (!cache_ptr->second.isAlive()) { continue; } } - addresses[sha3( address )] = address; + addresses[sha3(address)] = address; } - addresses.erase( addresses.begin(), addresses.lower_bound( _begin ) ); - if ( addresses.size() > _maxResults ) { - assert( numeric_limits< long >::max() >= long( _maxResults ) ); - auto next_ptr = std::next( addresses.begin(), static_cast< long >( _maxResults ) ); + addresses.erase(addresses.begin(), addresses.lower_bound(_begin)); + if (addresses.size() > _maxResults) { + STATE_CHECK(numeric_limits::max() >= long(_maxResults)); + auto next_ptr = std::next(addresses.begin(), static_cast< long >( _maxResults )); next = next_ptr->first; - addresses.erase( next_ptr, addresses.end() ); + addresses.erase(next_ptr, addresses.end()); } - return { addresses, next }; + return {addresses, next}; } -u256 const& State::requireAccountStartNonce() const { - if ( m_accountStartNonce == Invalid256 ) - BOOST_THROW_EXCEPTION( InvalidAccountStartNonceInState() ); +u256 const &State::requireAccountStartNonce() const { + if (m_accountStartNonce == Invalid256) + BOOST_THROW_EXCEPTION(InvalidAccountStartNonceInState()); return m_accountStartNonce; } -void State::noteAccountStartNonce( u256 const& _actual ) { - if ( m_accountStartNonce == Invalid256 ) +void State::noteAccountStartNonce(u256 const &_actual) { + if (m_accountStartNonce == Invalid256) m_accountStartNonce = _actual; - else if ( m_accountStartNonce != _actual ) - BOOST_THROW_EXCEPTION( IncorrectAccountStartNonceInState() ); + else if (m_accountStartNonce != _actual) + BOOST_THROW_EXCEPTION(IncorrectAccountStartNonceInState()); } void State::removeEmptyAccounts() { - for ( auto& i : m_cache ) - if ( i.second.isDirty() && i.second.isEmpty() ) + for (auto &i: m_cache) + if (i.second.isDirty() && i.second.isEmpty()) i.second.kill(); } -eth::Account const* State::account( Address const& _a ) const { - return const_cast< State* >( this )->account( _a ); +eth::Account const *State::account(Address const &_a) const { + return const_cast< State * >( this )->account(_a); } -eth::Account* State::account( Address const& _address ) { - auto it = m_cache.find( _address ); - if ( it != m_cache.end() ) +eth::Account *State::account(Address const &_address) { + auto it = m_cache.find(_address); + if (it != m_cache.end()) return &it->second; - if ( m_nonExistingAccountsCache.count( _address ) ) + if (m_nonExistingAccountsCache.count(_address)) return nullptr; // Populate basic info. bytes stateBack; { - boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr ); + boost::shared_lock lock(*x_db_ptr); - if ( !checkVersion() ) { + if (!checkVersion()) { cerror << "Current state version is " << m_currentVersion << " but stored version is " << *m_storedVersion; - BOOST_THROW_EXCEPTION( AttemptToReadFromStateInThePast() ); + BOOST_THROW_EXCEPTION(AttemptToReadFromStateInThePast()); } - stateBack = asBytes( m_db_ptr->lookup( _address ) ); + stateBack = asBytes(m_db_ptr->lookup(_address)); } - if ( stateBack.empty() ) { - m_nonExistingAccountsCache.insert( _address ); + if (stateBack.empty()) { + m_nonExistingAccountsCache.insert(_address); return nullptr; } clearCacheIfTooLarge(); - RLP state( stateBack ); - u256 nonce = state[0].toInt< u256 >(); - u256 balance = state[1].toInt< u256 >(); - h256 codeHash = state[2].toInt< u256 >(); - s256 storageUsed = state[3].toInt< s256 >(); + RLP state(stateBack); + u256 nonce = state[0].toInt(); + u256 balance = state[1].toInt(); + h256 codeHash = state[2].toInt(); + s256 storageUsed = state[3].toInt(); // version is 0 if absent from RLP - auto const version = state[4] ? state[4].toInt< u256 >() : 0; + auto const version = state[4] ? state[4].toInt() : 0; - auto i = m_cache.emplace( std::piecewise_construct, std::forward_as_tuple( _address ), - std::forward_as_tuple( nonce, balance, dev::eth::StorageRoot( EmptyTrie ), codeHash, - version, dev::eth::Account::Changedness::Unchanged, storageUsed ) ); - m_unchangedCacheEntries.push_back( _address ); + auto i = m_cache.emplace(std::piecewise_construct, std::forward_as_tuple(_address), + std::forward_as_tuple(nonce, balance, dev::eth::StorageRoot(EmptyTrie), codeHash, + version, dev::eth::Account::Changedness::Unchanged, storageUsed)); + m_unchangedCacheEntries.push_back(_address); return &i.first->second; } void State::clearCacheIfTooLarge() const { // TODO: Find a good magic number - while ( m_unchangedCacheEntries.size() > 1000 ) { + while (m_unchangedCacheEntries.size() > 1000) { // Remove a random element // FIXME: Do not use random device as the engine. The random device should be only used to // seed other engine. - size_t const randomIndex = std::uniform_int_distribution< size_t >( - 0, m_unchangedCacheEntries.size() - 1 )( dev::s_fixedHashEngine ); + size_t const randomIndex = std::uniform_int_distribution( + 0, m_unchangedCacheEntries.size() - 1)(dev::s_fixedHashEngine); Address const addr = m_unchangedCacheEntries[randomIndex]; - swap( m_unchangedCacheEntries[randomIndex], m_unchangedCacheEntries.back() ); + swap(m_unchangedCacheEntries[randomIndex], m_unchangedCacheEntries.back()); m_unchangedCacheEntries.pop_back(); - auto cacheEntry = m_cache.find( addr ); - if ( cacheEntry != m_cache.end() && !cacheEntry->second.isDirty() ) - m_cache.erase( cacheEntry ); + auto cacheEntry = m_cache.find(addr); + if (cacheEntry != m_cache.end() && !cacheEntry->second.isDirty()) + m_cache.erase(cacheEntry); } } -void State::commit( dev::eth::CommitBehaviour _commitBehaviour ) { - if ( _commitBehaviour == dev::eth::CommitBehaviour::RemoveEmptyAccounts ) +void State::commit(dev::eth::CommitBehaviour _commitBehaviour) { + STATE_CHECK(!m_levelDBSnap) + if (_commitBehaviour == dev::eth::CommitBehaviour::RemoveEmptyAccounts) removeEmptyAccounts(); { - if ( !m_db_write_lock ) { - BOOST_THROW_EXCEPTION( AttemptToWriteToNotLockedStateObject() ); + if (!m_db_write_lock) { + BOOST_THROW_EXCEPTION(AttemptToWriteToNotLockedStateObject()); } - boost::upgrade_to_unique_lock< boost::shared_mutex > lock( *m_db_write_lock ); - if ( !checkVersion() ) { - BOOST_THROW_EXCEPTION( AttemptToWriteToStateInThePast() ); + boost::upgrade_to_unique_lock lock(*m_db_write_lock); + if (!checkVersion()) { + BOOST_THROW_EXCEPTION(AttemptToWriteToStateInThePast()); } - for ( auto const& addressAccountPair : m_cache ) { - const Address& address = addressAccountPair.first; - const eth::Account& account = addressAccountPair.second; + for (auto const &addressAccountPair: m_cache) { + const Address &address = addressAccountPair.first; + const eth::Account &account = addressAccountPair.second; - if ( account.isDirty() ) { - if ( !account.isAlive() ) { - m_db_ptr->kill( address ); - m_db_ptr->killAuxiliary( address, Auxiliary::CODE ); + if (account.isDirty()) { + if (!account.isAlive()) { + m_db_ptr->kill(address); + m_db_ptr->killAuxiliary(address, Auxiliary::CODE); - if ( StorageDestructionPatch::isEnabledInWorkingBlock() ) { - clearStorage( address ); + if (StorageDestructionPatch::isEnabledInWorkingBlock()) { + clearStorage(address); } } else { - RLPStream rlpStream( 4 ); + RLPStream rlpStream(4); - rlpStream << account.nonce() << account.balance() << u256( account.codeHash() ) + rlpStream << account.nonce() << account.balance() << u256(account.codeHash()) << account.storageUsed(); auto rawValue = rlpStream.out(); - m_db_ptr->insert( address, ref( rawValue ) ); + m_db_ptr->insert(address, ref(rawValue)); - for ( auto const& storageAddressValuePair : account.storageOverlay() ) { - const u256& storageAddress = storageAddressValuePair.first; - const u256& value = storageAddressValuePair.second; + for (auto const &storageAddressValuePair: account.storageOverlay()) { + const u256 &storageAddress = storageAddressValuePair.first; + const u256 &value = storageAddressValuePair.second; - m_db_ptr->insert( address, storageAddress, value ); + m_db_ptr->insert(address, storageAddress, value); } - if ( account.hasNewCode() ) { + if (account.hasNewCode()) { m_db_ptr->insertAuxiliary( - address, ref( account.code() ), Auxiliary::CODE ); + address, ref(account.code()), Auxiliary::CODE); } } } } - m_db_ptr->updateStorageUsage( totalStorageUsed_ ); - m_db_ptr->commit( std::to_string( ++*m_storedVersion ) ); + m_db_ptr->updateStorageUsage(totalStorageUsed_); + m_db_ptr->commit(std::to_string(++*m_storedVersion)); m_currentVersion = *m_storedVersion; } #ifdef HISTORIC_STATE - m_historicState.commitExternalChanges( m_cache ); + m_historicState.commitExternalChanges(m_cache); #endif m_changeLog.clear(); @@ -550,138 +566,133 @@ void State::commit( dev::eth::CommitBehaviour _commitBehaviour ) { } -bool State::addressInUse( Address const& _id ) const { - return !!account( _id ); +bool State::addressInUse(Address const &_id) const { + return !!account(_id); } -bool State::accountNonemptyAndExisting( Address const& _address ) const { - if ( eth::Account const* a = account( _address ) ) +bool State::accountNonemptyAndExisting(Address const &_address) const { + if (eth::Account const *a = account(_address)) return !a->isEmpty(); else return false; } -bool State::addressHasCode( Address const& _id ) const { - if ( auto a = account( _id ) ) +bool State::addressHasCode(Address const &_id) const { + if (auto a = account(_id)) return a->codeHash() != EmptySHA3; else return false; } -u256 State::balance( Address const& _id ) const { - if ( auto a = account( _id ) ) +u256 State::balance(Address const &_id) const { + if (auto a = account(_id)) return a->balance(); else return m_initial_funds; } -void State::incNonce( Address const& _addr ) { - if ( eth::Account* a = account( _addr ) ) { +void State::incNonce(Address const &_addr) { + if (eth::Account *a = account(_addr)) { auto oldNonce = a->nonce(); a->incNonce(); - m_changeLog.emplace_back( _addr, oldNonce ); + m_changeLog.emplace_back(_addr, oldNonce); } else // This is possible if a transaction has gas price 0. - createAccount( _addr, eth::Account( requireAccountStartNonce() + 1, m_initial_funds ) ); + createAccount(_addr, eth::Account(requireAccountStartNonce() + 1, m_initial_funds)); } -void State::setNonce( Address const& _addr, u256 const& _newNonce ) { - if ( eth::Account* a = account( _addr ) ) { +void State::setNonce(Address const &_addr, u256 const &_newNonce) { + if (eth::Account *a = account(_addr)) { auto oldNonce = a->nonce(); - a->setNonce( _newNonce ); - m_changeLog.emplace_back( _addr, oldNonce ); + a->setNonce(_newNonce); + m_changeLog.emplace_back(_addr, oldNonce); } else // This is possible when a contract is being created. - createAccount( _addr, eth::Account( _newNonce, m_initial_funds ) ); + createAccount(_addr, eth::Account(_newNonce, m_initial_funds)); } -void State::addBalance( Address const& _id, u256 const& _amount ) { - if ( eth::Account* a = account( _id ) ) { +void State::addBalance(Address const &_id, u256 const &_amount) { + if (eth::Account *a = account(_id)) { // Log empty account being touched. Empty touched accounts are cleared // after the transaction, so this event must be also reverted. // We only log the first touch (not dirty yet), and only for empty // accounts, as other accounts does not matter. // TODO: to save space we can combine this event with Balance by having // Balance and Balance+Touch events. - if ( !a->isDirty() && a->isEmpty() ) - m_changeLog.emplace_back( Change::Touch, _id ); + if (!a->isDirty() && a->isEmpty()) + m_changeLog.emplace_back(Change::Touch, _id); // Increase the account balance. This also is done for value 0 to mark // the account as dirty. Dirty account are not removed from the cache // and are cleared if empty at the end of the transaction. - a->addBalance( _amount ); + a->addBalance(_amount); } else - createAccount( _id, eth::Account( requireAccountStartNonce(), m_initial_funds + _amount ) ); + createAccount(_id, eth::Account(requireAccountStartNonce(), m_initial_funds + _amount)); - if ( _amount ) - m_changeLog.emplace_back( Change::Balance, _id, _amount ); + if (_amount) + m_changeLog.emplace_back(Change::Balance, _id, _amount); } -void State::subBalance( Address const& _addr, u256 const& _value ) { - if ( _value == 0 ) +void State::subBalance(Address const &_addr, u256 const &_value) { + if (_value == 0) return; - eth::Account* a = account( _addr ); - if ( !a || a->balance() < _value ) + eth::Account *a = account(_addr); + if (!a || a->balance() < _value) // TODO: I expect this never happens. - BOOST_THROW_EXCEPTION( eth::NotEnoughCash() ); + BOOST_THROW_EXCEPTION(eth::NotEnoughCash()); // Fall back to addBalance(). - addBalance( _addr, 0 - _value ); + addBalance(_addr, 0 - _value); } -void State::setBalance( Address const& _addr, u256 const& _value ) { - eth::Account* a = account( _addr ); +void State::setBalance(Address const &_addr, u256 const &_value) { + eth::Account *a = account(_addr); u256 original = a ? a->balance() : 0; // Fall back to addBalance(). - addBalance( _addr, _value - original ); + addBalance(_addr, _value - original); } -void State::createContract( Address const& _address ) { - createAccount( _address, { requireAccountStartNonce(), m_initial_funds } ); +void State::createContract(Address const &_address) { + createAccount(_address, {requireAccountStartNonce(), m_initial_funds}); } -void State::createAccount( Address const& _address, eth::Account const&& _account ) { - assert( !addressInUse( _address ) && "Account already exists" ); - m_cache[_address] = std::move( _account ); - m_nonExistingAccountsCache.erase( _address ); - m_changeLog.emplace_back( Change::Create, _address ); +void State::createAccount(Address const &_address, eth::Account const &&_account) { + STATE_CHECK(!addressInUse(_address) && "Account already exists"); + m_cache[_address] = std::move(_account); + m_nonExistingAccountsCache.erase(_address); + m_changeLog.emplace_back(Change::Create, _address); } -void State::kill( Address _addr ) { - if ( auto a = account( _addr ) ) +void State::kill(Address _addr) { + if (auto a = account(_addr)) a->kill(); // If the account is not in the db, nothing to kill. } -std::map< h256, std::pair< u256, u256 > > State::storage( const Address& _contract ) const { - boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr ); - return storage_WITHOUT_LOCK( _contract ); +std::map > State::storage(const Address &_contract) const { + SharedDBGuard lock(*this); + return storage_WITHOUT_LOCK(_contract); } -std::map< h256, std::pair< u256, u256 > > State::storage_WITHOUT_LOCK( - const Address& _contract ) const { - if ( !checkVersion() ) { - cerror << "Current state version is " << m_currentVersion << " but stored version is " - << *m_storedVersion; - BOOST_THROW_EXCEPTION( AttemptToReadFromStateInThePast() ); - } +std::map > State::storage_WITHOUT_LOCK( + const Address &_contract) const { - std::map< h256, std::pair< u256, u256 > > storage; - for ( auto const& addressValuePair : m_db_ptr->storage( _contract ) ) { - u256 const& address = addressValuePair.first; - u256 const& value = addressValuePair.second; - storage[sha3( address )] = { address, value }; + std::map > storage; + for (auto const &addressValuePair: m_db_ptr->storage(_contract)) { + u256 const &address = addressValuePair.first; + u256 const &value = addressValuePair.second; + storage[sha3(address)] = {address, value}; } - for ( auto const& addressAccountPair : m_cache ) { - Address const& accountAddress = addressAccountPair.first; - eth::Account const& account = addressAccountPair.second; - if ( account.isDirty() && accountAddress == _contract ) { - for ( auto const& addressValuePair : account.storageOverlay() ) { - storage[sha3( addressValuePair.first )] = addressValuePair; + for (auto const &addressAccountPair: m_cache) { + Address const &accountAddress = addressAccountPair.first; + eth::Account const &account = addressAccountPair.second; + if (account.isDirty() && accountAddress == _contract) { + for (auto const &addressValuePair: account.storageOverlay()) { + storage[sha3(addressValuePair.first)] = addressValuePair; } } } @@ -691,65 +702,62 @@ std::map< h256, std::pair< u256, u256 > > State::storage_WITHOUT_LOCK( return storage; } -u256 State::getNonce( Address const& _addr ) const { - if ( auto a = account( _addr ) ) +u256 State::getNonce(Address const &_addr) const { + if (auto a = account(_addr)) return a->nonce(); else return m_accountStartNonce; } -u256 State::storage( Address const& _id, u256 const& _key ) const { - if ( eth::Account const* acc = account( _id ) ) { - auto memoryIterator = acc->storageOverlay().find( _key ); - if ( memoryIterator != acc->storageOverlay().end() ) +u256 State::storage(Address const &_id, u256 const &_key) const { + if (eth::Account const *acc = account(_id)) { + auto memoryIterator = acc->storageOverlay().find(_key); + if (memoryIterator != acc->storageOverlay().end()) return memoryIterator->second; - memoryIterator = acc->originalStorageCache().find( _key ); - if ( memoryIterator != acc->originalStorageCache().end() ) + memoryIterator = acc->originalStorageCache().find(_key); + if (memoryIterator != acc->originalStorageCache().end()) return memoryIterator->second; // Not in the storage cache - go to the DB. - boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr ); - if ( !checkVersion() ) { - BOOST_THROW_EXCEPTION( AttemptToReadFromStateInThePast() ); - } - u256 value = m_db_ptr->lookup( _id, _key ); - acc->setStorageCache( _key, value ); + SharedDBGuard lock(*this); + u256 value = m_db_ptr->lookup(_id, _key); + acc->setStorageCache(_key, value); return value; } else return 0; } -void State::setStorage( Address const& _contract, u256 const& _key, u256 const& _value ) { - dev::u256 _currentValue = storage( _contract, _key ); +void State::setStorage(Address const &_contract, u256 const &_key, u256 const &_value) { + dev::u256 _currentValue = storage(_contract, _key); - m_changeLog.emplace_back( _contract, _key, _currentValue ); - m_cache[_contract].setStorage( _key, _value ); + m_changeLog.emplace_back(_contract, _key, _currentValue); + m_cache[_contract].setStorage(_key, _value); int count = 0; - if ( ( _value > 0 && _currentValue > 0 ) || ( _value == 0 && _currentValue == 0 ) ) { + if ((_value > 0 && _currentValue > 0) || (_value == 0 && _currentValue == 0)) { count = 0; } else { - count = ( _value == 0 ? -1 : 1 ); + count = (_value == 0 ? -1 : 1); } storageUsage[_contract] += count * 32; currentStorageUsed_ += count * 32; - if ( totalStorageUsed_ + currentStorageUsed_ > contractStorageLimit_ ) { - BOOST_THROW_EXCEPTION( dev::StorageOverflow() << errinfo_comment( _contract.hex() ) ); + if (totalStorageUsed_ + currentStorageUsed_ > contractStorageLimit_) { + BOOST_THROW_EXCEPTION(dev::StorageOverflow() << errinfo_comment(_contract.hex())); } } void State::clearStorageValue( - Address const& _contract, u256 const& _key, u256 const& _currentValue ) { - m_changeLog.emplace_back( _contract, _key, _currentValue ); - m_cache[_contract].setStorage( _key, 0 ); + Address const &_contract, u256 const &_key, u256 const &_currentValue) { + m_changeLog.emplace_back(_contract, _key, _currentValue); + m_cache[_contract].setStorage(_key, 0); int count; - if ( _currentValue == 0 ) { + if (_currentValue == 0) { count = 0; } else { count = -1; @@ -760,19 +768,17 @@ void State::clearStorageValue( } -u256 State::originalStorageValue( Address const& _contract, u256 const& _key ) const { - if ( Account const* acc = account( _contract ) ) { - auto memoryPtr = acc->originalStorageCache().find( _key ); - if ( memoryPtr != acc->originalStorageCache().end() ) { +u256 State::originalStorageValue(Address const &_contract, u256 const &_key) const { + if (Account const *acc = account(_contract)) { + auto memoryPtr = acc->originalStorageCache().find(_key); + if (memoryPtr != acc->originalStorageCache().end()) { return memoryPtr->second; } - boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr ); - if ( !checkVersion() ) { - BOOST_THROW_EXCEPTION( AttemptToReadFromStateInThePast() ); - } - u256 value = m_db_ptr->lookup( _contract, _key ); - acc->setStorageCache( _key, value ); + + SharedDBGuard lock(*this); + u256 value = m_db_ptr->lookup(_contract, _key); + acc->setStorageCache(_key, value); return value; } else { return 0; @@ -780,100 +786,97 @@ u256 State::originalStorageValue( Address const& _contract, u256 const& _key ) c } -void State::clearStorage( Address const& _contract ) { +void State::clearStorage(Address const &_contract) { // only clear storage if the storage used is not 0 cdebug << "Self-destructing" << _contract; - Account* acc = account( _contract ); + Account *acc = account(_contract); dev::s256 accStorageUsed = acc->storageUsed(); - if ( accStorageUsed == 0 && storageUsage[_contract] == 0 ) { + if (accStorageUsed == 0 && storageUsage[_contract] == 0) { return; } // clearStorage is called from functions that already hold a read // or write lock over the state Therefore, we can use // storage_WITHOUT_LOCK() here - for ( auto const& hashPairPair : storage_WITHOUT_LOCK( _contract ) ) { - auto const& key = hashPairPair.second.first; - auto const& value = hashPairPair.second.first; + for (auto const &hashPairPair: storage_WITHOUT_LOCK(_contract)) { + auto const &key = hashPairPair.second.first; + auto const &value = hashPairPair.second.first; // Set storage to zero in state cache - clearStorageValue( _contract, key, value ); + clearStorageValue(_contract, key, value); // Set storage to zero in the account storage cache // we have lots of caches, some of them may be unneeded // will analyze this more in future releases - acc->setStorageCache( key, 0 ); + acc->setStorageCache(key, 0); /* The corresponding key/value pair needs to be cleared in database Inserting ZERO deletes the key during commit at the end of transaction see OverlayDB::commitStorageValues() */ - h256 ZERO( 0 ); - m_db_ptr->insert( _contract, key, ZERO ); + h256 ZERO(0); + m_db_ptr->insert(_contract, key, ZERO); } - totalStorageUsed_ -= ( accStorageUsed + storageUsage[_contract] ); - acc->updateStorageUsage( -accStorageUsed ); + totalStorageUsed_ -= (accStorageUsed + storageUsage[_contract]); + acc->updateStorageUsage(-accStorageUsed); } -bytes const& State::code( Address const& _addr ) const { - eth::Account const* a = account( _addr ); - if ( !a || a->codeHash() == EmptySHA3 ) +bytes const &State::code(Address const &_addr) const { + eth::Account const *a = account(_addr); + if (!a || a->codeHash() == EmptySHA3) return NullBytes; - if ( a->code().empty() ) { + if (a->code().empty()) { // Load the code from the backend. - eth::Account* mutableAccount = const_cast< eth::Account* >( a ); - boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr ); - if ( !checkVersion() ) { - BOOST_THROW_EXCEPTION( AttemptToReadFromStateInThePast() ); - } - mutableAccount->noteCode( m_db_ptr->lookupAuxiliary( _addr, Auxiliary::CODE ) ); - eth::CodeSizeCache::instance().store( a->codeHash(), a->code().size() ); + eth::Account *mutableAccount = const_cast< eth::Account * >( a ); + SharedDBGuard lock(*this); + mutableAccount->noteCode(m_db_ptr->lookupAuxiliary(_addr, Auxiliary::CODE)); + eth::CodeSizeCache::instance().store(a->codeHash(), a->code().size()); } return a->code(); } -void State::setCode( Address const& _address, bytes&& _code, u256 const& _version ) { +void State::setCode(Address const &_address, bytes &&_code, u256 const &_version) { // rollback assumes that overwriting of the code never happens // (not allowed in contract creation logic in Executive) - assert( !addressHasCode( _address ) ); - m_changeLog.emplace_back( _address, code( _address ) ); - m_cache[_address].setCode( std::move( _code ), _version ); + STATE_CHECK(!addressHasCode(_address)); + m_changeLog.emplace_back(_address, code(_address)); + m_cache[_address].setCode(std::move(_code), _version); } -void State::setCode( const Address& _address, const bytes& _code, u256 const& _version ) { - setCode( _address, bytes( _code ), _version ); +void State::setCode(const Address &_address, const bytes &_code, u256 const &_version) { + setCode(_address, bytes(_code), _version); } -h256 State::codeHash( Address const& _a ) const { - if ( eth::Account const* a = account( _a ) ) +h256 State::codeHash(Address const &_a) const { + if (eth::Account const *a = account(_a)) return a->codeHash(); else return EmptySHA3; } -size_t State::codeSize( Address const& _a ) const { - if ( eth::Account const* a = account( _a ) ) { - if ( a->hasNewCode() ) +size_t State::codeSize(Address const &_a) const { + if (eth::Account const *a = account(_a)) { + if (a->hasNewCode()) return a->code().size(); - auto& codeSizeCache = eth::CodeSizeCache::instance(); + auto &codeSizeCache = eth::CodeSizeCache::instance(); h256 codeHash = a->codeHash(); - if ( codeSizeCache.contains( codeHash ) ) - return codeSizeCache.get( codeHash ); + if (codeSizeCache.contains(codeHash)) + return codeSizeCache.get(codeHash); else { - size_t size = code( _a ).size(); - codeSizeCache.store( codeHash, size ); + size_t size = code(_a).size(); + codeSizeCache.store(codeHash, size); return size; } } else return 0; } -u256 State::version( const Address& _contract ) const { - Account const* a = account( _contract ); +u256 State::version(const Address &_contract) const { + Account const *a = account(_contract); return a ? a->version() : 0; } @@ -881,86 +884,100 @@ size_t State::savepoint() const { return m_changeLog.size(); } -void State::rollback( size_t _savepoint ) { - assert( _savepoint <= m_changeLog.size() ); - while ( _savepoint != m_changeLog.size() ) { - auto& change = m_changeLog.back(); - eth::Account* account_ptr = this->account( change.address ); - assert( account_ptr ); - auto& account = *( account_ptr ); +void State::rollback(size_t _savepoint) { + STATE_CHECK(_savepoint <= m_changeLog.size()); + while (_savepoint != m_changeLog.size()) { + auto &change = m_changeLog.back(); + eth::Account *account_ptr = this->account(change.address); + STATE_CHECK(account_ptr); + auto &account = *(account_ptr); // Public State API cannot be used here because it will add another // change log entry. - switch ( change.kind ) { - case Change::Storage: - if ( ContractStoragePatch::isEnabledInWorkingBlock() ) { - rollbackStorageChange( change, account ); - } else { - account.setStorage( change.key, change.value ); - } - break; - case Change::StorageRoot: - account.setStorageRoot( change.value ); - break; - case Change::Balance: - account.addBalance( 0 - change.value ); - break; - case Change::Nonce: - account.setNonce( change.value ); - break; - case Change::Create: - m_cache.erase( change.address ); - break; - case Change::Code: - account.resetCode(); - break; - case Change::Touch: - account.untouch(); - m_unchangedCacheEntries.emplace_back( change.address ); - break; + switch (change.kind) { + case Change::Storage: + if (ContractStoragePatch::isEnabledInWorkingBlock()) { + rollbackStorageChange(change, account); + } else { + account.setStorage(change.key, change.value); + } + break; + case Change::StorageRoot: + account.setStorageRoot(change.value); + break; + case Change::Balance: + account.addBalance(0 - change.value); + break; + case Change::Nonce: + account.setNonce(change.value); + break; + case Change::Create: + m_cache.erase(change.address); + break; + case Change::Code: + account.resetCode(); + break; + case Change::Touch: + account.untouch(); + m_unchangedCacheEntries.emplace_back(change.address); + break; } m_changeLog.pop_back(); } clearFileStorageCache(); - if ( !ContractStoragePatch::isEnabledInWorkingBlock() ) { + if (!ContractStoragePatch::isEnabledInWorkingBlock()) { resetStorageChanges(); } } -void State::updateToLatestVersion() { +void State::clearCachesAndUpdateToLatestVersion() { m_changeLog.clear(); m_cache.clear(); m_unchangedCacheEntries.clear(); m_nonExistingAccountsCache.clear(); + if (m_levelDBSnap) { + // we do not do any locks/version checks in snaps + return; + } + + { - boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr ); + boost::shared_lock lock(*x_db_ptr); m_currentVersion = *m_storedVersion; } } State State::createStateReadOnlyCopy() const { - State stateCopy = State( *this ); - stateCopy.m_db_read_lock.emplace( *stateCopy.x_db_ptr ); - stateCopy.updateToLatestVersion(); + State stateCopy = State(*this); + stateCopy.m_db_read_lock.emplace(*stateCopy.x_db_ptr); + stateCopy.clearCachesAndUpdateToLatestVersion(); + return stateCopy; +} + +State State::createStateSnapCopy(std::shared_ptr &_snap) const { + State stateCopy = State(*this); + stateCopy.useLevelDBSnap(_snap); return stateCopy; } + State State::createStateModifyCopy() const { - State stateCopy = State( *this ); - stateCopy.m_db_write_lock.emplace( *stateCopy.x_db_ptr ); - stateCopy.updateToLatestVersion(); + State stateCopy = State(*this); + stateCopy.m_db_write_lock.emplace(*stateCopy.x_db_ptr); + stateCopy.clearCachesAndUpdateToLatestVersion(); return stateCopy; } State State::createStateModifyCopyAndPassLock() { - if ( m_db_write_lock ) { - boost::upgrade_lock< boost::shared_mutex > lock; - lock.swap( *m_db_write_lock ); + STATE_CHECK(!m_levelDBSnap) + if (m_db_write_lock) { + boost::upgrade_lock lock; + lock.swap(*m_db_write_lock); m_db_write_lock = boost::none; - State stateCopy = State( *this ); - stateCopy.m_db_write_lock = boost::upgrade_lock< boost::shared_mutex >(); - stateCopy.m_db_write_lock->swap( lock ); + State stateCopy = State(*this); + stateCopy.m_db_write_lock = boost::upgrade_lock(); + stateCopy.m_db_write_lock->swap(lock); return stateCopy; } else { return createStateModifyCopy(); @@ -968,33 +985,35 @@ State State::createStateModifyCopyAndPassLock() { } void State::releaseWriteLock() { + STATE_CHECK(!m_levelDBSnap) m_db_write_lock = boost::none; } State State::createNewCopyWithLocks() { + STATE_CHECK(!m_levelDBSnap) State copy; - if ( m_db_write_lock ) + if (m_db_write_lock) copy = createStateModifyCopyAndPassLock(); else - copy = State( *this ); - if ( m_db_read_lock ) - copy.m_db_read_lock.emplace( *copy.x_db_ptr ); - copy.updateToLatestVersion(); + copy = State(*this); + if (m_db_read_lock) + copy.m_db_read_lock.emplace(*copy.x_db_ptr); + copy.clearCachesAndUpdateToLatestVersion(); return copy; } bool State::connected() const { - if ( m_db_ptr ) { + if (m_db_ptr) { return m_db_ptr->connected(); } return false; } bool State::empty() const { - if ( m_cache.empty() ) { - if ( m_db_ptr ) { - boost::shared_lock< boost::shared_mutex > lock( *x_db_ptr ); - if ( m_db_ptr->empty() ) { + STATE_CHECK(!m_levelDBSnap) + if (m_cache.empty()) { + if (m_db_ptr) { + if (m_db_ptr->empty()) { return true; } } else { @@ -1004,19 +1023,26 @@ bool State::empty() const { return false; } -std::pair< ExecutionResult, TransactionReceipt > State::execute( EnvInfo const& _envInfo, - eth::ChainOperationParams const& _chainParams, Transaction const& _t, Permanence _p, - OnOpFunc const& _onOp ) { +std::pair State::execute(EnvInfo const &_envInfo, + eth::ChainOperationParams const &_chainParams, + Transaction const &_t, Permanence _p, + OnOpFunc const &_onOp) { // Create and initialize the executive. This will throw fairly cheaply and quickly if the // transaction is bad in any way. // HACK 0 here is for gasPrice // TODO Not sure that 1st 0 as timestamp is acceptable here - Executive e( *this, _envInfo, _chainParams, 0, 0, _p != Permanence::Committed ); + + if (m_levelDBSnap) { + // snaps only support read only calls + STATE_CHECK(_p == Permanence::Reverted); + } + + Executive e(*this, _envInfo, _chainParams, 0, 0, _p != Permanence::Committed); ExecutionResult res; - e.setResultRecipient( res ); + e.setResultRecipient(res); - bool isCacheEnabled = RevertableFSPatch::isEnabledWhen( _envInfo.committedBlockTimestamp() ); - resetOverlayFS( isCacheEnabled ); + bool isCacheEnabled = RevertableFSPatch::isEnabledWhen(_envInfo.committedBlockTimestamp()); + resetOverlayFS(isCacheEnabled); auto onOp = _onOp; #if ETH_VMTRACE @@ -1024,109 +1050,109 @@ std::pair< ExecutionResult, TransactionReceipt > State::execute( EnvInfo const& onOp = e.simpleTrace(); #endif u256 const startGasUsed = _envInfo.gasUsed(); - bool const statusCode = executeTransaction( e, _t, onOp ); + bool const statusCode = executeTransaction(e, _t, onOp); std::string strRevertReason; - if ( res.excepted == dev::eth::TransactionException::RevertInstruction ) { - strRevertReason = skutils::eth::call_error_message_2_str( res.output ); - if ( strRevertReason.empty() ) + if (res.excepted == dev::eth::TransactionException::RevertInstruction) { + strRevertReason = skutils::eth::call_error_message_2_str(res.output); + if (strRevertReason.empty()) strRevertReason = "EVM revert instruction without description message"; std::string strOut = "Error message from State::execute(): " + strRevertReason; cerror << strOut; } bool removeEmptyAccounts = false; - switch ( _p ) { - case Permanence::Reverted: - case Permanence::CommittedWithoutState: - resetStorageChanges(); - m_cache.clear(); - break; - case Permanence::Committed: { - if ( account( _t.from() ) != nullptr && account( _t.from() )->code() == bytes() ) { - totalStorageUsed_ += currentStorageUsed_; - updateStorageUsage(); - } - // TODO: review logic|^ - - h256 shaLastTx = _t.sha3(); // _t.hasSignature() ? _t.sha3() : _t.sha3( - // dev::eth::WithoutSignature ); - this->m_db_ptr->setLastExecutedTransactionHash( shaLastTx ); - // std::cout << "--- saving \"safeLastExecutedTransactionHash\" = " << - // shaLastTx.hex() << "\n"; - - TransactionReceipt receipt = - _envInfo.number() >= _chainParams.byzantiumForkBlock ? - TransactionReceipt( statusCode, startGasUsed + e.gasUsed(), e.logs() ) : - TransactionReceipt( EmptyTrie, startGasUsed + e.gasUsed(), e.logs() ); - receipt.setRevertReason( strRevertReason ); - m_db_ptr->addReceiptToPartials( receipt ); - m_fs_ptr->commit(); - - removeEmptyAccounts = _envInfo.number() >= _chainParams.EIP158ForkBlock; - commit( removeEmptyAccounts ? dev::eth::CommitBehaviour::RemoveEmptyAccounts : - dev::eth::CommitBehaviour::KeepEmptyAccounts ); + switch (_p) { + case Permanence::Reverted: + case Permanence::CommittedWithoutState: + resetStorageChanges(); + m_cache.clear(); + break; + case Permanence::Committed: { + if (account(_t.from()) != nullptr && account(_t.from())->code() == bytes()) { + totalStorageUsed_ += currentStorageUsed_; + updateStorageUsage(); + } + // TODO: review logic|^ + + h256 shaLastTx = _t.sha3(); // _t.hasSignature() ? _t.sha3() : _t.sha3( + // dev::eth::WithoutSignature ); + this->m_db_ptr->setLastExecutedTransactionHash(shaLastTx); + // std::cout << "--- saving \"safeLastExecutedTransactionHash\" = " << + // shaLastTx.hex() << "\n"; + + TransactionReceipt receipt = + _envInfo.number() >= _chainParams.byzantiumForkBlock ? + TransactionReceipt(statusCode, startGasUsed + e.gasUsed(), e.logs()) : + TransactionReceipt(EmptyTrie, startGasUsed + e.gasUsed(), e.logs()); + receipt.setRevertReason(strRevertReason); + m_db_ptr->addReceiptToPartials(receipt); + m_fs_ptr->commit(); + + removeEmptyAccounts = _envInfo.number() >= _chainParams.EIP158ForkBlock; + commit(removeEmptyAccounts ? dev::eth::CommitBehaviour::RemoveEmptyAccounts : + dev::eth::CommitBehaviour::KeepEmptyAccounts); - break; - } - case Permanence::Uncommitted: - resetStorageChanges(); - break; + break; + } + case Permanence::Uncommitted: + resetStorageChanges(); + break; } TransactionReceipt receipt = - _envInfo.number() >= _chainParams.byzantiumForkBlock ? - TransactionReceipt( statusCode, startGasUsed + e.gasUsed(), e.logs() ) : - TransactionReceipt( EmptyTrie, startGasUsed + e.gasUsed(), e.logs() ); - receipt.setRevertReason( strRevertReason ); + _envInfo.number() >= _chainParams.byzantiumForkBlock ? + TransactionReceipt(statusCode, startGasUsed + e.gasUsed(), e.logs()) : + TransactionReceipt(EmptyTrie, startGasUsed + e.gasUsed(), e.logs()); + receipt.setRevertReason(strRevertReason); - return make_pair( res, receipt ); + return make_pair(res, receipt); } /// @returns true when normally halted; false when exceptionally halted; throws when internal VM /// exception occurred. bool State::executeTransaction( - eth::Executive& _e, eth::Transaction const& _t, eth::OnOpFunc const& _onOp ) { + eth::Executive &_e, eth::Transaction const &_t, eth::OnOpFunc const &_onOp) { size_t const savept = savepoint(); try { - _e.initialize( _t ); + _e.initialize(_t); - if ( !_e.execute() ) - _e.go( _onOp ); + if (!_e.execute()) + _e.go(_onOp); return _e.finalize(); - } catch ( dev::eth::RevertInstruction const& re ) { - rollback( savept ); + } catch (dev::eth::RevertInstruction const &re) { + rollback(savept); throw; - } catch ( Exception const& ) { - rollback( savept ); + } catch (Exception const &) { + rollback(savept); throw; } } -void State::rollbackStorageChange( const Change& _change, eth::Account& _acc ) { - dev::u256 _currentValue = storage( _change.address, _change.key ); +void State::rollbackStorageChange(const Change &_change, eth::Account &_acc) { + dev::u256 _currentValue = storage(_change.address, _change.key); int count = 0; - if ( ( _change.value > 0 && _currentValue > 0 ) || - ( _change.value == 0 && _currentValue == 0 ) ) { + if ((_change.value > 0 && _currentValue > 0) || + (_change.value == 0 && _currentValue == 0)) { count = 0; } else { - count = ( _change.value == 0 ? -1 : 1 ); + count = (_change.value == 0 ? -1 : 1); } storageUsage[_change.address] += count * 32; currentStorageUsed_ += count * 32; - _acc.setStorage( _change.key, _change.value ); + _acc.setStorage(_change.key, _change.value); } void State::updateStorageUsage() { - for ( const auto& [_address, _value] : storageUsage ) { - if ( auto a = account( _address ) ) - a->updateStorageUsage( _value ); + for (const auto &[_address, _value]: storageUsage) { + if (auto a = account(_address)) + a->updateStorageUsage(_value); } resetStorageChanges(); } -dev::s256 State::storageUsed( const dev::Address& _addr ) const { - if ( auto a = account( _addr ) ) { +dev::s256 State::storageUsed(const dev::Address &_addr) const { + if (auto a = account(_addr)) { return a->storageUsed(); } else { return 0; @@ -1137,67 +1163,73 @@ bool State::checkVersion() const { return *m_storedVersion == m_currentVersion; } -std::ostream& skale::operator<<( std::ostream& _out, State const& _s ) { - _out << cc::debug( "--- Cache ---" ) << std::endl; - std::set< Address > d; - for ( auto i : _s.m_cache ) - d.insert( i.first ); +void State::useLevelDBSnap(std::shared_ptr &_snap) { + //STATE_CHECK(_snap); + m_levelDBSnap = _snap; +} + + +std::ostream &skale::operator<<(std::ostream &_out, State const &_s) { + _out << cc::debug("--- Cache ---") << std::endl; + std::set
d; + for (auto i: _s.m_cache) + d.insert(i.first); - for ( auto i : d ) { - auto it = _s.m_cache.find( i ); - eth::Account* cache = it != _s.m_cache.end() ? &it->second : nullptr; - assert( cache ); + for (auto i: d) { + auto it = _s.m_cache.find(i); + eth::Account *cache = it != _s.m_cache.end() ? &it->second : nullptr; + STATE_CHECK(cache); - if ( cache && !cache->isAlive() ) - _out << cc::debug( "XXX " ) << i << std::endl; + if (cache && !cache->isAlive()) + _out << cc::debug("XXX ") << i << std::endl; else { - string lead = cc::debug( " + " ); - if ( cache ) - lead = cc::debug( " . " ); + string lead = cc::debug(" + "); + if (cache) + lead = cc::debug(" . "); stringstream contout; - if ( ( cache && cache->codeHash() == EmptySHA3 ) || ( !cache ) ) { - std::map< u256, u256 > mem; - std::set< u256 > back; - std::set< u256 > delta; - std::set< u256 > cached; - if ( cache ) - for ( auto const& j : cache->storageOverlay() ) { - if ( ( !mem.count( j.first ) && j.second ) || - ( mem.count( j.first ) && mem.at( j.first ) != j.second ) ) { + if ((cache && cache->codeHash() == EmptySHA3) || (!cache)) { + std::map mem; + std::set back; + std::set delta; + std::set cached; + if (cache) + for (auto const &j: cache->storageOverlay()) { + if ((!mem.count(j.first) && j.second) || + (mem.count(j.first) && mem.at(j.first) != j.second)) { mem[j.first] = j.second; - delta.insert( j.first ); - } else if ( j.second ) - cached.insert( j.first ); + delta.insert(j.first); + } else if (j.second) + cached.insert(j.first); } - if ( !delta.empty() ) - lead = ( lead == " . " ) ? "*.* " : "*** "; + if (!delta.empty()) + lead = (lead == " . ") ? "*.* " : "*** "; contout << " @:"; - if ( cache && cache->hasNewCode() ) - contout << cc::debug( " $" ) << toHex( cache->code() ); + if (cache && cache->hasNewCode()) + contout << cc::debug(" $") << toHex(cache->code()); else - contout << cc::debug( " $" ) << ( cache ? cache->codeHash() : dev::h256( 0 ) ); + contout << cc::debug(" $") << (cache ? cache->codeHash() : dev::h256(0)); - for ( auto const& j : mem ) - if ( j.second ) + for (auto const &j: mem) + if (j.second) contout << std::endl - << ( delta.count( j.first ) ? - back.count( j.first ) ? " * " : " + " : - cached.count( j.first ) ? " . " : - " " ) - << std::hex << nouppercase << std::setw( 64 ) << j.first << ": " - << std::setw( 0 ) << j.second; + << (delta.count(j.first) ? + back.count(j.first) ? " * " : " + " : + cached.count(j.first) ? " . " : + " ") + << std::hex << nouppercase << std::setw(64) << j.first << ": " + << std::setw(0) << j.second; else contout << std::endl - << cc::debug( "XXX " ) << std::hex << nouppercase - << std::setw( 64 ) << j.first << ""; + << cc::debug("XXX ") << std::hex << nouppercase + << std::setw(64) << j.first << ""; } else - contout << cc::debug( " [SIMPLE]" ); - _out << lead << i << cc::debug( ": " ) << std::dec - << ( cache ? cache->nonce() : u256( 0 ) ) << cc::debug( " #:" ) - << ( cache ? cache->balance() : u256( 0 ) ) << contout.str() << std::endl; + contout << cc::debug(" [SIMPLE]"); + _out << lead << i << cc::debug(": ") << std::dec + << (cache ? cache->nonce() : u256(0)) << cc::debug(" #:") + << (cache ? cache->balance() : u256(0)) << contout.str() << std::endl; } } return _out; diff --git a/libskale/State.h b/libskale/State.h index 94c7118ea..47552fb24 100644 --- a/libskale/State.h +++ b/libskale/State.h @@ -42,7 +42,7 @@ #include "OverlayDB.h" #include "OverlayFS.h" #include - +#include namespace std { template <> @@ -158,6 +158,26 @@ using ChangeLog = std::vector< Change >; * The changelog is managed by savepoint(), rollback() and commit() methods. */ class State { + + // this class locks read lock only if the state is not snap + // snaps do not need locking + class SharedDBGuard { + const State& m_state; + public: + explicit SharedDBGuard( const State& _state ) : m_state( _state ) { + if ( m_state.m_levelDBSnap ) + return; + m_state.x_db_ptr->lock_shared(); + } + + + ~SharedDBGuard() { + if ( m_state.m_levelDBSnap ) + return; + m_state.x_db_ptr->unlock_shared(); + } + }; + public: using AddressMap = std::map< dev::h256, dev::Address >; @@ -362,6 +382,11 @@ class State { /// No one can change state while returned object exists. State createStateReadOnlyCopy() const; + + // create State read-only copy based on snap. + // this copy does not require any locking for state reads + State createStateSnapCopy(std::shared_ptr &_snap) const; + /// Create State copy to modify data. State createStateModifyCopy() const; @@ -401,7 +426,7 @@ class State { private: - void updateToLatestVersion(); + void clearCachesAndUpdateToLatestVersion(); explicit State( dev::u256 const& _accountStartNonce, skale::OverlayDB const& _db, #ifdef HISTORIC_STATE @@ -453,6 +478,8 @@ class State { public: bool checkVersion() const; + void useLevelDBSnap( std::shared_ptr &_snap); + #ifdef HISTORIC_STATE void populateHistoricStateFromSkaleState(); void populateHistoricStateBatchFromSkaleState( @@ -496,6 +523,8 @@ class State { dev::s256 totalStorageUsed_ = 0; dev::s256 currentStorageUsed_ = 0; + std::shared_ptr m_levelDBSnap; + #ifdef HISTORIC_STATE dev::eth::HistoricState m_historicState; @@ -516,6 +545,8 @@ class State { return pDB; } std::shared_ptr< OverlayFS > fs() { return m_fs_ptr; } + + }; std::ostream& operator<<( std::ostream& _out, State const& _s ); From 5b467a7e4fbff8c14dada10d4ad02a8b1a0b825e Mon Sep 17 00:00:00 2001 From: Stan Kladko <13399135+kladkogex@users.noreply.github.com> Date: Fri, 24 May 2024 19:26:20 +0100 Subject: [PATCH 006/314] 1545 added test --- .../hardhat/scripts/calls_perf_test.js | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 test/historicstate/hardhat/scripts/calls_perf_test.js diff --git a/test/historicstate/hardhat/scripts/calls_perf_test.js b/test/historicstate/hardhat/scripts/calls_perf_test.js new file mode 100644 index 000000000..0f2b2e72e --- /dev/null +++ b/test/historicstate/hardhat/scripts/calls_perf_test.js @@ -0,0 +1,213 @@ +// We require the Hardhat Runtime Environment explicitly here. This is optional +// but useful for running the script in a standalone fashion through `node