Skip to content

Commit

Permalink
Merge branch 'v3.18.0' into enhancement/SKALED-1693-func-tests-on-bot…
Browse files Browse the repository at this point in the history 
…h-builds
  • Loading branch information
@dimalit
dimalit authored Jan 18, 2024
2 parents bde0c84 + 46bbdbd commit bd6fa93
Show file tree
Hide file tree
Showing 6 changed files with 150 additions and 38 deletions.
66 changes: 51 additions & 15 deletions libdevcore/LevelDB.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@
#include "Assertions.h"
#include "Log.h"
#include <libdevcore/microprofile.h>
#include <secp256k1_sha256.h>

namespace dev {
namespace db {

unsigned c_maxOpenLeveldbFiles = 25;

const size_t LevelDB::BATCH_CHUNK_SIZE = 10000;

namespace {
inline leveldb::Slice toLDBSlice( Slice _slice ) {
return leveldb::Slice( _slice.data(), _slice.size() );
Expand Down Expand Up @@ -213,7 +214,6 @@ void LevelDB::forEach( std::function< bool( Slice, Slice ) > f ) const {
}
}


void LevelDB::forEachWithPrefix(
std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
cnote << "Iterating over the LevelDB prefix: " << _prefix;
Expand All @@ -238,21 +238,23 @@ h256 LevelDB::hashBase() const {
if ( it == nullptr ) {
BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
}

secp256k1_sha256_t ctx;
secp256k1_sha256_initialize( &ctx );
for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
std::string key_ = it->key().ToString();
std::string value_ = it->value().ToString();
std::string keyTmp = it->key().ToString();
std::string valueTmp = it->value().ToString();
// HACK! For backward compatibility! When snapshot could happen between update of two nodes
// - it would lead to stateRoot mismatch
// TODO Move this logic to separate "compatiliblity layer"!
if ( key_ == "pieceUsageBytes" )
if ( keyTmp == "pieceUsageBytes" )
continue;
std::string key_value = key_ + value_;
const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
bytesConstRef str_key_value( usc.data(), usc.size() );
secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
std::string keyValue = keyTmp + valueTmp;
const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
bytesConstRef strKeyValue( usc.data(), usc.size() );
secp256k1_sha256_write( &ctx, strKeyValue.data(), strKeyValue.size() );
}

h256 hash;
secp256k1_sha256_finalize( &ctx, hash.data() );
return hash;
Expand All @@ -263,23 +265,57 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
if ( it == nullptr ) {
BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
}

secp256k1_sha256_t ctx;
secp256k1_sha256_initialize( &ctx );
for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
if ( it->key()[0] == _prefix ) {
std::string key_ = it->key().ToString();
std::string value_ = it->value().ToString();
std::string key_value = key_ + value_;
const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
bytesConstRef str_key_value( usc.data(), usc.size() );
secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
std::string keyTmp = it->key().ToString();
std::string valueTmp = it->value().ToString();
std::string keyValue = keyTmp + valueTmp;
const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
bytesConstRef strKeyValue( usc.data(), usc.size() );
secp256k1_sha256_write( &ctx, strKeyValue.data(), strKeyValue.size() );
}
}
h256 hash;
secp256k1_sha256_finalize( &ctx, hash.data() );
return hash;
}

bool LevelDB::hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const {
std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
if ( it == nullptr ) {
BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
}

if ( lastHashedKey != "start" )
it->Seek( lastHashedKey );
else
it->SeekToFirst();

for ( size_t counter = 0; it->Valid() && counter < BATCH_CHUNK_SIZE; it->Next() ) {
std::string keyTmp = it->key().ToString();
std::string valueTmp = it->value().ToString();
// HACK! For backward compatibility! When snapshot could happen between update of two nodes
// - it would lead to stateRoot mismatch
// TODO Move this logic to separate "compatiliblity layer"!
if ( keyTmp == "pieceUsageBytes" )
continue;
std::string keyValue = keyTmp + valueTmp;
const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
bytesConstRef strKeyValue( usc.data(), usc.size() );
secp256k1_sha256_write( ctx, strKeyValue.data(), strKeyValue.size() );
++counter;
}

if ( it->Valid() ) {
lastHashedKey = it->key().ToString();
return true;
} else
return false;
}

void LevelDB::doCompaction() const {
m_db->CompactRange( nullptr, nullptr );
}
Expand Down
6 changes: 6 additions & 0 deletions libdevcore/LevelDB.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
#include <leveldb/write_batch.h>
#include <boost/filesystem.hpp>

#include <secp256k1_sha256.h>

namespace dev {
namespace db {
class LevelDB : public DatabaseFace {
Expand Down Expand Up @@ -59,6 +61,8 @@ class LevelDB : public DatabaseFace {
h256 hashBase() const override;
h256 hashBaseWithPrefix( char _prefix ) const;

bool hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const;

void doCompaction() const;

// Return the total count of key deletes since the start
Expand All @@ -74,6 +78,8 @@ class LevelDB : public DatabaseFace {
leveldb::WriteOptions const m_writeOptions;
leveldb::Options m_options;
boost::filesystem::path const m_path;

static const size_t BATCH_CHUNK_SIZE;
};

} // namespace db
Expand Down
24 changes: 18 additions & 6 deletions libskale/SnapshotManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,13 +445,25 @@ void SnapshotManager::computeDatabaseHash(
BOOST_THROW_EXCEPTION( InvalidPath( _dbDir ) );
}

std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
dev::db::LevelDB::defaultSnapshotDBOptions() ) );
dev::h256 hash_volume = m_db->hashBase();
cnote << _dbDir << " hash is: " << hash_volume << std::endl;
secp256k1_sha256_t dbCtx;
secp256k1_sha256_initialize( &dbCtx );

secp256k1_sha256_write( ctx, hash_volume.data(), hash_volume.size );
std::string lastHashedKey = "start";
bool isContinue = true;

while ( isContinue ) {
std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
dev::db::LevelDB::defaultSnapshotDBOptions() ) );

isContinue = m_db->hashBasePartially( &dbCtx, lastHashedKey );
}

dev::h256 dbHash;
secp256k1_sha256_finalize( &dbCtx, dbHash.data() );
cnote << _dbDir << " hash is: " << dbHash << std::endl;

secp256k1_sha256_write( ctx, dbHash.data(), dbHash.size );
} catch ( const fs::filesystem_error& ex ) {
std::throw_with_nested( CannotRead( ex.path1() ) );
}
Expand Down
9 changes: 9 additions & 0 deletions libweb3jsonrpc/Skale.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@ nlohmann::json Skale::impl_skale_getSnapshot( const nlohmann::json& joRequest, C

// TODO check
unsigned blockNumber = joRequest["blockNumber"].get< unsigned >();
if ( blockNumber != m_client.getLatestSnapshotBlockNumer() ) {
joResponse["error"] = "Invalid snapshot block number requested - it might be deleted.";
return joResponse;
}

// exit if too early
if ( currentSnapshotBlockNumber >= 0 ) {
Expand Down Expand Up @@ -366,6 +370,11 @@ Json::Value Skale::skale_getSnapshotSignature( unsigned blockNumber ) {
if ( chainParams.nodeInfo.keyShareName.empty() || chainParams.nodeInfo.sgxServerUrl.empty() )
throw jsonrpc::JsonRpcException( "Snapshot signing is not enabled" );

if ( blockNumber != this->m_client.getLatestSnapshotBlockNumer() ) {
throw jsonrpc::JsonRpcException(
"Invalid snapshot block number requested - it might be deleted." );
}

try {
dev::h256 snapshot_hash = this->m_client.getSnapshotHash( blockNumber );
if ( !snapshot_hash )
Expand Down
19 changes: 8 additions & 11 deletions skaled/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,9 +438,9 @@ bool tryDownloadSnapshot( std::shared_ptr< SnapshotManager >& snapshotManager,
try {
snapshotManager->computeSnapshotHash( blockNumber, true );
} catch ( const std::exception& ) {
std::throw_with_nested(
std::runtime_error( cc::fatal( "FATAL:" ) + " " +
cc::error( "Exception while computing snapshot hash " ) ) );
std::throw_with_nested( std::runtime_error(
std::string( "FATAL:" ) +
std::string( " Exception while computing snapshot hash " ) ) );
}

dev::h256 calculated_hash = snapshotManager->getSnapshotHash( blockNumber );
Expand All @@ -449,18 +449,15 @@ bool tryDownloadSnapshot( std::shared_ptr< SnapshotManager >& snapshotManager,
successfullDownload = true;
if ( isRegularSnapshot ) {
snapshotManager->restoreSnapshot( blockNumber );
std::cout << cc::success( "Snapshot restore success for block " )
<< cc::u( to_string( blockNumber ) ) << std::endl;
std::cout << "Snapshot restore success for block " << to_string( blockNumber )
<< std::endl;
}
return successfullDownload;
} else {
clog( VerbosityWarning, "tryDownloadSnapshot" )
<< cc::notice(
"Downloaded snapshot with incorrect hash! Incoming "
"hash " )
<< cc::notice( votedHash.first.hex() )
<< cc::notice( " is not equal to calculated hash " )
<< cc::notice( calculated_hash.hex() ) << cc::notice( "Will try again" );
<< "Downloaded snapshot with incorrect hash! Incoming hash "
<< votedHash.first.hex() << " is not equal to calculated hash "
<< calculated_hash.hex() << " Will try again";
if ( isRegularSnapshot )
snapshotManager->cleanup();
else
Expand Down
64 changes: 58 additions & 6 deletions test/unittests/libweb3core/LevelDBHash.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,44 +9,96 @@ BOOST_AUTO_TEST_SUITE( LevelDBHashBase )
BOOST_AUTO_TEST_CASE( hash ) {
dev::TransientDirectory td;

std::vector< std::pair< std::string, std::string > > randomKeysValues(123);

dev::h256 hash;
{
std::unique_ptr< dev::db::LevelDB > db( new dev::db::LevelDB( td.path() ) );
BOOST_REQUIRE( db );

db->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
db->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );

for ( size_t i = 0; i < 123; ++i ) {
std::string key = std::to_string( 43 + i );
std::string value = std::to_string( i );
std::string key = dev::h256::random().hex();
std::string value = dev::h256::random().hex();
db->insert( dev::db::Slice(key), dev::db::Slice(value) );

randomKeysValues[i] = { key, value };
}

hash = db->hashBase();
}

boost::filesystem::remove_all( td.path() );
BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );

dev::h256 hash_same;
{
std::unique_ptr< dev::db::LevelDB > db_copy( new dev::db::LevelDB( td.path() ) );
BOOST_REQUIRE( db_copy );

for ( size_t i = 0; i < 123; ++i ) {
std::string key = std::to_string( 43 + i );
std::string value = std::to_string( i );
std::string key = randomKeysValues[i].first;
std::string value = randomKeysValues[i].second;
db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
}
db_copy->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );

hash_same = db_copy->hashBase();
}

BOOST_REQUIRE( hash == hash_same );

boost::filesystem::remove_all( td.path() );
BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );

dev::h256 hashPartially;
{
{
std::unique_ptr< dev::db::LevelDB > db_copy( new dev::db::LevelDB( td.path() ) );
BOOST_REQUIRE( db_copy );

for ( size_t i = 0; i < 123; ++i ) {
std::string key = randomKeysValues[i].first;
std::string value = randomKeysValues[i].second;
db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
}

db_copy->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
}

secp256k1_sha256_t dbCtx;
secp256k1_sha256_initialize( &dbCtx );

std::string lastHashedKey = "start";
bool isContinue = true;
while ( isContinue ) {
std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( td.path(),
dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
dev::db::LevelDB::defaultSnapshotDBOptions() ) );

isContinue = m_db->hashBasePartially( &dbCtx, lastHashedKey );
}

secp256k1_sha256_finalize( &dbCtx, hashPartially.data() );
}

BOOST_REQUIRE( hash == hashPartially );

boost::filesystem::remove_all( td.path() );
BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );

dev::h256 hash_diff;
{
std::unique_ptr< dev::db::LevelDB > db_diff( new dev::db::LevelDB( td.path() ) );
BOOST_REQUIRE( db_diff );

for ( size_t i = 0; i < 123; ++i ) {
std::string key = std::to_string( 42 + i );
std::string value = std::to_string( i );
std::string key = dev::h256::random().hex();
std::string value = dev::h256::random().hex();
db_diff->insert( dev::db::Slice(key), dev::db::Slice(value) );
}

Expand Down

0 comments on commit bd6fa93

Please sign in to comment.