From f08f134af4a265bb66e6c742aa62222213045f64 Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Wed, 10 Jan 2024 16:24:19 +0000
Subject: [PATCH 01/12] #1588 change snapshot hash computation

---
 libdevcore/LevelDB.cpp                     | 23 ++++++++++++++-
 libdevcore/LevelDB.h                       |  5 ++++
 libskale/SnapshotManager.cpp               | 25 ++++++++++++----
 test/unittests/libweb3core/LevelDBHash.cpp | 33 ++++++++++++++++++++++
 4 files changed, 79 insertions(+), 7 deletions(-)

diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp
index 2896cf4d0..cb6fba758 100644
--- a/libdevcore/LevelDB.cpp
+++ b/libdevcore/LevelDB.cpp
@@ -22,7 +22,6 @@
 #include "Assertions.h"
 #include "Log.h"
 #include <libdevcore/microprofile.h>
-#include <secp256k1_sha256.h>
 
 namespace dev {
 namespace db {
@@ -263,6 +262,7 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
@@ -280,6 +280,27 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
     return hash;
 }
 
+void LevelDB::hashBasePartially(
+    secp256k1_sha256_t* ctx, const std::string& start, const std::string& finish ) const {
+    std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
+    if ( it == nullptr ) {
+        BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
+    }
+    for ( it->Seek( start ); it->Valid() && it->key().ToString() < finish; it->Next() ) {
+        std::string key_ = it->key().ToString();
+        std::string value_ = it->value().ToString();
+        // HACK! For backward compatibility! When snapshot could happen between update of two nodes
+        // - it would lead to stateRoot mismatch
+        // TODO Move this logic to separate "compatiliblity layer"!
+        if ( key_ == "pieceUsageBytes" )
+            continue;
+        std::string key_value = key_ + value_;
+        const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
+        bytesConstRef str_key_value( usc.data(), usc.size() );
+        secp256k1_sha256_write( ctx, str_key_value.data(), str_key_value.size() );
+    }
+}
+
 void LevelDB::doCompaction() const {
     m_db->CompactRange( nullptr, nullptr );
 }
diff --git a/libdevcore/LevelDB.h b/libdevcore/LevelDB.h
index 0a597c51e..470f318dd 100644
--- a/libdevcore/LevelDB.h
+++ b/libdevcore/LevelDB.h
@@ -26,6 +26,8 @@
 #include <leveldb/write_batch.h>
 #include <boost/filesystem.hpp>
 
+#include <secp256k1_sha256.h>
+
 namespace dev {
 namespace db {
 class LevelDB : public DatabaseFace {
@@ -59,6 +61,9 @@ class LevelDB : public DatabaseFace {
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 
+    void hashBasePartially(
+        secp256k1_sha256_t* ctx, const std::string& start, const std::string& finish ) const;
+
     void doCompaction() const;
 
     // Return the total count of key deletes  since the start
diff --git a/libskale/SnapshotManager.cpp b/libskale/SnapshotManager.cpp
index d3c3bc04e..04c7cd0be 100644
--- a/libskale/SnapshotManager.cpp
+++ b/libskale/SnapshotManager.cpp
@@ -445,13 +445,26 @@ void SnapshotManager::computeDatabaseHash(
         BOOST_THROW_EXCEPTION( InvalidPath( _dbDir ) );
     }
 
-    std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
-        dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
-        dev::db::LevelDB::defaultSnapshotDBOptions() ) );
-    dev::h256 hash_volume = m_db->hashBase();
-    cnote << _dbDir << " hash is: " << hash_volume << std::endl;
+    std::array< std::string, 17 > lexographicKeysSegments = { "0", "1", "2", "3", "4", "5", "6",
+        "7", "8", "9", "a", "b", "c", "d", "e", "f", "g" };
 
-    secp256k1_sha256_write( ctx, hash_volume.data(), hash_volume.size );
+    secp256k1_sha256_t dbCtx;
+    secp256k1_sha256_initialize( &dbCtx );
+
+    for ( size_t i = 0; i < lexographicKeysSegments.size() - 1; ++i ) {
+        std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
+            dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
+            dev::db::LevelDB::defaultSnapshotDBOptions() ) );
+
+        m_db->hashBasePartially(
+            &dbCtx, lexographicKeysSegments[i], lexographicKeysSegments[i + 1] );
+    }
+
+    dev::h256 dbHash;
+    secp256k1_sha256_finalize( &dbCtx, dbHash.data() );
+    cnote << _dbDir << " hash is: " << dbHash << std::endl;
+
+    secp256k1_sha256_write( ctx, dbHash.data(), dbHash.size );
 } catch ( const fs::filesystem_error& ex ) {
     std::throw_with_nested( CannotRead( ex.path1() ) );
 }
diff --git a/test/unittests/libweb3core/LevelDBHash.cpp b/test/unittests/libweb3core/LevelDBHash.cpp
index 67dde80cb..bbecf1249 100644
--- a/test/unittests/libweb3core/LevelDBHash.cpp
+++ b/test/unittests/libweb3core/LevelDBHash.cpp
@@ -39,6 +39,39 @@ BOOST_AUTO_TEST_CASE( hash ) {
 
     BOOST_REQUIRE( hash == hash_same );
 
+    dev::h256 hashPartially;
+    {
+        {
+            std::unique_ptr< dev::db::LevelDB > db_copy( new dev::db::LevelDB( td.path() ) );
+            BOOST_REQUIRE( db_copy );
+
+            for ( size_t i = 0; i < 123; ++i ) {
+                std::string key = std::to_string( 43 + i );
+                std::string value = std::to_string( i );
+                db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
+            }
+        }
+
+        std::array< std::string, 17 > lexographicKeysSegments = { "0", "1", "2", "3", "4", "5", "6",
+                                                                  "7", "8", "9", "a", "b", "c", "d",
+                                                                  "e", "f", "g" };
+
+        secp256k1_sha256_t dbCtx;
+        secp256k1_sha256_initialize( &dbCtx );
+
+        for (size_t i = 0; i < lexographicKeysSegments.size() - 1; ++i) {
+            std::unique_ptr< dev::db::LevelDB > db( new dev::db::LevelDB( td.path(),
+                dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
+                dev::db::LevelDB::defaultSnapshotDBOptions() ) );
+
+            db->hashBasePartially( &dbCtx, lexographicKeysSegments[i], lexographicKeysSegments[i + 1] );
+        }
+
+        secp256k1_sha256_finalize( &dbCtx, hashPartially.data() );
+    }
+
+    BOOST_REQUIRE( hash == hashPartially );
+
     dev::h256 hash_diff;
     {
         std::unique_ptr< dev::db::LevelDB > db_diff( new dev::db::LevelDB( td.path() ) );

From 7469839b079e9deab6c6b24d813d71c732e84231 Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Thu, 11 Jan 2024 13:19:17 +0000
Subject: [PATCH 02/12] #1588 fix new snapshot hash calculation

---
 libdevcore/LevelDB.cpp                     |  1 -
 libskale/SnapshotManager.cpp               |  7 ++--
 skaled/main.cpp                            |  2 +-
 test/unittests/libweb3core/LevelDBHash.cpp | 42 ++++++++++++++++------
 4 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp
index cb6fba758..91791b554 100644
--- a/libdevcore/LevelDB.cpp
+++ b/libdevcore/LevelDB.cpp
@@ -212,7 +212,6 @@ void LevelDB::forEach( std::function< bool( Slice, Slice ) > f ) const {
     }
 }
 
-
 void LevelDB::forEachWithPrefix(
     std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
     cnote << "Iterating over the LevelDB prefix: " << _prefix;
diff --git a/libskale/SnapshotManager.cpp b/libskale/SnapshotManager.cpp
index 04c7cd0be..58d0afbe0 100644
--- a/libskale/SnapshotManager.cpp
+++ b/libskale/SnapshotManager.cpp
@@ -445,8 +445,11 @@ void SnapshotManager::computeDatabaseHash(
         BOOST_THROW_EXCEPTION( InvalidPath( _dbDir ) );
     }
 
-    std::array< std::string, 17 > lexographicKeysSegments = { "0", "1", "2", "3", "4", "5", "6",
-        "7", "8", "9", "a", "b", "c", "d", "e", "f", "g" };
+    //    std::array< std::string, 23 > lexographicKeysSegments = { "0", "1", "2", "3", "4", "5",
+    //    "6",
+    //        "7", "8", "9", "A", "B", "C", "D", "E", "F", "a", "b", "c", "d", "e", "f", "{" };
+    std::array< std::string, 11 > lexographicKeysSegments = { "0", "2", "4", "6", "8", "A", "F",
+        "a", "c", "e", "{" };
 
     secp256k1_sha256_t dbCtx;
     secp256k1_sha256_initialize( &dbCtx );
diff --git a/skaled/main.cpp b/skaled/main.cpp
index 381cda01f..27469a750 100644
--- a/skaled/main.cpp
+++ b/skaled/main.cpp
@@ -460,7 +460,7 @@ bool tryDownloadSnapshot( std::shared_ptr< SnapshotManager >& snapshotManager,
                            "hash " )
                     << cc::notice( votedHash.first.hex() )
                     << cc::notice( " is not equal to calculated hash " )
-                    << cc::notice( calculated_hash.hex() ) << cc::notice( "Will try again" );
+                    << cc::notice( calculated_hash.hex() ) << cc::notice( " Will try again" );
                 if ( isRegularSnapshot )
                     snapshotManager->cleanup();
                 else
diff --git a/test/unittests/libweb3core/LevelDBHash.cpp b/test/unittests/libweb3core/LevelDBHash.cpp
index bbecf1249..0e77f86b9 100644
--- a/test/unittests/libweb3core/LevelDBHash.cpp
+++ b/test/unittests/libweb3core/LevelDBHash.cpp
@@ -9,36 +9,51 @@ BOOST_AUTO_TEST_SUITE( LevelDBHashBase )
 BOOST_AUTO_TEST_CASE( hash ) {
     dev::TransientDirectory td;
 
+    std::vector< std::pair< std::string, std::string > > randomKeysValues(123);
+
     dev::h256 hash;
     {
         std::unique_ptr< dev::db::LevelDB > db( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db );
 
+        db->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+        db->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 43 + i );
-            std::string value = std::to_string( i );
+            std::string key = dev::h256::random().hex();
+            std::string value = dev::h256::random().hex();
             db->insert( dev::db::Slice(key), dev::db::Slice(value) );
+
+            randomKeysValues[i] = { key, value };
         }
 
         hash = db->hashBase();
     }
 
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
     dev::h256 hash_same;
     {
         std::unique_ptr< dev::db::LevelDB > db_copy( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db_copy );
 
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 43 + i );
-            std::string value = std::to_string( i );
+            std::string key = randomKeysValues[i].first;
+            std::string value = randomKeysValues[i].second;
             db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
         }
+        db_copy->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+        db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
 
         hash_same = db_copy->hashBase();
     }
 
     BOOST_REQUIRE( hash == hash_same );
 
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
     dev::h256 hashPartially;
     {
         {
@@ -46,15 +61,17 @@ BOOST_AUTO_TEST_CASE( hash ) {
             BOOST_REQUIRE( db_copy );
 
             for ( size_t i = 0; i < 123; ++i ) {
-                std::string key = std::to_string( 43 + i );
-                std::string value = std::to_string( i );
+                std::string key = randomKeysValues[i].first;
+                std::string value = randomKeysValues[i].second;
                 db_copy->insert( dev::db::Slice(key), dev::db::Slice(value) );
             }
+
+            db_copy->insert( dev::db::Slice( "PieceUsageBytes" ), dev::db::Slice( "123456789" ) );
+            db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
         }
 
-        std::array< std::string, 17 > lexographicKeysSegments = { "0", "1", "2", "3", "4", "5", "6",
-                                                                  "7", "8", "9", "a", "b", "c", "d",
-                                                                  "e", "f", "g" };
+        std::array< std::string, 11 > lexographicKeysSegments = { "0", "2", "4", "6", "8", "A",
+                                                                  "F", "a", "c", "e", "{" };
 
         secp256k1_sha256_t dbCtx;
         secp256k1_sha256_initialize( &dbCtx );
@@ -72,14 +89,17 @@ BOOST_AUTO_TEST_CASE( hash ) {
 
     BOOST_REQUIRE( hash == hashPartially );
 
+    boost::filesystem::remove_all( td.path() );
+    BOOST_REQUIRE( !boost::filesystem::exists( td.path() ) );
+
     dev::h256 hash_diff;
     {
         std::unique_ptr< dev::db::LevelDB > db_diff( new dev::db::LevelDB( td.path() ) );
         BOOST_REQUIRE( db_diff );
 
         for ( size_t i = 0; i < 123; ++i ) {
-            std::string key = std::to_string( 42 + i );
-            std::string value = std::to_string( i );
+            std::string key = dev::h256::random().hex();
+            std::string value = dev::h256::random().hex();
             db_diff->insert( dev::db::Slice(key), dev::db::Slice(value) );
         }
 

From 1bf2cd7bc15e2a0a71d5601a406ef6ae56091573 Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Fri, 12 Jan 2024 13:46:21 +0000
Subject: [PATCH 03/12] #1588 change snapshot hash computation

---
 libdevcore/LevelDB.cpp                     | 11 +++++++----
 libskale/SnapshotManager.cpp               | 12 +++++++-----
 test/unittests/libweb3core/LevelDBHash.cpp |  9 +++++++--
 3 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp
index 91791b554..0025cf022 100644
--- a/libdevcore/LevelDB.cpp
+++ b/libdevcore/LevelDB.cpp
@@ -236,6 +236,7 @@ h256 LevelDB::hashBase() const {
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
@@ -251,6 +252,7 @@ h256 LevelDB::hashBase() const {
         bytesConstRef str_key_value( usc.data(), usc.size() );
         secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
     }
+
     h256 hash;
     secp256k1_sha256_finalize( &ctx, hash.data() );
     return hash;
@@ -285,6 +287,7 @@ void LevelDB::hashBasePartially(
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     for ( it->Seek( start ); it->Valid() && it->key().ToString() < finish; it->Next() ) {
         std::string key_ = it->key().ToString();
         std::string value_ = it->value().ToString();
@@ -293,10 +296,10 @@ void LevelDB::hashBasePartially(
         // TODO Move this logic to separate "compatiliblity layer"!
         if ( key_ == "pieceUsageBytes" )
             continue;
-        std::string key_value = key_ + value_;
-        const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
-        bytesConstRef str_key_value( usc.data(), usc.size() );
-        secp256k1_sha256_write( ctx, str_key_value.data(), str_key_value.size() );
+        std::string keyValue = key_ + value_;
+        const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+        bytesConstRef strKeyValue( usc.data(), usc.size() );
+        secp256k1_sha256_write( ctx, strKeyValue.data(), strKeyValue.size() );
     }
 }
 
diff --git a/libskale/SnapshotManager.cpp b/libskale/SnapshotManager.cpp
index 58d0afbe0..c6e495c74 100644
--- a/libskale/SnapshotManager.cpp
+++ b/libskale/SnapshotManager.cpp
@@ -445,11 +445,13 @@ void SnapshotManager::computeDatabaseHash(
         BOOST_THROW_EXCEPTION( InvalidPath( _dbDir ) );
     }
 
-    //    std::array< std::string, 23 > lexographicKeysSegments = { "0", "1", "2", "3", "4", "5",
-    //    "6",
-    //        "7", "8", "9", "A", "B", "C", "D", "E", "F", "a", "b", "c", "d", "e", "f", "{" };
-    std::array< std::string, 11 > lexographicKeysSegments = { "0", "2", "4", "6", "8", "A", "F",
-        "a", "c", "e", "{" };
+    std::array< std::string, 17 > lexographicKeysSegments = { std::string( 1, char( 0 ) ),
+        std::string( 1, char( 16 ) ), std::string( 1, char( 32 ) ), std::string( 1, char( 48 ) ),
+        std::string( 1, char( 64 ) ), std::string( 1, char( 80 ) ), std::string( 1, char( 96 ) ),
+        std::string( 1, char( 112 ) ), std::string( 1, char( 128 ) ), std::string( 1, char( 144 ) ),
+        std::string( 1, char( 160 ) ), std::string( 1, char( 176 ) ), std::string( 1, char( 192 ) ),
+        std::string( 1, char( 208 ) ), std::string( 1, char( 224 ) ), std::string( 1, char( 240 ) ),
+        std::string( 1000, char( 255 ) ) };
 
     secp256k1_sha256_t dbCtx;
     secp256k1_sha256_initialize( &dbCtx );
diff --git a/test/unittests/libweb3core/LevelDBHash.cpp b/test/unittests/libweb3core/LevelDBHash.cpp
index 0e77f86b9..f1140f8a1 100644
--- a/test/unittests/libweb3core/LevelDBHash.cpp
+++ b/test/unittests/libweb3core/LevelDBHash.cpp
@@ -70,8 +70,13 @@ BOOST_AUTO_TEST_CASE( hash ) {
             db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
         }
 
-        std::array< std::string, 11 > lexographicKeysSegments = { "0", "2", "4", "6", "8", "A",
-                                                                  "F", "a", "c", "e", "{" };
+        std::array< std::string, 17 > lexographicKeysSegments = { std::string( 1, char( 0 ) ),
+            std::string( 1, char( 16 ) ), std::string( 1, char( 32 ) ), std::string( 1, char( 48 ) ),
+            std::string( 1, char( 64 ) ), std::string( 1, char( 80 ) ), std::string( 1, char( 96 ) ),
+            std::string( 1, char( 112 ) ), std::string( 1, char( 128 ) ), std::string( 1, char( 144 ) ),
+            std::string( 1, char( 160 ) ), std::string( 1, char( 176 ) ), std::string( 1, char( 192 ) ),
+            std::string( 1, char( 208 ) ), std::string( 1, char( 224 ) ), std::string( 1, char( 240 ) ),
+            std::string( 1000, char( 255 ) ) };
 
         secp256k1_sha256_t dbCtx;
         secp256k1_sha256_initialize( &dbCtx );

From 9431fab06e454694cb569948b70fab2cabb00cf3 Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Fri, 12 Jan 2024 18:53:42 +0000
Subject: [PATCH 04/12] IS-902 check block number to download snapshot

---
 libweb3jsonrpc/Skale.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libweb3jsonrpc/Skale.cpp b/libweb3jsonrpc/Skale.cpp
index 763c3af6c..40904ac46 100644
--- a/libweb3jsonrpc/Skale.cpp
+++ b/libweb3jsonrpc/Skale.cpp
@@ -158,6 +158,10 @@ nlohmann::json Skale::impl_skale_getSnapshot( const nlohmann::json& joRequest, C
 
     // TODO check
     unsigned blockNumber = joRequest["blockNumber"].get< unsigned >();
+    if ( blockNumber != m_client.getLatestSnapshotBlockNumer() ) {
+        joResponse["error"] = "Wrong snapshot block number requested.";
+        return joResponse;
+    }
 
     // exit if too early
     if ( currentSnapshotBlockNumber >= 0 ) {

From aab2420511525adef6cbea434af66573425f339b Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Tue, 16 Jan 2024 13:40:30 +0000
Subject: [PATCH 05/12] IS 902 handle wrong snapshot block number request for
 signatures

---
 libweb3jsonrpc/Skale.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/libweb3jsonrpc/Skale.cpp b/libweb3jsonrpc/Skale.cpp
index 40904ac46..492e25fc8 100644
--- a/libweb3jsonrpc/Skale.cpp
+++ b/libweb3jsonrpc/Skale.cpp
@@ -159,7 +159,7 @@ nlohmann::json Skale::impl_skale_getSnapshot( const nlohmann::json& joRequest, C
     // TODO check
     unsigned blockNumber = joRequest["blockNumber"].get< unsigned >();
     if ( blockNumber != m_client.getLatestSnapshotBlockNumer() ) {
-        joResponse["error"] = "Wrong snapshot block number requested.";
+        joResponse["error"] = "Invalid snapshot block number requested - it might be deleted.";
         return joResponse;
     }
 
@@ -370,6 +370,11 @@ Json::Value Skale::skale_getSnapshotSignature( unsigned blockNumber ) {
     if ( chainParams.nodeInfo.keyShareName.empty() || chainParams.nodeInfo.sgxServerUrl.empty() )
         throw jsonrpc::JsonRpcException( "Snapshot signing is not enabled" );
 
+    if ( blockNumber != this->m_client.getLatestSnapshotBlockNumer() ) {
+        throw jsonrpc::JsonRpcException(
+            "Invalid snapshot block number requested - it might be deleted." );
+    }
+
     try {
         dev::h256 snapshot_hash = this->m_client.getSnapshotHash( blockNumber );
         if ( !snapshot_hash )

From c4cffb1abd48a3005a1f1d522d3ec39563cb30ea Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Tue, 16 Jan 2024 15:34:46 +0000
Subject: [PATCH 06/12] #1588 change snapshot hash computation

---
 libdevcore/LevelDB.cpp                     | 15 +++++++++++++--
 libdevcore/LevelDB.h                       |  2 +-
 libskale/SnapshotManager.cpp               | 15 ++++-----------
 test/unittests/libweb3core/LevelDBHash.cpp | 15 ++++-----------
 4 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp
index 0025cf022..dda5756ef 100644
--- a/libdevcore/LevelDB.cpp
+++ b/libdevcore/LevelDB.cpp
@@ -282,13 +282,18 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
 }
 
 void LevelDB::hashBasePartially(
-    secp256k1_sha256_t* ctx, const std::string& start, const std::string& finish ) const {
+    secp256k1_sha256_t* ctx, std::string& lastHashedKey, size_t batchSize ) const {
     std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
 
-    for ( it->Seek( start ); it->Valid() && it->key().ToString() < finish; it->Next() ) {
+    if ( lastHashedKey != "start" )
+        it->Seek( lastHashedKey );
+    else
+        it->SeekToFirst();
+
+    for ( size_t counter = 0; it->Valid() && counter < batchSize; it->Next() ) {
         std::string key_ = it->key().ToString();
         std::string value_ = it->value().ToString();
         // HACK! For backward compatibility! When snapshot could happen between update of two nodes
@@ -300,7 +305,13 @@ void LevelDB::hashBasePartially(
         const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
         bytesConstRef strKeyValue( usc.data(), usc.size() );
         secp256k1_sha256_write( ctx, strKeyValue.data(), strKeyValue.size() );
+        ++counter;
     }
+
+    if ( it->Valid() )
+        lastHashedKey = it->key().ToString();
+    else
+        lastHashedKey = "stop";
 }
 
 void LevelDB::doCompaction() const {
diff --git a/libdevcore/LevelDB.h b/libdevcore/LevelDB.h
index 470f318dd..01461f4c7 100644
--- a/libdevcore/LevelDB.h
+++ b/libdevcore/LevelDB.h
@@ -62,7 +62,7 @@ class LevelDB : public DatabaseFace {
     h256 hashBaseWithPrefix( char _prefix ) const;
 
     void hashBasePartially(
-        secp256k1_sha256_t* ctx, const std::string& start, const std::string& finish ) const;
+        secp256k1_sha256_t* ctx, std::string& lastHashedKey, size_t batchSize = 10000 ) const;
 
     void doCompaction() const;
 
diff --git a/libskale/SnapshotManager.cpp b/libskale/SnapshotManager.cpp
index c6e495c74..5e6e5660a 100644
--- a/libskale/SnapshotManager.cpp
+++ b/libskale/SnapshotManager.cpp
@@ -445,24 +445,17 @@ void SnapshotManager::computeDatabaseHash(
         BOOST_THROW_EXCEPTION( InvalidPath( _dbDir ) );
     }
 
-    std::array< std::string, 17 > lexographicKeysSegments = { std::string( 1, char( 0 ) ),
-        std::string( 1, char( 16 ) ), std::string( 1, char( 32 ) ), std::string( 1, char( 48 ) ),
-        std::string( 1, char( 64 ) ), std::string( 1, char( 80 ) ), std::string( 1, char( 96 ) ),
-        std::string( 1, char( 112 ) ), std::string( 1, char( 128 ) ), std::string( 1, char( 144 ) ),
-        std::string( 1, char( 160 ) ), std::string( 1, char( 176 ) ), std::string( 1, char( 192 ) ),
-        std::string( 1, char( 208 ) ), std::string( 1, char( 224 ) ), std::string( 1, char( 240 ) ),
-        std::string( 1000, char( 255 ) ) };
-
     secp256k1_sha256_t dbCtx;
     secp256k1_sha256_initialize( &dbCtx );
 
-    for ( size_t i = 0; i < lexographicKeysSegments.size() - 1; ++i ) {
+    std::string finishKey = "start";
+
+    while ( finishKey != "stop" ) {
         std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
             dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
             dev::db::LevelDB::defaultSnapshotDBOptions() ) );
 
-        m_db->hashBasePartially(
-            &dbCtx, lexographicKeysSegments[i], lexographicKeysSegments[i + 1] );
+        m_db->hashBasePartially( &dbCtx, finishKey );
     }
 
     dev::h256 dbHash;
diff --git a/test/unittests/libweb3core/LevelDBHash.cpp b/test/unittests/libweb3core/LevelDBHash.cpp
index f1140f8a1..0756d0678 100644
--- a/test/unittests/libweb3core/LevelDBHash.cpp
+++ b/test/unittests/libweb3core/LevelDBHash.cpp
@@ -70,23 +70,16 @@ BOOST_AUTO_TEST_CASE( hash ) {
             db_copy->insert( dev::db::Slice( "ppieceUsageBytes" ), dev::db::Slice( "123456789" ) );
         }
 
-        std::array< std::string, 17 > lexographicKeysSegments = { std::string( 1, char( 0 ) ),
-            std::string( 1, char( 16 ) ), std::string( 1, char( 32 ) ), std::string( 1, char( 48 ) ),
-            std::string( 1, char( 64 ) ), std::string( 1, char( 80 ) ), std::string( 1, char( 96 ) ),
-            std::string( 1, char( 112 ) ), std::string( 1, char( 128 ) ), std::string( 1, char( 144 ) ),
-            std::string( 1, char( 160 ) ), std::string( 1, char( 176 ) ), std::string( 1, char( 192 ) ),
-            std::string( 1, char( 208 ) ), std::string( 1, char( 224 ) ), std::string( 1, char( 240 ) ),
-            std::string( 1000, char( 255 ) ) };
-
         secp256k1_sha256_t dbCtx;
         secp256k1_sha256_initialize( &dbCtx );
 
-        for (size_t i = 0; i < lexographicKeysSegments.size() - 1; ++i) {
-            std::unique_ptr< dev::db::LevelDB > db( new dev::db::LevelDB( td.path(),
+        std::string finishKey = "start";
+        while ( finishKey != "stop" ) {
+            std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( td.path(),
                 dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
                 dev::db::LevelDB::defaultSnapshotDBOptions() ) );
 
-            db->hashBasePartially( &dbCtx, lexographicKeysSegments[i], lexographicKeysSegments[i + 1] );
+            m_db->hashBasePartially( &dbCtx, finishKey, 10 );
         }
 
         secp256k1_sha256_finalize( &dbCtx, hashPartially.data() );

From 99a645a241646e1658ad108e7c1a70500e4e6e3a Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Wed, 17 Jan 2024 11:59:23 +0000
Subject: [PATCH 07/12] fix tests

---
 .github/workflows/test.yml | 180 +++++++++++++++++++------------------
 1 file changed, 91 insertions(+), 89 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 88d95350a..f3943c70b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -166,55 +166,57 @@ jobs:
           ccache --show-stats
       - name: Testeth verbosity 1
         run : |
+          mkdir -p /tmp/tests/
+          rm -rf /tmp/tests/*
           #first run with verbosity 1. If test fails, rerun with verbosity 4
           cd build/test
           export NO_NTP_CHECK=1
-          export NO_ULIMIT_CHECK=1          
+          export NO_ULIMIT_CHECK=1
           # we specifically run each test for easier log review          
-          ./testeth -t BlockchainTests -- --express && touch /tmp/BlockchainTestsPassed
-          ./testeth -t TransitionTests  -- --express && touch /tmp/TransitionTestsPassed
-          ./testeth -t TransactionTests -- --express && touch /tmp/TransactionTestsPassed
-          ./testeth -t VMTests -- --express && touch /tmp/VMTestsPassed
-          ./testeth -t LevelDBTests -- --express && touch /tmp/LevelDBTestsPassed
-          ./testeth -t CoreLibTests -- --express && touch /tmp/CoreLibTestsPassed
-          ./testeth -t RlpTests -- --express && touch /tmp/RlpTestsPassed
-          ./testeth -t SharedSpaceTests -- --express && touch /tmp/SharedSpaceTestsPassed
-          ./testeth -t EthashTests -- --express && touch /tmp/EthashTestsPassed
-          ./testeth -t SealEngineTests -- --express && touch /tmp/SealEngineTestsPassed
-          ./testeth -t DifficultyTests -- --express && touch /tmp/DifficultyTestsPassed
-          ./testeth -t BlockSuite -- --express && touch /tmp/BlockSuitePassed
-          ./testeth -t BlockChainMainNetworkSuite -- --express && touch /tmp/BlockChainMainNetworkSuitePassed
-          ./testeth -t BlockChainFrontierSuite -- --express && touch /tmp/BlockChainFrontierSuitePassed
-          ./testeth -t BlockQueueSuite -- --express && touch /tmp/BlockQueueSuitePassed
-          ./testeth -t ClientBase -- --express && touch /tmp/ClientBasePassed
-          ./testeth -t EstimateGas -- --express && touch /tmp/EstimateGasPassed
-          ./testeth -t getHistoricNodesData -- --express && touch /tmp/getHistoricNodesDataPassed
-          ./testeth -t ExtVmSuite -- --express && touch /tmp/ExtVmSuitePassed
-          ./testeth -t GasPricer -- --express && touch /tmp/GasPricerPassed
-          ./testeth -t BasicTests -- --express && touch /tmp/BasicTestsPassed
-          ./testeth -t InstanceMonitorSuite -- --express && touch /tmp/InstanceMonitorSuitePassed
-          ./testeth -t PrecompiledTests -- --express && touch /tmp/PrecompiledTestsPassed
-          ./testeth -t SkaleHostSuite -- --express && touch /tmp/SkaleHostSuitePassed
-          ./testeth -t StateUnitTests -- --express && touch /tmp/StateUnitTestsPassed
-          ./testeth -t libethereum -- --express && touch /tmp/libethereumPassed
-          ./testeth -t TransactionQueueSuite -- --express && touch /tmp/TransactionQueueSuitePassed
-          ./testeth -t LegacyVMSuite -- --express && touch /tmp/LegacyVMSuitePassed
-          ./testeth -t SkaleInterpreterSuite -- --express && touch /tmp/SkaleInterpreterSuitePassed
-          ./testeth -t SnapshotSigningTestSuite -- --express && touch /tmp/SnapshotSigningTestSuitePassed
-          ./testeth -t SkUtils -- --express && touch /tmp/SkUtilsPassed
-          ./testeth -t BlockChainTestSuite -- --express && touch /tmp/BlockChainTestSuitePassed
-          ./testeth -t TestHelperSuite -- --express && touch /tmp/TestHelperSuitePassed
-          ./testeth -t LevelDBHashBase -- --express && touch /tmp/LevelDBHashBasePassed
-          ./testeth -t memDB -- --express && touch /tmp/memDBPassed
-          ./testeth -t OverlayDBTests -- --express && touch /tmp/OverlayDBTestsPassed
-          ./testeth -t AccountHolderTest -- --express && touch /tmp/AccountHolderTestPassed
-          ./testeth -t ClientTests -- --express && touch /tmp/ClientTestsPassed
-          ./testeth -t JsonRpcSuite  -- --express && touch /tmp/JsonRpcSuitePassed 
-          ./testeth -t SingleConsensusTests  -- --express && touch /tmp/SingleConsensusTestsPassed
-          ./testeth -t ConsensusTests  -- --express && touch /tmp/ConsensusTestsPassed
-          sudo ./testeth -t BtrfsTestSuite -- --all && touch /tmp/BtrfsTestSuitePassed
-          sudo ./testeth -t HashSnapshotTestSuite -- --all && touch /tmp/HashSnapshotTestSuitePassed
-          sudo ./testeth -t ClientSnapshotsSuite -- --all && touch /tmp/ClientSnapshotsSuitePassed
+          ./testeth -t BlockchainTests -- --express && touch /tmp/tests/BlockchainTestsPassed
+          ./testeth -t TransitionTests  -- --express && touch /tmp/tests/TransitionTestsPassed
+          ./testeth -t TransactionTests -- --express && touch /tmp/tests/TransactionTestsPassed
+          ./testeth -t VMTests -- --express && touch /tmp/tests/VMTestsPassed
+          ./testeth -t LevelDBTests -- --express && touch /tmp/tests/LevelDBTestsPassed
+          ./testeth -t CoreLibTests -- --express && touch /tmp/tests/CoreLibTestsPassed
+          ./testeth -t RlpTests -- --express && touch /tmp/tests/RlpTestsPassed
+          ./testeth -t SharedSpaceTests -- --express && touch /tmp/tests/SharedSpaceTestsPassed
+          ./testeth -t EthashTests -- --express && touch /tmp/tests/EthashTestsPassed
+          ./testeth -t SealEngineTests -- --express && touch /tmp/tests/SealEngineTestsPassed
+          ./testeth -t DifficultyTests -- --express && touch /tmp/tests/DifficultyTestsPassed
+          ./testeth -t BlockSuite -- --express && touch /tmp/tests/BlockSuitePassed
+          ./testeth -t BlockChainMainNetworkSuite -- --express && touch /tmp/tests/BlockChainMainNetworkSuitePassed
+          ./testeth -t BlockChainFrontierSuite -- --express && touch /tmp/tests/BlockChainFrontierSuitePassed
+          ./testeth -t BlockQueueSuite -- --express && touch /tmp/tests/BlockQueueSuitePassed
+          ./testeth -t ClientBase -- --express && touch /tmp/tests/ClientBasePassed
+          ./testeth -t EstimateGas -- --express && touch /tmp/tests/EstimateGasPassed
+          ./testeth -t getHistoricNodesData -- --express && touch /tmp/tests/getHistoricNodesDataPassed
+          ./testeth -t ExtVmSuite -- --express && touch /tmp/tests/ExtVmSuitePassed
+          ./testeth -t GasPricer -- --express && touch /tmp/tests/GasPricerPassed
+          ./testeth -t BasicTests -- --express && touch /tmp/tests/BasicTestsPassed
+          ./testeth -t InstanceMonitorSuite -- --express && touch /tmp/tests/InstanceMonitorSuitePassed
+          ./testeth -t PrecompiledTests -- --express && touch /tmp/tests/PrecompiledTestsPassed
+          ./testeth -t SkaleHostSuite -- --express && touch /tmp/tests/SkaleHostSuitePassed
+          ./testeth -t StateUnitTests -- --express && touch /tmp/tests/StateUnitTestsPassed
+          ./testeth -t libethereum -- --express && touch /tmp/tests/libethereumPassed
+          ./testeth -t TransactionQueueSuite -- --express && touch /tmp/tests/TransactionQueueSuitePassed
+          ./testeth -t LegacyVMSuite -- --express && touch /tmp/tests/LegacyVMSuitePassed
+          ./testeth -t SkaleInterpreterSuite -- --express && touch /tmp/tests/SkaleInterpreterSuitePassed
+          ./testeth -t SnapshotSigningTestSuite -- --express && touch /tmp/tests/SnapshotSigningTestSuitePassed
+          ./testeth -t SkUtils -- --express && touch /tmp/tests/SkUtilsPassed
+          ./testeth -t BlockChainTestSuite -- --express && touch /tmp/tests/BlockChainTestSuitePassed
+          ./testeth -t TestHelperSuite -- --express && touch /tmp/tests/TestHelperSuitePassed
+          ./testeth -t LevelDBHashBase -- --express && touch /tmp/tests/LevelDBHashBasePassed
+          ./testeth -t memDB -- --express && touch /tmp/tests/memDBPassed
+          ./testeth -t OverlayDBTests -- --express && touch /tmp/tests/OverlayDBTestsPassed
+          ./testeth -t AccountHolderTest -- --express && touch /tmp/tests/AccountHolderTestPassed
+          ./testeth -t ClientTests -- --express && touch /tmp/tests/ClientTestsPassed
+          ./testeth -t JsonRpcSuite  -- --express && touch /tmp/tests/JsonRpcSuitePassed 
+          ./testeth -t SingleConsensusTests  -- --express && touch /tmp/tests/SingleConsensusTestsPassed
+          ./testeth -t ConsensusTests  -- --express && touch /tmp/tests/ConsensusTestsPassed
+          sudo ./testeth -t BtrfsTestSuite -- --all && touch /tmp/tests/BtrfsTestSuitePassed
+          sudo ./testeth -t HashSnapshotTestSuite -- --all && touch /tmp/tests/HashSnapshotTestSuitePassed
+          sudo ./testeth -t ClientSnapshotsSuite -- --all && touch /tmp/tests/ClientSnapshotsSuitePassed
           cd ..
       - name: Testeth verbosity 4
         run : |
@@ -222,50 +224,50 @@ jobs:
           cd build/test      
           export NO_NTP_CHECK=1
           export NO_ULIMIT_CHECK=1          
-          ls /tmp/BlockchainTestsPassed || ./testeth -t BlockchainTests -- --express --verbosity 4
-          ls /tmp/TransitionTestsPassed || ./testeth -t TransitionTests -- --express --verbosity 4
-          ls /tmp/TransactionTestsPassed || ./testeth -t TransactionTests -- --express --verbosity 4
-          ls /tmp/VMTestsPassed || ./testeth -t VMTests -- --express --verbosity 4
-          ls /tmp/LevelDBTestsPassed || ./testeth -t LevelDBTests -- --express --verbosity 4
-          ls /tmp/CoreLibTestsPassed || ./testeth -t CoreLibTests -- --express --verbosity 4
-          ls /tmp/RlpTestsPassed || ./testeth -t RlpTests -- --express --verbosity 4
-          ls /tmp/SharedSpaceTestsPassed || ./testeth -t SharedSpaceTests -- --express --verbosity 4
-          ls /tmp/EthashTestsPassed || ./testeth -t EthashTests -- --express --verbosity 4
-          ls /tmp/SealEngineTestsPassed || ./testeth -t SealEngineTests -- --express --verbosity 4
-          ls /tmp/DifficultyTestsPassed || ./testeth -t DifficultyTests -- --express --verbosity 4
-          ls /tmp/BlockSuitePassed || ./testeth -t BlockSuite -- --express --verbosity 4
-          ls /tmp/BlockChainMainNetworkSuitePassed || ./testeth -t BlockChainMainNetworkSuite -- --express --verbosity 4
-          ls /tmp/BlockChainFrontierSuitePassed || ./testeth -t BlockChainFrontierSuite -- --express --verbosity 4
-          ls /tmp/BlockQueueSuitePassed || ./testeth -t BlockQueueSuite -- --express --verbosity 4
-          ls /tmp/ClientBasePassed || ./testeth -t ClientBase -- --express --verbosity 4
-          ls /tmp/EstimateGasPassed || ./testeth -t EstimateGas -- --express --verbosity 4
-          ls /tmp/getHistoricNodesDataPassed || ./testeth -t getHistoricNodesData -- --express --verbosity 4
-          ls /tmp/ExtVmSuitePassed || ./testeth -t ExtVmSuite -- --express --verbosity 4
-          ls /tmp/GasPricerPassed || ./testeth -t GasPricer -- --express --verbosity 4
-          ls /tmp/BasicTestsPassed || ./testeth -t BasicTests -- --express --verbosity 4
-          ls /tmp/InstanceMonitorSuitePassed || ./testeth -t InstanceMonitorSuite -- --express --verbosity 4
-          ls /tmp/PrecompiledTestsPassed || ./testeth -t PrecompiledTests -- --express --verbosity 4
-          ls /tmp/SkaleHostSuitePassed || ./testeth -t SkaleHostSuite -- --express --verbosity 4
-          ls /tmp/StateUnitTestsPassed || ./testeth -t StateUnitTests -- --express --verbosity 4
-          ls /tmp/libethereumPassed || ./testeth -t libethereum -- --express --verbosity 4
-          ls /tmp/TransactionQueueSuitePassed || ./testeth -t TransactionQueueSuite -- --express --verbosity 4
-          ls /tmp/LegacyVMSuitePassed || ./testeth -t LegacyVMSuite -- --express --verbosity 4
-          ls /tmp/SkaleInterpreterSuitePassed || ./testeth -t SkaleInterpreterSuite -- --express --verbosity 4
-          ls /tmp/SnapshotSigningTestSuitePassed || ./testeth -t SnapshotSigningTestSuite -- --express --verbosity 4
-          ls /tmp/SkUtilsPassed || ./testeth -t SkUtils -- --express --verbosity 4
-          ls /tmp/BlockChainTestSuitePassed || ./testeth -t BlockChainTestSuite -- --express --verbosity 4
-          ls /tmp/TestHelperSuitePassed || ./testeth -t TestHelperSuite -- --express --verbosity 4
-          ls /tmp/LevelDBHashBasePassed || ./testeth -t LevelDBHashBase -- --express --verbosity 4
-          ls /tmp/memDBPassed || ./testeth -t memDB -- --express --verbosity 4
-          ls /tmp/OverlayDBTestsPassed || ./testeth -t OverlayDBTests -- --express --verbosity 4
-          ls /tmp/AccountHolderTestPassed || ./testeth -t AccountHolderTest -- --express --verbosity 4
-          ls /tmp/ClientTestsPassed || ./testeth -t ClientTests -- --express --verbosity 4
-          ls /tmp/JsonRpcSuitePassed || ./testeth -t JsonRpcSuite  -- --express --verbosity 4 
-          ls /tmp/SingleConsensusTestsPassed || ./testeth -t SingleConsensusTests  -- --express --verbosity 4
-          ls /tmp/ConsensusTestsPassed || ./testeth -t ConsensusTests  -- --express --verbosity 4
-          ls /tmp/BtrfsTestSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t BtrfsTestSuite -- --all --verbosity 4
-          ls /tmp/HashSnapshotTestSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t HashSnapshotTestSuite -- --all --verbosity 4
-          ls /tmp/ClientSnapshotsSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t ClientSnapshotsSuite -- --all --verbosity 4
+          ls /tmp/tests/BlockchainTestsPassed || ./testeth -t BlockchainTests -- --express --verbosity 4
+          ls /tmp/tests/TransitionTestsPassed || ./testeth -t TransitionTests -- --express --verbosity 4
+          ls /tmp/tests/TransactionTestsPassed || ./testeth -t TransactionTests -- --express --verbosity 4
+          ls /tmp/tests/VMTestsPassed || ./testeth -t VMTests -- --express --verbosity 4
+          ls /tmp/tests/LevelDBTestsPassed || ./testeth -t LevelDBTests -- --express --verbosity 4
+          ls /tmp/tests/CoreLibTestsPassed || ./testeth -t CoreLibTests -- --express --verbosity 4
+          ls /tmp/tests/RlpTestsPassed || ./testeth -t RlpTests -- --express --verbosity 4
+          ls /tmp/tests/SharedSpaceTestsPassed || ./testeth -t SharedSpaceTests -- --express --verbosity 4
+          ls /tmp/tests/EthashTestsPassed || ./testeth -t EthashTests -- --express --verbosity 4
+          ls /tmp/tests/SealEngineTestsPassed || ./testeth -t SealEngineTests -- --express --verbosity 4
+          ls /tmp/tests/DifficultyTestsPassed || ./testeth -t DifficultyTests -- --express --verbosity 4
+          ls /tmp/tests/BlockSuitePassed || ./testeth -t BlockSuite -- --express --verbosity 4
+          ls /tmp/tests/BlockChainMainNetworkSuitePassed || ./testeth -t BlockChainMainNetworkSuite -- --express --verbosity 4
+          ls /tmp/tests/BlockChainFrontierSuitePassed || ./testeth -t BlockChainFrontierSuite -- --express --verbosity 4
+          ls /tmp/tests/BlockQueueSuitePassed || ./testeth -t BlockQueueSuite -- --express --verbosity 4
+          ls /tmp/tests/ClientBasePassed || ./testeth -t ClientBase -- --express --verbosity 4
+          ls /tmp/tests/EstimateGasPassed || ./testeth -t EstimateGas -- --express --verbosity 4
+          ls /tmp/tests/getHistoricNodesDataPassed || ./testeth -t getHistoricNodesData -- --express --verbosity 4
+          ls /tmp/tests/ExtVmSuitePassed || ./testeth -t ExtVmSuite -- --express --verbosity 4
+          ls /tmp/tests/GasPricerPassed || ./testeth -t GasPricer -- --express --verbosity 4
+          ls /tmp/tests/BasicTestsPassed || ./testeth -t BasicTests -- --express --verbosity 4
+          ls /tmp/tests/InstanceMonitorSuitePassed || ./testeth -t InstanceMonitorSuite -- --express --verbosity 4
+          ls /tmp/tests/PrecompiledTestsPassed || ./testeth -t PrecompiledTests -- --express --verbosity 4
+          ls /tmp/tests/SkaleHostSuitePassed || ./testeth -t SkaleHostSuite -- --express --verbosity 4
+          ls /tmp/tests/StateUnitTestsPassed || ./testeth -t StateUnitTests -- --express --verbosity 4
+          ls /tmp/tests/libethereumPassed || ./testeth -t libethereum -- --express --verbosity 4
+          ls /tmp/tests/TransactionQueueSuitePassed || ./testeth -t TransactionQueueSuite -- --express --verbosity 4
+          ls /tmp/tests/LegacyVMSuitePassed || ./testeth -t LegacyVMSuite -- --express --verbosity 4
+          ls /tmp/tests/SkaleInterpreterSuitePassed || ./testeth -t SkaleInterpreterSuite -- --express --verbosity 4
+          ls /tmp/tests/SnapshotSigningTestSuitePassed || ./testeth -t SnapshotSigningTestSuite -- --express --verbosity 4
+          ls /tmp/tests/SkUtilsPassed || ./testeth -t SkUtils -- --express --verbosity 4
+          ls /tmp/tests/BlockChainTestSuitePassed || ./testeth -t BlockChainTestSuite -- --express --verbosity 4
+          ls /tmp/tests/TestHelperSuitePassed || ./testeth -t TestHelperSuite -- --express --verbosity 4
+          ls /tmp/tests/LevelDBHashBasePassed || ./testeth -t LevelDBHashBase -- --express --verbosity 4
+          ls /tmp/tests/memDBPassed || ./testeth -t memDB -- --express --verbosity 4
+          ls /tmp/tests/OverlayDBTestsPassed || ./testeth -t OverlayDBTests -- --express --verbosity 4
+          ls /tmp/tests/AccountHolderTestPassed || ./testeth -t AccountHolderTest -- --express --verbosity 4
+          ls /tmp/tests/ClientTestsPassed || ./testeth -t ClientTests -- --express --verbosity 4
+          ls /tmp/tests/JsonRpcSuitePassed || ./testeth -t JsonRpcSuite  -- --express --verbosity 4 
+          ls /tmp/tests/SingleConsensusTestsPassed || ./testeth -t SingleConsensusTests  -- --express --verbosity 4
+          ls /tmp/tests/ConsensusTestsPassed || ./testeth -t ConsensusTests  -- --express --verbosity 4
+          ls /tmp/tests/BtrfsTestSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t BtrfsTestSuite -- --all --verbosity 4
+          ls /tmp/tests/HashSnapshotTestSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t HashSnapshotTestSuite -- --all --verbosity 4
+          ls /tmp/tests/ClientSnapshotsSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t ClientSnapshotsSuite -- --all --verbosity 4
           cd ..
           
       - name: Create lcov report

From 886c695a710aa358d378a75e80bde0ea8018b451 Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Wed, 17 Jan 2024 11:59:28 +0000
Subject: [PATCH 08/12] fix tests

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f3943c70b..aa3a0e340 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -167,7 +167,7 @@ jobs:
       - name: Testeth verbosity 1
         run : |
           mkdir -p /tmp/tests/
-          rm -rf /tmp/tests/*
+          sudo rm -rf /tmp/tests/*
           #first run with verbosity 1. If test fails, rerun with verbosity 4
           cd build/test
           export NO_NTP_CHECK=1

From 7c67a04a6a4801130b34041e309eed7d4a64b6e8 Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Thu, 18 Jan 2024 11:30:59 +0000
Subject: [PATCH 09/12] #1588 rename variables

---
 libskale/SnapshotManager.cpp               | 6 +++---
 test/unittests/libweb3core/LevelDBHash.cpp | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/libskale/SnapshotManager.cpp b/libskale/SnapshotManager.cpp
index 5e6e5660a..8989b4274 100644
--- a/libskale/SnapshotManager.cpp
+++ b/libskale/SnapshotManager.cpp
@@ -448,14 +448,14 @@ void SnapshotManager::computeDatabaseHash(
     secp256k1_sha256_t dbCtx;
     secp256k1_sha256_initialize( &dbCtx );
 
-    std::string finishKey = "start";
+    std::string lastHashedKey = "start";
 
-    while ( finishKey != "stop" ) {
+    while ( lastHashedKey != "stop" ) {
         std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
             dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
             dev::db::LevelDB::defaultSnapshotDBOptions() ) );
 
-        m_db->hashBasePartially( &dbCtx, finishKey );
+        m_db->hashBasePartially( &dbCtx, lastHashedKey );
     }
 
     dev::h256 dbHash;
diff --git a/test/unittests/libweb3core/LevelDBHash.cpp b/test/unittests/libweb3core/LevelDBHash.cpp
index 0756d0678..1eec5366e 100644
--- a/test/unittests/libweb3core/LevelDBHash.cpp
+++ b/test/unittests/libweb3core/LevelDBHash.cpp
@@ -73,13 +73,13 @@ BOOST_AUTO_TEST_CASE( hash ) {
         secp256k1_sha256_t dbCtx;
         secp256k1_sha256_initialize( &dbCtx );
 
-        std::string finishKey = "start";
-        while ( finishKey != "stop" ) {
+        std::string lastHashedKey = "start";
+        while ( lastHashedKey != "stop" ) {
             std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( td.path(),
                 dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
                 dev::db::LevelDB::defaultSnapshotDBOptions() ) );
 
-            m_db->hashBasePartially( &dbCtx, finishKey, 10 );
+            m_db->hashBasePartially( &dbCtx, lastHashedKey, 10 );
         }
 
         secp256k1_sha256_finalize( &dbCtx, hashPartially.data() );

From b8cdd8d5851f666eac80f1d5f14d3fc06804cea0 Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Thu, 18 Jan 2024 12:00:32 +0000
Subject: [PATCH 10/12] #1588 rename variables and remove colored logs

---
 libdevcore/LevelDB.cpp | 34 +++++++++++++++++-----------------
 skaled/main.cpp        | 19 ++++++++-----------
 2 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp
index dda5756ef..214c49522 100644
--- a/libdevcore/LevelDB.cpp
+++ b/libdevcore/LevelDB.cpp
@@ -240,17 +240,17 @@ h256 LevelDB::hashBase() const {
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
-        std::string key_ = it->key().ToString();
-        std::string value_ = it->value().ToString();
+        std::string keyTmp = it->key().ToString();
+        std::string valueTmp = it->value().ToString();
         // HACK! For backward compatibility! When snapshot could happen between update of two nodes
         // - it would lead to stateRoot mismatch
         // TODO Move this logic to separate "compatiliblity layer"!
-        if ( key_ == "pieceUsageBytes" )
+        if ( keyTmp == "pieceUsageBytes" )
             continue;
-        std::string key_value = key_ + value_;
-        const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
-        bytesConstRef str_key_value( usc.data(), usc.size() );
-        secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
+        std::string keyValue = keyTmp + valueTmp;
+        const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+        bytesConstRef strKeyValue( usc.data(), usc.size() );
+        secp256k1_sha256_write( &ctx, strKeyValue.data(), strKeyValue.size() );
     }
 
     h256 hash;
@@ -268,12 +268,12 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
         if ( it->key()[0] == _prefix ) {
-            std::string key_ = it->key().ToString();
-            std::string value_ = it->value().ToString();
-            std::string key_value = key_ + value_;
-            const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
-            bytesConstRef str_key_value( usc.data(), usc.size() );
-            secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
+            std::string keyTmp = it->key().ToString();
+            std::string valueTmp = it->value().ToString();
+            std::string keyValue = keyTmp + valueTmp;
+            const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+            bytesConstRef strKeyValue( usc.data(), usc.size() );
+            secp256k1_sha256_write( &ctx, strKeyValue.data(), strKeyValue.size() );
         }
     }
     h256 hash;
@@ -294,14 +294,14 @@ void LevelDB::hashBasePartially(
         it->SeekToFirst();
 
     for ( size_t counter = 0; it->Valid() && counter < batchSize; it->Next() ) {
-        std::string key_ = it->key().ToString();
-        std::string value_ = it->value().ToString();
+        std::string keyTmp = it->key().ToString();
+        std::string valueTmp = it->value().ToString();
         // HACK! For backward compatibility! When snapshot could happen between update of two nodes
         // - it would lead to stateRoot mismatch
         // TODO Move this logic to separate "compatiliblity layer"!
-        if ( key_ == "pieceUsageBytes" )
+        if ( keyTmp == "pieceUsageBytes" )
             continue;
-        std::string keyValue = key_ + value_;
+        std::string keyValue = keyTmp + valueTmp;
         const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
         bytesConstRef strKeyValue( usc.data(), usc.size() );
         secp256k1_sha256_write( ctx, strKeyValue.data(), strKeyValue.size() );
diff --git a/skaled/main.cpp b/skaled/main.cpp
index 27469a750..b0ad1aca8 100644
--- a/skaled/main.cpp
+++ b/skaled/main.cpp
@@ -438,9 +438,9 @@ bool tryDownloadSnapshot( std::shared_ptr< SnapshotManager >& snapshotManager,
             try {
                 snapshotManager->computeSnapshotHash( blockNumber, true );
             } catch ( const std::exception& ) {
-                std::throw_with_nested(
-                    std::runtime_error( cc::fatal( "FATAL:" ) + " " +
-                                        cc::error( "Exception while computing snapshot hash " ) ) );
+                std::throw_with_nested( std::runtime_error(
+                    std::string( "FATAL:" ) +
+                    std::string( " Exception while computing snapshot hash " ) ) );
             }
 
             dev::h256 calculated_hash = snapshotManager->getSnapshotHash( blockNumber );
@@ -449,18 +449,15 @@ bool tryDownloadSnapshot( std::shared_ptr< SnapshotManager >& snapshotManager,
                 successfullDownload = true;
                 if ( isRegularSnapshot ) {
                     snapshotManager->restoreSnapshot( blockNumber );
-                    std::cout << cc::success( "Snapshot restore success for block " )
-                              << cc::u( to_string( blockNumber ) ) << std::endl;
+                    std::cout << "Snapshot restore success for block " << to_string( blockNumber )
+                              << std::endl;
                 }
                 return successfullDownload;
             } else {
                 clog( VerbosityWarning, "tryDownloadSnapshot" )
-                    << cc::notice(
-                           "Downloaded snapshot with incorrect hash! Incoming "
-                           "hash " )
-                    << cc::notice( votedHash.first.hex() )
-                    << cc::notice( " is not equal to calculated hash " )
-                    << cc::notice( calculated_hash.hex() ) << cc::notice( " Will try again" );
+                    << "Downloaded snapshot with incorrect hash! Incoming hash "
+                    << votedHash.first.hex() << " is not equal to calculated hash "
+                    << calculated_hash.hex() << " Will try again";
                 if ( isRegularSnapshot )
                     snapshotManager->cleanup();
                 else

From ac6490e83f6bc6a8327f459fd9d164b766006dfd Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Thu, 18 Jan 2024 12:13:13 +0000
Subject: [PATCH 11/12] #1588 move batchCHunkSize to class field

---
 libdevcore/LevelDB.cpp                     | 7 ++++---
 libdevcore/LevelDB.h                       | 5 +++--
 test/unittests/libweb3core/LevelDBHash.cpp | 2 +-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp
index 214c49522..2fb094469 100644
--- a/libdevcore/LevelDB.cpp
+++ b/libdevcore/LevelDB.cpp
@@ -28,6 +28,8 @@ namespace db {
 
 unsigned c_maxOpenLeveldbFiles = 25;
 
+const size_t LevelDB::BATCH_CHUNK_SIZE = 10000;
+
 namespace {
 inline leveldb::Slice toLDBSlice( Slice _slice ) {
     return leveldb::Slice( _slice.data(), _slice.size() );
@@ -281,8 +283,7 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
     return hash;
 }
 
-void LevelDB::hashBasePartially(
-    secp256k1_sha256_t* ctx, std::string& lastHashedKey, size_t batchSize ) const {
+void LevelDB::hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const {
     std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
@@ -293,7 +294,7 @@ void LevelDB::hashBasePartially(
     else
         it->SeekToFirst();
 
-    for ( size_t counter = 0; it->Valid() && counter < batchSize; it->Next() ) {
+    for ( size_t counter = 0; it->Valid() && counter < BATCH_CHUNK_SIZE; it->Next() ) {
         std::string keyTmp = it->key().ToString();
         std::string valueTmp = it->value().ToString();
         // HACK! For backward compatibility! When snapshot could happen between update of two nodes
diff --git a/libdevcore/LevelDB.h b/libdevcore/LevelDB.h
index 01461f4c7..7a316f99c 100644
--- a/libdevcore/LevelDB.h
+++ b/libdevcore/LevelDB.h
@@ -61,8 +61,7 @@ class LevelDB : public DatabaseFace {
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 
-    void hashBasePartially(
-        secp256k1_sha256_t* ctx, std::string& lastHashedKey, size_t batchSize = 10000 ) const;
+    void hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const;
 
     void doCompaction() const;
 
@@ -79,6 +78,8 @@ class LevelDB : public DatabaseFace {
     leveldb::WriteOptions const m_writeOptions;
     leveldb::Options m_options;
     boost::filesystem::path const m_path;
+
+    static const size_t BATCH_CHUNK_SIZE;
 };
 
 }  // namespace db
diff --git a/test/unittests/libweb3core/LevelDBHash.cpp b/test/unittests/libweb3core/LevelDBHash.cpp
index 1eec5366e..d3324fa60 100644
--- a/test/unittests/libweb3core/LevelDBHash.cpp
+++ b/test/unittests/libweb3core/LevelDBHash.cpp
@@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE( hash ) {
                 dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
                 dev::db::LevelDB::defaultSnapshotDBOptions() ) );
 
-            m_db->hashBasePartially( &dbCtx, lastHashedKey, 10 );
+            m_db->hashBasePartially( &dbCtx, lastHashedKey );
         }
 
         secp256k1_sha256_finalize( &dbCtx, hashPartially.data() );

From a79c45da42865e5723e031d41146dfe2d88f464b Mon Sep 17 00:00:00 2001
From: Oleh Nikolaiev <oleg@skalelabs.com>
Date: Thu, 18 Jan 2024 12:25:18 +0000
Subject: [PATCH 12/12] #1588 bool flag to continue db hash computation

---
 libdevcore/LevelDB.cpp                     | 9 +++++----
 libdevcore/LevelDB.h                       | 2 +-
 libskale/SnapshotManager.cpp               | 5 +++--
 test/unittests/libweb3core/LevelDBHash.cpp | 5 +++--
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp
index 2fb094469..b568d85da 100644
--- a/libdevcore/LevelDB.cpp
+++ b/libdevcore/LevelDB.cpp
@@ -283,7 +283,7 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
     return hash;
 }
 
-void LevelDB::hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const {
+bool LevelDB::hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const {
     std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
@@ -309,10 +309,11 @@ void LevelDB::hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashe
         ++counter;
     }
 
-    if ( it->Valid() )
+    if ( it->Valid() ) {
         lastHashedKey = it->key().ToString();
-    else
-        lastHashedKey = "stop";
+        return true;
+    } else
+        return false;
 }
 
 void LevelDB::doCompaction() const {
diff --git a/libdevcore/LevelDB.h b/libdevcore/LevelDB.h
index 7a316f99c..5c314e8bf 100644
--- a/libdevcore/LevelDB.h
+++ b/libdevcore/LevelDB.h
@@ -61,7 +61,7 @@ class LevelDB : public DatabaseFace {
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 
-    void hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const;
+    bool hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const;
 
     void doCompaction() const;
 
diff --git a/libskale/SnapshotManager.cpp b/libskale/SnapshotManager.cpp
index 8989b4274..8729d2a74 100644
--- a/libskale/SnapshotManager.cpp
+++ b/libskale/SnapshotManager.cpp
@@ -449,13 +449,14 @@ void SnapshotManager::computeDatabaseHash(
     secp256k1_sha256_initialize( &dbCtx );
 
     std::string lastHashedKey = "start";
+    bool isContinue = true;
 
-    while ( lastHashedKey != "stop" ) {
+    while ( isContinue ) {
         std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( _dbDir.string(),
             dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
             dev::db::LevelDB::defaultSnapshotDBOptions() ) );
 
-        m_db->hashBasePartially( &dbCtx, lastHashedKey );
+        isContinue = m_db->hashBasePartially( &dbCtx, lastHashedKey );
     }
 
     dev::h256 dbHash;
diff --git a/test/unittests/libweb3core/LevelDBHash.cpp b/test/unittests/libweb3core/LevelDBHash.cpp
index d3324fa60..f7c444dcd 100644
--- a/test/unittests/libweb3core/LevelDBHash.cpp
+++ b/test/unittests/libweb3core/LevelDBHash.cpp
@@ -74,12 +74,13 @@ BOOST_AUTO_TEST_CASE( hash ) {
         secp256k1_sha256_initialize( &dbCtx );
 
         std::string lastHashedKey = "start";
-        while ( lastHashedKey != "stop" ) {
+        bool isContinue = true;
+        while ( isContinue ) {
             std::unique_ptr< dev::db::LevelDB > m_db( new dev::db::LevelDB( td.path(),
                 dev::db::LevelDB::defaultSnapshotReadOptions(), dev::db::LevelDB::defaultWriteOptions(),
                 dev::db::LevelDB::defaultSnapshotDBOptions() ) );
 
-            m_db->hashBasePartially( &dbCtx, lastHashedKey );
+            isContinue = m_db->hashBasePartially( &dbCtx, lastHashedKey );
         }
 
         secp256k1_sha256_finalize( &dbCtx, hashPartially.data() );