Merge pull request #1791 from skalenetwork/v3.18.0

V3.18.0 to develop

.github/workflows/functional-tests.yml

@@ -24,6 +24,7 @@
       with:
         token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
         repository: skalenetwork/skale-ci-integration_tests
+        ref: v3.18.0
         submodules: recursive
     - name: Set up Node 
       uses: actions/[email protected]
@@ -65,8 +66,8 @@
     - name: skaled+load_python+all
       run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+load_python+all
 
-    # - name: skaled+load_js+run_angry_cats
-    #   run: SKALED_PROVIDER=skaled_providers/endpoint_by_container ./run_tests.sh skaled+load_js+run_angry_cats
+    - name: skaled+load_js+run_angry_cats
+      run: SKALED_PROVIDER=skaled_providers/endpoint_by_container ./run_tests.sh skaled+load_js+run_angry_cats
 
     - name: skaled+internals+test_snapshot_api
       run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+internals+test_snapshot_api

.github/workflows/publish.yml

@@ -150,6 +150,7 @@ jobs:
           export VERSION=$(cat VERSION)
           export VERSION=$(bash ./scripts/calculate_version.sh $BRANCH $VERSION)
           echo "::set-env name=VERSION::$VERSION"
+          echo "::set-env name=VERSION_ORIG::$VERSION"
           echo "Version $VERSION"
           ( test $BRANCH = "stable" && export PRERELEASE=false ) || export PRERELEASE=true
           echo "PRERELEASE=$PRERELEASE" >> $GITHUB_ENV
@@ -191,6 +192,7 @@ jobs:
           echo "Branch $BRANCH"
           export VERSION=$VERSION-historic
           echo "::set-env name=VERSION::$VERSION"
+          echo "::set-env name=VERSION_HISTORIC::$VERSION"
           echo "Version $VERSION"
           export RELEASE=true
           echo "::set-env name=RELEASE::$RELEASE"
@@ -214,12 +216,21 @@ jobs:
           asset_name: skaled-debug-historic
           asset_content_type: application/octet-stream
     outputs:
-      version: ${{ env.VERSION }}
+      version_orig: ${{ env.VERSION_ORIG }}
+      version_historic: ${{ env.VERSION_HISTORIC }}
 
   functional-tests:
     uses: ./.github/workflows/functional-tests.yml
     name: Functional testing for build
     needs: [build]
     with:
-      version: ${{ needs.build.outputs.version }}
+      version: ${{ needs.build.outputs.version_orig }}
+    secrets: inherit
+
+  functional-tests-historic:
+    uses: ./.github/workflows/functional-tests.yml
+    name: Functional testing for build
+    needs: [build]
+    with:
+      version: ${{ needs.build.outputs.version_historic }}
     secrets: inherit

VERSION

@@ -1 +1 @@
-3.17.1
+3.18.0

deps/build.sh

@@ -1223,7 +1223,7 @@ then
 			#		#CMAKE_ARGS_FOR_LIB_WEB_SOCKETS="$CMAKE_ARGS_FOR_LIB_WEB_SOCKETS -DLWS_WITH_LIBEV=ON"
 			#		LWS_WITH_LIBEV=ON
 			#	fi
-			#else
+		#else
 			##	#CMAKE_ARGS_FOR_LIB_WEB_SOCKETS="$CMAKE_ARGS_FOR_LIB_WEB_SOCKETS -DLWS_WITH_LIBEV=OFF"
 			#	echo " "
 			#fi
@@ -1383,7 +1383,7 @@ then
 			if [ ! -f "boost_1_68_0.tar.bz2" ];
 			then
 				echo -e "${COLOR_INFO}downloading it${COLOR_DOTS}...${COLOR_RESET}"
-				eval "$WGET" https://boostorg.jfrog.io/artifactory/main/release/1.68.0/source/boost_1_68_0.tar.bz2
+				eval "$WGET" https://sourceforge.net/projects/boost/files/boost/1.68.0/boost_1_68_0.tar.bz2
 			fi
 			echo -e "${COLOR_INFO}unpacking it${COLOR_DOTS}...${COLOR_RESET}"
                         eval tar -xf boost_1_68_0.tar.bz2

libdevcore/CMakeLists.txt

@@ -15,8 +15,9 @@ add_library(devcore ${sources} ${headers})
 add_dependencies(devcore secp256k1)
 
 target_compile_options( devcore PRIVATE
-    -Wno-error=deprecated-copy -Wno-error=unused-result -Wno-error=unused-parameter -Wno-error=unused-variable -Wno-error=maybe-uninitialized
-    )
+    -Wno-error=deprecated-copy -Wno-error=unused-result -Wno-error=unused-parameter
+    -Wno-error=unused-variable -Wno-error=maybe-uninitialized -Wno-error=class-memaccess
+)
 
 # Needed to prevent including system-level boost headers:
 target_include_directories(devcore SYSTEM PUBLIC ${Boost_INCLUDE_DIR} PRIVATE ../utils)

libdevcore/Common.cpp

@@ -47,14 +47,18 @@ void ExitHandler::exitHandler( int s ) {
 }
 
 void ExitHandler::exitHandler( int nSignalNo, ExitHandler::exit_code_t ec ) {
-    std::string strMessagePrefix = ExitHandler::shouldExit() ?
-                                       cc::error( "\nStop flag was already raised on. " ) +
-                                           cc::fatal( "WILL FORCE TERMINATE." ) +
-                                           cc::error( " Caught (second) signal. " ) :
-                                       cc::error( "\nCaught (first) signal. " );
-    std::cerr << strMessagePrefix << cc::error( skutils::signal::signal2str( nSignalNo ) )
-              << "\n\n";
-    std::cerr.flush();
+    std::string strMessagePrefix;
+    if ( nSignalNo > 0 ) {
+        strMessagePrefix = ( ExitHandler::shouldExit() && s_nStopSignal > 0 ) ?
+                               string( "\nStop flag was already raised on. " ) +
+                                   "WILL FORCE TERMINATE." + " Caught (second) signal. " :
+                               "\nCaught (first) signal. ";
+    } else {
+        strMessagePrefix = ExitHandler::shouldExit() ?
+                               string( "\nInternal exit requested while already exiting. " ) :
+                               "\nInternal exit initiated. ";
+    }
+    std::cerr << strMessagePrefix << skutils::signal::signal2str( nSignalNo ) << "\n\n";
 
     switch ( nSignalNo ) {
     case SIGINT:
@@ -102,10 +106,10 @@ void ExitHandler::exitHandler( int nSignalNo, ExitHandler::exit_code_t ec ) {
             auto start_time = std::chrono::steady_clock::now();
 
             std::thread( [nSignalNo, start_time]() {
-                std::cerr << ( "\n" + cc::fatal( "SELF-KILL:" ) + " " + cc::error( "Will sleep " ) +
-                               cc::size10( ExitHandler::KILL_TIMEOUT ) +
-                               cc::error( " seconds before force exit..." ) + "\n\n" );
-                std::cerr.flush();
+                std::cerr << ( "\n" + string( "SELF-KILL:" ) + " " + "Will sleep " +
+                                 cc::size10( ExitHandler::KILL_TIMEOUT ) +
+                                 " seconds before force exit..." ) +
+                                 "\n\n";
 
                 clog( VerbosityInfo, "exit" ) << "THREADS timer started";
 
@@ -133,11 +137,10 @@ void ExitHandler::exitHandler( int nSignalNo, ExitHandler::exit_code_t ec ) {
                     std::this_thread::sleep_for( 100ms );
                 }
 
-                std::cerr << ( "\n" + cc::fatal( "SELF-KILL:" ) + " " +
-                               cc::error( "Will force exit after sleeping " ) +
+                std::cerr << ( "\n" + string( "SELF-KILL:" ) + " " +
+                               "Will force exit after sleeping " +
                                cc::size10( ExitHandler::KILL_TIMEOUT ) + cc::error( " second(s)" ) +
                                "\n\n" );
-                std::cerr.flush();
 
                 // TODO deduplicate this with main() before return
                 ExitHandler::exit_code_t ec = ExitHandler::requestedExitCode();
@@ -153,9 +156,9 @@ void ExitHandler::exitHandler( int nSignalNo, ExitHandler::exit_code_t ec ) {
 
     // nice exit here:
 
-    if ( ExitHandler::shouldExit() ) {
-        std::cerr << ( "\n" + cc::fatal( "SIGNAL-HANDLER:" ) + " " +
-                       cc::error( "Will force exit now..." ) + "\n\n" );
+    // TODO deduplicate with first if()
+    if ( ExitHandler::shouldExit() && s_nStopSignal > 0 && nSignalNo > 0 ) {
+        std::cerr << ( "\n" + string( "SIGNAL-HANDLER:" ) + " " + "Will force exit now...\n\n" );
         _exit( 13 );
     }
 

libdevcore/LevelDB.cpp

@@ -22,13 +22,14 @@
 #include "Assertions.h"
 #include "Log.h"
 #include <libdevcore/microprofile.h>
-#include <secp256k1_sha256.h>
 
 namespace dev {
 namespace db {
 
 unsigned c_maxOpenLeveldbFiles = 25;
 
+const size_t LevelDB::BATCH_CHUNK_SIZE = 10000;
+
 namespace {
 inline leveldb::Slice toLDBSlice( Slice _slice ) {
     return leveldb::Slice( _slice.data(), _slice.size() );
@@ -213,7 +214,6 @@ void LevelDB::forEach( std::function< bool( Slice, Slice ) > f ) const {
     }
 }
 
-
 void LevelDB::forEachWithPrefix(
     std::string& _prefix, std::function< bool( Slice, Slice ) > f ) const {
     cnote << "Iterating over the LevelDB prefix: " << _prefix;
@@ -238,21 +238,23 @@ h256 LevelDB::hashBase() const {
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
-        std::string key_ = it->key().ToString();
-        std::string value_ = it->value().ToString();
+        std::string keyTmp = it->key().ToString();
+        std::string valueTmp = it->value().ToString();
         // HACK! For backward compatibility! When snapshot could happen between update of two nodes
         // - it would lead to stateRoot mismatch
         // TODO Move this logic to separate "compatiliblity layer"!
-        if ( key_ == "pieceUsageBytes" )
+        if ( keyTmp == "pieceUsageBytes" )
             continue;
-        std::string key_value = key_ + value_;
-        const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
-        bytesConstRef str_key_value( usc.data(), usc.size() );
-        secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
+        std::string keyValue = keyTmp + valueTmp;
+        const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+        bytesConstRef strKeyValue( usc.data(), usc.size() );
+        secp256k1_sha256_write( &ctx, strKeyValue.data(), strKeyValue.size() );
     }
+
     h256 hash;
     secp256k1_sha256_finalize( &ctx, hash.data() );
     return hash;
@@ -263,23 +265,57 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
     if ( it == nullptr ) {
         BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
     }
+
     secp256k1_sha256_t ctx;
     secp256k1_sha256_initialize( &ctx );
     for ( it->SeekToFirst(); it->Valid(); it->Next() ) {
         if ( it->key()[0] == _prefix ) {
-            std::string key_ = it->key().ToString();
-            std::string value_ = it->value().ToString();
-            std::string key_value = key_ + value_;
-            const std::vector< uint8_t > usc( key_value.begin(), key_value.end() );
-            bytesConstRef str_key_value( usc.data(), usc.size() );
-            secp256k1_sha256_write( &ctx, str_key_value.data(), str_key_value.size() );
+            std::string keyTmp = it->key().ToString();
+            std::string valueTmp = it->value().ToString();
+            std::string keyValue = keyTmp + valueTmp;
+            const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+            bytesConstRef strKeyValue( usc.data(), usc.size() );
+            secp256k1_sha256_write( &ctx, strKeyValue.data(), strKeyValue.size() );
         }
     }
     h256 hash;
     secp256k1_sha256_finalize( &ctx, hash.data() );
     return hash;
 }
 
+bool LevelDB::hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const {
+    std::unique_ptr< leveldb::Iterator > it( m_db->NewIterator( m_readOptions ) );
+    if ( it == nullptr ) {
+        BOOST_THROW_EXCEPTION( DatabaseError() << errinfo_comment( "null iterator" ) );
+    }
+
+    if ( lastHashedKey != "start" )
+        it->Seek( lastHashedKey );
+    else
+        it->SeekToFirst();
+
+    for ( size_t counter = 0; it->Valid() && counter < BATCH_CHUNK_SIZE; it->Next() ) {
+        std::string keyTmp = it->key().ToString();
+        std::string valueTmp = it->value().ToString();
+        // HACK! For backward compatibility! When snapshot could happen between update of two nodes
+        // - it would lead to stateRoot mismatch
+        // TODO Move this logic to separate "compatiliblity layer"!
+        if ( keyTmp == "pieceUsageBytes" )
+            continue;
+        std::string keyValue = keyTmp + valueTmp;
+        const std::vector< uint8_t > usc( keyValue.begin(), keyValue.end() );
+        bytesConstRef strKeyValue( usc.data(), usc.size() );
+        secp256k1_sha256_write( ctx, strKeyValue.data(), strKeyValue.size() );
+        ++counter;
+    }
+
+    if ( it->Valid() ) {
+        lastHashedKey = it->key().ToString();
+        return true;
+    } else
+        return false;
+}
+
 void LevelDB::doCompaction() const {
     m_db->CompactRange( nullptr, nullptr );
 }

libdevcore/LevelDB.h

@@ -26,6 +26,8 @@
 #include <leveldb/write_batch.h>
 #include <boost/filesystem.hpp>
 
+#include <secp256k1_sha256.h>
+
 namespace dev {
 namespace db {
 class LevelDB : public DatabaseFace {
@@ -59,6 +61,8 @@ class LevelDB : public DatabaseFace {
     h256 hashBase() const override;
     h256 hashBaseWithPrefix( char _prefix ) const;
 
+    bool hashBasePartially( secp256k1_sha256_t* ctx, std::string& lastHashedKey ) const;
+
     void doCompaction() const;
 
     // Return the total count of key deletes  since the start
@@ -74,6 +78,8 @@ class LevelDB : public DatabaseFace {
     leveldb::WriteOptions const m_writeOptions;
     leveldb::Options m_options;
     boost::filesystem::path const m_path;
+
+    static const size_t BATCH_CHUNK_SIZE;
 };
 
 }  // namespace db

libethcore/ChainOperationParams.h

@@ -175,7 +175,10 @@ struct SChain {
     time_t verifyDaSigsPatchTimestamp = 0;
     time_t storageDestructionPatchTimestamp = 0;
     time_t powCheckPatchTimestamp = 0;
+    time_t precompiledConfigPatchTimestamp = 0;
+    time_t pushZeroPatchTimestamp = 0;
     time_t skipInvalidTransactionsPatchTimestamp = 0;
+    time_t correctForkInPowPatchTimestamp = 0;
 
     SChain() {
         name = "TestChain";

libethereum/Block.cpp

@@ -351,7 +351,7 @@ pair< TransactionReceipts, bool > Block::sync(
                                                                     // caller if we hit the limit
 
     for ( Transaction& transaction : transactions ) {
-        transaction.checkOutExternalGas( _bc.chainParams().externalGasDifficulty );
+        transaction.checkOutExternalGas( _bc.chainParams(), _bc.number() );
     }
 
     assert( _bc.currentHash() == m_currentBlock.parentHash() );
@@ -631,8 +631,7 @@ u256 Block::enact( VerifiedBlockRef const& _block, BlockChain const& _bc ) {
             //            << " (state #"
             //                 << state().getNonce( tr.from() ) << ") value = " << tr.value() <<
             //                 endl;
-            const_cast< Transaction& >( tr ).checkOutExternalGas(
-                _bc.chainParams().externalGasDifficulty );
+            const_cast< Transaction& >( tr ).checkOutExternalGas( _bc.chainParams(), _bc.number() );
             execute( _bc.lastBlockHashes(), tr );
             // cerr << "Now: "
             // << "State #" << state().getNonce( tr.from() ) << endl;

libethereum/ChainParams.cpp

@@ -266,11 +266,25 @@ ChainParams ChainParams::loadConfig(
                                        sChainObj.at( "powCheckPatchTimestamp" ).get_int64() :
                                        0;
 
+        s.precompiledConfigPatchTimestamp =
+            sChainObj.count( "precompiledConfigPatchTimestamp" ) ?
+                sChainObj.at( "precompiledConfigPatchTimestamp" ).get_int64() :
+                0;
+
+        s.pushZeroPatchTimestamp = sChainObj.count( "pushZeroPatchTimestamp" ) ?
+                                       sChainObj.at( "pushZeroPatchTimestamp" ).get_int64() :
+                                       0;
+
         s.skipInvalidTransactionsPatchTimestamp =
             sChainObj.count( "skipInvalidTransactionsPatchTimestamp" ) ?
                 sChainObj.at( "skipInvalidTransactionsPatchTimestamp" ).get_int64() :
                 0;
 
+        s.correctForkInPowPatchTimestamp =
+            sChainObj.count( "correctForkInPowPatchTimestamp" ) ?
+                sChainObj.at( "correctForkInPowPatchTimestamp" ).get_int64() :
+                0;
+
         if ( sChainObj.count( "nodeGroups" ) ) {
             std::vector< NodeGroup > nodeGroups;
             for ( const auto& nodeGroupConf : sChainObj["nodeGroups"].get_obj() ) {
@@ -284,9 +298,12 @@ ChainParams ChainParams::loadConfig(
                 auto groupNodesObj = nodeGroupObj["nodes"].get_obj();
                 for ( const auto& groupNodeConf : groupNodesObj ) {
                     auto groupNodeConfObj = groupNodeConf.second.get_array();
-                    u256 sChainIndex = groupNodeConfObj[0].get_uint64();
-                    u256 id = groupNodeConfObj[1].get_uint64();
-                    std::string publicKey = groupNodeConfObj[2].get_str();
+                    u256 sChainIndex = groupNodeConfObj.at( 0 ).get_uint64();
+                    u256 id = groupNodeConfObj.at( 1 ).get_uint64();
+                    std::string publicKey = groupNodeConfObj.at( 2 ).get_str();
+                    if ( publicKey.empty() ) {
+                        BOOST_THROW_EXCEPTION( std::runtime_error( "Empty public key in config" ) );
+                    }
                     groupNodes.push_back( { id, sChainIndex, publicKey } );
                 }
                 std::sort( groupNodes.begin(), groupNodes.end(),