Merge pull request #1734 from skalenetwork/1664_eth_trace_current

1664 Implement tracing

.github/workflows/clang-format-check.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - uses: DoozyX/clang-format-lint-action@v0.14
+    - uses: DoozyX/clang-format-lint-action@v0.16.2
       with:
         source: '.'
         exclude: './CMakeFiles ./cmake ./deps ./build ./skaled_ssl_test ./newer_lcov'

.github/workflows/test.yml

@@ -34,6 +34,7 @@ jobs:
           ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       - name: checkout
         uses: actions/checkout@v2
+
       - name: Cache apt packages
         uses: actions/cache@v2
         with:
@@ -117,12 +118,11 @@ jobs:
           sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 9
       - name: Configure ccache cache size, zero ccache counters and print ccache stats before start
         run: |
-          ccache --max-size=15G
+          ccache --max-size=30G
           ccache -z
           ccache --show-stats
       - name: Build dependencies
-        run: |
-          
+        run: |          
           export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
           export CC=gcc-9
           export CXX=g++-9
@@ -134,32 +134,73 @@ jobs:
           rm -f ./libwebsockets-from-git.tar.gz
           ./build.sh DEBUG=1 PARALLEL_COUNT=$(nproc)
           cd ..
-      - name: Configure all
+      - name: Print ccache stats for deps
+        run: |
+          ccache --show-stats
+      - name: Configure all as historic
+        run: |
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" &&
+          export CC=gcc-9 &&
+          export CXX=g++-9 &&
+          export TARGET=all &&
+          export CMAKE_BUILD_TYPE=Debug &&
+          export CODE_COVERAGE=ON &&
+          mkdir -p build &&
+          cd build &&
+          # -DCMAKE_C_FLAGS=-O3 -DCMAKE_CXX_FLAGS=-O3
+          cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCOVERAGE=$CODE_COVERAGE -DHISTORIC_STATE=1 ..
+          cd ..
+      - name: Build all historic
         run: |
           export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
           export CMAKE_BUILD_TYPE=Debug
           export CODE_COVERAGE=ON
-          mkdir -p build
           cd build
-          # -DCMAKE_C_FLAGS=-O3 -DCMAKE_CXX_FLAGS=-O3
-          cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCOVERAGE=$CODE_COVERAGE ..
+          make skaled testeth -j$(nproc)                              
           cd ..
-      - name: Print ccache stats for deps
-        run: |
+      - name: Print ccache stats after full historic build
+        run : |
           ccache --show-stats
-      - name: Build all
+      - name: Use Node.js 19
+        uses: actions/setup-node@v1
+        with:
+          node-version: 19
+      - name: run historic tests        
+        run: |
+          build/skaled/skaled --config test/historicstate/configs/basic_config.json& 
+          SKALED_PID=$! &&                  
+          cd test/historicstate/hardhat/ &&
+          npm install --save-dev typescript ts-node @types/node @types/mocha &&          
+          npx hardhat check &&
+          sleep 60 &&
+          npx hardhat run scripts/trace.ts --network skaled &&
+          kill $SKALED_PID
+      - name: Configure all
         run: |
           export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
           export CMAKE_BUILD_TYPE=Debug
           export CODE_COVERAGE=ON
+          mkdir -p build
           cd build
-          make testeth -j$(nproc)
+          # -DCMAKE_C_FLAGS=-O3 -DCMAKE_CXX_FLAGS=-O3
+          cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCOVERAGE=$CODE_COVERAGE ..
+          cd ..
+      - name: Build all
+        run: |
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" &&
+          export CC=gcc-9 &&
+          export CXX=g++-9 &&
+          export TARGET=all &&
+          export CMAKE_BUILD_TYPE=Debug &&
+          export CODE_COVERAGE=ON &&
+          cd build &&
+          make skaled testeth -j$(nproc) &&          
           cd ..
       - name: Print ccache stats after full build
         run : |
@@ -283,30 +324,5 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.info
 
-      - name: Configure all as historic
-        run: |
-          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
-          export CC=gcc-9
-          export CXX=g++-9
-          export TARGET=all
-          export CMAKE_BUILD_TYPE=Debug
-          export CODE_COVERAGE=ON
-          mkdir -p build
-          cd build
-          # -DCMAKE_C_FLAGS=-O3 -DCMAKE_CXX_FLAGS=-O3
-          cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCOVERAGE=$CODE_COVERAGE -DHISTORIC_STATE=1 ..
-          cd ..
-      - name: Build all historic
-        run: |
-          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
-          export CC=gcc-9
-          export CXX=g++-9
-          export TARGET=all
-          export CMAKE_BUILD_TYPE=Debug
-          export CODE_COVERAGE=ON
-          cd build
-          make testeth -j$(nproc)          
-          cd ..
-      - name: Print ccache stats after full historic build
-        run : |
-          ccache --show-stats
+
+

.gitmodules

@@ -15,3 +15,6 @@
 [submodule "cmake/cable"]
 	path = cmake/cable
 	url = https://github.com/ethereum/cable.git
+[submodule "test/historicstate/hardhat/tracely"]
+	path = test/historicstate/hardhat/tracely
+	url = https://github.com/DenrianWeiss/tracely

README.md

@@ -60,7 +60,7 @@ If you have already cloned the repo and forgot to pass `--recurse-submodules`, e
 sudo apt update
 sudo apt install autoconf build-essential cmake libprocps-dev libtool texinfo wget yasm flex bison btrfs-progs python3 python3-pip gawk git vim doxygen 
 sudo apt install make build-essential cmake pkg-config libgnutls28-dev libssl-dev unzip zlib1g-dev libgcrypt20-dev docker.io gcc-9 g++-9 gperf clang-format-11 gnutls-dev
-sudo apt install nettle-dev libhiredis-dev redis-server google-perftools libgoogle-perftools-dev lcov
+sudo apt install nettle-dev libhiredis-dev redis-server google-perftools libgoogle-perftools-dev lcov sudo apt-get install libv8-dev
 ```
 
 

VERSION

@@ -1 +1 @@
-3.17.2
+3.17.2

docs/tracing.md

@@ -0,0 +1,61 @@
+# Tracing API
+
+## API calls
+
+SKALE tracing API implements the following Geth tracing API calls
+
+```angular2html
+
+  debug_traceTransaction
+  debug_traceCall
+  debug_traceBlockByNumber
+  debug_traceBlockByHash
+```
+
+The calls a fully compatible with Geth API. If there is 
+an incompatibility, its a bug.
+
+Geth API is documented here
+
+https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-debug
+
+Also see here for live examples
+
+https://www.quicknode.com/docs/ethereum/debug_traceTransaction
+https://www.quicknode.com/docs/ethereum/debug_traceBlockByNumber
+https://www.quicknode.com/docs/ethereum/debug_traceBlockByHash
+https://www.quicknode.com/docs/ethereum/debug_traceCall
+
+
+## Tracer config and types implemented
+
+All tracer config options documented here are implemented
+
+https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-debug#traceconfig
+
+The following Geth Tracer types are implemented:
+
+* "4byteTracer"
+* "callTracer"
+* "prestateTracer"
+* "noopTracer"
+
+In addition the following Parity tracer is implemented
+
+* replayTracer
+
+See here for documentation of replayTracer
+
+https://openethereum.github.io/JSONRPC-trace-module
+https://www.quicknode.com/docs/ethereum/trace_replayTransaction
+https://docs.alchemy.com/reference/trace-replaytransaction
+
+Note, that we do not implement Parity "trace_replayTransaction"
+API call. Instead, "replayTracer" parameter needs to be 
+passed to Geth API calls.
+
+
+## All Tracer 
+
+* allTracer has beeen added to help QA, it prints results of all supported traces at once 
+

libdevcore/CommonData.h

@@ -203,6 +203,16 @@ inline std::string toCompactHexPrefixed( u256 _val, unsigned _min = 0 ) {
     return toHexPrefixed( toCompactBigEndian( _val, _min ) );
 }
 
+
+inline std::string toHex( u256 _val ) {
+    return toHex( toBigEndian( _val ) );
+}
+
+inline std::string toHexPrefixed( u256 _val ) {
+    return toHexPrefixed( toBigEndian( _val ) );
+}
+
+
 // Algorithms for string and string-like collections.
 
 /// Escapes a string into the C-string representation.

libethereum/Block.cpp

@@ -793,24 +793,39 @@ u256 Block::enact( VerifiedBlockRef const& _block, BlockChain const& _bc ) {
 
 
 #ifdef HISTORIC_STATE
-ExecutionResult Block::executeHistoricCall(
-    LastBlockHashesFace const& _lh, Transaction const& _t ) {
-    auto p = Permanence::Reverted;
-
+ExecutionResult Block::executeHistoricCall( LastBlockHashesFace const& _lh, Transaction const& _t,
+    std::shared_ptr< AlethStandardTrace > _tracer, uint64_t _transactionIndex ) {
     auto onOp = OnOpFunc();
 
+    if ( _tracer ) {
+        onOp = _tracer->functionToExecuteOnEachOperation();
+    }
+
+
     if ( isSealed() )
         BOOST_THROW_EXCEPTION( InvalidOperationOnSealedBlock() );
 
     // Uncommitting is a non-trivial operation - only do it once we've verified as much of the
     // transaction as possible.
     uncommitToSeal();
 
-    EnvInfo const envInfo{ info(), _lh, gasUsed(), m_sealEngine->chainParams().chainID };
-    std::pair< ExecutionResult, TransactionReceipt > resultReceipt =
-        m_state.mutableHistoricState().execute( envInfo, *m_sealEngine, _t, p, onOp );
+    u256 const gasUsed =
+        _transactionIndex ? receipt( _transactionIndex - 1 ).cumulativeGasUsed() : 0;
 
-    return resultReceipt.first;
+    EnvInfo const envInfo{ info(), _lh, gasUsed, m_sealEngine->chainParams().chainID };
+
+    if ( _tracer ) {
+        HistoricState stateBefore( m_state.mutableHistoricState() );
+        auto resultReceipt = m_state.mutableHistoricState().execute(
+            envInfo, *m_sealEngine, _t, skale::Permanence::Uncommitted, onOp );
+        HistoricState stateAfter( m_state.mutableHistoricState() );
+        _tracer->finalizeTrace( resultReceipt.first, stateBefore, stateAfter );
+        return resultReceipt.first;
+    } else {
+        auto resultReceipt = m_state.mutableHistoricState().execute(
+            envInfo, *m_sealEngine, _t, skale::Permanence::Reverted, onOp );
+        return resultReceipt.first;
+    }
 }
 #endif
 

libethereum/Block.h

@@ -35,7 +35,9 @@
 #include <libethcore/Counter.h>
 #include <libethcore/Exceptions.h>
 #include <libskale/State.h>
-
+#ifdef HISTORIC_STATE
+#include <libhistoric/AlethStandardTrace.h>
+#endif
 #include "Account.h"
 #include "GasPricer.h"
 #include "Transaction.h"
@@ -214,9 +216,9 @@ class Block {
     ExecutionResult execute( LastBlockHashesFace const& _lh, Transaction const& _t,
         skale::Permanence _p = skale::Permanence::Committed, OnOpFunc const& _onOp = OnOpFunc() );
 
-
 #ifdef HISTORIC_STATE
-    ExecutionResult executeHistoricCall( LastBlockHashesFace const& _lh, Transaction const& _t );
+    ExecutionResult executeHistoricCall( LastBlockHashesFace const& _lh, Transaction const& _t,
+        std::shared_ptr< AlethStandardTrace > _tracer, uint64_t _transactionIndex );
 #endif
 
 

libethereum/CMakeLists.txt

@@ -10,7 +10,7 @@ target_include_directories( ethereum PRIVATE "${UTILS_INCLUDE_DIR}" ${SKUTILS_IN
         ${CMAKE_SOURCE_DIR}/libconsensus/jsoncpp/include
         ${CMAKE_SOURCE_DIR}/libconsensus/spdlog/include
         ${CMAKE_SOURCE_DIR}/libconsensus/libjson/include)
-target_link_libraries( ethereum PUBLIC evm ethcore p2p devcrypto devcore skale PRIVATE skutils Snappy::snappy
+target_link_libraries( ethereum PUBLIC evm ethcore p2p devcrypto devcore skale PRIVATE historic skutils Snappy::snappy
         jsoncpp # ${CMAKE_SOURCE_DIR}/libconsensus/jsoncpp/build/src/lib_json/libjsoncpp.a
         Boost::fiber Boost::context Boost::chrono
         batched-io