Merge pull request #170 from okx/zjg/basedev-v2.60.0-beta10

Merge upstream from v2.60.0 beta10

.github/scripts/cpu_monitor.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# Configuration
+THRESHOLD=80
+MEASUREMENTS_FILE="/tmp/cpu_measurements.txt"
+MONITOR_INTERVAL=5  # seconds
+PROCESS_NAME="cdk-erigon"
+DETAILED_LOG="/tmp/cpu_detailed.log"
+
+# Function to get CPU usage for all matching processes
+get_process_cpu() {
+    # Clear previous detailed log
+    > "$DETAILED_LOG"
+
+    # Get PIDs of cdk-erigon processes
+    pids=$(pgrep -f "[c]dk-erigon")
+
+    if [ -n "$pids" ]; then
+        # Use top in batch mode for each PID to get current CPU usage
+        for pid in $pids; do
+            # Get process command
+            if [[ "$OSTYPE" == "darwin"* ]]; then
+                cmd=$(ps -p $pid -o command=)
+                cpu=$(top -l 1 -pid $pid | tail -1 | awk '{print $3}')
+            else
+                cmd=$(ps -p $pid -o cmd=)
+                cpu=$(top -b -n 1 -p $pid | tail -1 | awk '{print $9}')
+            fi
+            # Get current CPU usage
+            echo "$pid $cpu $cmd" >> "$DETAILED_LOG"
+        done
+    fi
+
+    # Sum total CPU usage
+    total_cpu=$(awk '{sum += $2} END {printf "%.1f", sum}' "$DETAILED_LOG")
+
+    # Return 0 if no process found
+    if [ -z "$total_cpu" ]; then
+        echo "0.0"
+    else
+        echo "$total_cpu"
+    fi
+}
+
+# Function to show current process details
+show_process_details() {
+    if [ -s "$DETAILED_LOG" ]; then
+        echo "Individual process details:"
+        printf "%-10s %-8s %-s\n" "PID" "CPU%" "Command"
+        echo "----------------------------------------"
+        while read -r line; do
+            pid=$(echo "$line" | awk '{print $1}')
+            cpu=$(echo "$line" | awk '{print $2}')
+            cmd=$(echo "$line" | cut -d' ' -f3-)
+            printf "%-10s %-8.1f %-s\n" "$pid" "$cpu" "$cmd"
+        done < "$DETAILED_LOG"
+        echo "----------------------------------------"
+    else
+        echo "No $PROCESS_NAME processes found"
+    fi
+}
+
+# Function to analyze CPU measurements
+analyze_cpu() {
+    if [ -f "$MEASUREMENTS_FILE" ]; then
+        # Calculate statistics
+        avg_cpu=$(awk '{ sum += $1 } END { print sum/NR }' "$MEASUREMENTS_FILE")
+        avg_cpu_rounded=$(printf "%.1f" "$avg_cpu")
+        max_cpu=$(awk 'BEGIN{max=0} {if($1>max) max=$1} END{print max}' "$MEASUREMENTS_FILE")
+        measurement_count=$(wc -l < "$MEASUREMENTS_FILE")
+
+        echo ""
+        echo "=== CPU Usage Analysis for all $PROCESS_NAME processes ==="
+        echo "Number of measurements: $measurement_count"
+        echo "Average Combined CPU Usage: $avg_cpu_rounded%"
+        echo "Peak Combined CPU Usage: $max_cpu%"
+        echo "Threshold: $THRESHOLD%"
+
+        # Get final process details for the report
+        echo ""
+        echo "Final process state:"
+        show_process_details
+
+        # Compare with threshold
+        if [ "$(echo "$avg_cpu > $THRESHOLD" | bc -l)" -eq 1 ]; then
+            echo ""
+            echo "ERROR: Average CPU usage ($avg_cpu_rounded%) exceeded threshold of $THRESHOLD%"
+            cleanup_and_exit 1
+        else
+            echo ""
+            echo "SUCCESS: CPU usage ($avg_cpu_rounded%) is within threshold of $THRESHOLD%"
+            cleanup_and_exit 0
+        fi
+    else
+        echo "ERROR: No CPU measurements found at $MEASUREMENTS_FILE"
+        cleanup_and_exit 1
+    fi
+}
+
+# Function to clean up and exit
+cleanup_and_exit() {
+    exit_code=$1
+    rm -f "$DETAILED_LOG"
+    exit $exit_code
+}
+
+# Function to handle interruption
+handle_interrupt() {
+    echo ""
+    echo "Monitoring interrupted. Analyzing collected data..."
+    analyze_cpu
+}
+
+# Set up trap for various signals
+trap handle_interrupt TERM INT
+
+# Clear measurements file
+> "$MEASUREMENTS_FILE"
+> "$DETAILED_LOG"
+
+echo "Starting CPU monitoring for all '$PROCESS_NAME' processes"
+echo "Storing measurements in $MEASUREMENTS_FILE"
+echo "Monitoring interval: ${MONITOR_INTERVAL}s"
+echo "Press Ctrl+C to stop monitoring and see analysis"
+echo ""
+
+# Start monitoring loop
+while true; do
+    # Get CPU usage for all matching processes
+    cpu_usage=$(get_process_cpu)
+    echo "$cpu_usage" >> "$MEASUREMENTS_FILE"
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - Combined CPU Usage: $cpu_usage%"
+    show_process_details
+    echo ""
+    sleep "$MONITOR_INTERVAL"
+done

.github/workflows/ci_zkevm.yml

@@ -116,6 +116,23 @@ jobs:
         run: | 
           kurtosis run --enclave cdk-v1 --image-download always . '{"args": {"data_availability_mode": "${{ matrix.da-mode }}", "cdk_erigon_node_image": "cdk-erigon:local"}}'
 
+      - name: Run process with CPU monitoring
+        working-directory: ./cdk-erigon
+        run: |
+          # Start monitoring in background
+          bash ./.github/scripts/cpu_monitor.sh &
+          monitor_pid=$!
+
+          # Wait for 30 seconds
+          sleep 30
+
+          # Stop monitoring and get analysis
+          kill -TERM $monitor_pid
+          wait $monitor_pid || {
+            echo "CPU usage exceeded threshold!"
+            exit 1
+          }
+
       - name: Monitor verified batches
         working-directory: ./kurtosis-cdk
         shell: bash

.gitignore

@@ -113,6 +113,7 @@ vendor
 
 # X Layer
 data/
+test/sqlite
 start.sh
 xlayerconfig-mainnet.yaml
 xlayerconfig-testnet.yaml

Makefile

@@ -194,6 +194,9 @@ lint:
 	@./erigon-lib/tools/golangci_lint.sh
 	@./erigon-lib/tools/mod_tidy_check.sh
 
+cpu_monitor:
+	@.github/scripts/cpu_monitor.sh
+
 ## clean:                             cleans the go cache, build dir, libmdbx db dir
 clean:
 	go clean -cache

README.md

@@ -208,6 +208,15 @@ Useful config entries:
 - `zkevm.sync-limit`: This will ensure the network only syncs to a given block height.
 - `debug.timers`: This will enable debug timers in the logs to help with performance tuning. Recording timings of witness generation, etc. at INFO level.
 
+Metrics and pprof configuration flags:
+
+- `metrics:` Enables or disables the metrics collection. Set to true to enable.
+- `metrics.addr`: The address on which the metrics server will listen. Default is "0.0.0.0".
+- `metrics.port`: The port on which the metrics server will listen. Default is 6060.
+- `pprof`: Enables or disables the pprof profiling. Set to true to enable.
+- `pprof.addr`: The address on which the pprof server will listen. Default is "0.0.0.0".
+- `pprof.port`: The port on which the pprof server will listen. Default is 6061.
+
 ***
 
 

cmd/rpcdaemon/cli/config_zkevm.go

@@ -2,11 +2,12 @@ package cli
 
 import (
 	"fmt"
-	"github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer"
+
+	"github.com/ledgerwatch/erigon/zk/datastream/server"
 	"github.com/ledgerwatch/log/v3"
 )
 
-func StartDataStream(server *datastreamer.StreamServer) error {
+func StartDataStream(server server.StreamServer) error {
 	if server == nil {
 		// no stream server to start, we might not have the right flags set to create one
 		return nil

cmd/rpcdaemon/health/check_time.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/ledgerwatch/erigon-lib/common/hexutil"
+
 	"github.com/ledgerwatch/erigon/rpc"
 )
 
@@ -20,13 +22,13 @@ func checkTime(
 	if err != nil {
 		return err
 	}
-	timestamp := 0
+	timestamp := uint64(0)
 	if ts, ok := i["timestamp"]; ok {
-		if cs, ok := ts.(uint64); ok {
-			timestamp = int(cs)
+		if cs, ok := ts.(hexutil.Uint64); ok {
+			timestamp = cs.Uint64()
 		}
 	}
-	if timestamp < seconds {
+	if timestamp < uint64(seconds) {
 		return fmt.Errorf("%w: got ts: %d, need: %d", errTimestampTooOld, timestamp, seconds)
 	}
 

cmd/rpcdaemon/health/health_test.go

@@ -245,7 +245,7 @@ func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
 			netApiResponse: hexutil.Uint(1),
 			netApiError:    nil,
 			ethApiBlockResult: map[string]interface{}{
-				"timestamp": uint64(time.Now().Add(-10 * time.Second).Unix()),
+				"timestamp": hexutil.Uint64(time.Now().Add(-10 * time.Second).Unix()),
 			},
 			ethApiBlockError:    nil,
 			ethApiSyncingResult: false,
@@ -264,7 +264,7 @@ func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
 			netApiResponse: hexutil.Uint(1),
 			netApiError:    nil,
 			ethApiBlockResult: map[string]interface{}{
-				"timestamp": uint64(time.Now().Add(-1 * time.Hour).Unix()),
+				"timestamp": hexutil.Uint64(time.Now().Add(-1 * time.Hour).Unix()),
 			},
 			ethApiBlockError:    nil,
 			ethApiSyncingResult: false,
@@ -283,7 +283,7 @@ func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
 			netApiResponse: hexutil.Uint(1),
 			netApiError:    nil,
 			ethApiBlockResult: map[string]interface{}{
-				"timestamp": uint64(time.Now().Add(1 * time.Hour).Unix()),
+				"timestamp": hexutil.Uint64(time.Now().Add(1 * time.Hour).Unix()),
 			},
 			ethApiBlockError:    nil,
 			ethApiSyncingResult: false,
@@ -319,7 +319,7 @@ func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
 			netApiResponse: hexutil.Uint(10),
 			netApiError:    nil,
 			ethApiBlockResult: map[string]interface{}{
-				"timestamp": uint64(time.Now().Add(1 * time.Second).Unix()),
+				"timestamp": hexutil.Uint64(time.Now().Add(1 * time.Second).Unix()),
 			},
 			ethApiBlockError:    nil,
 			ethApiSyncingResult: false,

cmd/utils/flags.go

@@ -758,6 +758,16 @@ var (
 		Usage: "Seal the batch immediately when detecting a counter overflow",
 		Value: false,
 	}
+	MockWitnessGeneration = cli.BoolFlag{
+		Name:  "zkevm.mock-witness-generation",
+		Usage: "Mock the witness generation",
+		Value: false,
+	}
+	WitnessContractInclusion = cli.StringFlag{
+		Name:  "zkevm.witness-contract-inclusion",
+		Usage: "Contracts that will have all of their storage added to the witness every time",
+		Value: "",
+	}
 	ACLPrintHistory = cli.IntFlag{
 		Name:  "acl.print-history",
 		Usage: "Number of entries to print from the ACL history on node start up",

core/state/intra_block_state.go

@@ -430,6 +430,11 @@ func (sdb *IntraBlockState) SeenAccount(addr libcommon.Address) bool {
 	return ok
 }
 
+func (sdb *IntraBlockState) IsDirtyJournal(addr libcommon.Address) bool {
+	_, ok := sdb.journal.dirties[addr]
+	return ok
+}
+
 func (sdb *IntraBlockState) HasLiveState(addr libcommon.Address, key *libcommon.Hash) bool {
 	if stateObject := sdb.stateObjects[addr]; stateObject != nil {
 		if _, ok := stateObject.originStorage[*key]; ok {

core/state/trie_db.go

@@ -885,7 +885,7 @@ func (tds *TrieDbState) GetTrieHash() common.Hash {
 	return tds.t.Hash()
 }
 
-func (tds *TrieDbState) ResolveSMTRetainList() (*trie.RetainList, error) {
+func (tds *TrieDbState) ResolveSMTRetainList(inclusion map[libcommon.Address][]libcommon.Hash) (*trie.RetainList, error) {
 	// Aggregating the current buffer, if any
 	if tds.currentBuffer != nil {
 		if tds.aggregateBuffer == nil {
@@ -967,6 +967,16 @@ func (tds *TrieDbState) ResolveSMTRetainList() (*trie.RetainList, error) {
 		keys = append(keys, smtPath)
 	}
 
+	for address, slots := range inclusion {
+		for _, slot := range slots {
+			smtPath, err := getSMTPath(address.String(), slot.String())
+			if err != nil {
+				return nil, err
+			}
+			keys = append(keys, smtPath)
+		}
+	}
+
 	rl := trie.NewRetainList(0)
 
 	for _, key := range keys {

core/vm/contracts_zkevm.go

@@ -395,18 +395,26 @@ func (c *bigModExp_zkevm) Run(input []byte) ([]byte, error) {
 		baseLen = new(big.Int).SetBytes(getData(input, 0, 32)).Uint64()
 		expLen  = new(big.Int).SetBytes(getData(input, 32, 32)).Uint64()
 		modLen  = new(big.Int).SetBytes(getData(input, 64, 32)).Uint64()
+		base = big.NewInt(0)
+		exp = big.NewInt(0)
+		mod = big.NewInt(0)
 	)
-	if len(input) > 96 {
-		input = input[96:]
-	} else {
-		input = input[:0]
+
+	if len(input) >= 96 + int(baseLen) {
+		base = new(big.Int).SetBytes(getData(input, 96, uint64(baseLen)))
+	}
+	if len(input) >= 96 + int(baseLen) + int(expLen) {
+		exp = new(big.Int).SetBytes(getData(input, 96 + uint64(baseLen), uint64(expLen)))
+	}
+	if len(input) >= 96 + int(baseLen) + int(expLen) + int(modLen) {
+		mod = new(big.Int).SetBytes(getData(input, 96 + uint64(baseLen) + uint64(expLen), uint64(modLen)))
+	}
+	if len(input) < 96 + int(baseLen) + int(expLen) + int(modLen) {
+		input = common.LeftPadBytes(input, 96 + int(baseLen) + int(expLen) + int(modLen))
 	}
 
 	// Retrieve the operands and execute the exponentiation
 	var (
-		base       = new(big.Int).SetBytes(getData(input, 0, baseLen))
-		exp        = new(big.Int).SetBytes(getData(input, baseLen, expLen))
-		mod        = new(big.Int).SetBytes(getData(input, baseLen+expLen, modLen))
 		v          []byte
 		baseBitLen = base.BitLen()
 		expBitLen  = exp.BitLen()

core/vm/evmtypes/evmtypes.go

@@ -84,6 +84,7 @@ type IntraBlockState interface {
 	SetState(common.Address, *common.Hash, uint256.Int)
 	HasLiveAccount(addr common.Address) bool
 	SeenAccount(addr common.Address) bool
+	IsDirtyJournal(addr common.Address) bool
 	HasLiveState(addr common.Address, key *common.Hash) bool
 
 	GetTransientState(addr common.Address, key common.Hash) uint256.Int

core/vm/instructions_zkevm_test.go

@@ -211,3 +211,5 @@ func (ibs TestIntraBlockState) Prepare(rules *chain.Rules, sender, coinbase comm
 func (ibs TestIntraBlockState) Selfdestruct6780(common.Address) {}
 
 func (ibs TestIntraBlockState) SetDisableBalanceInc(disable bool) {}
+
+func (ibs TestIntraBlockState) IsDirtyJournal(addr common.Address) bool { return false }

docs/endpoints/endpoints.md

@@ -200,6 +200,8 @@ If the endpoint is not in the list below, it means this specific endpoint is not
 - zkevm_getL2BlockInfoTree
 - zkevm_getLatestGlobalExitRoot
 - zkevm_getProverInput
+- zkevm_getRollupAddress
+- zkevm_getRollupManagerAddress
 - zkevm_getVersionHistory
 - zkevm_getWitness
 - zkevm_isBlockConsolidated