Merge pull request #192 from okx/zjg/basedev-v2.61.0-RC1

Merge upstream v2.61.0 rc1

.github/actions/setup-kurtosis/action.yml

@@ -0,0 +1,74 @@
+
+name: "Setup Kurtosis"
+description: "Setup Kurtosis CDK for tests"
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout cdk-erigon
+      uses: actions/checkout@v4
+      with:
+        path: cdk-erigon
+
+    - name: Checkout kurtosis-cdk
+      uses: actions/checkout@v4
+      with:
+        repository: 0xPolygon/kurtosis-cdk
+        ref: v0.2.24
+        path: kurtosis-cdk
+
+    - name: Install Kurtosis CDK tools
+      uses: ./kurtosis-cdk/.github/actions/setup-kurtosis-cdk
+
+    - name: Install Foundry
+      uses: foundry-rs/foundry-toolchain@v1
+
+    - name: Install polycli
+      shell: bash
+      run: |
+        tmp_dir=$(mktemp -d) && curl -L https://github.com/0xPolygon/polygon-cli/releases/download/v0.1.48/polycli_v0.1.48_linux_amd64.tar.gz | tar -xz -C "$tmp_dir" && mv "$tmp_dir"/* /usr/local/bin/polycli && rm -rf "$tmp_dir"
+        sudo chmod +x /usr/local/bin/polycli
+        /usr/local/bin/polycli version
+
+    - name: Install yq
+      shell: bash
+      run: |
+        sudo curl -L https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64 -o /usr/local/bin/yq
+        sudo chmod +x /usr/local/bin/yq
+        /usr/local/bin/yq --version
+        
+    - name: Build docker image
+      working-directory: ./cdk-erigon
+      shell: bash
+      run: docker build -t cdk-erigon:local --file Dockerfile .
+
+    - name: Remove unused flags
+      working-directory: ./kurtosis-cdk
+      shell: bash
+      run: |
+        sed -i '/zkevm.sequencer-batch-seal-time:/d' templates/cdk-erigon/config.yml
+        sed -i '/zkevm.sequencer-non-empty-batch-seal-time:/d' templates/cdk-erigon/config.yml
+        sed -i '/zkevm\.sequencer-initial-fork-id/d' ./templates/cdk-erigon/config.yml
+        sed -i '/sentry.drop-useless-peers:/d' templates/cdk-erigon/config.yml
+        sed -i '/zkevm\.pool-manager-url/d' ./templates/cdk-erigon/config.yml
+        sed -i '$a\zkevm.disable-virtual-counters: true' ./templates/cdk-erigon/config.yml
+        sed -i '/zkevm.l2-datastreamer-timeout:/d' templates/cdk-erigon/config.yml
+
+    - name: Create params.yml overrides
+      working-directory: ./kurtosis-cdk
+      shell: bash
+      run: |
+        echo 'args:' > params.yml
+        echo '  cdk_erigon_node_image: cdk-erigon:local' >> params.yml
+        echo '  el-1-geth-lighthouse: ethpandaops/lighthouse@sha256:4902d9e4a6b6b8d4c136ea54f0e51582a32f356f3dec7194a1adee13ed2d662e' >> params.yml
+        /usr/local/bin/yq -i '.args.data_availability_mode = "${{ matrix.da-mode }}"' params.yml
+        sed -i 's/"londonBlock": [0-9]\+/"londonBlock": 0/' ./templates/cdk-erigon/chainspec.json
+        sed -i 's/"normalcyBlock": [0-9]\+/"normalcyBlock": 0/' ./templates/cdk-erigon/chainspec.json
+        sed -i 's/"shanghaiTime": [0-9]\+/"shanghaiTime": 0/' ./templates/cdk-erigon/chainspec.json
+        sed -i 's/"cancunTime": [0-9]\+/"cancunTime": 0/' ./templates/cdk-erigon/chainspec.json
+        sed -i '/"terminalTotalDifficulty"/d' ./templates/cdk-erigon/chainspec.json
+
+    - name: Deploy Kurtosis CDK package
+      working-directory: ./kurtosis-cdk
+      shell: bash
+      run: |
+        kurtosis run --enclave cdk-v1 --args-file params.yml --image-download always . '{"args": {"erigon_strict_mode": false, "cdk_erigon_node_image": "cdk-erigon:local"}}'

.github/scripts/test_resequence.sh

@@ -50,7 +50,7 @@ wait_for_l1_batch() {
             current_batch=$(cast logs --rpc-url "$(kurtosis port print cdk-v1 el-1-geth-lighthouse rpc)" --address 0x1Fe038B54aeBf558638CA51C91bC8cCa06609e91 --from-block 0 --json | jq -r '.[] | select(.topics[0] == "0x3e54d0825ed78523037d00a81759237eb436ce774bd546993ee67a1b67b6e766") | .topics[1]' | tail -n 1 | sed 's/^0x//')
             current_batch=$((16#$current_batch))
         elif [ "$batch_type" = "verified" ]; then
-            current_batch=$(cast rpc zkevm_verifiedBatchNumber --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-node-001 rpc)" | sed 's/^"//;s/"$//')
+            current_batch=$(cast rpc zkevm_verifiedBatchNumber --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc)" | sed 's/^"//;s/"$//')
         else
             echo "Invalid batch type. Use 'virtual' or 'verified'."
             return 1
@@ -121,7 +121,7 @@ kurtosis service exec cdk-v1 cdk-erigon-sequencer-001 "nohup cdk-erigon --pprof=
 sleep 30
 
 echo "Running loadtest using polycli"
-/usr/local/bin/polycli loadtest --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-node-001 rpc)" --private-key "0x12d7de8621a77640c9241b2595ba78ce443d05e94090365ab3bb5e19df82c625" --verbosity 600 --requests 2000 --rate-limit 500  --mode uniswapv3 --legacy
+/usr/local/bin/polycli loadtest --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc)" --private-key "0x12d7de8621a77640c9241b2595ba78ce443d05e94090365ab3bb5e19df82c625" --verbosity 600 --requests 2000 --rate-limit 500  --mode uniswapv3 --legacy
 
 echo "Waiting for batch virtualization"
 if ! wait_for_l1_batch 600 "virtual"; then
@@ -174,13 +174,13 @@ echo "Getting block hash from sequencer"
 sequencer_hash=$(cast block $comparison_block --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-sequencer-001 rpc)" | grep "hash" | awk '{print $2}')
 
 # wait for block to be available on sync node
-if ! wait_for_l2_block_number $comparison_block "$(kurtosis port print cdk-v1 cdk-erigon-node-001 rpc)"; then
+if ! wait_for_l2_block_number $comparison_block "$(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc)"; then
     echo "Failed to wait for batch verification"
     exit 1
 fi
 
 echo "Getting block hash from node"
-node_hash=$(cast block $comparison_block --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-node-001 rpc)" | grep "hash" | awk '{print $2}')
+node_hash=$(cast block $comparison_block --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc)" | grep "hash" | awk '{print $2}')
 
 echo "Sequencer block hash: $sequencer_hash"
 echo "Node block hash: $node_hash"

.github/workflows/ci_zkevm.yml

@@ -3,9 +3,14 @@ on:
   push:
     branches:
       - dev
+      - zkevm
+      - zkevm-2.60
   pull_request:
     branches:
       - dev
+      - zkevm
+      - zkevm-2.60
+      - stable*
     types:
       - opened
       - reopened
@@ -17,51 +22,58 @@ concurrency:
   group: ${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
 jobs:
-#  tests:
-#    strategy:
-#      matrix:
-#        os: [ ubuntu-22.04, macos-14-xlarge ] # list of os: https://github.com/actions/virtual-environments
-#    runs-on: ${{ matrix.os }}
-#    timeout-minutes: ${{ matrix.os == 'macos-14-xlarge' && 40 || 30 }}
-#
-#    steps:
-#      - uses: actions/checkout@v3
-#      - uses: actions/setup-go@v4
-#        with:
-#          go-version: '1.21'
-#      - name: Install dependencies on Linux
+  tests:
+    if: false
+    strategy:
+      max-parallel: 2
+      fail-fast: true
+      matrix:
+        os: [ ubuntu-22.04, macos-14-xlarge ] # list of os: https://github.com/actions/virtual-environments
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: ${{ matrix.os == 'macos-14-xlarge' && 40 || 30 }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: Install dependencies on Linux
+        if: runner.os == 'Linux'
+        run: sudo apt update && sudo apt install build-essential
+
+      - name: Build
+        run: make BUILD_TAGS=nosqlite,noboltdb,nosilkworm cdk-erigon
+
+      - name: Reproducible build test
+        run: |
+          make cdk-erigon
+          shasum -a256 ./build/bin/cdk-erigon > erigon1.sha256
+          make cdk-erigon
+          shasum -a256 ./build/bin/cdk-erigon > erigon2.sha256
+          if ! cmp -s erigon1.sha256 erigon2.sha256; then
+            echo >&2 "Reproducible build broken"; cat erigon1.sha256; cat erigon2.sha256; exit 1
+          fi
+
+#      - name: Lint
 #        if: runner.os == 'Linux'
-#        run: sudo apt update && sudo apt install build-essential
-#
-#      - name: Build
-#        run: make BUILD_TAGS=nosqlite,noboltdb,nosilkworm all
-#
-#      - name: Reproducible build test
-#        run: |
-#          make cdk-erigon
-#          shasum -a256 ./build/bin/cdk-erigon > erigon1.sha256
-#          make cdk-erigon
-#          shasum -a256 ./build/bin/cdk-erigon > erigon2.sha256
-#          if ! cmp -s erigon1.sha256 erigon2.sha256; then
-#            echo >&2 "Reproducible build broken"; cat erigon1.sha256; cat erigon2.sha256; exit 1
-#          fi
-#
-##      - name: Lint
-##        if: runner.os == 'Linux'
-##        uses: golangci/golangci-lint-action@v4
-##        with:
-##          version: v1.54
-#
-#      - name: Test
-#        run: make test
-#
-#      - name: SonarCloud Scan
-#        uses: SonarSource/sonarcloud-github-action@master
-#        if: ${{ matrix.os == 'ubuntu-20.04' }}
-#        env:
-#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
-#          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+#        uses: golangci/golangci-lint-action@v4
+#        with:
+#          version: v1.54
+
+      - name: Test
+        run: make test
+
+      - name: SonarCloud Scan
+        uses: SonarSource/sonarcloud-github-action@master
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
   kurtosis-cdk:
     if: true
@@ -72,49 +84,9 @@ jobs:
     steps:
       - name: Checkout cdk-erigon
         uses: actions/checkout@v4
-        with:
-          path: cdk-erigon
-
-      - name: Checkout kurtosis-cdk
-        uses: actions/checkout@v4
-        with:
-          repository: 0xPolygon/kurtosis-cdk
-          ref: v0.2.12
-          path: kurtosis-cdk
-
-      - name: Install Kurtosis CDK tools
-        uses: ./kurtosis-cdk/.github/actions/setup-kurtosis-cdk
+      - name: Setup kurtosis
+        uses: ./.github/actions/setup-kurtosis
 
-      - name: Install Foundry
-        uses: foundry-rs/foundry-toolchain@v1
-
-      - name: Install yq
-        run: |
-          sudo curl -L https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64 -o /usr/local/bin/yq
-          sudo chmod +x /usr/local/bin/yq
-          /usr/local/bin/yq --version
-
-      - name: Build docker image
-        working-directory: ./cdk-erigon
-        run: docker build -t cdk-erigon:local --file Dockerfile .
-
-      - name: Remove unused flags
-        working-directory: ./kurtosis-cdk
-        run: |
-          sed -i '/zkevm.sequencer-batch-seal-time:/d' templates/cdk-erigon/config.yml
-          sed -i '/zkevm.sequencer-non-empty-batch-seal-time:/d' templates/cdk-erigon/config.yml
-          sed -i '/sentry.drop-useless-peers:/d' templates/cdk-erigon/config.yml
-          sed -i '/zkevm.l2-datastreamer-timeout:/d' templates/cdk-erigon/config.yml
-      - name: Configure Kurtosis CDK
-        working-directory: ./kurtosis-cdk
-        run: |
-          /usr/local/bin/yq -i '.args.data_availability_mode = "${{ matrix.da-mode }}"' params.yml
-          /usr/local/bin/yq -i '.args.cdk_erigon_node_image = "cdk-erigon:local"' params.yml
-
-      - name: Deploy Kurtosis CDK package
-        working-directory: ./kurtosis-cdk
-        run: | 
-          kurtosis run --enclave cdk-v1 --image-download always . '{"args": {"data_availability_mode": "${{ matrix.da-mode }}", "cdk_erigon_node_image": "cdk-erigon:local"}}'
 
       - name: Run process with CPU monitoring
         working-directory: ./cdk-erigon
@@ -136,7 +108,7 @@ jobs:
       - name: Monitor verified batches
         working-directory: ./kurtosis-cdk
         shell: bash
-        run: timeout 900s .github/scripts/monitor-verified-batches.sh --rpc-url $(kurtosis port print cdk-v1 cdk-erigon-node-001 rpc) --target 20 --timeout 900
+        run: timeout 900s .github/scripts/monitor-verified-batches.sh --enclave zdk-v1 --rpc-url $(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc) --target 20 --timeout 900
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
@@ -146,9 +118,8 @@ jobs:
           kurtosis files download cdk-v1 bridge-config-artifact
           echo "BRIDGE_ADDRESS=$(/usr/local/bin/yq '.NetworkConfig.PolygonBridgeAddress' bridge-config-artifact/bridge-config.toml)" >> $GITHUB_ENV
           echo "ETH_RPC_URL=$(kurtosis port print cdk-v1 el-1-geth-lighthouse rpc)" >> $GITHUB_ENV
-          echo "L2_RPC_URL=$(kurtosis port print cdk-v1 cdk-erigon-node-001 rpc)" >> $GITHUB_ENV
           echo "BRIDGE_API_URL=$(kurtosis port print cdk-v1 zkevm-bridge-service-001 rpc)" >> $GITHUB_ENV
-
+          echo "L2_RPC_URL=$(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc)" >> $GITHUB_ENV
       - name: Clone bridge repository
         run:  git clone --recurse-submodules -j8 https://github.com/0xPolygonHermez/zkevm-bridge-service.git -b develop  bridge
 
@@ -186,7 +157,7 @@ jobs:
         run: |
           mkdir -p ci_logs
           cd ci_logs
-          kurtosis service logs cdk-v1 cdk-erigon-node-001 --all > cdk-erigon-node-001.log
+          kurtosis service logs cdk-v1 cdk-erigon-rpc-001 --all > cdk-erigon-rpc-001.log
           kurtosis service logs cdk-v1 cdk-erigon-sequencer-001 --all > cdk-erigon-sequencer-001.log
           kurtosis service logs cdk-v1 zkevm-agglayer-001 --all > zkevm-agglayer-001.log
           kurtosis service logs cdk-v1 zkevm-prover-001 --all > zkevm-prover-001.log
@@ -211,62 +182,12 @@ jobs:
 
       - name: Checkout kurtosis-cdk
         uses: actions/checkout@v4
-        with:
-          repository: 0xPolygon/kurtosis-cdk
-          ref: v0.2.12
-          path: kurtosis-cdk
-
-      - name: Install Kurtosis CDK tools
-        uses: ./kurtosis-cdk/.github/actions/setup-kurtosis-cdk
-
-      - name: Install Foundry
-        uses: foundry-rs/foundry-toolchain@v1
+      - name: Setup kurtosis
+        uses: ./.github/actions/setup-kurtosis
 
-      - name: Install yq
-        run: |
-          sudo curl -L https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64 -o /usr/local/bin/yq
-          sudo chmod +x /usr/local/bin/yq
-          /usr/local/bin/yq --version
-
-      - name: Install polycli
-        run: |
-          tmp_dir=$(mktemp -d) && curl -L https://github.com/0xPolygon/polygon-cli/releases/download/v0.1.48/polycli_v0.1.48_linux_amd64.tar.gz | tar -xz -C "$tmp_dir" && mv "$tmp_dir"/* /usr/local/bin/polycli && rm -rf "$tmp_dir"
-          sudo chmod +x /usr/local/bin/polycli
-          /usr/local/bin/polycli version
-
-      - name: Build docker image
-        working-directory: ./cdk-erigon
-        run: docker build -t cdk-erigon:local --file Dockerfile .
-
-      - name: Modify cdk-erigon flags
-        working-directory: ./kurtosis-cdk
-        run: |
-          sed -i '/zkevm.sequencer-batch-seal-time:/d' templates/cdk-erigon/config.yml
-          sed -i '/zkevm.sequencer-non-empty-batch-seal-time:/d' templates/cdk-erigon/config.yml
-          sed -i '/zkevm\.sequencer-initial-fork-id/d' ./templates/cdk-erigon/config.yml
-          sed -i '/sentry.drop-useless-peers:/d' templates/cdk-erigon/config.yml
-          sed -i '/zkevm\.pool-manager-url/d' ./templates/cdk-erigon/config.yml
-          sed -i '$a\zkevm.disable-virtual-counters: true' ./templates/cdk-erigon/config.yml
-          sed -i '/zkevm.l2-datastreamer-timeout:/d' templates/cdk-erigon/config.yml
-
-
-      - name: Configure Kurtosis CDK
-        working-directory: ./kurtosis-cdk
-        run: |
-          sed -i 's/"londonBlock": [0-9]\+/"londonBlock": 0/' ./templates/cdk-erigon/chainspec.json
-          sed -i 's/"normalcyBlock": [0-9]\+/"normalcyBlock": 0/' ./templates/cdk-erigon/chainspec.json
-          sed -i 's/"shanghaiTime": [0-9]\+/"shanghaiTime": 0/' ./templates/cdk-erigon/chainspec.json
-          sed -i 's/"cancunTime": [0-9]\+/"cancunTime": 0/' ./templates/cdk-erigon/chainspec.json
-          sed -i '/"terminalTotalDifficulty"/d' ./templates/cdk-erigon/chainspec.json
-          
-      - name: Deploy Kurtosis CDK package
-        working-directory: ./kurtosis-cdk
-        run: |
-           kurtosis run --enclave cdk-v1 --image-download always . '{"args": {"erigon_strict_mode": false, "cdk_erigon_node_image": "cdk-erigon:local"}}'
-  
       - name: Dynamic gas fee tx load test
         working-directory: ./kurtosis-cdk
-        run: /usr/local/bin/polycli loadtest --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-node-001 rpc)" --private-key "0x12d7de8621a77640c9241b2595ba78ce443d05e94090365ab3bb5e19df82c625" --verbosity 700 --requests 500 --rate-limit 50  --mode uniswapv3
+        run: /usr/local/bin/polycli loadtest --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc)" --private-key "0x12d7de8621a77640c9241b2595ba78ce443d05e94090365ab3bb5e19df82c625" --verbosity 700 --requests 500 --rate-limit 50  --mode uniswapv3 --legacy
 
       - name: Upload logs
         uses: actions/upload-artifact@v3
@@ -280,7 +201,7 @@ jobs:
         run: |
           mkdir -p ci_logs
           cd ci_logs
-          kurtosis service logs cdk-v1 cdk-erigon-node-001 --all > cdk-erigon-node-001.log
+          kurtosis service logs cdk-v1 cdk-erigon-rpc-001 --all > cdk-erigon-rpc-001.log
           kurtosis service logs cdk-v1 cdk-erigon-sequencer-001 --all > cdk-erigon-sequencer-001.log
 
       - name: Upload logs

.github/workflows/doc-rpc.yml

@@ -2,12 +2,12 @@ name: RPC endpoint doc
 on:
   push:
     branches:
+      - dev
       - zkevm
-      - stable*
   pull_request:
     branches:
+      - dev
       - zkevm
-      - stable*
     types:
       - opened
       - reopened