Fix several sync issues

.github/workflows/test-e2e.yml

@@ -13,6 +13,7 @@ jobs:
       matrix:
         go-version: [ 1.21.x ]
         goarch: [ "amd64" ]
+        test: ["e2e", "edge", "multirollup"]
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
@@ -24,4 +25,4 @@ jobs:
       env:
         GOARCH: ${{ matrix.goarch }}
     - name: Test
-      run: make test-full
+      run: make test-${{ matrix.test }}

.gitignore

@@ -18,3 +18,4 @@
 
 config/config.mainnet.toml
 config/config.testnet.toml
+**__debug**

Makefile

@@ -1,45 +1,44 @@
 include version.mk
 
 DOCKER_COMPOSE := docker-compose -f docker-compose.yml
-DOCKER_COMPOSE_STATE_DB := zkevm-state-db
-DOCKER_COMPOSE_POOL_DB := zkevm-pool-db
-DOCKER_COMPOSE_RPC_DB := zkevm-rpc-db
-DOCKER_COMPOSE_BRIDGE_DB := zkevm-bridge-db
-DOCKER_COMPOSE_ZKEVM_NODE := zkevm-node
+DOCKER_COMPOSE_DB := zkevm-db
+DOCKER_COMPOSE_ZKEVM_NODE-1 := zkevm-node-1
+DOCKER_COMPOSE_ZKEVM_NODE-2 := zkevm-node-2
 DOCKER_COMPOSE_ZKEVM_NODE_V1TOV2 := zkevm-node-v1tov2
 DOCKER_COMPOSE_ZKEVM_AGGREGATOR_V1TOV2 := zkevm-aggregator-v1tov2
 DOCKER_COMPOSE_L1_NETWORK := zkevm-mock-l1-network
 DOCKER_COMPOSE_L1_NETWORK_V1TOV2 := zkevm-v1tov2-l1-network
-DOCKER_COMPOSE_ZKPROVER := zkevm-prover
+DOCKER_COMPOSE_ZKPROVER-1 := zkevm-prover-1
+DOCKER_COMPOSE_ZKPROVER-2 := zkevm-prover-2
 DOCKER_COMPOSE_ZKPROVER_V1TOV2 := zkevm-prover-v1tov2
-DOCKER_COMPOSE_BRIDGE := zkevm-bridge-service
+DOCKER_COMPOSE_BRIDGE-1 := zkevm-bridge-service-1
+DOCKER_COMPOSE_BRIDGE-2 := zkevm-bridge-service-2
 DOCKER_COMPOSE_BRIDGE_V1TOV2 := zkevm-bridge-service-v1tov2
 
-RUN_STATE_DB := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_STATE_DB)
-RUN_POOL_DB := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_POOL_DB)
-RUN_BRIDGE_DB := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_BRIDGE_DB)
-RUN_DBS := ${RUN_BRIDGE_DB} && ${RUN_STATE_DB} && ${RUN_POOL_DB}
-RUN_NODE := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKEVM_NODE)
+RUN_DB := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_DB)
+RUN_NODE_1 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKEVM_NODE-1)
+RUN_NODE_2 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKEVM_NODE-2)
 RUN_NODE_V1TOV2 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKEVM_NODE_V1TOV2)
 RUN_AGGREGATOR_V1TOV2 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKEVM_AGGREGATOR_V1TOV2)
 RUN_L1_NETWORK := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_L1_NETWORK)
 RUN_L1_NETWORK_V1TOV2 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_L1_NETWORK_V1TOV2)
-RUN_ZKPROVER := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKPROVER)
+RUN_ZKPROVER_1 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKPROVER-1)
+RUN_ZKPROVER_2 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKPROVER-2)
 RUN_ZKPROVER_V1TOV2 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_ZKPROVER_V1TOV2)
-RUN_BRIDGE := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_BRIDGE)
+RUN_BRIDGE_1 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_BRIDGE-1)
+RUN_BRIDGE_2 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_BRIDGE-2)
 RUN_BRIDGE_V1TOV2 := $(DOCKER_COMPOSE) up -d $(DOCKER_COMPOSE_BRIDGE_V1TOV2)
 
-STOP_NODE_DB := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_NODE_DB) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_NODE_DB)
-STOP_BRIDGE_DB := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_BRIDGE_DB) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_BRIDGE_DB)
-STOP_DBS := ${STOP_NODE_DB} && ${STOP_BRIDGE_DB}
-STOP_NODE := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_ZKEVM_NODE) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_ZKEVM_NODE)
+STOP_DB := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_DB) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_DB)
+STOP_NODE := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_ZKEVM_NODE-1) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_ZKEVM_NODE-1)
 STOP_NODE_V1TOV2 := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_ZKEVM_NODE_V1TOV2) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_ZKEVM_NODE_V1TOV2)
 STOP_AGGREGATOR_V1TOV2 := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_ZKEVM_AGGREGATOR_V1TOV2) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_ZKEVM_AGGREGATOR_V1TOV2)
 STOP_NETWORK := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_L1_NETWORK) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_L1_NETWORK)
 STOP_NETWORK_V1TOV2 := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_L1_NETWORK_V1TOV2) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_L1_NETWORK_V1TOV2)
-STOP_ZKPROVER := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_ZKPROVER) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_ZKPROVER)
+STOP_ZKPROVER_1 := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_ZKPROVER-1) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_ZKPROVER-1)
+STOP_ZKPROVER_2 := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_ZKPROVER-2) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_ZKPROVER-2)
 STOP_ZKPROVER_V1TOV2 := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_ZKPROVER_V1TOV2) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_ZKPROVER_V1TOV2)
-STOP_BRIDGE := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_BRIDGE) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_BRIDGE)
+STOP_BRIDGE := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_BRIDGE-1) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_BRIDGE-1)
 STOP_BRIDGE_V1TOV2 := $(DOCKER_COMPOSE) stop $(DOCKER_COMPOSE_BRIDGE_V1TOV2) && $(DOCKER_COMPOSE) rm -f $(DOCKER_COMPOSE_BRIDGE_V1TOV2)
 STOP := $(DOCKER_COMPOSE) down --remove-orphans
 
@@ -71,9 +70,9 @@ install-git-hooks: ## Moves hook files to the .git/hooks directory
 
 .PHONY: test
 test: ## Runs only short tests without checking race conditions
-	$(STOP_BRIDGE_DB) || true
-	$(RUN_BRIDGE_DB); sleep 3
-	trap '$(STOP_BRIDGE_DB)' EXIT; go test --cover -short -p 1 ./...
+	$(STOP_DB) || true
+	$(RUN_DB); sleep 3
+	trap '$(STOP_DB)' EXIT; go test --cover -short -p 1 ./...
 
 .PHONY: install-linter
 install-linter: ## Installs the linter
@@ -83,33 +82,17 @@ install-linter: ## Installs the linter
 build-docker: ## Builds a docker image with the zkevm bridge binary
 	docker build -t zkevm-bridge-service -f ./Dockerfile .
 
-.PHONY: run-db-node
-run-db-node: ## Runs the node database
-	$(RUN_NODE_DB)
+.PHONY: run-db
+run-db: ## Runs the node database
+	$(RUN_DB)
 
-.PHONY: stop-db-node
-stop-db-node: ## Stops the node database
-	$(STOP_NODE_DB)
-
-.PHONY: run-db-bridge
-run-db-bridge: ## Runs the node database
-	$(RUN_BRIDGE_DB)
-
-.PHONY: stop-db-bridge
-stop-db-bridge: ## Stops the node database
-	$(STOP_BRIDGE_DB)
-
-.PHONY: run-dbs
-run-dbs: ## Runs the node database
-	$(RUN_DBS)
-
-.PHONY: stop-dbs
-stop-dbs: ## Stops the node database
-	$(STOP_DBS)
+.PHONY: stop-db
+stop-db: ## Stops the node database
+	$(STOP_DB)
 
 .PHONY: run-node
 run-node: ## Runs the node
-	$(RUN_NODE)
+	$(RUN_NODE_1)
 
 .PHONY: stop-node
 stop-node: ## Stops the node
@@ -149,11 +132,11 @@ stop-network-v1tov2: ## Stops the l1 network
 
 .PHONY: run-prover
 run-prover: ## Runs the zk prover
-	$(RUN_ZKPROVER)
+	$(RUN_ZKPROVER_1)
 
 .PHONY: stop-prover
 stop-prover: ## Stops the zk prover
-	$(STOP_ZKPROVER)
+	$(STOP_ZKPROVER_1)
 
 .PHONY: run-prover-v1tov2
 run-prover-v1tov2: ## Runs the zk prover
@@ -165,7 +148,7 @@ stop-prover-v1tov2: ## Stops the zk prover
 
 .PHONY: run-bridge
 run-bridge: ## Runs the bridge service
-	$(RUN_BRIDGE)
+	$(RUN_BRIDGE_1)
 
 .PHONY: stop-bridge
 stop-bridge: ## Stops the bridge service
@@ -188,18 +171,26 @@ restart: stop run ## Executes `make stop` and `make run` commands
 
 .PHONY: run
 run: stop ## runs all services
-	$(RUN_DBS)
+	$(RUN_DB)
 	$(RUN_L1_NETWORK)
 	sleep 5
-	$(RUN_ZKPROVER)
+	$(RUN_ZKPROVER_1)
 	sleep 3
-	$(RUN_NODE)
-	sleep 7
-	$(RUN_BRIDGE)
+	$(RUN_NODE_1)
+	sleep 25
+	$(RUN_BRIDGE_1)
+
+.PHONY: run-2Rollups
+run-2Rollups: run
+	$(RUN_ZKPROVER_2)
+	sleep 3
+	$(RUN_NODE_2)
+	sleep 25
+	$(RUN_BRIDGE_2)
 
 .PHONY: run-v1tov2
 run-v1tov2: stop ## runs all services
-	$(RUN_DBS)
+	$(RUN_DB)
 	$(RUN_L1_NETWORK_V1TOV2)
 	sleep 5
 	$(RUN_ZKPROVER_V1TOV2)
@@ -224,9 +215,9 @@ stop-mockserver: ## Stops the mock bridge service
 
 .PHONY: bench
 bench: ## benchmark test
-	$(STOP_BRIDGE_DB) || true
-	$(RUN_BRIDGE_DB); sleep 3
-	trap '$(STOP_BRIDGE_DB)' EXIT; go test -run=NOTEST -timeout=30m -bench=Small ./test/benchmark/...
+	$(STOP_DB) || true
+	$(RUN_DB); sleep 3
+	trap '$(STOP_DB)' EXIT; go test -run=NOTEST -timeout=30m -bench=Small ./test/benchmark/...
 
 .PHONY: bench-full
 bench-full: export ZKEVM_BRIDGE_DATABASE_PORT = 5432
@@ -236,8 +227,8 @@ bench-full: ## benchmark full test
 	go test -run=NOTEST -bench=Medium . && \
 	go test -run=NOTEST -timeout=30m -bench=Large .
 
-.PHONY: test-full
-test-full: build-docker stop run ## Runs all tests checking race conditions
+.PHONY: test-e2e
+test-e2e: build-docker stop run ## Runs all tests checking race conditions
 	sleep 3
 	trap '$(STOP)' EXIT; MallocNanoZone=0 go test -v -failfast -race -p 1 -timeout 2400s ./test/e2e/... -count 1 -tags='e2e'
 
@@ -246,6 +237,11 @@ test-edge: build-docker stop run ## Runs all tests checking race conditions
 	sleep 3
 	trap '$(STOP)' EXIT; MallocNanoZone=0 go test -v -failfast -race -p 1 -timeout 2400s ./test/e2e/... -count 1 -tags='edge'
 
+.PHONY: test-multirollup
+test-multirollup: build-docker stop run-2Rollups ## Runs all tests checking race conditions
+	sleep 3
+	trap '$(STOP)' EXIT; MallocNanoZone=0 go test -v -failfast -race -p 1 -timeout 2400s ./test/e2e/... -count 1 -tags='multirollup'
+
 .PHONY: validate
 validate: lint build test-full ## Validates the whole integrity of the code base
 

claimtxman/claimtxman.go

@@ -85,6 +85,7 @@ func (tm *ClaimTxManager) Start() {
 	for {
 		select {
 		case <-tm.ctx.Done():
+			ticker.Stop()
 			return
 		case netID := <-tm.chSynced:
 			if netID == tm.l2NetworkID && !tm.synced {
@@ -154,46 +155,52 @@ func (tm *ClaimTxManager) processDepositStatus(ger *etherman.GlobalExitRoot, dbT
 			return err
 		}
 		for _, deposit := range deposits {
-			if tm.l2NetworkID != deposit.DestinationNetwork {
-				log.Infof("Ignoring deposit: %d: dest_net: %d, we are:%d", deposit.DepositCount, deposit.DestinationNetwork, tm.l2NetworkID)
-				continue
-			}
-
-			claimHash, err := tm.bridgeService.GetDepositStatus(tm.ctx, deposit.DepositCount, deposit.DestinationNetwork)
+			claimHash, err := tm.bridgeService.GetDepositStatus(tm.ctx, deposit.DepositCount, deposit.OriginalNetwork, deposit.DestinationNetwork)
 			if err != nil {
 				log.Errorf("error getting deposit status for deposit %d. Error: %v", deposit.DepositCount, err)
 				return err
 			}
+
 			if len(claimHash) > 0 || deposit.LeafType == LeafTypeMessage && !tm.isDepositMessageAllowed(deposit) {
 				log.Infof("Ignoring deposit: %d, leafType: %d, claimHash: %s, deposit.OriginalAddress: %s", deposit.DepositCount, deposit.LeafType, claimHash, deposit.OriginalAddress.String())
 				continue
 			}
 
+			if tm.l2NetworkID != deposit.DestinationNetwork {
+				log.Debugf("Ignoring deposit: %d", deposit.DepositCount)
+				continue
+			}
+
 			log.Infof("create the claim tx for the deposit %d", deposit.DepositCount)
 			ger, proof, rollupProof, err := tm.bridgeService.GetClaimProof(deposit.DepositCount, deposit.NetworkID, dbTx)
 			if err != nil {
 				log.Errorf("error getting Claim Proof for deposit %d. Error: %v", deposit.DepositCount, err)
 				return err
 			}
+
 			var (
 				mtProof       [mtHeight][keyLen]byte
 				mtRollupProof [mtHeight][keyLen]byte
 			)
+
 			for i := 0; i < mtHeight; i++ {
 				mtProof[i] = proof[i]
 				mtRollupProof[i] = rollupProof[i]
 			}
+
 			tx, err := tm.l2Node.BuildSendClaim(tm.ctx, deposit, mtProof, mtRollupProof,
 				&etherman.GlobalExitRoot{
 					ExitRoots: []common.Hash{
 						ger.ExitRoots[0],
 						ger.ExitRoots[1],
 					}}, 1, 1, 1, tm.rollupID,
 				tm.auth)
+
 			if err != nil {
 				log.Errorf("error BuildSendClaim tx for deposit %d. Error: %v", deposit.DepositCount, err)
 				return err
 			}
+
 			if err = tm.addClaimTx(deposit.DepositCount, tm.auth.From, tx.To(), nil, tx.Data(), dbTx); err != nil {
 				log.Errorf("error adding claim tx for deposit %d. Error: %v", deposit.DepositCount, err)
 				return err

claimtxman/interfaces.go

@@ -24,5 +24,5 @@ type storageInterface interface {
 
 type bridgeServiceInterface interface {
 	GetClaimProof(depositCnt, networkID uint, dbTx pgx.Tx) (*etherman.GlobalExitRoot, [][bridgectrl.KeyLen]byte, [][bridgectrl.KeyLen]byte, error)
-	GetDepositStatus(ctx context.Context, depositCount uint, destNetworkID uint) (string, error)
+	GetDepositStatus(ctx context.Context, depositCount uint, originNetworkID, destNetworkID uint) (string, error)
 }

cmd/run.go

@@ -95,7 +95,9 @@ func start(ctx *cli.Context) error {
 		log.Error(err)
 		return err
 	}
-	rollupID := l1Etherman.GetRollupID()
+
+	// TODO: this works because the service only supports one L2 network
+	rollupID := l2Ethermans[0].GetRollupID()
 	bridgeService := server.NewBridgeService(c.BridgeServer, c.BridgeController.Height, networkIDs, apiStorage, rollupID)
 	err = server.RunServer(c.BridgeServer, bridgeService)
 	if err != nil {
@@ -153,17 +155,23 @@ func setupLog(c log.Config) {
 }
 
 func newEthermans(c *config.Config) (*etherman.Client, []*etherman.Client, error) {
-	l1Etherman, err := etherman.NewClient(c.Etherman, c.NetworkConfig.PolygonBridgeAddress, c.NetworkConfig.PolygonZkEVMGlobalExitRootAddress, c.NetworkConfig.PolygonRollupManagerAddress, c.NetworkConfig.PolygonZkEvmAddress)
+	l1Etherman, err := etherman.NewL1Client(c.Etherman.L1URL, c.NetworkConfig.PolygonBridgeAddress, c.NetworkConfig.PolygonZkEVMGlobalExitRootAddress, c.NetworkConfig.PolygonRollupManagerAddress, c.NetworkConfig.PolygonZkEvmAddress)
 	if err != nil {
 		log.Error("L1 etherman error: ", err)
 		return nil, nil, err
 	}
 	if len(c.L2PolygonBridgeAddresses) != len(c.Etherman.L2URLs) {
 		log.Fatal("environment configuration error. zkevm bridge addresses and zkevm node urls mismatch")
 	}
+	if len(c.L2PolygonBridgeAddresses) != 1 {
+		return nil, nil, fmt.Errorf(
+			"the bridge service only supports working with a single L2, but %d were provided",
+			len(c.L2PolygonBridgeAddresses),
+		)
+	}
 	var l2Ethermans []*etherman.Client
-	for i, addr := range c.L2PolygonBridgeAddresses {
-		l2Etherman, err := etherman.NewL2Client(c.Etherman.L2URLs[i], addr)
+	for i, bridgeAddr := range c.L2PolygonBridgeAddresses {
+		l2Etherman, err := etherman.NewL2Client(c.Etherman.L2URLs[i], bridgeAddr)
 		if err != nil {
 			log.Error("L2 etherman ", i, c.Etherman.L2URLs[i], ", error: ", err)
 			return l1Etherman, nil, err

db/pgstorage/migrations/0009.sql

@@ -0,0 +1,15 @@
+-- +migrate Up
+
+ALTER TABLE sync.deposit
+ADD COLUMN IF NOT EXISTS origin_rollup_id BIGINT DEFAULT 0;
+
+ALTER TABLE sync.claim DROP CONSTRAINT claim_pkey;
+ALTER TABLE sync.claim ADD PRIMARY KEY (index, rollup_index, mainnet_flag);
+
+-- +migrate Down
+
+ALTER TABLE sync.claim DROP CONSTRAINT claim_pkey;
+ALTER TABLE sync.claim ADD PRIMARY KEY (network_id, index);
+
+ALTER TABLE sync.deposit
+DROP COLUMN IF EXISTS origin_rollup_id;