feat(permissionless batches): batch production toolkit and operator recovery

build/dockerfiles/coordinator-api.Dockerfile.dockerignore

@@ -4,3 +4,5 @@ docs/
 l2geth/
 rpc-gateway/
 *target/*
+
+permissionless-batches/conf/

build/dockerfiles/coordinator-cron.Dockerfile.dockerignore

@@ -4,3 +4,5 @@ docs/
 l2geth/
 rpc-gateway/
 *target/*
+
+permissionless-batches/conf/

build/dockerfiles/db_cli.Dockerfile.dockerignore

@@ -4,3 +4,5 @@ docs/
 l2geth/
 rpc-gateway/
 *target/*
+
+permissionless-batches/conf/

build/dockerfiles/gas_oracle.Dockerfile.dockerignore

@@ -1,5 +1,8 @@
 assets/
+contracts/
 docs/
 l2geth/
 rpc-gateway/
-*target/*
+*target/*
+
+permissionless-batches/conf/

build/dockerfiles/recovery_permissionless_batches.Dockerfile

@@ -0,0 +1,30 @@
+# Download Go dependencies
+FROM scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03 as base
+
+WORKDIR /src
+COPY go.work* ./
+COPY ./rollup/go.* ./rollup/
+COPY ./common/go.* ./common/
+COPY ./coordinator/go.* ./coordinator/
+COPY ./database/go.* ./database/
+COPY ./tests/integration-test/go.* ./tests/integration-test/
+COPY ./bridge-history-api/go.* ./bridge-history-api/
+RUN go mod download -x
+
+# Build rollup_relayer
+FROM base as builder
+
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    cd /src/rollup/cmd/permissionless_batches/ && CGO_LDFLAGS="-ldl" go build -v -p 4 -o /bin/rollup_relayer
+
+# Pull rollup_relayer into a second stage deploy ubuntu container
+FROM ubuntu:20.04
+
+RUN apt update && apt install vim netcat-openbsd net-tools curl ca-certificates -y
+
+ENV CGO_LDFLAGS="-ldl"
+
+COPY --from=builder /bin/rollup_relayer /bin/
+WORKDIR /app
+ENTRYPOINT ["rollup_relayer"]

build/dockerfiles/recovery_permissionless_batches.Dockerfile.dockerignore

@@ -0,0 +1,8 @@
+assets/
+contracts/
+docs/
+l2geth/
+rpc-gateway/
+*target/*
+
+permissionless-batches/conf/

build/dockerfiles/rollup_relayer.Dockerfile.dockerignore

@@ -1,5 +1,8 @@
 assets/
+contracts/
 docs/
 l2geth/
 rpc-gateway/
-*target/*
+*target/*
+
+permissionless-batches/conf/

go.work

@@ -7,4 +7,5 @@ use (
 	./database
 	./rollup
 	./tests/integration-test
+	//../go-ethereum
 )

permissionless-batches/README.md

@@ -0,0 +1,36 @@
+# Permissionless Batches
+Permissionless batches aka enforced batches is a feature that provides guarantee to users that they can exit Scroll even if the operator is down or censoring. 
+It allows anyone to take over and submit a batch (permissionless batch submission) together with a proof after a certain time period has passed without a batch being finalized on L1. 
+
+Once permissionless batch mode is activated, the operator can no longer submit batches in a permissioned way. Only the security council can deactivate permissionless batch mode and reinstate the operator as the only batch submitter.
+There are two types of situations to consider:
+- `Permissionless batch mode is activated:` This means that finalization halted for some time. Now anyone can submit batches utilizing the [batch production toolkit](#batch-production-toolkit).
+- `Permissionless batch mode is deactivated:` This means that the security council has decided to reinstate the operator as the only batch submitter. The operator needs to [recover](#operator-recovery) the sequencer and relayer to resume batch submission and the valid L2 chain.
+
+
+## Pre-requisites
+- install instructions
+- download stuff for coordinator
+
+
+## Batch production toolkit
+1. l2geth recovery 
+2. l2geth block production 
+3. relayer in permissionless mode with proving etc
+
+### Proving service
+```
+"l2geth": {
+    "endpoint": ""
+  }
+```
+
+
+## Operator recovery
+- l2geth recovery and relayer recovery
+
+### Relayer
+```
+l2_config.endpoint
+```
+

permissionless-batches/docker-compose.yml

@@ -0,0 +1,119 @@
+name: permissionless-batches
+
+services:
+  relayer-batch-production:
+    build:
+      context: ../
+      dockerfile: build/dockerfiles/recovery_permissionless_batches.Dockerfile
+    container_name: permissionless-batches-relayer
+    volumes:
+      - ./conf/relayer:/app/conf
+    command: "--config /app/conf/config.json"
+    profiles:
+      - batch-production-submission
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: postgres:17.0
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_USER: postgres
+      POSTGRES_DB: scroll
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+    volumes:
+      - db_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+  coordinator:
+    build:
+      context: ../
+      dockerfile: build/dockerfiles/coordinator-api.Dockerfile
+    volumes:
+      - ./conf/coordinator/:/app/conf
+    command: "--config /app/conf/config.json --http.port 8390 --verbosity 5"
+    profiles:
+      - proving
+    depends_on:
+      db:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8390/coordinator/v1/challenge"]
+      interval: 1s
+      timeout: 1s
+      retries: 10
+      start_period: 5m
+
+  coordinator-cron:
+    build:
+      context: ../
+      dockerfile: build/dockerfiles/coordinator-cron.Dockerfile
+    volumes:
+      - ./conf/coordinator/:/app/conf
+    command: "--config /app/conf/config.json --verbosity 3"
+    profiles:
+      - proving
+    depends_on:
+      db:
+        condition: service_healthy
+
+
+  proving-service-chunk:
+    image: scrolltech/sdk-cloud-prover:sindri-v0.0.5
+    platform: linux/amd64
+    command: "--config /app/config.json"
+    profiles:
+      - proving
+    environment:
+          PROVER_NAME_PREFIX: "sindri_chunk"
+          CIRCUIT_TYPE: 1 # 1 for chunk proving
+          N_WORKERS: 1
+    volumes:
+      - ./conf/proving-service/chunk/:/app/
+      - ./conf/proving-service/config.json:/app/config.json
+    depends_on:
+      coordinator:
+        condition: service_healthy
+
+  proving-service-batch:
+    image: scrolltech/sdk-cloud-prover:sindri-v0.0.5
+    platform: linux/amd64
+    command: "--config /app/config.json"
+    profiles:
+      - proving
+    environment:
+      PROVER_NAME_PREFIX: "sindri_batch"
+      CIRCUIT_TYPE: 2 # 2 for batch proving
+      N_WORKERS: 1
+    volumes:
+      - ./conf/proving-service/batch/:/app
+      - ./conf/proving-service/config.json:/app/config.json
+    depends_on:
+      coordinator:
+        condition: service_healthy
+
+  proving-service-bundle:
+    image: scrolltech/sdk-cloud-prover:sindri-v0.0.5
+    platform: linux/amd64
+    command: "--config /app/config.json"
+    profiles:
+      - proving
+    environment:
+      PROVER_NAME_PREFIX: "sindri_bundle"
+      CIRCUIT_TYPE: 3 # 3 for bundle proving
+      N_WORKERS: 1
+    volumes:
+      - ./conf/proving-service/bundle/:/app
+      - ./conf/proving-service/config.json:/app/config.json
+    depends_on:
+      coordinator:
+        condition: service_healthy
+
+volumes:
+    db_data:

rollup/cmd/permissionless_batches/app/app.go

@@ -0,0 +1,111 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/scroll-tech/go-ethereum/ethclient"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+
+	"scroll-tech/common/database"
+	"scroll-tech/common/observability"
+	"scroll-tech/common/utils"
+	"scroll-tech/common/version"
+	"scroll-tech/rollup/internal/config"
+	"scroll-tech/rollup/internal/controller/permissionless_batches"
+	"scroll-tech/rollup/internal/controller/watcher"
+)
+
+var app *cli.App
+
+func init() {
+	// Set up rollup-relayer app info.
+	app = cli.NewApp()
+	app.Action = action
+	app.Name = "permissionless-batches"
+	app.Usage = "The Scroll Rollup Relayer for permissionless batch production"
+	app.Version = version.Version
+	app.Flags = append(app.Flags, utils.CommonFlags...)
+	app.Flags = append(app.Flags, utils.RollupRelayerFlags...)
+	app.Commands = []*cli.Command{}
+	app.Before = func(ctx *cli.Context) error {
+		return utils.LogSetup(ctx)
+	}
+}
+
+func action(ctx *cli.Context) error {
+	// Load config file.
+	cfgFile := ctx.String(utils.ConfigFileFlag.Name)
+	cfg, err := config.NewConfig(cfgFile)
+	if err != nil {
+		log.Crit("failed to load config file", "config file", cfgFile, "error", err)
+	}
+
+	subCtx, cancel := context.WithCancel(ctx.Context)
+	defer cancel()
+
+	// Make sure the required fields are set.
+	if cfg.RecoveryConfig.L1BlockHeight == 0 {
+		return fmt.Errorf("L1 block height must be specified")
+	}
+	if cfg.RecoveryConfig.LatestFinalizedBatch == 0 {
+		return fmt.Errorf("latest finalized batch must be specified")
+	}
+
+	// init db connection
+	db, err := database.InitDB(cfg.DBConfig)
+	if err != nil {
+		log.Crit("failed to init db connection", "err", err)
+	}
+	defer func() {
+		if err = database.CloseDB(db); err != nil {
+			log.Crit("failed to close db connection", "error", err)
+		}
+	}()
+
+	registry := prometheus.DefaultRegisterer
+	observability.Server(ctx, db)
+
+	genesisPath := ctx.String(utils.Genesis.Name)
+	genesis, err := utils.ReadGenesis(genesisPath)
+	if err != nil {
+		log.Crit("failed to read genesis", "genesis file", genesisPath, "error", err)
+	}
+
+	chunkProposer := watcher.NewChunkProposer(subCtx, cfg.L2Config.ChunkProposerConfig, genesis.Config, db, registry)
+	batchProposer := watcher.NewBatchProposer(subCtx, cfg.L2Config.BatchProposerConfig, genesis.Config, db, registry)
+	bundleProposer := watcher.NewBundleProposer(subCtx, cfg.L2Config.BundleProposerConfig, genesis.Config, db, registry)
+
+	// Init l2geth connection
+	l2client, err := ethclient.Dial(cfg.L2Config.Endpoint)
+	if err != nil {
+		return fmt.Errorf("failed to connect to L2geth at RPC=%s: %w", cfg.L2Config.Endpoint, err)
+	}
+
+	l2Watcher := watcher.NewL2WatcherClient(subCtx, l2client, cfg.L2Config.Confirmations, cfg.L2Config.L2MessageQueueAddress, cfg.L2Config.WithdrawTrieRootSlot, genesis.Config, db, registry)
+
+	recovery := permissionless_batches.NewRecovery(subCtx, cfg, genesis, db, chunkProposer, batchProposer, bundleProposer, l2Watcher)
+
+	if recovery.RecoveryNeeded() {
+		if err = recovery.Run(); err != nil {
+			return fmt.Errorf("failed to run recovery: %w", err)
+		}
+		log.Info("Success! You're ready to generate proofs!")
+	} else {
+		// TODO: implement batch submission if proofs are available
+		log.Info("TODO: Batch submission")
+	}
+
+	return nil
+}
+
+// Run rollup relayer cmd instance.
+func Run() {
+	if err := app.Run(os.Args); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

rollup/cmd/permissionless_batches/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "scroll-tech/rollup/cmd/permissionless_batches/app"
+
+func main() {
+	app.Run()
+}