Clay/feature/merge optismism v1.9.3

.github/CODEOWNERS

@@ -1,43 +1,37 @@
-# Packages
-/packages/contracts-bedrock     @ethereum-optimism/contract-reviewers
-/packages/sdk                   @ethereum-optimism/devxpod
+# Monorepo - default to go-reviewers
+* @ethereum-optimism/go-reviewers
+
+# OP Stack general
+/bedrock-devnet @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+/op-alt-da      @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+/op-batcher     @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+/op-chain-ops   @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+/op-e2e         @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+/op-node        @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+/op-proposer    @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+/op-wheel       @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+/ops-bedrock    @ethereum-optimism/op-stack @ethereum-optimism/go-reviewers
+
+# Expert areas
+/op-node/rollup @ethereum-optimism/consensus @ethereum-optimism/go-reviewers
+
+/op-supervisor  @ethereum-optimism/interop @ethereum-optimism/go-reviewers
 
-# Bedrock codebases
-/bedrock-devnet @ethereum-optimism/go-reviewers
-/cannon         @ethereum-optimism/go-reviewers
-/op-batcher     @ethereum-optimism/go-reviewers
-/op-bootnode    @ethereum-optimism/go-reviewers
-/op-chain-ops   @ethereum-optimism/go-reviewers
-/op-challenger  @ethereum-optimism/go-reviewers
-/op-dispute-mon  @ethereum-optimism/go-reviewers
-/op-e2e         @ethereum-optimism/go-reviewers
-/op-node        @ethereum-optimism/go-reviewers
-/op-node/rollup @protolambda @ajsutton
-/op-alt-da      @ethereum-optimism/go-reviewers
-/op-preimage    @ethereum-optimism/go-reviewers
-/op-program     @ethereum-optimism/go-reviewers
-/op-proposer    @ethereum-optimism/go-reviewers
-/op-service     @ethereum-optimism/go-reviewers
-/op-supervisor  @ethereum-optimism/go-reviewers
-/op-wheel       @ethereum-optimism/go-reviewers
-/ops-bedrock    @ethereum-optimism/go-reviewers
-/op-conductor   @0x00101010 @zhwrd @mslipper
+/op-conductor   @ethereum-optimism/op-conductor @ethereum-optimism/go-reviewers
+
+/cannon                @ethereum-optimism/proofs @ethereum-optimism/go-reviewers
+/op-challenger         @ethereum-optimism/proofs @ethereum-optimism/go-reviewers
+/op-dispute-mon        @ethereum-optimism/proofs @ethereum-optimism/go-reviewers
+/op-preimage           @ethereum-optimism/proofs @ethereum-optimism/go-reviewers
+/op-program            @ethereum-optimism/proofs @ethereum-optimism/go-reviewers
+/op-e2e/actions/proofs @ethereum-optimism/proofs @ethereum-optimism/go-reviewers
+/op-e2e/faultproofs    @ethereum-optimism/proofs @ethereum-optimism/go-reviewers
 
 # Ops
 /.circleci       @ethereum-optimism/monorepo-ops-reviewers
 /.github         @ethereum-optimism/monorepo-ops-reviewers
 /ops             @ethereum-optimism/monorepo-ops-reviewers
 /docker-bake.hcl @ethereum-optimism/monorepo-ops-reviewers
 
-# Misc
-/proxyd           @ethereum-optimism/infra-reviewers
-/infra            @ethereum-optimism/infra-reviewers
-/specs            @ethereum-optimism/contract-reviewers @ethereum-optimism/go-reviewers
-
-# Don't add owners if only package.json is updated
-/packages/*/package.json
-/*/package.json
-
-# JavaScript Releases
-/packages/*/CHANGELOG.md @ethereum-optimism/release-managers
-/*/CHANGELOG.md          @ethereum-optimism/release-managers
+# Contracts
+/packages/contracts-bedrock     @ethereum-optimism/contract-reviewers

.github/workflows/tag-service.yml

@@ -36,6 +36,9 @@ on:
         type: boolean
         default: false
 
+permissions:
+  contents: write
+
 jobs:
   release:
     runs-on: ubuntu-latest

.vscode/extensions.json

@@ -1,7 +1,7 @@
 {
   "recommendations": [
     "editorconfig.editorconfig",
-    "juanblanco.solidity",
+    "nomicfoundation.hardhat-solidity",
     "golang.go"
   ]
 }

Makefile

@@ -133,23 +133,25 @@ reproducible-prestate:   ## Builds reproducible-prestate binary
 	make -C ./op-program reproducible-prestate
 .PHONY: reproducible-prestate
 
-# Include any files required for the devnet to build and run. This appears to be the only one that's actually needed.
-DEVNET_CANNON_PRESTATE_FILES := op-program/bin/prestate-proof.json op-program/bin/prestate.json
+# Include any files required for the devnet to build and run.
+DEVNET_CANNON_PRESTATE_FILES := op-program/bin/prestate-proof.json op-program/bin/prestate.json op-program/bin/prestate-proof-mt.json op-program/bin/prestate-mt.bin.gz
+
 
 $(DEVNET_CANNON_PRESTATE_FILES):
 	make cannon-prestate
+	make cannon-prestate-mt
 
 cannon-prestate: op-program cannon ## Generates prestate using cannon and op-program
-	./cannon/bin/cannon load-elf --path op-program/bin/op-program-client.elf --out op-program/bin/prestate.json --meta op-program/bin/meta.json
-	./cannon/bin/cannon run --proof-at '=0' --stop-at '=1' --input op-program/bin/prestate.json --meta op-program/bin/meta.json --proof-fmt 'op-program/bin/%d.json' --output ""
+	./cannon/bin/cannon load-elf --type singlethreaded --path op-program/bin/op-program-client.elf --out op-program/bin/prestate.json --meta op-program/bin/meta.json
+	./cannon/bin/cannon run --proof-at '=0'  --stop-at '=1' --input op-program/bin/prestate.json --meta op-program/bin/meta.json --proof-fmt 'op-program/bin/%d.json' --output ""
 	mv op-program/bin/0.json op-program/bin/prestate-proof.json
 .PHONY: cannon-prestate
 
 cannon-prestate-mt: op-program cannon ## Generates prestate using cannon and op-program in the multithreaded cannon format
-	./cannon/bin/cannon load-elf --type mt --path op-program/bin/op-program-client.elf --out op-program/bin/prestate-mt.json --meta op-program/bin/meta-mt.json
-	./cannon/bin/cannon run --type mt --proof-at '=0' --stop-at '=1' --input op-program/bin/prestate-mt.json --meta op-program/bin/meta-mt.json --proof-fmt 'op-program/bin/%d-mt.json' --output ""
+	./cannon/bin/cannon load-elf --type multithreaded --path op-program/bin/op-program-client.elf --out op-program/bin/prestate-mt.bin.gz --meta op-program/bin/meta-mt.json
+	./cannon/bin/cannon run --proof-at '=0' --stop-at '=1' --input op-program/bin/prestate-mt.bin.gz --meta op-program/bin/meta-mt.json --proof-fmt 'op-program/bin/%d-mt.json' --output ""
 	mv op-program/bin/0-mt.json op-program/bin/prestate-proof-mt.json
-.PHONY: cannon-prestate
+.PHONY: cannon-prestate-mt
 
 mod-tidy: ## Cleans up unused dependencies in Go modules
 	# Below GOPRIVATE line allows mod-tidy to be run immediately after

bedrock-devnet/devnet/__init__.py

@@ -11,6 +11,8 @@
 from multiprocessing import Process, Queue
 import concurrent.futures
 from collections import namedtuple
+# This import is necessary for devnet logs to be shown.
+from . import log_setup
 
 
 pjoin = os.path.join

cannon/.gitignore

@@ -13,3 +13,4 @@ state.json
 *.pprof
 *.out
 bin
+multicannon/embeds/cannon*

cannon/Makefile

@@ -4,24 +4,39 @@ VERSION ?= v0.0.0
 
 LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT)
 LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
-LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-program/version.Version=$(VERSION)
-LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-program/version.Meta=$(VERSION_META)
+LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/cannon/multicannon/version.Version=$(VERSION)
+LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/cannon/multicannon/version.Meta=$(VERSION_META)
 LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
 
 # Use the old Apple linker to workaround broken xcode - https://github.com/golang/go/issues/65169
 ifeq ($(shell uname),Darwin)
 	FUZZLDFLAGS := -ldflags=-extldflags=-Wl,-ld_classic
 endif
 
-cannon:
-	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/cannon .
+cannon-impl:
+	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/cannon-impl .
+
+cannon-embeds: cannon-impl
+	@cp bin/cannon-impl ./multicannon/embeds/cannon-0
+	@cp bin/cannon-impl ./multicannon/embeds/cannon-1
+
+cannon: cannon-embeds
+	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/cannon ./multicannon/
 
 clean:
 	rm -rf bin
 
 elf:
 	make -C ./testdata/example elf
 
+sanitize-program:
+	@if ! { mips-linux-gnu-objdump -d -j .text $$GUEST_PROGRAM | awk '{print $3}' | grep -Ew -m1 '(bgezal|bltzal)'; }; then \
+		echo "guest program is sanitized for unsupported instructions"; \
+	else \
+		echo "found unsupported instructions in the guest program"; \
+		exit 1; \
+	fi
+
 contract:
 	cd ../packages/contracts-bedrock && forge build
 

cannon/README.md

@@ -30,7 +30,7 @@ make cannon
 
 # Transform MIPS op-program client binary into first VM state.
 # This outputs state.json (VM state) and meta.json (for debug symbols).
-./bin/cannon load-elf --path=../op-program/bin/op-program-client.elf
+./bin/cannon load-elf --type singlethreaded --path=../op-program/bin/op-program-client.elf
 
 # Run cannon emulator (with example inputs)
 # Note that the server-mode op-program command is passed into cannon (after the --),

cannon/cmd/load_elf.go

@@ -4,31 +4,34 @@ import (
 	"debug/elf"
 	"fmt"
 
-	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
-	"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
 	"github.com/urfave/cli/v2"
 
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
+	"github.com/ethereum-optimism/optimism/cannon/serialize"
+	openum "github.com/ethereum-optimism/optimism/op-service/enum"
+	"github.com/ethereum-optimism/optimism/op-service/ioutil"
 	"github.com/ethereum-optimism/optimism/op-service/jsonutil"
 )
 
 var (
+	LoadELFVMTypeFlag = &cli.StringFlag{
+		Name:     "type",
+		Usage:    "VM type to create state for. Valid options: " + openum.EnumString(stateVersions()),
+		Required: true,
+	}
 	LoadELFPathFlag = &cli.PathFlag{
 		Name:      "path",
 		Usage:     "Path to 32-bit big-endian MIPS ELF file",
 		TakesFile: true,
 		Required:  true,
 	}
-	LoadELFPatchFlag = &cli.StringSliceFlag{
-		Name:     "patch",
-		Usage:    "Type of patching to do",
-		Value:    cli.NewStringSlice("go", "stack"),
-		Required: false,
-	}
 	LoadELFOutFlag = &cli.PathFlag{
 		Name:     "out",
-		Usage:    "Output path to write JSON state to. State is dumped to stdout if set to -. Not written if empty.",
+		Usage:    "Output path to write state to. State is dumped to stdout if set to '-'. Not written if empty. Use file extension '.bin', '.bin.gz', or '.json' for binary, compressed binary, or JSON formats.",
 		Value:    "state.json",
 		Required: false,
 	}
@@ -40,74 +43,88 @@ var (
 	}
 )
 
+func stateVersions() []string {
+	vers := make([]string, len(versions.StateVersionTypes))
+	for i, v := range versions.StateVersionTypes {
+		vers[i] = v.String()
+	}
+	return vers
+}
+
 func LoadELF(ctx *cli.Context) error {
+	elfPath := ctx.Path(LoadELFPathFlag.Name)
+	elfProgram, err := elf.Open(elfPath)
+	if err != nil {
+		return fmt.Errorf("failed to open ELF file %q: %w", elfPath, err)
+	}
+	if elfProgram.Machine != elf.EM_MIPS {
+		return fmt.Errorf("ELF is not big-endian MIPS R3000, but got %q", elfProgram.Machine.String())
+	}
+
 	var createInitialState func(f *elf.File) (mipsevm.FPVMState, error)
-	var writeState func(path string, state mipsevm.FPVMState) error
 
-	if vmType, err := vmTypeFromString(ctx); err != nil {
+	var patcher = program.PatchStack
+	ver, err := versions.ParseStateVersion(ctx.String(LoadELFVMTypeFlag.Name))
+	if err != nil {
 		return err
-	} else if vmType == cannonVMType {
+	}
+	switch ver {
+	case versions.VersionSingleThreaded:
 		createInitialState = func(f *elf.File) (mipsevm.FPVMState, error) {
 			return program.LoadELF(f, singlethreaded.CreateInitialState)
 		}
-		writeState = func(path string, state mipsevm.FPVMState) error {
-			return jsonutil.WriteJSON[*singlethreaded.State](path, state.(*singlethreaded.State), OutFilePerm)
+		patcher = func(state mipsevm.FPVMState) error {
+			err := program.PatchGoGC(elfProgram, state)
+			if err != nil {
+				return err
+			}
+			return program.PatchStack(state)
 		}
-	} else if vmType == mtVMType {
+	case versions.VersionMultiThreaded:
 		createInitialState = func(f *elf.File) (mipsevm.FPVMState, error) {
 			return program.LoadELF(f, multithreaded.CreateInitialState)
 		}
-		writeState = func(path string, state mipsevm.FPVMState) error {
-			return jsonutil.WriteJSON[*multithreaded.State](path, state.(*multithreaded.State), OutFilePerm)
-		}
-	} else {
-		return fmt.Errorf("invalid VM type: %q", vmType)
-	}
-	elfPath := ctx.Path(LoadELFPathFlag.Name)
-	elfProgram, err := elf.Open(elfPath)
-	if err != nil {
-		return fmt.Errorf("failed to open ELF file %q: %w", elfPath, err)
-	}
-	if elfProgram.Machine != elf.EM_MIPS {
-		return fmt.Errorf("ELF is not big-endian MIPS R3000, but got %q", elfProgram.Machine.String())
+	default:
+		return fmt.Errorf("unsupported state version: %d (%s)", ver, ver.String())
 	}
+
 	state, err := createInitialState(elfProgram)
 	if err != nil {
 		return fmt.Errorf("failed to load ELF data into VM state: %w", err)
 	}
-	for _, typ := range ctx.StringSlice(LoadELFPatchFlag.Name) {
-		switch typ {
-		case "stack":
-			err = program.PatchStack(state)
-		case "go":
-			err = program.PatchGo(elfProgram, state)
-		default:
-			return fmt.Errorf("unrecognized form of patching: %q", typ)
-		}
-		if err != nil {
-			return fmt.Errorf("failed to apply patch %s: %w", typ, err)
-		}
+	err = patcher(state)
+	if err != nil {
+		return fmt.Errorf("failed to patch state: %w", err)
 	}
 	meta, err := program.MakeMetadata(elfProgram)
 	if err != nil {
 		return fmt.Errorf("failed to compute program metadata: %w", err)
 	}
-	if err := jsonutil.WriteJSON[*program.Metadata](ctx.Path(LoadELFMetaFlag.Name), meta, OutFilePerm); err != nil {
+	if err := jsonutil.WriteJSON[*program.Metadata](meta, ioutil.ToStdOutOrFileOrNoop(ctx.Path(LoadELFMetaFlag.Name), OutFilePerm)); err != nil {
 		return fmt.Errorf("failed to output metadata: %w", err)
 	}
-	return writeState(ctx.Path(LoadELFOutFlag.Name), state)
+
+	// Ensure the state is written with appropriate version information
+	versionedState, err := versions.NewFromState(state)
+	if err != nil {
+		return fmt.Errorf("failed to create versioned state: %w", err)
+	}
+	return serialize.Write(ctx.Path(LoadELFOutFlag.Name), versionedState, OutFilePerm)
 }
 
-var LoadELFCommand = &cli.Command{
-	Name:        "load-elf",
-	Usage:       "Load ELF file into Cannon JSON state",
-	Description: "Load ELF file into Cannon JSON state, optionally patch out functions",
-	Action:      LoadELF,
-	Flags: []cli.Flag{
-		VMTypeFlag,
-		LoadELFPathFlag,
-		LoadELFPatchFlag,
-		LoadELFOutFlag,
-		LoadELFMetaFlag,
-	},
+func CreateLoadELFCommand(action cli.ActionFunc) *cli.Command {
+	return &cli.Command{
+		Name:        "load-elf",
+		Usage:       "Load ELF file into Cannon state",
+		Description: "Load ELF file into Cannon state",
+		Action:      action,
+		Flags: []cli.Flag{
+			LoadELFVMTypeFlag,
+			LoadELFPathFlag,
+			LoadELFOutFlag,
+			LoadELFMetaFlag,
+		},
+	}
 }
+
+var LoadELFCommand = CreateLoadELFCommand(LoadELF)