Merge branch 'develop' into feat/cancun

.cargo/config.toml

@@ -0,0 +1,4 @@
+[build]
+# https://github.com/rust-lang/rust/pull/124129
+# https://github.com/dtolnay/linkme/pull/88
+rustflags = ["-Z", "linker-features=-lld"]

.dockerignore

@@ -0,0 +1,28 @@
+# this is loosely based on `docker init`'s rust template.
+
+**/.DS_Store
+**/.classpath
+**/.dockerignore
+# **/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/secrets.dev.yaml
+**/values.dev.yaml
+/bin
+/target
+LICENSE
+README.md

.github/workflows/docker_build.yml

@@ -12,24 +12,15 @@ on:
 
 jobs:
   docker:
-    name: Build and run leader and worker docker images for regression check
+    name: Regression test docker images
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Build leader docker container
-        run: |
-          docker build --progress plain -t leader:${{ github.ref_name }} -f leader.Dockerfile .
-
-      - name: Run leader docker container
-        run: |
-          docker run --rm leader:${{ github.ref_name }} --help
-
-      - name: Build worker docker container
-        run: |
-          docker build --progress plain -t worker:${{ github.ref_name }} -f worker.Dockerfile .
-
-      - name: Run worker docker container
-        run: |
-          docker run --rm worker:${{ github.ref_name }} --help
+      - run: |
+          docker build --progress=plain --build-arg=PROFILE=dev --tag scratch .
+          docker run --rm --init --entrypoint leader scratch --help
+          docker run --rm --init --entrypoint worker scratch --help
+          docker run --rm --init --entrypoint rpc scratch --help
+          docker run --rm --init --entrypoint verifier scratch --help

.github/workflows/docker_build_push.yml

@@ -8,8 +8,6 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME_LEADER: ${{ github.repository }}-leader
-  IMAGE_NAME_WORKER: ${{ github.repository }}-worker
 
 jobs:
   docker:
@@ -34,50 +32,24 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract metadata (tags, labels) for Leader Docker
-        id: meta_leader
-        uses: docker/metadata-action@v5
-        with:
-          images: | 
-            name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LEADER }}
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Push to GitHub Container Registry - Leader
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: ./leader.Dockerfile
-          push: true
-          # platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta_leader.outputs.tags }}
-          labels: ${{ steps.meta_leader.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Extract metadata (tags, labels) for Worker Docker
-        id: meta_worker
+      - name: Extract metadata (tags, labels)
+        id: meta
         uses: docker/metadata-action@v5
         with:
           images: |
-            name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME_WORKER }}
+            name=${{ env.REGISTRY }}/${{ github.repository }}
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
-      - name: Push to GitHub Container Registry - Worker
+      - name: Push to GitHub Container Registry
         uses: docker/build-push-action@v3
         with:
-          context: .
-          file: ./worker.Dockerfile
           push: true
           # platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta_worker.outputs.tags }}
-          labels: ${{ steps.meta_worker.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

Cargo.toml

@@ -1,18 +1,20 @@
 [workspace]
-members = ["mpt_trie",
-    "smt_trie",
-    "proof_gen",
-    "trace_decoder",
+members = [
+    "compat",
     "evm_arithmetization",
+    "mpt_trie",
     "proc_macro",
-    "zero_bin/leader",
-    "zero_bin/worker",
+    "proof_gen",
+    "smt_trie",
+    "trace_decoder",
     "zero_bin/common",
+    "zero_bin/leader",
     "zero_bin/ops",
-    "zero_bin/verifier",
+    "zero_bin/prover",
     "zero_bin/rpc",
-    "zero_bin/prover", 
-    "compat"]
+    "zero_bin/verifier",
+    "zero_bin/worker",
+]
 resolver = "2"
 
 [workspace.package]
@@ -24,7 +26,7 @@ keywords = ["cryptography", "STARK", "plonky2", "ethereum", "zk"]
 categories = ["cryptography::cryptocurrencies"]
 
 [workspace.dependencies]
-alloy = { git = "https://github.com/alloy-rs/alloy", tag='v0.1.1', default-features = false, features = [
+alloy = { git = "https://github.com/alloy-rs/alloy", tag = 'v0.1.1', default-features = false, features = [
     "consensus",
     "reqwest",
     "json-rpc",
@@ -36,7 +38,7 @@ alloy = { git = "https://github.com/alloy-rs/alloy", tag='v0.1.1', default-featu
     "providers",
     "transports",
     "transport-http",
-    "rpc-types-debug"
+    "rpc-types-debug",
 ] }
 anyhow = "1.0.86"
 async-stream = "0.3.5"

Dockerfile

@@ -0,0 +1,100 @@
+# syntax=docker/dockerfile:1
+# This is loosely based on `docker init`'s rust template.
+# For a completely clean build, run something like this:
+# ```
+# docker build --build-arg=PROFILE=dev --no-cache
+# ```
+
+#############
+# Build stage
+#############
+# - `/src` is the repo directory.
+# - `/artifacts` is $CARGO_TARGET_DIR.
+# - `/output` is where the binaries go.
+
+ARG BUILD_BASE=rustlang/rust:nightly-bullseye-slim
+FROM ${BUILD_BASE} AS build
+
+# Install build dependencies.
+RUN apt-get update && apt-get install -y \
+    # for jemalloc
+    libjemalloc-dev \
+    libjemalloc2 \
+    make \
+    # for openssl
+    libssl-dev \
+    pkg-config \
+    # clean the image
+    && rm -rf /var/lib/apt/lists/*
+
+ARG PROFILE=release
+# forward the docker argument so that the script below can read it
+ENV PROFILE=${PROFILE}
+
+# Build the application.
+RUN \
+    # mount the repository so we don't have to COPY it in
+    --mount=type=bind,source=.,target=/src \
+    # cache artifacts and the cargo registry to speed up subsequent builds
+    --mount=type=cache,target=/artifacts \
+    --mount=type=cache,target=/usr/local/cargo/registry/ \
+    # run the build
+    <<EOF
+set -eux
+
+# need to change workdir instead of using --manifest-path because we need
+# .cargo/config.toml
+cd /src
+
+# use the cache mount
+# (we will not be able to to write to e.g `/src/target` because it is bind-mounted)
+CARGO_TARGET_DIR=/artifacts cargo build --locked "--profile=${PROFILE}" --all
+
+# narrow the find call to SUBDIR because if we just copy out all executables
+# we will break the cache invariant
+if [ "$PROFILE" = "dev" ]; then
+    SUBDIR=debug # edge case
+else
+    SUBDIR=$PROFILE
+fi
+
+# maxdepth because binaries are in the root
+# - other folders contain build scripts etc.
+mkdir /output
+find "/artifacts/$SUBDIR" \
+    -maxdepth 1 \
+    -type f \
+    -executable \
+    -not -name '*.so' \
+    -exec cp '{}' /output \; \
+    -print
+
+EOF
+
+##################
+# Final executable
+##################
+FROM debian:bullseye-slim AS final
+
+# Install runtime dependencies.
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    libjemalloc2 \
+    && rm -rf /var/lib/apt/lists/*
+
+# this keeps this build target agnostic to the build profile
+COPY --from=build ["/output/rpc", "/output/leader", "/output/worker", "/output/verifier", "/usr/local/bin/"]
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    user
+USER user
+

docs/arithmetization/framework.tex

@@ -82,7 +82,7 @@ \subsection{Range-checks}
 \subsubsection{What to range-check?}
 One can note that every element that ever appears on the stack has been pushed. Therefore, enforcing a range-check on pushed elements is enough to range-check all elements on the stack. Similarly, all elements in memory must have been written prior, and therefore it is enough to range-check memory writes. However, range-checking the PUSH and MSTORE opcodes is not sufficient.
 \begin{enumerate}
-  \item Pushes and memory writes for ``MSTORE\_32BYTES'' are range-checked in ``BytePackingStark''.
+  \item Pushes and memory writes for ``MSTORE\_32BYTES'' are range-checked in ``BytePackingStark'', except PUSH operations happening in privileged mode. See \ref{push_general_view}.
   \item Syscalls, exceptions and prover inputs are range-checked in ``ArithmeticStark''.
   \item The inputs and outputs of binary and ternary arithmetic operations are range-checked in ``ArithmeticStark''.
   \item The inputs' bits of logic operations are checked to be either 1 or 0 in ``LogicStark''. Since ``LogicStark'' only deals with bitwise operations, this is enough to have range-checked outputs as well.

docs/arithmetization/tables/cpu.tex

@@ -70,4 +70,6 @@ \subsubsection{CPU columns}
     \item  \texttt{Stack}: \texttt{stack\_inv}, \texttt{stack\_inv\_aux} and \texttt{stack\_inv\_aux\_2} are used by popping-only (resp. pushing-only) instructions to check if the stack is empty after (resp. was empty
 before) the instruction. \texttt{stack\_len\_bounds\_ aux} is used to check that the stack doesn't overflow in user mode. We use the last four columns to prevent conflicts with the other general columns.
 See \ref{stackhandling} for more details.
+    \label{push_general_view}
+    \item  \texttt{Push}: \texttt{is\_not\_kernel} is used to skip range-checking the output of a PUSH operation when we are in privileged mode, as the kernel code is known and trusted.
 \end{itemize}

evm_arithmetization/src/byte_packing/byte_packing_stark.rs

@@ -175,8 +175,7 @@ impl<F: RichField + Extendable<D>, const D: usize> BytePackingStark<F, D> {
         ops: Vec<BytePackingOp>,
         min_rows: usize,
     ) -> Vec<[F; NUM_COLUMNS]> {
-        let base_len: usize = ops.iter().map(|op| usize::from(!op.bytes.is_empty())).sum();
-        let num_rows = core::cmp::max(base_len.max(BYTE_RANGE_MAX), min_rows).next_power_of_two();
+        let num_rows = core::cmp::max(ops.len().max(BYTE_RANGE_MAX), min_rows).next_power_of_two();
         let mut rows = Vec::with_capacity(num_rows);
 
         for op in ops {