Merge pull request #20 from lambdaclass/update-main

Update main

.cargo/config.toml

@@ -0,0 +1,13 @@
+[env]
+CAIRO_NATIVE_RUNTIME_LIBRARY = "./libcairo_native_runtime.a"
+LLVM_SYS_191_PREFIX = "/usr/lib/llvm-19/"
+MLIR_SYS_190_PREFIX = "/usr/lib/llvm-19/"
+TABLEGEN_190_PREFIX = "/usr/lib/llvm-19/"
+
+# Use `lld` for linking instead of `ld`, since we run out of memory while linking with `ld` on
+# 16-cores linux machines, see:
+# https://nnethercote.github.io/perf-book/build-configuration.html#linking.
+# TODO: remove this once `rust` stabilizes `lld` as the default linker, currently only on nightly:
+# https://github.com/rust-lang/rust/issues/39915#issuecomment-618726211
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-Clink-arg=-fuse-ld=lld"]

.github/actions/bootstrap/action.yml

@@ -0,0 +1,7 @@
+runs:
+  using: "composite"
+  steps:
+    - name: Install rust.
+      uses: ./.github/actions/install_rust
+    - name: Install cairo native.
+      uses: ./.github/actions/setup_native_deps

.github/actions/install_rust/action.yml

@@ -1,27 +1,9 @@
-# Installs rust toolchain with the version defined in .github/actions/install_rust/rust_version.txt.
-inputs:
-  components:
-    description: "Optional: which additional components to install."
-    required: false
-    type: string
-    default: ''
+name: Bootsrap rust installation
+description: Setup rust environment and its components, also caching the build results.
 
 runs:
   using: "composite"
   steps:
-    - name: Define toolchain
-      run: echo "DEFAULT_TOOLCHAIN=$(cat .github/actions/install_rust/rust_version.txt)" >> $GITHUB_ENV
-      shell: bash
-
-    - name: install rust toolchain with given components
-      if: "${{ inputs.components != '' }}"
-      uses: dtolnay/rust-toolchain@master
-      with:
-        toolchain: ${{ env.DEFAULT_TOOLCHAIN }}
-        components: ${{ inputs.components }}
-
-    - name: install rust toolchain without given components
-      if: "${{ inputs.components == '' }}"
-      uses: dtolnay/rust-toolchain@master
+    - uses: moonrepo/setup-rust@v1
       with:
-        toolchain: ${{ env.DEFAULT_TOOLCHAIN }}
+        cache-base: main(-v[0-9].*)?

.github/actions/setup_native_deps/action.yml

@@ -0,0 +1,16 @@
+name: "Setup Cairo Native Dependencies"
+description: "Sets up LLVM and GMP libraries"
+
+outputs:
+  cairo-native-runtime-library:
+    description: "The path to the cairo native runtime library"
+    value: ${{ steps.set-env-vars.outputs.cairo-native-runtime-library }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Cairo Native Runtime Dependencies
+      id: set-runtime-deps
+      shell: bash
+      run: |
+        sudo ./scripts/dependencies.sh

.github/workflows/blockifier_ci.yml

@@ -23,90 +23,38 @@ on:
       # Other than code-related changes, all changes related to the native-blockifier build-and-push
       # process should trigger the build (e.g., changes to the Dockerfile, build scripts, etc.).
       - '.github/workflows/blockifier_ci.yml'
+      - '.github/workflows/upload_artifacts_workflow.yml'
       - 'build_native_in_docker.sh'
       - 'Cargo.lock'
       - 'Cargo.toml'
       - 'crates/blockifier/**'
       - 'crates/native_blockifier/**'
       - 'scripts/build_native_blockifier.sh'
+      - 'scripts/dependencies.sh'
       - 'scripts/install_build_tools.sh'
       - 'scripts/sequencer-ci.Dockerfile'
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  featureless-build:
+  feature-combo-builds:
     runs-on: starkware-ubuntu-20-04-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
-      - uses: Swatinem/rust-cache@v2
         with:
-          prefix-key: "v0-rust-ubuntu-20.04"
+          # required to clone native as a gitsubmodule
+          submodules: recursive
+          fetch-depth: 0
+      - uses: ./.github/actions/bootstrap
+      # No features - build blockifier without features activated by dependencies in the workspace.
       - run: cargo build -p blockifier
       - run: cargo test -p blockifier
-
-  native-blockifier-artifacts-push:
-    runs-on: starkware-ubuntu-20-04-medium
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
-
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v1-rust-ubuntu-20.04"
-
-      - name: Build native blockifier
-        run: ./build_native_in_docker.sh scripts/build_native_blockifier.sh
-
-        # Commit hash on pull request event would be the head commit of the branch.
-      - name: Get commit hash prefix for PR update
-        if: ${{ github.event_name == 'pull_request' }}
-        env:
-          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
-        run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV
-
-        # On push event (to main, for example) we should take the commit post-push.
-      - name: Get commit hash prefix for merge
-        if: ${{ github.event_name != 'pull_request' }}
-        env:
-          COMMIT_SHA: ${{ github.event.after }}
-        run: echo "SHORT_HASH=${COMMIT_SHA:0:7}" >> $GITHUB_ENV
-
-      # Rename is required; see https://pyo3.rs/v0.19.2/building_and_distribution#manual-builds.
-      - name: Rename shared object
-        run: |
-          mv \
-            target/release/libnative_blockifier.so \
-            target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so
-
-      # Check if the user has the required permission to upload the artifact.
-      - name: Get User Permission
-        id: checkAccess
-        uses: actions-cool/check-user-permission@v2
-        with:
-          require: write
-          username: ${{ github.triggering_actor }}
-
-      - name: Check User Permission
-        if: steps.checkAccess.outputs.require-result == 'false'
-        run: |
-          echo "${{ github.triggering_actor }} does not have permissions on this repo."
-          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
-          echo "Job originally triggered by ${{ github.actor }}"
-          exit 1
-
-      - name: Authenticate with GCS
-        uses: "google-github-actions/auth@v2"
-        with:
-          credentials_json: ${{ secrets.SA_NATIVE_BLOCKIFIER_ARTIFACTS_BUCKET_WRITER_ACCESS_KEY }}
-
-      - name: Upload binary to GCP
-        id: upload_file
-        uses: "google-github-actions/upload-cloud-storage@v2"
-        with:
-          path: "target/release/native_blockifier.pypy39-pp73-x86_64-linux-gnu.so"
-          destination: "native_blockifier_artifacts/${{ env.SHORT_HASH }}/release/"
+      # transaction_serde is not activated by any workspace crate; test the build.
+      - run: cargo build -p blockifier --features transaction_serde
+      - run: cargo test -p blockifier --features transaction_serde
+      # cairo_native is not activated by any workspace crate; test the build.
+      - run: cargo build -p blockifier --features cairo_native
+      - run: cargo test -p blockifier --features cairo_native

.github/workflows/blockifier_compiled_cairo.yml

@@ -1,44 +1,31 @@
 name: Blockifier-Compiled-Cairo
 
 on:
-  push:
-    branches:
-      - main
-      - main-v[0-9].**
-    tags:
-      - v[0-9].**
-    paths:
-      - '.github/workflows/blockifier_compiled_cairo.yml'
-      - 'crates/blockifier/feature_contracts/**'
-      - 'crates/blockifier/src/test_utils/cairo_compile.rs'
-      - 'crates/blockifier/tests/feature_contracts_compatibility_test.rs'
-      - 'crates/blockifier/tests/requirements.txt'
   pull_request:
     types:
       - opened
       - reopened
       - synchronize
     paths:
+      - 'Cargo.toml'
       - '.github/workflows/blockifier_compiled_cairo.yml'
       - 'crates/blockifier/feature_contracts/**'
       - 'crates/blockifier/src/test_utils/cairo_compile.rs'
       - 'crates/blockifier/tests/feature_contracts_compatibility_test.rs'
       - 'crates/blockifier/tests/requirements.txt'
+      - 'scripts/dependencies.sh'
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   verify_cairo_file_dependencies:
     runs-on: starkware-ubuntu-20-04-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v0-rust-ubuntu-20.04"
+      - uses: ./.github/actions/bootstrap
 
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5
@@ -70,6 +57,11 @@ jobs:
         with:
           toolchain: nightly-2023-07-05
 
+      - name: install toolchain for cairo steps test contract compilation (old compiler tag)
+        uses: actions-rs/toolchain@master
+        with:
+          toolchain: nightly-2024-04-29
+
       - name: Verify cairo contract recompilation (both cairo versions).
         run:
           cd sequencer &&

.github/workflows/blockifier_post-merge.yml

@@ -8,18 +8,15 @@ on:
       - '.github/workflows/blockifier_post-merge.yml'
       - 'crates/blockifier/**'
       - 'crates/native_blockifier/**'
+      - 'scripts/dependencies.sh'
 
 jobs:
   if_merged:
     if: github.event.pull_request.merged == true
     runs-on: starkware-ubuntu-20-04-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
-      - uses: Noelware/[email protected]
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v0-rust-ubuntu-20.04"
+      - uses: ./.github/actions/bootstrap
 
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5

.github/workflows/blockifier_reexecution_ci.yml

@@ -0,0 +1,44 @@
+name: Blockifier-Reexecution-CI
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - auto_merge_enabled
+      - edited
+    paths:
+      # Other than code-related changes, all changes related to the blockifier should trigger the build.
+      - '.github/workflows/blockifier_reexecution_ci.yml'
+      - 'Cargo.lock'
+      - 'Cargo.toml'
+      - 'crates/blockifier/**'
+      - 'crates/blockifier_reexecution/**'
+      - 'scripts/dependencies.sh'
+      - 'scripts/install_build_tools.sh'
+      - 'scripts/sequencer-ci.Dockerfile'
+
+# On PR events, cancel existing CI runs on this same PR for this workflow.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.job }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  blockifier_reexecution:
+    runs-on: starkware-ubuntu-latest-medium
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/bootstrap
+      # Download the blockifier re-execution test data.
+      - id: auth
+        uses: "google-github-actions/auth@v2"
+        with:
+          credentials_json: ${{ secrets.SA_REEXECUTION_ARTIFACTS_READ_ACCESS_KEY }}
+      - uses: 'google-github-actions/setup-gcloud@v2'
+      - run: echo "REEXECUTION_INPUT_FILES_PREFIX=$(cat ./crates/blockifier_reexecution/resources/offline_reexecution_files_prefix)" >> $GITHUB_ENV
+      - run: gcloud storage cp -r gs://reexecution_artifacts/$REEXECUTION_INPUT_FILES_PREFIX/resources/* ./crates/blockifier_reexecution/resources/
+      # Run blockifier re-execution tests.
+      - run: cargo test --release -p blockifier_reexecution -- --include-ignored
+      # Compile the rpc-tests, without running them.
+      - run: cargo test -p blockifier_reexecution --features blockifier_regression_https_testing --no-run