From fc2eb5292bdc14efb5ee96c421956392681cd2d8 Mon Sep 17 00:00:00 2001 From: enitrat Date: Sat, 28 Sep 2024 22:59:10 +0200 Subject: [PATCH 01/10] dev: use native in ci --- .github/workflows/ci.yml | 53 ++++++++++++---- .github/workflows/test.yml | 113 ++++++++++++++++++++++++++++++++- .gitignore | 2 + scripts/setup_cairo_native.sh | 115 ++++++++++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 12 deletions(-) create mode 100644 scripts/setup_cairo_native.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92fd8de84..f858a28ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,12 +24,19 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.81.0 + - uses: Swatinem/rust-cache@v2 + - name: Set up Scarb uses: software-mansion/setup-scarb@v1 - name: Build contracts run: scarb build -p contracts + upload-artifacts: + runs-on: ubuntu-latest + needs: build + steps: - name: Upload artifacts uses: actions/upload-artifact@v3 with: @@ -47,15 +54,44 @@ jobs: - run: scarb build - run: scarb test - ef-tests: + ef-tests: # trunk-ignore(actionlint/runner-label) - runs-on: ubuntu-latest-32-cores - needs: build + runs-on: ubuntu-latest-16-cores + needs: [build] steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache cairo-native setup + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + ./libcairo_native_runtime.a + key: ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', 'scripts/setup_cairo_native.sh') }} + + - name: Make setup script executable + run: chmod +x ./scripts/setup_cairo_native.sh + + - name: Setup Cairo Native + run: sudo ./scripts/setup_cairo_native.sh + + - name: Set Environment Variables + run: | + echo "MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "TABLEGEN_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$(pwd)/libcairo_native_runtime.a" >> $GITHUB_ENV + - name: Checkout ef-tests uses: actions/checkout@v4 with: repository: kkrt-labs/ef-tests + ref: feat/cairo-native - name: Checkout local skip file uses: actions/checkout@v4 @@ -68,11 +104,6 @@ jobs: - name: Replace the skip files run: mv skip-file/blockchain-tests-skip.yml blockchain-tests-skip.yml - - name: Rust cache - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: "true" - - name: Setup run: | mkdir -p build/common @@ -88,8 +119,8 @@ jobs: path: ./build/v1 - name: Move Cairo1Helpers - run: - mv build/v1/contracts_Cairo1Helpers.compiled_contract_class.json + run: | + mv build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ build/common/cairo1_helpers.json - name: Set up Python @@ -100,7 +131,7 @@ jobs: - name: Run tests run: | set -o pipefail - make ef-test-v1 | tee data.txt + make ef-test-v1-native | tee data.txt set +o pipefail - name: Retrieve ef-tests execution resources diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 32c1a450a..da38c8bca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,16 @@ concurrency: cancel-in-progress: true jobs: + + build: + # trunk-ignore(actionlint/runner-label) + runs-on: ubuntu-latest-16-cores + steps: + - uses: actions/checkout@v4 + - uses: foundry-rs/setup-snfoundry@v3 + - uses: software-mansion/setup-scarb@v1 + - run: scarb build + tests: # trunk-ignore(actionlint/runner-label) runs-on: ubuntu-latest-16-cores @@ -17,5 +27,106 @@ jobs: - uses: foundry-rs/setup-snfoundry@v3 - uses: software-mansion/setup-scarb@v1 - run: scarb fmt --check - - run: scarb build - run: scarb test + + ef-tests: + # trunk-ignore(actionlint/runner-label) + runs-on: ubuntu-latest-16-cores + needs: [build] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache cairo-native setup + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + ./libcairo_native_runtime.a + key: ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', 'scripts/setup_cairo_native.sh') }} + + - name: Make setup script executable + run: chmod +x ./scripts/setup_cairo_native.sh + + - name: Setup Cairo Native + run: sudo ./scripts/setup_cairo_native.sh + + - name: Set Environment Variables + run: | + echo "MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "TABLEGEN_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$(pwd)/libcairo_native_runtime.a" >> $GITHUB_ENV + + - name: Checkout ef-tests + uses: actions/checkout@v4 + with: + repository: kkrt-labs/ef-tests + ref: feat/cairo-native + + - name: Checkout local skip file + uses: actions/checkout@v4 + with: + sparse-checkout: | + blockchain-tests-skip.yml + sparse-checkout-cone-mode: false + path: skip-file + + - name: Replace the skip files + run: mv skip-file/blockchain-tests-skip.yml blockchain-tests-skip.yml + + - name: Setup + run: | + mkdir -p build/common + make setup setup-kakarot-v0 + + - name: Install nextest + uses: taiki-e/install-action@nextest + + - name: Download Kakarot-SSJ build artifacts in v1 + uses: actions/download-artifact@v3 + with: + name: kakarot-ssj-build + path: ./build/v1 + + - name: Move Cairo1Helpers + run: | + mv build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ + build/common/cairo1_helpers.json + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.10.14 + + - name: Run tests + run: | + set -o pipefail + make ef-test-v1-native | tee data.txt + set +o pipefail + + - name: Retrieve ef-tests execution resources + run: python scripts/compute_resources.py + env: + KAKAROT_VERSION: v1 + + - name: Upload resources + uses: actions/upload-artifact@v3 + with: + path: resources + name: resources + + - name: Generate blockchain-tests-skip.yml file + if: github.event_name == 'workflow_dispatch' + run: make generate-skip-file + + - name: Upload skip file + if: github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v3 + with: + path: blockchain-tests-skip.yml + name: blockchain-tests-skip diff --git a/.gitignore b/.gitignore index 0469dd3ad..bcdcc22c4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ target node_modules/ *.snfoundry_cache cache/ + +scripts/libcairo_native_runtime.a diff --git a/scripts/setup_cairo_native.sh b/scripts/setup_cairo_native.sh new file mode 100644 index 000000000..8321cab9a --- /dev/null +++ b/scripts/setup_cairo_native.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +install_essential_deps_linux() { + apt-get update -y + apt-get install -y \ + curl \ + jq \ + ripgrep \ + wget \ + ca-certificates \ + gnupg \ + git +} + +setup_llvm_deps() { + case "$(uname)" in + Darwin) + brew update + brew install llvm@19 + + LIBRARY_PATH=/opt/homebrew/lib + MLIR_SYS_190_PREFIX="$(brew --prefix llvm@19)" + LLVM_SYS_191_PREFIX="$MLIR_SYS_190_PREFIX" + TABLEGEN_190_PREFIX="$MLIR_SYS_190_PREFIX" + + export LIBRARY_PATH + export MLIR_SYS_190_PREFIX + export LLVM_SYS_191_PREFIX + export TABLEGEN_190_PREFIX + ;; + Linux) + export DEBIAN_FRONTEND=noninteractive + export TZ=America/New_York + + CODENAME=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) + [ -z "$CODENAME" ] && { echo "Error: Unable to determine OS codename"; exit 1; } + + echo "deb http://apt.llvm.org/$CODENAME/ llvm-toolchain-$CODENAME-19 main" > /etc/apt/sources.list.d/llvm-19.list + echo "deb-src http://apt.llvm.org/$CODENAME/ llvm-toolchain-$CODENAME-19 main" >> /etc/apt/sources.list.d/llvm-19.list + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + + apt-get update && apt-get upgrade -y + apt-get install -y llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools + + MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/ + LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/ + TABLEGEN_190_PREFIX=/usr/lib/llvm-19/ + + export MLIR_SYS_190_PREFIX + export LLVM_SYS_191_PREFIX + export TABLEGEN_190_PREFIX + ;; + *) + echo "Error: Unsupported operating system" + exit 1 + ;; + esac + + # GitHub Actions specific + [ -n "$GITHUB_ACTIONS" ] && { + echo "MLIR_SYS_190_PREFIX=$MLIR_SYS_190_PREFIX" >> $GITHUB_ENV + echo "LLVM_SYS_191_PREFIX=$LLVM_SYS_191_PREFIX" >> $GITHUB_ENV + echo "TABLEGEN_190_PREFIX=$TABLEGEN_190_PREFIX" >> $GITHUB_ENV + } +} + +install_rust() { + if command -v cargo >/dev/null 2>&1; then + echo "Rust is already installed with cargo available in PATH." + return 0 + fi + + echo "cargo not found. Installing Rust..." + if ! curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.81.0 --no-modify-path; then + echo >&2 "Failed to install Rust. Aborting." + return 1 + fi + + # shellcheck disable=SC1090 + source "$HOME/.cargo/env" || { + echo >&2 "Failed to source Rust environment. Aborting." + return 1 + } + + echo "Rust installed successfully." +} + +install_cairo_native_runtime() { + install_rust || { echo "Error: Failed to install Rust"; exit 1; } + + git clone https://github.com/lambdaclass/cairo_native.git + pushd ./cairo_native || exit 1 + make deps + make runtime + popd || exit 1 + + rm -rf ./cairo_native + + export CAIRO_NATIVE_RUNTIME_LIBRARY="$(pwd)/libcairo_native_runtime.a" + + echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$CAIRO_NATIVE_RUNTIME_LIBRARY" + + [ -n "$GITHUB_ACTIONS" ] && echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$CAIRO_NATIVE_RUNTIME_LIBRARY" >> $GITHUB_ENV +} + +main() { + [ "$(uname)" = "Linux" ] && install_essential_deps_linux + + setup_llvm_deps + install_cairo_native_runtime + + echo "LLVM and Cairo native runtime dependencies installed successfully." +} + +main "$@" From 4b73c097a5cb54602cf8b2d1fbf9bdee96994568 Mon Sep 17 00:00:00 2001 From: enitrat Date: Sat, 28 Sep 2024 23:00:30 +0200 Subject: [PATCH 02/10] remove outdated gas reports --- .github/workflows/gas_reports.yml | 43 ------------------------------ .github/workflows/gas_snapshot.yml | 34 ----------------------- 2 files changed, 77 deletions(-) delete mode 100644 .github/workflows/gas_reports.yml delete mode 100644 .github/workflows/gas_snapshot.yml diff --git a/.github/workflows/gas_reports.yml b/.github/workflows/gas_reports.yml deleted file mode 100644 index 6200d6329..000000000 --- a/.github/workflows/gas_reports.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Compare Snapshot - -on: pull_request - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - pull-requests: write - -jobs: - compare-snapshot: - permissions: write-all - # trunk-ignore(actionlint/runner-label) - runs-on: ubuntu-latest-16-cores - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - - name: Set up Scarb - uses: software-mansion/setup-scarb@v1 - - - name: Run compare_snapshot script - id: run-script - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - continue-on-error: true - run: | - result=$(python scripts/compare_snapshot.py 2>&1) - status=$(echo "$result" | tail -n 1 | awk -F ':' '{print $1}') - echo "$result" >> "$GITHUB_STEP_SUMMARY" - echo "status=$status" >> "$GITHUB_OUTPUT" - - - name: Exit step based on status - if: steps.run-script.outputs.status == 'ERROR' - run: exit 1 diff --git a/.github/workflows/gas_snapshot.yml b/.github/workflows/gas_snapshot.yml deleted file mode 100644 index 3e8ffa6cf..000000000 --- a/.github/workflows/gas_snapshot.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Generate and Upload Gas Snapshot - -permissions: read-all - -on: - push: - branches: - - main - workflow_dispatch: {} - -jobs: - build: - # trunk-ignore(actionlint/runner-label) - runs-on: ubuntu-latest-16-cores - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - name: Set up Scarb - uses: software-mansion/setup-scarb@v1 - - - name: Generate gas snapshot - run: python scripts/gen_snapshot.py - - - name: Upload gas snapshot to GitHub Artifacts - uses: actions/upload-artifact@v2 - with: - name: gas-snapshot - path: gas_snapshot.json From 490ea1b41b443958f9b3a19fa370e0af9b095ef2 Mon Sep 17 00:00:00 2001 From: enitrat Date: Sat, 28 Sep 2024 23:02:01 +0200 Subject: [PATCH 03/10] fix fmt --- .github/workflows/ci.yml | 10 +- .github/workflows/test.yml | 31 ++++-- scripts/setup_cairo_native.sh | 187 ++++++++++++++++++---------------- 3 files changed, 123 insertions(+), 105 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f858a28ff..256039b79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,10 +33,6 @@ jobs: - name: Build contracts run: scarb build -p contracts - upload-artifacts: - runs-on: ubuntu-latest - needs: build - steps: - name: Upload artifacts uses: actions/upload-artifact@v3 with: @@ -54,7 +50,7 @@ jobs: - run: scarb build - run: scarb test - ef-tests: + ef-tests: # trunk-ignore(actionlint/runner-label) runs-on: ubuntu-latest-16-cores needs: [build] @@ -72,7 +68,9 @@ jobs: ~/.cargo/git/db/ target/ ./libcairo_native_runtime.a - key: ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', 'scripts/setup_cairo_native.sh') }} + key: + ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', + 'scripts/setup_cairo_native.sh') }} - name: Make setup script executable run: chmod +x ./scripts/setup_cairo_native.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da38c8bca..3a84e33b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,15 +9,26 @@ concurrency: cancel-in-progress: true jobs: - build: - # trunk-ignore(actionlint/runner-label) - runs-on: ubuntu-latest-16-cores + runs-on: ubuntu-latest + env: + CI_COMMIT_MESSAGE: CI Formatting Auto Commit + CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} CI steps: - - uses: actions/checkout@v4 - - uses: foundry-rs/setup-snfoundry@v3 - - uses: software-mansion/setup-scarb@v1 - - run: scarb build + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Scarb + uses: software-mansion/setup-scarb@v1 + + - name: Build contracts + run: scarb build -p contracts + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: kakarot-ssj-build + path: target/dev tests: # trunk-ignore(actionlint/runner-label) @@ -29,7 +40,7 @@ jobs: - run: scarb fmt --check - run: scarb test - ef-tests: + ef-tests: # trunk-ignore(actionlint/runner-label) runs-on: ubuntu-latest-16-cores needs: [build] @@ -47,7 +58,9 @@ jobs: ~/.cargo/git/db/ target/ ./libcairo_native_runtime.a - key: ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', 'scripts/setup_cairo_native.sh') }} + key: + ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', + 'scripts/setup_cairo_native.sh') }} - name: Make setup script executable run: chmod +x ./scripts/setup_cairo_native.sh diff --git a/scripts/setup_cairo_native.sh b/scripts/setup_cairo_native.sh index 8321cab9a..ef81af9fc 100644 --- a/scripts/setup_cairo_native.sh +++ b/scripts/setup_cairo_native.sh @@ -1,115 +1,122 @@ #!/bin/bash install_essential_deps_linux() { - apt-get update -y - apt-get install -y \ - curl \ - jq \ - ripgrep \ - wget \ - ca-certificates \ - gnupg \ - git + apt-get update -y + apt-get install -y \ + curl \ + jq \ + ripgrep \ + wget \ + ca-certificates \ + gnupg \ + git } setup_llvm_deps() { - case "$(uname)" in - Darwin) - brew update - brew install llvm@19 - - LIBRARY_PATH=/opt/homebrew/lib - MLIR_SYS_190_PREFIX="$(brew --prefix llvm@19)" - LLVM_SYS_191_PREFIX="$MLIR_SYS_190_PREFIX" - TABLEGEN_190_PREFIX="$MLIR_SYS_190_PREFIX" - - export LIBRARY_PATH - export MLIR_SYS_190_PREFIX - export LLVM_SYS_191_PREFIX - export TABLEGEN_190_PREFIX - ;; - Linux) - export DEBIAN_FRONTEND=noninteractive - export TZ=America/New_York - - CODENAME=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) - [ -z "$CODENAME" ] && { echo "Error: Unable to determine OS codename"; exit 1; } - - echo "deb http://apt.llvm.org/$CODENAME/ llvm-toolchain-$CODENAME-19 main" > /etc/apt/sources.list.d/llvm-19.list - echo "deb-src http://apt.llvm.org/$CODENAME/ llvm-toolchain-$CODENAME-19 main" >> /etc/apt/sources.list.d/llvm-19.list - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - - - apt-get update && apt-get upgrade -y - apt-get install -y llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools - - MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/ - LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/ - TABLEGEN_190_PREFIX=/usr/lib/llvm-19/ - - export MLIR_SYS_190_PREFIX - export LLVM_SYS_191_PREFIX - export TABLEGEN_190_PREFIX - ;; - *) - echo "Error: Unsupported operating system" - exit 1 - ;; - esac - - # GitHub Actions specific - [ -n "$GITHUB_ACTIONS" ] && { - echo "MLIR_SYS_190_PREFIX=$MLIR_SYS_190_PREFIX" >> $GITHUB_ENV - echo "LLVM_SYS_191_PREFIX=$LLVM_SYS_191_PREFIX" >> $GITHUB_ENV - echo "TABLEGEN_190_PREFIX=$TABLEGEN_190_PREFIX" >> $GITHUB_ENV - } + case "$(uname)" in + Darwin) + brew update + brew install llvm@19 + + LIBRARY_PATH=/opt/homebrew/lib + MLIR_SYS_190_PREFIX="$(brew --prefix llvm@19)" + LLVM_SYS_191_PREFIX="${MLIR_SYS_190_PREFIX}" + TABLEGEN_190_PREFIX="${MLIR_SYS_190_PREFIX}" + + export LIBRARY_PATH + export MLIR_SYS_190_PREFIX + export LLVM_SYS_191_PREFIX + export TABLEGEN_190_PREFIX + ;; + Linux) + export DEBIAN_FRONTEND=noninteractive + export TZ=America/New_York + + CODENAME=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) + [[ -z ${CODENAME} ]] && { + echo "Error: Unable to determine OS codename" + exit 1 + } + + echo "deb http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-19 main" >/etc/apt/sources.list.d/llvm-19.list + echo "deb-src http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-19 main" >>/etc/apt/sources.list.d/llvm-19.list + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + + apt-get update && apt-get upgrade -y + apt-get install -y llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools + + MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/ + LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/ + TABLEGEN_190_PREFIX=/usr/lib/llvm-19/ + + export MLIR_SYS_190_PREFIX + export LLVM_SYS_191_PREFIX + export TABLEGEN_190_PREFIX + ;; + *) + echo "Error: Unsupported operating system" + exit 1 + ;; + esac + + # GitHub Actions specific + [[ -n ${GITHUB_ACTIONS} ]] && { + echo "MLIR_SYS_190_PREFIX=${MLIR_SYS_190_PREFIX}" >>"${GITHUB_ENV}" + echo "LLVM_SYS_191_PREFIX=${LLVM_SYS_191_PREFIX}" >>"${GITHUB_ENV}" + echo "TABLEGEN_190_PREFIX=${TABLEGEN_190_PREFIX}" >>"${GITHUB_ENV}" + } } install_rust() { - if command -v cargo >/dev/null 2>&1; then - echo "Rust is already installed with cargo available in PATH." - return 0 - fi - - echo "cargo not found. Installing Rust..." - if ! curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.81.0 --no-modify-path; then - echo >&2 "Failed to install Rust. Aborting." - return 1 - fi - - # shellcheck disable=SC1090 - source "$HOME/.cargo/env" || { - echo >&2 "Failed to source Rust environment. Aborting." - return 1 - } - - echo "Rust installed successfully." + if command -v cargo >/dev/null 2>&1; then + echo "Rust is already installed with cargo available in PATH." + return 0 + fi + + echo "cargo not found. Installing Rust..." + if ! curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.81.0 --no-modify-path; then + echo >&2 "Failed to install Rust. Aborting." + return 1 + fi + + # shellcheck disable=SC1090 + source "${HOME}/.cargo/env" || { + echo >&2 "Failed to source Rust environment. Aborting." + return 1 + } + + echo "Rust installed successfully." } install_cairo_native_runtime() { - install_rust || { echo "Error: Failed to install Rust"; exit 1; } + install_rust || { + echo "Error: Failed to install Rust" + exit 1 + } - git clone https://github.com/lambdaclass/cairo_native.git - pushd ./cairo_native || exit 1 - make deps - make runtime - popd || exit 1 + git clone https://github.com/lambdaclass/cairo_native.git + pushd ./cairo_native || exit 1 + make deps + make runtime + popd || exit 1 - rm -rf ./cairo_native + rm -rf ./cairo_native - export CAIRO_NATIVE_RUNTIME_LIBRARY="$(pwd)/libcairo_native_runtime.a" + CAIRO_NATIVE_RUNTIME_LIBRARY="$(pwd)/libcairo_native_runtime.a" + export CAIRO_NATIVE_RUNTIME_LIBRARY - echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$CAIRO_NATIVE_RUNTIME_LIBRARY" + echo "CAIRO_NATIVE_RUNTIME_LIBRARY=${CAIRO_NATIVE_RUNTIME_LIBRARY}" - [ -n "$GITHUB_ACTIONS" ] && echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$CAIRO_NATIVE_RUNTIME_LIBRARY" >> $GITHUB_ENV + [[ -n ${GITHUB_ACTIONS} ]] && echo "CAIRO_NATIVE_RUNTIME_LIBRARY=${CAIRO_NATIVE_RUNTIME_LIBRARY}" >>"${GITHUB_ENV}" } main() { - [ "$(uname)" = "Linux" ] && install_essential_deps_linux + [[ "$(uname)" == "Linux" ]] && install_essential_deps_linux - setup_llvm_deps - install_cairo_native_runtime + setup_llvm_deps + install_cairo_native_runtime - echo "LLVM and Cairo native runtime dependencies installed successfully." + echo "LLVM and Cairo native runtime dependencies installed successfully." } main "$@" From 73d47846ca67fbed0bedfc3e8e302e2fbd7ecff5 Mon Sep 17 00:00:00 2001 From: enitrat Date: Sun, 29 Sep 2024 00:13:01 +0200 Subject: [PATCH 04/10] fix runtime location --- .github/workflows/test.yml | 6 ++++++ scripts/setup_cairo_native.sh | 1 + 2 files changed, 7 insertions(+) mode change 100644 => 100755 scripts/setup_cairo_native.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a84e33b4..64a41b727 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,6 +68,12 @@ jobs: - name: Setup Cairo Native run: sudo ./scripts/setup_cairo_native.sh + # Add this step to verify the file exists + - name: Verify libcairo_native_runtime.a + run: | + ls -l libcairo_native_runtime.a || echo "File not found" + pwd + - name: Set Environment Variables run: | echo "MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV diff --git a/scripts/setup_cairo_native.sh b/scripts/setup_cairo_native.sh old mode 100644 new mode 100755 index ef81af9fc..495babd7f --- a/scripts/setup_cairo_native.sh +++ b/scripts/setup_cairo_native.sh @@ -98,6 +98,7 @@ install_cairo_native_runtime() { pushd ./cairo_native || exit 1 make deps make runtime + cp libcairo_native_runtime.a ../libcairo_native_runtime.a popd || exit 1 rm -rf ./cairo_native From 125c63f7ee66b75134d41e78cb797f7c5be69d4d Mon Sep 17 00:00:00 2001 From: enitrat Date: Sun, 29 Sep 2024 00:39:29 +0200 Subject: [PATCH 05/10] avoid clearing the ssj checkout --- .github/workflows/test.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64a41b727..80019b1f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,12 +68,6 @@ jobs: - name: Setup Cairo Native run: sudo ./scripts/setup_cairo_native.sh - # Add this step to verify the file exists - - name: Verify libcairo_native_runtime.a - run: | - ls -l libcairo_native_runtime.a || echo "File not found" - pwd - - name: Set Environment Variables run: | echo "MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV @@ -86,6 +80,7 @@ jobs: with: repository: kkrt-labs/ef-tests ref: feat/cairo-native + path: ef-tests # Check out to a subdirectory to avoid cleaning the kakarot-ssj directory - name: Checkout local skip file uses: actions/checkout@v4 @@ -95,11 +90,10 @@ jobs: sparse-checkout-cone-mode: false path: skip-file - - name: Replace the skip files - run: mv skip-file/blockchain-tests-skip.yml blockchain-tests-skip.yml - - - name: Setup + - name: Setup ef-tests run: | + mv skip-file/blockchain-tests-skip.yml ef-tests/blockchain-tests-skip.yml + cd ef-tests mkdir -p build/common make setup setup-kakarot-v0 @@ -122,6 +116,12 @@ jobs: with: python-version: 3.10.14 + # Add this step to verify the file exists + - name: Verify libcairo_native_runtime.a + run: | + echo $CAIRO_NATIVE_RUNTIME_LIBRARY + ls -l $CAIRO_NATIVE_RUNTIME_LIBRARY + - name: Run tests run: | set -o pipefail From 2c18549304c8a6aa4e47ba1102358fef57744abb Mon Sep 17 00:00:00 2001 From: enitrat Date: Sun, 29 Sep 2024 01:04:58 +0200 Subject: [PATCH 06/10] use correct working dir --- .github/workflows/test.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80019b1f1..325928459 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,12 +104,12 @@ jobs: uses: actions/download-artifact@v3 with: name: kakarot-ssj-build - path: ./build/v1 + path: ef-tests/build/v1 - name: Move Cairo1Helpers run: | - mv build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ - build/common/cairo1_helpers.json + mv ef-tests/build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ + ef-tests/build/common/cairo1_helpers.json - name: Set up Python uses: actions/setup-python@v4 @@ -123,12 +123,14 @@ jobs: ls -l $CAIRO_NATIVE_RUNTIME_LIBRARY - name: Run tests + working-directory: ef-tests run: | set -o pipefail make ef-test-v1-native | tee data.txt set +o pipefail - name: Retrieve ef-tests execution resources + working-directory: ef-tests run: python scripts/compute_resources.py env: KAKAROT_VERSION: v1 @@ -136,16 +138,17 @@ jobs: - name: Upload resources uses: actions/upload-artifact@v3 with: - path: resources + path: ef-tests/resources name: resources - name: Generate blockchain-tests-skip.yml file if: github.event_name == 'workflow_dispatch' + working-directory: ef-tests run: make generate-skip-file - name: Upload skip file if: github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@v3 with: - path: blockchain-tests-skip.yml + path: ef-tests/blockchain-tests-skip.yml name: blockchain-tests-skip From aff6d9638aa84c437eb666211dcbc1717abcfba6 Mon Sep 17 00:00:00 2001 From: enitrat Date: Sun, 29 Sep 2024 11:04:24 +0200 Subject: [PATCH 07/10] update stack size when running rust ef-tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 325928459..f933a7fe1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -126,7 +126,7 @@ jobs: working-directory: ef-tests run: | set -o pipefail - make ef-test-v1-native | tee data.txt + RUST_MIN_STACK=1342177280 make ef-test-v1-native | tee data.txt set +o pipefail - name: Retrieve ef-tests execution resources From ba4297cdc2d58a21fd8bac526b7aab43c6da0420 Mon Sep 17 00:00:00 2001 From: enitrat Date: Sun, 29 Sep 2024 21:57:42 +0200 Subject: [PATCH 08/10] update workflows --- .github/workflows/ci.yml | 29 ++++++++++++------- .github/workflows/test.yml | 2 +- scripts/setup_cairo_native.sh | 54 ++++++++++++++++++++++++++--------- 3 files changed, 61 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 256039b79..ff476c8a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,7 @@ jobs: with: repository: kkrt-labs/ef-tests ref: feat/cairo-native + path: ef-tests # Check out to a subdirectory to avoid cleaning the kakarot-ssj directory - name: Checkout local skip file uses: actions/checkout@v4 @@ -99,11 +100,10 @@ jobs: sparse-checkout-cone-mode: false path: skip-file - - name: Replace the skip files - run: mv skip-file/blockchain-tests-skip.yml blockchain-tests-skip.yml - - - name: Setup + - name: Setup ef-tests run: | + mv skip-file/blockchain-tests-skip.yml ef-tests/blockchain-tests-skip.yml + cd ef-tests mkdir -p build/common make setup setup-kakarot-v0 @@ -114,25 +114,33 @@ jobs: uses: actions/download-artifact@v3 with: name: kakarot-ssj-build - path: ./build/v1 + path: ef-tests/build/v1 - name: Move Cairo1Helpers run: | - mv build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ - build/common/cairo1_helpers.json + mv ef-tests/build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ + ef-tests/build/common/cairo1_helpers.json - name: Set up Python uses: actions/setup-python@v4 with: python-version: 3.10.14 + # Add this step to verify the file exists + - name: Verify libcairo_native_runtime.a + run: | + echo $CAIRO_NATIVE_RUNTIME_LIBRARY + ls -l $CAIRO_NATIVE_RUNTIME_LIBRARY + - name: Run tests + working-directory: ef-tests run: | set -o pipefail - make ef-test-v1-native | tee data.txt + RUST_MIN_STACK=1342177280 make ef-test-v1-native | tee data.txt set +o pipefail - name: Retrieve ef-tests execution resources + working-directory: ef-tests run: python scripts/compute_resources.py env: KAKAROT_VERSION: v1 @@ -140,18 +148,19 @@ jobs: - name: Upload resources uses: actions/upload-artifact@v3 with: - path: resources + path: ef-tests/resources name: resources - name: Generate blockchain-tests-skip.yml file if: github.event_name == 'workflow_dispatch' + working-directory: ef-tests run: make generate-skip-file - name: Upload skip file if: github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@v3 with: - path: blockchain-tests-skip.yml + path: ef-tests/blockchain-tests-skip.yml name: blockchain-tests-skip resources: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f933a7fe1..31c638db6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,7 +80,7 @@ jobs: with: repository: kkrt-labs/ef-tests ref: feat/cairo-native - path: ef-tests # Check out to a subdirectory to avoid cleaning the kakarot-ssj directory + path: ef-tests # Check out to a subdirectory to avoid cleaning the kakarot-ssj directory - name: Checkout local skip file uses: actions/checkout@v4 diff --git a/scripts/setup_cairo_native.sh b/scripts/setup_cairo_native.sh index 495babd7f..a95d30196 100755 --- a/scripts/setup_cairo_native.sh +++ b/scripts/setup_cairo_native.sh @@ -32,18 +32,30 @@ setup_llvm_deps() { export DEBIAN_FRONTEND=noninteractive export TZ=America/New_York + # shellcheck disable=SC2312 CODENAME=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) - [[ -z ${CODENAME} ]] && { + if [[ -z ${CODENAME} ]]; then echo "Error: Unable to determine OS codename" exit 1 - } + fi + # shellcheck disable=SC2312 echo "deb http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-19 main" >/etc/apt/sources.list.d/llvm-19.list echo "deb-src http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-19 main" >>/etc/apt/sources.list.d/llvm-19.list - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + # shellcheck disable=SC2312 + if ! wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -; then + echo "Error: Failed to add LLVM GPG key" + exit 1 + fi - apt-get update && apt-get upgrade -y - apt-get install -y llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools + if ! apt-get update && apt-get upgrade -y; then + echo "Error: Failed to update and upgrade packages" + exit 1 + fi + if ! apt-get install -y llvm-19 llvm-19-dev llvm-19-runtime clang-19 clang-tools-19 lld-19 libpolly-19-dev libmlir-19-dev mlir-19-tools; then + echo "Error: Failed to install LLVM packages" + exit 1 + fi MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/ LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/ @@ -52,6 +64,12 @@ setup_llvm_deps() { export MLIR_SYS_190_PREFIX export LLVM_SYS_191_PREFIX export TABLEGEN_190_PREFIX + + { + echo "MLIR_SYS_190_PREFIX=${MLIR_SYS_190_PREFIX}" + echo "LLVM_SYS_191_PREFIX=${LLVM_SYS_191_PREFIX}" + echo "TABLEGEN_190_PREFIX=${TABLEGEN_190_PREFIX}" + } >>"${GITHUB_ENV}" ;; *) echo "Error: Unsupported operating system" @@ -61,9 +79,11 @@ setup_llvm_deps() { # GitHub Actions specific [[ -n ${GITHUB_ACTIONS} ]] && { - echo "MLIR_SYS_190_PREFIX=${MLIR_SYS_190_PREFIX}" >>"${GITHUB_ENV}" - echo "LLVM_SYS_191_PREFIX=${LLVM_SYS_191_PREFIX}" >>"${GITHUB_ENV}" - echo "TABLEGEN_190_PREFIX=${TABLEGEN_190_PREFIX}" >>"${GITHUB_ENV}" + { + echo "MLIR_SYS_190_PREFIX=${MLIR_SYS_190_PREFIX}" + echo "LLVM_SYS_191_PREFIX=${LLVM_SYS_191_PREFIX}" + echo "TABLEGEN_190_PREFIX=${TABLEGEN_190_PREFIX}" + } >>"${GITHUB_ENV}" } } @@ -74,16 +94,19 @@ install_rust() { fi echo "cargo not found. Installing Rust..." + # shellcheck disable=SC2312 if ! curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.81.0 --no-modify-path; then echo >&2 "Failed to install Rust. Aborting." return 1 fi - # shellcheck disable=SC1090 - source "${HOME}/.cargo/env" || { - echo >&2 "Failed to source Rust environment. Aborting." + # shellcheck disable=SC1091 + if [[ -f "${HOME}/.cargo/env" ]]; then + . "${HOME}/.cargo/env" + else + echo >&2 "Failed to find Rust environment file. Aborting." return 1 - } + fi echo "Rust installed successfully." } @@ -103,7 +126,11 @@ install_cairo_native_runtime() { rm -rf ./cairo_native - CAIRO_NATIVE_RUNTIME_LIBRARY="$(pwd)/libcairo_native_runtime.a" + CAIRO_NATIVE_RUNTIME_LIBRARY=$(pwd)/libcairo_native_runtime.a + if [[ -z ${CAIRO_NATIVE_RUNTIME_LIBRARY} ]]; then + echo "Error: Failed to set CAIRO_NATIVE_RUNTIME_LIBRARY" + exit 1 + fi export CAIRO_NATIVE_RUNTIME_LIBRARY echo "CAIRO_NATIVE_RUNTIME_LIBRARY=${CAIRO_NATIVE_RUNTIME_LIBRARY}" @@ -112,6 +139,7 @@ install_cairo_native_runtime() { } main() { + # shellcheck disable=SC2312 [[ "$(uname)" == "Linux" ]] && install_essential_deps_linux setup_llvm_deps From 99d261f9d5ca397508e5e693502dda85cd8fbadf Mon Sep 17 00:00:00 2001 From: enitrat Date: Mon, 30 Sep 2024 11:20:02 +0200 Subject: [PATCH 09/10] simplify workflows by avoiding re-building runtimes --- .github/workflows/ci.yml | 135 +------------------------------- .github/workflows/common.yml | 143 ++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 132 +------------------------------ scripts/setup_cairo_native.sh | 33 +++++++- 4 files changed, 178 insertions(+), 265 deletions(-) create mode 100644 .github/workflows/common.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff476c8a0..65adf5a3e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,28 +16,7 @@ permissions: read-all jobs: build: - runs-on: ubuntu-latest - env: - CI_COMMIT_MESSAGE: CI Formatting Auto Commit - CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} CI - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@1.81.0 - - uses: Swatinem/rust-cache@v2 - - - name: Set up Scarb - uses: software-mansion/setup-scarb@v1 - - - name: Build contracts - run: scarb build -p contracts - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: kakarot-ssj-build - path: target/dev + uses: ./.github/workflows/common.yml#build tests-unit: # trunk-ignore(actionlint/runner-label) @@ -51,117 +30,7 @@ jobs: - run: scarb test ef-tests: - # trunk-ignore(actionlint/runner-label) - runs-on: ubuntu-latest-16-cores - needs: [build] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cairo-native setup - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - ./libcairo_native_runtime.a - key: - ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', - 'scripts/setup_cairo_native.sh') }} - - - name: Make setup script executable - run: chmod +x ./scripts/setup_cairo_native.sh - - - name: Setup Cairo Native - run: sudo ./scripts/setup_cairo_native.sh - - - name: Set Environment Variables - run: | - echo "MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV - echo "LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV - echo "TABLEGEN_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV - echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$(pwd)/libcairo_native_runtime.a" >> $GITHUB_ENV - - - name: Checkout ef-tests - uses: actions/checkout@v4 - with: - repository: kkrt-labs/ef-tests - ref: feat/cairo-native - path: ef-tests # Check out to a subdirectory to avoid cleaning the kakarot-ssj directory - - - name: Checkout local skip file - uses: actions/checkout@v4 - with: - sparse-checkout: | - blockchain-tests-skip.yml - sparse-checkout-cone-mode: false - path: skip-file - - - name: Setup ef-tests - run: | - mv skip-file/blockchain-tests-skip.yml ef-tests/blockchain-tests-skip.yml - cd ef-tests - mkdir -p build/common - make setup setup-kakarot-v0 - - - name: Install nextest - uses: taiki-e/install-action@nextest - - - name: Download Kakarot-SSJ build artifacts in v1 - uses: actions/download-artifact@v3 - with: - name: kakarot-ssj-build - path: ef-tests/build/v1 - - - name: Move Cairo1Helpers - run: | - mv ef-tests/build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ - ef-tests/build/common/cairo1_helpers.json - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.10.14 - - # Add this step to verify the file exists - - name: Verify libcairo_native_runtime.a - run: | - echo $CAIRO_NATIVE_RUNTIME_LIBRARY - ls -l $CAIRO_NATIVE_RUNTIME_LIBRARY - - - name: Run tests - working-directory: ef-tests - run: | - set -o pipefail - RUST_MIN_STACK=1342177280 make ef-test-v1-native | tee data.txt - set +o pipefail - - - name: Retrieve ef-tests execution resources - working-directory: ef-tests - run: python scripts/compute_resources.py - env: - KAKAROT_VERSION: v1 - - - name: Upload resources - uses: actions/upload-artifact@v3 - with: - path: ef-tests/resources - name: resources - - - name: Generate blockchain-tests-skip.yml file - if: github.event_name == 'workflow_dispatch' - working-directory: ef-tests - run: make generate-skip-file - - - name: Upload skip file - if: github.event_name == 'workflow_dispatch' - uses: actions/upload-artifact@v3 - with: - path: ef-tests/blockchain-tests-skip.yml - name: blockchain-tests-skip + uses: ./.github/workflows/common.yml#ef-tests resources: runs-on: ubuntu-latest diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml new file mode 100644 index 000000000..2e05bdf1a --- /dev/null +++ b/.github/workflows/common.yml @@ -0,0 +1,143 @@ +name: Common Workflow + +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + env: + CI_COMMIT_MESSAGE: CI Formatting Auto Commit + CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} CI + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Scarb + uses: software-mansion/setup-scarb@v1 + + - name: Build contracts + run: scarb build -p contracts + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: kakarot-ssj-build + path: target/dev + + ef-tests: + # trunk-ignore(actionlint/runner-label) + runs-on: ubuntu-latest-16-cores + needs: [build] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache cairo-native setup + id: cache-cairo-native + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + ./libcairo_native_runtime.a + key: ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', 'scripts/setup_cairo_native.sh') }} + + - name: Make setup script executable + run: chmod +x ./scripts/setup_cairo_native.sh + + - name: Setup Cairo Native + run: | + if [[ "${{ steps.cache-cairo-native.outputs.cache-hit }}" == 'true' ]]; then + sudo ./scripts/setup_cairo_native.sh -s + else + sudo ./scripts/setup_cairo_native.sh + fi + + - name: Set Environment Variables + run: | + echo "MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "TABLEGEN_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV + echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$(pwd)/libcairo_native_runtime.a" >> $GITHUB_ENV + + - name: Checkout ef-tests + uses: actions/checkout@v4 + with: + repository: kkrt-labs/ef-tests + ref: feat/cairo-native + path: ef-tests # Check out to a subdirectory to avoid cleaning the kakarot-ssj directory + + - name: Checkout local skip file + uses: actions/checkout@v4 + with: + sparse-checkout: | + blockchain-tests-skip.yml + sparse-checkout-cone-mode: false + path: skip-file + + - name: Setup ef-tests + run: | + mv skip-file/blockchain-tests-skip.yml ef-tests/blockchain-tests-skip.yml + cd ef-tests + mkdir -p build/common + make setup setup-kakarot-v0 + + - name: Install nextest + uses: taiki-e/install-action@nextest + + - name: Download Kakarot-SSJ build artifacts in v1 + uses: actions/download-artifact@v3 + with: + name: kakarot-ssj-build + path: ef-tests/build/v1 + + - name: Move Cairo1Helpers + run: | + mv ef-tests/build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ + ef-tests/build/common/cairo1_helpers.json + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.10.14 + + # Add this step to verify the file exists + - name: Verify libcairo_native_runtime.a + run: | + echo $CAIRO_NATIVE_RUNTIME_LIBRARY + ls -l $CAIRO_NATIVE_RUNTIME_LIBRARY + + - name: Run tests + working-directory: ef-tests + run: | + set -o pipefail + RUST_MIN_STACK=1342177280 make ef-test-v1-native | tee data.txt + set +o pipefail + + - name: Retrieve ef-tests execution resources + working-directory: ef-tests + run: python scripts/compute_resources.py + env: + KAKAROT_VERSION: v1 + + - name: Upload resources + uses: actions/upload-artifact@v3 + with: + path: ef-tests/resources + name: resources + + - name: Generate blockchain-tests-skip.yml file + if: github.event_name == 'workflow_dispatch' + working-directory: ef-tests + run: make generate-skip-file + + - name: Upload skip file + if: github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v3 + with: + path: ef-tests/blockchain-tests-skip.yml + name: blockchain-tests-skip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 31c638db6..3e3ff81c7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,25 +10,7 @@ concurrency: jobs: build: - runs-on: ubuntu-latest - env: - CI_COMMIT_MESSAGE: CI Formatting Auto Commit - CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} CI - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Scarb - uses: software-mansion/setup-scarb@v1 - - - name: Build contracts - run: scarb build -p contracts - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: kakarot-ssj-build - path: target/dev + uses: ./.github/workflows/common.yml#build tests: # trunk-ignore(actionlint/runner-label) @@ -41,114 +23,4 @@ jobs: - run: scarb test ef-tests: - # trunk-ignore(actionlint/runner-label) - runs-on: ubuntu-latest-16-cores - needs: [build] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cairo-native setup - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - ./libcairo_native_runtime.a - key: - ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', - 'scripts/setup_cairo_native.sh') }} - - - name: Make setup script executable - run: chmod +x ./scripts/setup_cairo_native.sh - - - name: Setup Cairo Native - run: sudo ./scripts/setup_cairo_native.sh - - - name: Set Environment Variables - run: | - echo "MLIR_SYS_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV - echo "LLVM_SYS_191_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV - echo "TABLEGEN_190_PREFIX=/usr/lib/llvm-19/" >> $GITHUB_ENV - echo "CAIRO_NATIVE_RUNTIME_LIBRARY=$(pwd)/libcairo_native_runtime.a" >> $GITHUB_ENV - - - name: Checkout ef-tests - uses: actions/checkout@v4 - with: - repository: kkrt-labs/ef-tests - ref: feat/cairo-native - path: ef-tests # Check out to a subdirectory to avoid cleaning the kakarot-ssj directory - - - name: Checkout local skip file - uses: actions/checkout@v4 - with: - sparse-checkout: | - blockchain-tests-skip.yml - sparse-checkout-cone-mode: false - path: skip-file - - - name: Setup ef-tests - run: | - mv skip-file/blockchain-tests-skip.yml ef-tests/blockchain-tests-skip.yml - cd ef-tests - mkdir -p build/common - make setup setup-kakarot-v0 - - - name: Install nextest - uses: taiki-e/install-action@nextest - - - name: Download Kakarot-SSJ build artifacts in v1 - uses: actions/download-artifact@v3 - with: - name: kakarot-ssj-build - path: ef-tests/build/v1 - - - name: Move Cairo1Helpers - run: | - mv ef-tests/build/v1/contracts_Cairo1Helpers.compiled_contract_class.json \ - ef-tests/build/common/cairo1_helpers.json - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.10.14 - - # Add this step to verify the file exists - - name: Verify libcairo_native_runtime.a - run: | - echo $CAIRO_NATIVE_RUNTIME_LIBRARY - ls -l $CAIRO_NATIVE_RUNTIME_LIBRARY - - - name: Run tests - working-directory: ef-tests - run: | - set -o pipefail - RUST_MIN_STACK=1342177280 make ef-test-v1-native | tee data.txt - set +o pipefail - - - name: Retrieve ef-tests execution resources - working-directory: ef-tests - run: python scripts/compute_resources.py - env: - KAKAROT_VERSION: v1 - - - name: Upload resources - uses: actions/upload-artifact@v3 - with: - path: ef-tests/resources - name: resources - - - name: Generate blockchain-tests-skip.yml file - if: github.event_name == 'workflow_dispatch' - working-directory: ef-tests - run: make generate-skip-file - - - name: Upload skip file - if: github.event_name == 'workflow_dispatch' - uses: actions/upload-artifact@v3 - with: - path: ef-tests/blockchain-tests-skip.yml - name: blockchain-tests-skip + uses: ./.github/workflows/common.yml#ef-tests diff --git a/scripts/setup_cairo_native.sh b/scripts/setup_cairo_native.sh index a95d30196..ef3c220a2 100755 --- a/scripts/setup_cairo_native.sh +++ b/scripts/setup_cairo_native.sh @@ -139,13 +139,42 @@ install_cairo_native_runtime() { } main() { + # New argument parsing + SKIP_RUNTIME=false + while getopts ":s" opt; do + case ${opt} in + s ) + SKIP_RUNTIME=true + ;; + \? ) + echo "Invalid option: $OPTARG" 1>&2 + exit 1 + ;; + esac + done + shift $((OPTIND -1)) + # shellcheck disable=SC2312 [[ "$(uname)" == "Linux" ]] && install_essential_deps_linux setup_llvm_deps - install_cairo_native_runtime - echo "LLVM and Cairo native runtime dependencies installed successfully." + if [ "$SKIP_RUNTIME" = false ]; then + install_cairo_native_runtime + else + echo "Skipping Cairo native runtime installation" + # Set the environment variable if the library file exists + if [ -f "$(pwd)/libcairo_native_runtime.a" ]; then + CAIRO_NATIVE_RUNTIME_LIBRARY=$(pwd)/libcairo_native_runtime.a + export CAIRO_NATIVE_RUNTIME_LIBRARY + echo "CAIRO_NATIVE_RUNTIME_LIBRARY=${CAIRO_NATIVE_RUNTIME_LIBRARY}" + [[ -n ${GITHUB_ACTIONS} ]] && echo "CAIRO_NATIVE_RUNTIME_LIBRARY=${CAIRO_NATIVE_RUNTIME_LIBRARY}" >>"${GITHUB_ENV}" + else + echo "Warning: libcairo_native_runtime.a not found. CAIRO_NATIVE_RUNTIME_LIBRARY not set." + fi + fi + + echo "LLVM and Cairo native runtime dependencies setup completed." } main "$@" From 15dbb4e3465b2301c32a40c31b8af7e73fbf57f6 Mon Sep 17 00:00:00 2001 From: enitrat Date: Mon, 30 Sep 2024 11:58:16 +0200 Subject: [PATCH 10/10] rework workflow structure --- .github/workflows/build.yml | 32 ++++++++++++++++ .github/workflows/ci.yml | 23 ++++++----- .../workflows/{common.yml => ef-tests.yml} | 38 ++++++------------- .github/workflows/test.yml | 22 +++++------ .github/workflows/tests-unit.yml | 26 +++++++++++++ scripts/setup_cairo_native.sh | 25 +++++++----- 6 files changed, 107 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/build.yml rename .github/workflows/{common.yml => ef-tests.yml} (83%) create mode 100644 .github/workflows/tests-unit.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..7cca4d20d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,32 @@ +name: Reusable Build Workflow + +on: + workflow_call: + inputs: + artifact-name: + required: true + type: string + +permissions: read-all + +jobs: + build: + runs-on: ubuntu-latest + env: + CI_COMMIT_MESSAGE: CI Formatting Auto Commit + CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} CI + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Scarb + uses: software-mansion/setup-scarb@v1 + + - name: Build contracts + run: scarb build -p contracts + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.artifact-name }} + path: target/dev diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65adf5a3e..b99ecd358 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,25 +16,24 @@ permissions: read-all jobs: build: - uses: ./.github/workflows/common.yml#build + uses: ./.github/workflows/build.yml + with: + artifact-name: kakarot-ssj-build tests-unit: - # trunk-ignore(actionlint/runner-label) - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v3 - - uses: foundry-rs/setup-snfoundry@v3 - - uses: software-mansion/setup-scarb@v1 - - run: scarb fmt --check - - run: scarb build - - run: scarb test + uses: ./.github/workflows/tests-unit.yml + with: + run-fmt-check: false ef-tests: - uses: ./.github/workflows/common.yml#ef-tests + uses: ./.github/workflows/ef-tests.yml + needs: [build] + with: + artifact-name: kakarot-ssj-build resources: runs-on: ubuntu-latest - needs: ef-tests + needs: [ef-tests] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/common.yml b/.github/workflows/ef-tests.yml similarity index 83% rename from .github/workflows/common.yml rename to .github/workflows/ef-tests.yml index 2e05bdf1a..5957d20fc 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/ef-tests.yml @@ -1,34 +1,18 @@ -name: Common Workflow +name: EF-Tests on: workflow_call: + inputs: + artifact-name: + required: true + type: string -jobs: - build: - runs-on: ubuntu-latest - env: - CI_COMMIT_MESSAGE: CI Formatting Auto Commit - CI_COMMIT_AUTHOR: ${{ github.event.repository.name }} CI - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Scarb - uses: software-mansion/setup-scarb@v1 +permissions: read-all - - name: Build contracts - run: scarb build -p contracts - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: kakarot-ssj-build - path: target/dev - - ef-tests: +jobs: + ef-tests: # trunk-ignore(actionlint/runner-label) runs-on: ubuntu-latest-16-cores - needs: [build] steps: - name: Checkout code uses: actions/checkout@v4 @@ -44,7 +28,9 @@ jobs: ~/.cargo/git/db/ target/ ./libcairo_native_runtime.a - key: ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', 'scripts/setup_cairo_native.sh') }} + key: + ${{ runner.os }}-cairo-native-${{ hashFiles('**/Cargo.lock', + 'scripts/setup_cairo_native.sh') }} - name: Make setup script executable run: chmod +x ./scripts/setup_cairo_native.sh @@ -92,7 +78,7 @@ jobs: - name: Download Kakarot-SSJ build artifacts in v1 uses: actions/download-artifact@v3 with: - name: kakarot-ssj-build + name: ${{ inputs.artifact-name }} path: ef-tests/build/v1 - name: Move Cairo1Helpers diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3e3ff81c7..cc095668a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,17 +10,17 @@ concurrency: jobs: build: - uses: ./.github/workflows/common.yml#build + uses: ./.github/workflows/build.yml + with: + artifact-name: kakarot-ssj-build - tests: - # trunk-ignore(actionlint/runner-label) - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v3 - - uses: foundry-rs/setup-snfoundry@v3 - - uses: software-mansion/setup-scarb@v1 - - run: scarb fmt --check - - run: scarb test + tests-unit: + uses: ./.github/workflows/tests-unit.yml + with: + run-fmt-check: true ef-tests: - uses: ./.github/workflows/common.yml#ef-tests + uses: ./.github/workflows/ef-tests.yml + needs: [build] + with: + artifact-name: kakarot-ssj-build diff --git a/.github/workflows/tests-unit.yml b/.github/workflows/tests-unit.yml new file mode 100644 index 000000000..2ff0ef28d --- /dev/null +++ b/.github/workflows/tests-unit.yml @@ -0,0 +1,26 @@ +name: Reusable Unit Tests Workflow + +on: + workflow_call: + inputs: + run-fmt-check: + type: boolean + default: false + required: false + +permissions: read-all + +jobs: + unit-tests: + runs-on: ubuntu-latest-16-cores # trunk-ignore(actionlint/runner-label) + steps: + - uses: actions/checkout@v4 + - uses: foundry-rs/setup-snfoundry@v3 + - uses: software-mansion/setup-scarb@v1 + + - name: Run format check + if: inputs.run-fmt-check + run: scarb fmt --check + + - name: Run tests + run: scarb test diff --git a/scripts/setup_cairo_native.sh b/scripts/setup_cairo_native.sh index ef3c220a2..8b74c612c 100755 --- a/scripts/setup_cairo_native.sh +++ b/scripts/setup_cairo_native.sh @@ -143,28 +143,33 @@ main() { SKIP_RUNTIME=false while getopts ":s" opt; do case ${opt} in - s ) - SKIP_RUNTIME=true - ;; - \? ) - echo "Invalid option: $OPTARG" 1>&2 - exit 1 - ;; + s) + SKIP_RUNTIME=true + ;; + \?) + echo "Invalid option: ${OPTARG}" 1>&2 + exit 1 + ;; + *) + echo "Error: Unhandled option" 1>&2 + exit 1 + ;; esac done - shift $((OPTIND -1)) + shift $((OPTIND - 1)) # shellcheck disable=SC2312 [[ "$(uname)" == "Linux" ]] && install_essential_deps_linux setup_llvm_deps - if [ "$SKIP_RUNTIME" = false ]; then + if [[ ${SKIP_RUNTIME} == false ]]; then install_cairo_native_runtime else echo "Skipping Cairo native runtime installation" # Set the environment variable if the library file exists - if [ -f "$(pwd)/libcairo_native_runtime.a" ]; then + # shellcheck disable=SC2312 + if [[ -f "$(pwd)/libcairo_native_runtime.a" ]]; then CAIRO_NATIVE_RUNTIME_LIBRARY=$(pwd)/libcairo_native_runtime.a export CAIRO_NATIVE_RUNTIME_LIBRARY echo "CAIRO_NATIVE_RUNTIME_LIBRARY=${CAIRO_NATIVE_RUNTIME_LIBRARY}"