build_linux.yml

name: build-linux

on:
  push:
    branches: [ master ]
  pull_request:

concurrency:
  # In master we want to run for every commit, in other branches — only for the last one
  group: ${{
    ( github.ref == 'refs/heads/master' &&
    format('{0}/{1}/{2}', github.workflow, github.ref, github.sha) )
    ||
    format('{0}/{1}', github.workflow, github.ref) }}
  cancel-in-progress: true

jobs:
  handle-syncwith:
    if: github.event_name == 'pull_request'
    name: Call Reusable SyncWith Handler
    uses: NilFoundation/ci-cd/.github/workflows/reusable-handle-syncwith.yml@v1.1.2
    with:
      ci-cd-ref: 'v1.1.2'
    secrets: inherit

  build-and-test-linux:
    name: Build zkLLVM, run local tests, prepare for integration testing
    runs-on: [ self-hosted, Linux, X64, aws_autoscaling ]
    needs:
      - handle-syncwith
    # Condition is needed to run it for push event (handle-syncwith is skipped)
    if: |
      always() && !cancelled() &&
      (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped')
    outputs:
      transpiler-artifact-name: ${{ steps.artifact-names.outputs.transpiler }}
      examples-artifact-name: ${{ steps.artifact-names.outputs.examples }}
      evm-targets: ${{ steps.get-targets.outputs.evm-targets }}
      prover-targets: ${{ steps.get-targets.outputs.prover-targets }}

    env:
      CONTAINER_TMP: /opt/
      HOST_TMP: /home/runner/work/_temp/
      DEBIAN_FRONTEND: noninteractive
      BOOST_VERSION: "1.78.0"
      INTEGRATION_TESTING_TARGETS: |
        arithmetics_cpp_example
        polynomial_cpp_example
        poseidon_cpp_example
        merkle_tree_poseidon_cpp_example
        uint_remainder_cpp
        uint_shift_left
        uint_bit_decomposition
        uint_bit_composition
        compare_eq_cpp
        private_input_cpp

    container:
      image: ubuntu:22.04
      volumes:
        - /home/runner/work/_temp/:/opt/

    steps:
      - name: Install dependencies
        run: |
          env && \
          apt update && \
          apt install -y \
            build-essential \
            libssl-dev \
            cmake \
            ninja-build \
            git \
            libicu-dev \
            curl \
            pkg-config

      - name: Print toolchain information
        run: |
          git --version
          cc --version
          cmake --version
          ninja --version

      - name: Checkout sources
        # We need full history, because during CMake config stage we are finding the nearest tag
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: false

      # Workaround: https://github.com/actions/checkout/issues/1169
      - name: Mark directory as safe
        run: |
          git config --system --add safe.directory $PWD

      - name: Checkout submodules
        run: |
          git submodule update --init --recursive --depth=1

      - name: Checkout modules to specified refs
        if: needs.handle-syncwith.outputs.prs-refs != ''
        uses: NilFoundation/ci-cd/actions/recursive-checkout@v1.2.1
        # TODO: figure out the mapping of volumes and use variable here, not hardcoded path
        with:
          paths: |
            /__w/zkLLVM/zkLLVM/**
            !/__w/zkLLVM/zkLLVM/
            !/__w/zkLLVM/zkLLVM/**/.git/**
          refs: ${{ needs.handle-syncwith.outputs.prs-refs }}

      - name: Clean index.lock files if checkout step was cancelled or failed
        if: cancelled() || failure()
        run: |
          find .git -name 'index.lock' -exec rm -v {} \;

      - name: Install boost
        uses: MarkusJx/install-boost@v2.4.1
        id: install-boost
        with:
            # A list of supported versions can be found here:
            # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json
            boost_version: ${{ env.BOOST_VERSION }}
            boost_install_dir: ${{ env.CONTAINER_TMP }}
            platform_version: 22.04
            toolset: gcc
            arch: x86

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Configure CMake
        env:
          BOOST_ROOT: "${{ steps.install-boost.outputs.BOOST_ROOT }}"
        run: |
          cmake . \
            -G "Ninja" \
            -B build \
            -DCMAKE_BUILD_TYPE=Release \
            -DBUILD_TESTS=TRUE \
            -DRSLANG_BUILD_EXTENDED=TRUE \
            -DRSLANG_BUILD_TOOLS=cargo \
            -DGENERATE_EVM_VERIFIER=TRUE

      - name: Build zkllvm
        run: |
          cmake --build build -t assigner clang transpiler

      - name: Build IR of the C++ examples
        run: |
          cmake --build build -t compile_cpp_examples
          ls -al ./build/examples/cpp

      - name: Build circuits(.crct) of the C++ examples
        run: |
          cmake --build build -t cpp_examples_generate_crct
          ls -al ./build/examples/cpp

      - name: Build assignment tables(.tbl) of the C++ examples
        run: |
          cmake --build build -t cpp_examples_generate_tbl_no_check
          ls -al ./build/examples/cpp

      - name: Copy assigner output from separate generation mode
        run: |
          bash ./tests/copy_assigner_outputs.sh build/separate_generatuon

      - name: Run simultaneous .tbl and .crct generation of the C++ examples
        run: |
          cmake --build build -t cpp_examples_generate_both
          ls -al ./build/examples/cpp

      - name: Copy assigner output from simultaneous generation mode
        run: |
          bash ./tests/copy_assigner_outputs.sh build/simultaneous_generation

      - name: Compare different assigner modes output
        run: |
          bash ./tests/compare_folders_content.sh build/separate_generatuon build/simultaneous_generation

      - name: Run size estimation for C++ examples
        run: |
          cmake --build build -t cpp_examples_estimate_size

      - name: Copy examples' circuits and assignments for uploading
        run: |
          chmod +x ./examples/copy_artifacts.sh
          ./examples/copy_artifacts.sh build/examples/cpp examples_output

      - name: Set aritfacts' names
        id: artifact-names
        run: |
          echo "transpiler=transpiler-output" >> $GITHUB_OUTPUT
          echo "examples=circuits-and-assignments" >> $GITHUB_OUTPUT

      - name: Upload examples' circuits and assignments artifact
        uses: actions/upload-artifact@v3
        with:
          name: ${{ steps.artifact-names.outputs.examples }}
          path: |
            examples_output

      - name: Compile tests as cpp code
        run: |
          cmake --build build -t all_tests_compile_as_cpp_code

      - name: Compile tests as circuits
        run: |
          cmake --build build -t all_tests_compile_as_circuits

      - name: Run tests as cpp code (expected res calculation)
        run: |
          cmake --build build -t all_tests_run_expected_res_calculation -j$(nproc)

      - name: Get transpiler output for integration testing
        run: |
          targets_str=$(echo "${{ env.INTEGRATION_TESTING_TARGETS }}" | awk 'NF {printf "-t %s_evm_verifier ", $0}' | sed 's/[[:space:]]*$//')
          cmake --build build ${targets_str}

      - name: Upload examples' circuits and assignments artifact
        uses: actions/upload-artifact@v3
        with:
          name: ${{ steps.artifact-names.outputs.transpiler }}
          path: |
            build/examples/cpp/transpiler_output_*

      - name: Run tests as circuits (real res calculation)
        run: |
          cmake --build build -t all_tests_assign_circuits -j$(nproc)

      - name: Compare expected and real test results
        run: |
          chmod +x ./tests/run_test_results_comparison.sh
          bash ./tests/run_test_results_comparison.sh

      - name: Run tests on faulty inputs
        run: |
          chmod +x ./tests/run_tests_faulty_input.sh
          bash ./tests/run_tests_faulty_input.sh pallas

      - name: Run assigner unit tests
        run: |
          cmake --build build -t check-crypto3-assigner

      - name: Build proof for the circuit of the C++ examples
        run: |
          cmake --build build -t prove_cpp_examples

      - name: Build recursive gen
        run: |
          cmake --build build -t recursive_gen

      - name: Build and run transpiler test
        run: |
          cmake --build build -t compile_and_run_transpiler_tests

      - name: Run recursive verifier tests
        run: |
          cmake --build build -t recursion

      - name: Build rslang
        run: |
          export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(pwd)/build/libs/circifier/llvm/lib"
          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/build/libs/circifier/llvm/lib" >> $GITHUB_ENV
          cmake --build build -t rslang

      - name: Build IR of the Rust examples
        run: |
          cmake --build build -t compile_rust_examples
          ls -al ./build/examples/rust/target/assigner-unknown-unknown/release/examples

      - name: Build circuits(.crct) of the Rust examples
        run: |
          cmake --build build -t rust_examples_generate_crct
          ls -al ./build/examples/cpp

      - name: Build assignment tables(.tbl) of the Rust examples
        run: |
          cmake --build build -t rust_examples_generate_tbl
          ls -al ./build/examples/cpp

      - name: Build proof for the circuit of the Rust examples
        run: |
          cmake --build build -t prove_rust_examples

      - name: Set targets for integration testing
        id: get-targets
        run: |
          targets_str=$(echo "${{ env.INTEGRATION_TESTING_TARGETS }}" | awk 'NF {print "transpiler_output_" $1}')
          echo "evm-targets<<EOF" >> $GITHUB_OUTPUT
          echo "${targets_str}" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

          echo "prover-targets<<EOF" >> $GITHUB_OUTPUT
          echo "${{ env.INTEGRATION_TESTING_TARGETS }}" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT


  verify-zkllvm-proofs:
    name: Verify built-it proofs with EVM-placeholder
    needs:
      - build-and-test-linux
      - handle-syncwith
    if: |
      always() && !cancelled() &&
      (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') &&
      needs.build-and-test-linux.result == 'success'
    uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
    with:
      artifact-name: ${{ needs.build-and-test-linux.outputs.transpiler-artifact-name }}
      evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
      refs: ${{ needs.handle-syncwith.outputs.prs-refs }}
      test-names: ${{ needs.build-and-test-linux.outputs.evm-targets }}


  generate-proofs:
    name: Generate proofs for cpp examples
    needs:
      - build-and-test-linux
      - handle-syncwith
    if: |
      always() && !cancelled() &&
      (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') &&
      needs.build-and-test-linux.result == 'success'
    uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@ef8cd9152b4bec871e7efdc1d6b606e445bad274
    with:
      artifact-name: ${{ needs.build-and-test-linux.outputs.examples-artifact-name }}
      # Update next line if you need new version of proof producer
      proof-producer-ref: ef8cd9152b4bec871e7efdc1d6b606e445bad274
      refs: ${{ needs.handle-syncwith.outputs.prs-refs }}
      targets: ${{ needs.build-and-test-linux.outputs.prover-targets }}


  merge-proofs-to-transpiler-output:
    runs-on: ubuntu-22.04
    needs:
      - generate-proofs
      - build-and-test-linux
    if: |
      always() && !cancelled() &&
      needs.generate-proofs.result == 'success' &&
      needs.build-and-test-linux.result == 'success'
    outputs:
      merged-artifact-name: ${{ steps.artifact-name.outputs.merged }}
    steps:
      - name: Download artifact with proofs
        uses: actions/download-artifact@v3
        with:
          name: ${{ needs.generate-proofs.outputs.artifact-name }}

      - name: Download artifact with transpiler output
        uses: actions/download-artifact@v3
        with:
          name: ${{ needs.build-and-test-linux.outputs.transpiler-artifact-name }}

      - name: Merge proofs into transpiler output
        run: |
          copy_failed=0
          while read dir; do
            base_name=${dir#./transpiler_output_}

            if [[ -d "$base_name" ]]; then
              if ! cp "${base_name}/proof.bin" "${dir}/"; then
                echo "Failed to copy proof.bin to ${dir}" >&2
                copy_failed=1
              else
                echo "proof.bin added to ${dir}"
              fi
            else
              echo "Error: No matching directory found for ${dir}" >&2
            fi
          done < <(find . -type d -name "transpiler_output_*")

          if [ $copy_failed -eq 1 ]; then
            echo "One or more copy operations failed."
            exit 1
          fi

      - name: Set aritfact name
        id: artifact-name
        run: |
          echo "merged=transpiler-output-merged-proofs" >> $GITHUB_OUTPUT

      - name: Upload merged artifact
        uses: actions/upload-artifact@v3
        with:
          name: ${{ steps.artifact-name.outputs.merged }}
          path: |
            transpiler_output_*


  verify-proof-producer-proofs:
    name: Verify proof-producer proofs with EVM-placeholder
    needs:
      - handle-syncwith
      - merge-proofs-to-transpiler-output
      - build-and-test-linux
    if: |
      always() && !cancelled() &&
      (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') &&
      needs.build-and-test-linux.result == 'success' &&
      needs.merge-proofs-to-transpiler-output.result == 'success'
    uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
    with:
      artifact-name: ${{ needs.merge-proofs-to-transpiler-output.outputs.merged-artifact-name }}
      evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d
      refs: ${{ needs.handle-syncwith.outputs.prs-refs }}
      test-names: ${{ needs.build-and-test-linux.outputs.evm-targets }}

  cleanup-integration-testing:
    # For now each artifact is 12 GB. Better clean it up to keep the space
    name: Clean up after integration testing
    needs:
      - verify-proof-producer-proofs
      - verify-zkllvm-proofs
      - merge-proofs-to-transpiler-output
      - build-and-test-linux
      - generate-proofs
    if: always()
    runs-on: ubuntu-22.04
    steps:
      - uses: geekyeggo/delete-artifact@v2
        with:
          name: |
            ${{ needs.merge-proofs-to-transpiler-output.outputs.merged-artifact-name }}
            ${{ needs.generate-proofs.outputs.artifact-name }}