Add runs without unsupported dialect to CI #4
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright (c) 2022-present, Trail of Bits, Inc. | |
| # All rights reserved. | |
| # | |
| # This source code is licensed in accordance with the terms specified in | |
| # the LICENSE file found in the root directory of this source tree. | |
| # | |
| name: "Test" | |
| on: | |
| pull_request: | |
| branches: | |
| - '*' | |
| env: | |
| LLVM_DISABLE_SYMBOLIZATION: 1 | |
| jobs: | |
| build: | |
| strategy: | |
| matrix: | |
| llvm-version: [18] | |
| image-version: [22.04] | |
| name: "Test" | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 60 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| env: | |
| CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/" | |
| TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake | |
| LLVM_EXTERNAL_LIT: "/usr/local/bin/lit" | |
| steps: | |
| - name: Clone the VAST repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Configure build | |
| run: cmake --preset ci | |
| - name: Build release | |
| run: cmake --build --preset ci-release -j $(nproc) | |
| - name: Package | |
| run: cpack --preset ci | |
| - name: Upload VAST build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: VAST | |
| path: ./builds/ci/package/* | |
| retention-days: 1 | |
| # | |
| # LLVM Test Suite | |
| # | |
| run_llvm_ts: | |
| name: "Run LLVM test suite" | |
| needs: build | |
| strategy: | |
| matrix: | |
| llvm-version: [18] | |
| image-version: [22.04] | |
| vast-target: ['hl', 'llvm', 'bin'] | |
| disable-unsup: ['ON', 'OFF'] | |
| exclude: | |
| - disable-unsup: 'ON' | |
| include: | |
| - vast-target: 'hl' | |
| disable-unsup: 'ON' | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 60 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| steps: | |
| - name: Fetch VAST artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: VAST | |
| - name: Unpack VAST | |
| run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1 | |
| - name: Export vast binaries | |
| run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH | |
| - name: Install test suite dependencies | |
| run: | | |
| apt-get update | |
| apt-get -y install clang | |
| pip3 install lit psutil | |
| - name: Clone test suite repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-llvm-test-suite | |
| ref: main | |
| path: llvm-test-suite | |
| fetch-depth: 1 | |
| - name: Run test-suite | |
| continue-on-error: true | |
| run: | | |
| mkdir llvm-test-suite/build && cd llvm-test-suite/build/ | |
| cmake ../ -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_C_COMPILER=$(readlink -f ../tools/compiler) \ | |
| -DCMAKE_CXX_COMPILER=$(readlink -f ../tools/compiler++) \ | |
| -C=../cmake/caches/O3.cmake \ | |
| -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \ | |
| -DTEST_SUITE_SUBDIRS=SingleSource \ | |
| -DVAST_TEST_SUITE_ENABLE=ON \ | |
| -DVAST_DISABLE_UNSUPPORTED=${{ matrix.disable-unsup }} \ | |
| -DVAST_TEST_SUITE_TARGET=${{ matrix.vast-target }} | |
| make -j $(nproc) -k | |
| - name: Collect run data (mlir) | |
| if: matrix.vast-target != 'bin' | |
| working-directory: ./llvm-test-suite/build | |
| run: python3 ../utils/mlir_compile_json_gen.py >> ../../results-${{ matrix.vast-target }}-${{ matrix.disable-unsup }}.json | |
| - name: Collect run data (bin) | |
| continue-on-error: true | |
| if: matrix.vast-target == 'bin' | |
| working-directory: ./llvm-test-suite/build | |
| run: lit --timeout=360 -v -o ../../results-${{ matrix.vast-target }}-${{ matrix.disable-unsup }}.json . | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./results-${{ matrix.vast-target }}-${{ matrix.disable-unsup }}.json | |
| name: results-llvm-ts-${{ matrix.vast-target }}-${{ matrix.disable-unsup }} | |
| eval_llvm_ts: | |
| name: "Eval LLVM Singlesource results" | |
| needs: run_llvm_ts | |
| strategy: | |
| matrix: | |
| image-version: [22.04] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| steps: | |
| - name: Fetch result artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-llvm-ts-* | |
| merge-multiple: true | |
| - name: Install evaluator dependencies | |
| run: pip3 install pandas scipy tabulate | |
| - name: Clone test suite repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-llvm-test-suite | |
| sparse-checkout: utils/vast_compare.py | |
| ref: main | |
| path: llvm-test-suite | |
| fetch-depth: 1 | |
| - name: Generate the results | |
| run: | | |
| python3 llvm-test-suite/utils/vast_compare.py \ | |
| --columns HighLevel,"HighLevel without Unsupported",LLVM,Binary \ | |
| --files results-hl-OFF.json,results-hl-ON.json,results-llvm-OFF.json,results-bin-OFF.json \ | |
| --output single-source-results | |
| - name: Post results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: llvm-test-suite-results | |
| path: | | |
| ./single-source-results.csv | |
| ./single-source-results.md | |
| # | |
| # SV-Comp bencmharks | |
| # | |
| run_svcomp: | |
| name: "Run SV-Comp test suite" | |
| needs: build | |
| strategy: | |
| matrix: | |
| llvm-version: [18] | |
| image-version: [22.04] | |
| vast-target: ['hl'] | |
| disable-unsup: [true, false] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 360 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| steps: | |
| - name: Fetch VAST artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: VAST | |
| - name: Unpack VAST | |
| run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1 | |
| - name: Export vast binaries | |
| run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH | |
| - name: Clone test suite repository | |
| run: git clone --depth=1 https://gitlab.com/sosy-lab/benchmarking/sv-benchmarks.git | |
| - name: Clone patch repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/sv-bench-patch | |
| ref: ci-version | |
| path: sv-bench-patch | |
| fetch-depth: 1 | |
| - name: Apply patch | |
| working-directory: sv-bench-patch | |
| run: ./apply | |
| - name: Run benchmarks | |
| if: ${{ !matrix.disable-unsup }} | |
| run: sh ./sv-bench-patch/compile-all.sh -t ${{ matrix.vast-target }} -d ./sv-benchmarks | |
| - name: Run benchmarks without unsupported | |
| if: ${{ matrix.disable-unsup }} | |
| run: sh ./sv-bench-patch/compile-all.sh -t ${{ matrix.vast-target }} --disable-unsup -d ./sv-benchmarks | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./stats | |
| name: results-sv-${{ matrix.vast-target }}-${{ matrix.disable-unsup }} | |
| eval_svcomp: | |
| name: "Eval SV-Comp test results" | |
| needs: run_svcomp | |
| strategy: | |
| matrix: | |
| image-version: [22.04] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| steps: | |
| - name: Fetch result artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-sv-* | |
| - name: Install evaluator dependencies | |
| run: pip3 install pandas scipy tabulate | |
| - name: Clone patch repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/sv-bench-patch | |
| sparse-checkout: make_table.py | |
| ref: ci-version | |
| path: sv-bench-patch | |
| fetch-depth: 1 | |
| - name: Generate the results | |
| run: | | |
| python3 sv-bench-patch/make_table.py \ | |
| --columns HighLevel,"HighLevel without Unsupported" \ | |
| --files results-sv-hl-false/results.txt,results-sv-hl-true/results.txt \ | |
| --output sv-comp-results | |
| - name: Post results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sv-comp-results | |
| path: ./sv-comp-results.md |