diff --git a/.github/workflows/llvm-ts-run.yml b/.github/workflows/llvm-ts-run.yml new file mode 100644 index 0000000000..cd0e0acbd9 --- /dev/null +++ b/.github/workflows/llvm-ts-run.yml @@ -0,0 +1,175 @@ +# +# Copyright (c) 2021-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +name: "LLVM test suite" + +on: + workflow_run: + workflows: [Build] + types: + - completed + branches: + - "master" + +jobs: + build: + name: "Build VAST" + strategy: + matrix: + llvm-version: [17] + image-version: [22.04] + + runs-on: ubuntu-${{ matrix.image-version }} + timeout-minutes: 60 + container: + image: + ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest + + env: + CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/" + TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake + LLVM_EXTERNAL_LIT: "/usr/local/bin/lit" + + steps: + - name: Clone the VAST repository + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 1 + + - name: Configure build + run: | + cmake --preset ninja-multi-default --toolchain ${TOOLCHAIN} \ + -DCMAKE_VERBOSE_MAKEFILE=True \ + -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} \ + -DLLVM_EXTERNAL_LIT=${LLVM_EXTERNAL_LIT} + + - name: Build release + run: | + cmake --build --preset ninja-rel -j $(nproc) + cpack -G TXZ --config ./builds/ninja-multi-default/CPackConfig.cmake + + - name: Upload VAST build artifact + uses: actions/upload-artifact@v4 + with: + name: VAST + path: ./builds/ninja-multi-default/package/* + retention-days: 1 + + test: + name: "Run LLVM test suite" + needs: build + strategy: + matrix: + llvm-version: [17] + image-version: [22.04] + vast-target: ['hl', 'llvm', 'bin'] + + + runs-on: ubuntu-${{ matrix.image-version }} + timeout-minutes: 60 + container: + image: + ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest + + steps: + - name: Fetch VAST artifact + uses: actions/download-artifact@v4 + with: + name: VAST + + - name: Unpack VAST + run: | + mkdir vast && tar -xf VAST-* -C vast --strip-components=1 + + - name: Export vast binaries + run: | + echo "${PWD}/vast/bin/" >> $GITHUB_PATH + + - name: Install test suite dependencies + run: | + apt-get update + apt-get -y install clang + pip3 install lit psutil + + - name: Clone test suite repository + uses: actions/checkout@v4 + with: + repository: trailofbits/vast-llvm-test-suite + ref: main + path: llvm-test-suite + fetch-depth: 1 + + - name: Run test-suite + continue-on-error: true + run: | + mkdir llvm-test-suite/build && cd llvm-test-suite/build/ + cmake ../ -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=$(readlink -f ../tools/compiler) \ + -DCMAKE_CXX_COMPILER=$(readlink -f ../tools/compiler++) \ + -C=../cmake/caches/O3.cmake \ + -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \ + -DTEST_SUITE_SUBDIRS=SingleSource \ + -DTEST_SUITE_VAST_ENABLE=ON \ + -DTEST_SUITE_VAST_TARGET=${{ matrix.vast-target }} + make -j $(nproc) -k + + - name: Collect run data (mlir) + if: matrix.vast-target != '' + working-directory: ./llvm-test-suite/build + run: python3 ../utils/mlir_compile_json_gen.py >> ../../results-${{ matrix.vast-target }}.json + + - name: Collect run data (bin) + continue-on-error: true + if: matrix.vast-target == '' + working-directory: ./llvm-test-suite/build + run: lit --timeout=60 -v -o ../../results.json . + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + path: ./results-${{ matrix.vast-target }}.json + name: results-${{ matrix.vast-target }} + + eval: + name: "Eval test results" + needs: test + strategy: + matrix: + image-version: [22.04] + runs-on: ubuntu-${{ matrix.image-version }} + steps: + - name: Fetch result artifacts + uses: actions/download-artifact@v4 + + - name: Display structure of downloaded files + run: ls -R + - name: Install evaluator dependencies + run: | + pip3 install pandas scipy tabulate + + - name: Clone test suite repository + uses: actions/checkout@v4 + with: + repository: trailofbits/vast-llvm-test-suite + sparse-checkout: utils/vast_compare.py + ref: main + path: llvm-test-suite + fetch-depth: 1 + + - name: Generate the results + run: | + python3 llvm-test-suite/utils/vast_compare.py --columns vast-hl,vast-llvm,vast-bin --files results-hl.json,results-llvm.json,results-bin.json + + - name: Post results as artifacts + uses: actions/upload-artifact@v4 + with: + name: final-result + path: | + ./results.csv + ./results.md diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 26295fcee5..9bf8b45ac3 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -10,7 +10,7 @@ name: "Pre Release" on: workflow_run: - workflows: ["Build"] + workflows: ["LLVM test suite"] types: - completed branches: @@ -69,6 +69,13 @@ jobs: - name: Build VAST Doc run: cmake --build --preset ninja-rel --target vast-doc + + - name: Fetch LLVM test suite results + uses: actions/download-artifact@v4 + with: + name: final-result + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} - name: Build Pages run: | diff --git a/CMakePresets.json b/CMakePresets.json index f3b83049db..990578a75e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -107,5 +107,31 @@ "displayName": "Test cxx-common-osx-deb", "configuration": "Debug" } + ], + "packagePresets": [ + { + "name": "default", + "configurePreset": "ninja-multi-default", + "generators": [ "TGZ" ] + } + ], + "workflowPresets": [ + { + "name": "release-package", + "steps": [ + { + "type": "configure", + "name": "ninja-multi-default" + }, + { + "type": "build", + "name": "ninja-rel" + }, + { + "type": "package", + "name": "default" + } + ] + } ] } diff --git a/www/mkdocs.yml b/www/mkdocs.yml index ede7d0b908..2c1edf12e7 100644 --- a/www/mkdocs.yml +++ b/www/mkdocs.yml @@ -23,6 +23,8 @@ nav: - Optimizer: Tools/vast-opt.md - Query: Tools/vast-query.md - REPL: Tools/vast-repl.md + - Benchmarks: + - LLVM Test Suite SingleSource: benchmarks/results.md - About: - 'License': 'statement.md' diff --git a/www/setup.sh b/www/setup.sh index af0dba4dff..97215949ac 100644 --- a/www/setup.sh +++ b/www/setup.sh @@ -40,6 +40,9 @@ cp -rv $(pwd)/CONTRIBUTING.md $dst/docs # Setup auto-generated docs cp -rv $build/docs $dst/docs/dialects +# Setup benchmark results +cp -rv $(pwd)/results.md $dst/docs + # Setup site assets cp -rv $(pwd)/www/assets $dst cp -rv $(pwd)/www/mkdocs.yml $dst