From 04255359fa767944916b35c8e8828889a3b17fa4 Mon Sep 17 00:00:00 2001 From: Robert Konicar Date: Tue, 27 Feb 2024 15:52:29 +0100 Subject: [PATCH 1/7] gh: Add llvm test suite workflow. --- .github/workflows/llvm-ts-run.yml | 189 ++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 .github/workflows/llvm-ts-run.yml diff --git a/.github/workflows/llvm-ts-run.yml b/.github/workflows/llvm-ts-run.yml new file mode 100644 index 0000000000..7baae88e55 --- /dev/null +++ b/.github/workflows/llvm-ts-run.yml @@ -0,0 +1,189 @@ +# +# Copyright (c) 2021-present, Trail of Bits, Inc. +# All rights reserved. +# +# This source code is licensed in accordance with the terms specified in +# the LICENSE file found in the root directory of this source tree. +# + +name: "LLVM test suite" + +on: + workflow_run: + workflows: [Build] + types: + - completed + branches: + - "master" + +jobs: + build: + name: "Build VAST" + strategy: + matrix: + llvm-version: [17] + image-version: [22.04] + + runs-on: ubuntu-${{ matrix.image-version }} + timeout-minutes: 60 + container: + image: + ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest + + env: + CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/" + TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake + LLVM_EXTERNAL_LIT: "/usr/local/bin/lit" + + steps: + - name: Clone the VAST repository + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 1 + + - name: Configure build + run: | + cmake --preset ninja-multi-default --toolchain ${TOOLCHAIN} \ + -DCMAKE_VERBOSE_MAKEFILE=True \ + -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} \ + -DLLVM_EXTERNAL_LIT=${LLVM_EXTERNAL_LIT} + + - name: Build release + run: | + cmake --build --preset ninja-rel -j $(nproc) + cpack -G TXZ --config ./builds/ninja-multi-default/CPackConfig.cmake + + - name: Upload VAST build artifact + uses: actions/upload-artifact@v4 + with: + name: VAST + path: ./builds/ninja-multi-default/package/* + retention-days: 1 + + test: + name: "Run LLVM test suite" + needs: build + strategy: + matrix: + llvm-version: [17] + image-version: [22.04] + vast-target: ['hl', 'llvm', ''] + + + runs-on: ubuntu-${{ matrix.image-version }} + timeout-minutes: 60 + container: + image: + ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest + + steps: + - name: Fetch VAST artifact + uses: actions/download-artifact@v4 + with: + name: VAST + + - name: Unpack VAST + run: | + mkdir vast && tar -xf VAST-* -C vast --strip-components=1 + + - name: Export vast binaries + run: | + echo "${PWD}/vast/bin/" >> $GITHUB_PATH + + - name: Install test suite dependencies + run: | + apt-get update + apt-get -y install clang + pip3 install lit psutil + + - name: Clone test suite repository + uses: actions/checkout@v4 + with: + repository: Jezurko/llvm-test-suite-vast + ref: main + path: llvm-test-suite + fetch-depth: 1 + + - name: Run test-suite + continue-on-error: true + run: | + mkdir llvm-test-suite/build && cd llvm-test-suite/build/ + cmake ../ -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=$(readlink -f ../tools/compiler) \ + -DCMAKE_CXX_COMPILER=$(readlink -f ../tools/compiler++) \ + -C=../cmake/caches/O3.cmake \ + -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \ + -DTEST_SUITE_SUBDIRS=SingleSource \ + -DTEST_SUITE_VAST_ENABLE=ON \ + -DTEST_SUITE_VAST_TARGET=${{ matrix.vast-target }} + make -j $(nproc) -k + + - name: Collect run data (mlir) + if: matrix.vast-target != '' + run: | + cd llvm-test-suite/build + python3 ../utils/mlir_compile_json_gen.py >> ../../results${{ matrix.vast-target }}.json + + - name: Collect run data (bin) + continue-on-error: true + if: matrix.vast-target == '' + run: | + cd llvm-test-suite/build + lit --timeout=60 -v -o ../../results.json . + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + path: ./results${{ matrix.vast-target }}.json + name: results${{ matrix.vast-target }} + + eval: + name: "Eval test results" + needs: test + strategy: + matrix: + image-version: [22.04] + runs-on: ubuntu-${{ matrix.image-version }} + steps: + - name: Fetch hl results artifact + uses: actions/download-artifact@v4 + with: + name: resultshl + + - name: Fetch llvm results artifact + uses: actions/download-artifact@v4 + with: + name: resultsllvm + + - name: Fetch binary results artifact + uses: actions/download-artifact@v4 + with: + name: results + + - name: Install evaluator dependencies + run: | + pip3 install pandas scipy tabulate + + - name: Clone test suite repository + uses: actions/checkout@v4 + with: + repository: Jezurko/llvm-test-suite-vast + sparse-checkout: | + utils/vast_compare.py + ref: main + path: llvm-test-suite + fetch-depth: 1 + + - name: Generate the results + run: | + python3 llvm-test-suite/utils/vast_compare.py --columns vast-hl,vast-llvm,vast-bin --files resultshl.json,resultsllvm.json,results.json + + - name: Post results as artifacts + uses: actions/upload-artifact@v4 + with: + name: final-result + path: | + ./results.csv + ./results.md + From 58d424b6a8dad892be6860d64f15e346f11b8909 Mon Sep 17 00:00:00 2001 From: Robert Konicar Date: Tue, 27 Feb 2024 21:13:05 +0100 Subject: [PATCH 2/7] gh: Make release workflow depend on test suite workflow. --- .github/workflows/prerelease.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 26295fcee5..9bf8b45ac3 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -10,7 +10,7 @@ name: "Pre Release" on: workflow_run: - workflows: ["Build"] + workflows: ["LLVM test suite"] types: - completed branches: @@ -69,6 +69,13 @@ jobs: - name: Build VAST Doc run: cmake --build --preset ninja-rel --target vast-doc + + - name: Fetch LLVM test suite results + uses: actions/download-artifact@v4 + with: + name: final-result + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} - name: Build Pages run: | From 03efcc6ed310a9113f42e11bf6fd55bc430b71ca Mon Sep 17 00:00:00 2001 From: Robert Konicar Date: Tue, 27 Feb 2024 21:13:44 +0100 Subject: [PATCH 3/7] www: Add LLVM test suite benchmarks results. --- www/mkdocs.yml | 2 ++ www/setup.sh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/www/mkdocs.yml b/www/mkdocs.yml index ede7d0b908..d7d33f5a99 100644 --- a/www/mkdocs.yml +++ b/www/mkdocs.yml @@ -23,6 +23,8 @@ nav: - Optimizer: Tools/vast-opt.md - Query: Tools/vast-query.md - REPL: Tools/vast-repl.md + - Benchmarks: + - LLVM Test Suite SingleSource: results.md - About: - 'License': 'statement.md' diff --git a/www/setup.sh b/www/setup.sh index af0dba4dff..97215949ac 100644 --- a/www/setup.sh +++ b/www/setup.sh @@ -40,6 +40,9 @@ cp -rv $(pwd)/CONTRIBUTING.md $dst/docs # Setup auto-generated docs cp -rv $build/docs $dst/docs/dialects +# Setup benchmark results +cp -rv $(pwd)/results.md $dst/docs + # Setup site assets cp -rv $(pwd)/www/assets $dst cp -rv $(pwd)/www/mkdocs.yml $dst From 238adb5326279ed4ca5ec4729cee1db6058e6a0e Mon Sep 17 00:00:00 2001 From: xlauko Date: Wed, 28 Feb 2024 10:15:29 +0100 Subject: [PATCH 4/7] gh: Update path to llvm-test-suite. --- .github/workflows/llvm-ts-run.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/llvm-ts-run.yml b/.github/workflows/llvm-ts-run.yml index 7baae88e55..2776c092e1 100644 --- a/.github/workflows/llvm-ts-run.yml +++ b/.github/workflows/llvm-ts-run.yml @@ -100,7 +100,7 @@ jobs: - name: Clone test suite repository uses: actions/checkout@v4 with: - repository: Jezurko/llvm-test-suite-vast + repository: trailofbits/vast-llvm-test-suite ref: main path: llvm-test-suite fetch-depth: 1 @@ -168,9 +168,8 @@ jobs: - name: Clone test suite repository uses: actions/checkout@v4 with: - repository: Jezurko/llvm-test-suite-vast - sparse-checkout: | - utils/vast_compare.py + repository: trailofbits/vast-llvm-test-suite + sparse-checkout: utils/vast_compare.py ref: main path: llvm-test-suite fetch-depth: 1 @@ -186,4 +185,3 @@ jobs: path: | ./results.csv ./results.md - From 34b7de783d20216b6ccac2316fb60e7f085f1d08 Mon Sep 17 00:00:00 2001 From: xlauko Date: Wed, 28 Feb 2024 10:28:19 +0100 Subject: [PATCH 5/7] gh: Clean up working directories. --- .github/workflows/llvm-ts-run.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/llvm-ts-run.yml b/.github/workflows/llvm-ts-run.yml index 2776c092e1..daa9a2d2a8 100644 --- a/.github/workflows/llvm-ts-run.yml +++ b/.github/workflows/llvm-ts-run.yml @@ -121,16 +121,14 @@ jobs: - name: Collect run data (mlir) if: matrix.vast-target != '' - run: | - cd llvm-test-suite/build - python3 ../utils/mlir_compile_json_gen.py >> ../../results${{ matrix.vast-target }}.json + working-directory: ./llvm-test-suite/build + run: python3 ../utils/mlir_compile_json_gen.py >> ../../results${{ matrix.vast-target }}.json - name: Collect run data (bin) continue-on-error: true if: matrix.vast-target == '' - run: | - cd llvm-test-suite/build - lit --timeout=60 -v -o ../../results.json . + working-directory: ./llvm-test-suite/build + run: lit --timeout=60 -v -o ../../results.json . - name: Upload results uses: actions/upload-artifact@v4 From 94184aecae2ed367a881930588567df2d77e03c1 Mon Sep 17 00:00:00 2001 From: xlauko Date: Wed, 28 Feb 2024 10:40:50 +0100 Subject: [PATCH 6/7] gh: Clean up results fetching. --- .github/workflows/llvm-ts-run.yml | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/.github/workflows/llvm-ts-run.yml b/.github/workflows/llvm-ts-run.yml index daa9a2d2a8..cd0e0acbd9 100644 --- a/.github/workflows/llvm-ts-run.yml +++ b/.github/workflows/llvm-ts-run.yml @@ -68,7 +68,7 @@ jobs: matrix: llvm-version: [17] image-version: [22.04] - vast-target: ['hl', 'llvm', ''] + vast-target: ['hl', 'llvm', 'bin'] runs-on: ubuntu-${{ matrix.image-version }} @@ -122,7 +122,7 @@ jobs: - name: Collect run data (mlir) if: matrix.vast-target != '' working-directory: ./llvm-test-suite/build - run: python3 ../utils/mlir_compile_json_gen.py >> ../../results${{ matrix.vast-target }}.json + run: python3 ../utils/mlir_compile_json_gen.py >> ../../results-${{ matrix.vast-target }}.json - name: Collect run data (bin) continue-on-error: true @@ -133,8 +133,8 @@ jobs: - name: Upload results uses: actions/upload-artifact@v4 with: - path: ./results${{ matrix.vast-target }}.json - name: results${{ matrix.vast-target }} + path: ./results-${{ matrix.vast-target }}.json + name: results-${{ matrix.vast-target }} eval: name: "Eval test results" @@ -144,21 +144,11 @@ jobs: image-version: [22.04] runs-on: ubuntu-${{ matrix.image-version }} steps: - - name: Fetch hl results artifact + - name: Fetch result artifacts uses: actions/download-artifact@v4 - with: - name: resultshl - - - name: Fetch llvm results artifact - uses: actions/download-artifact@v4 - with: - name: resultsllvm - - - name: Fetch binary results artifact - uses: actions/download-artifact@v4 - with: - name: results + - name: Display structure of downloaded files + run: ls -R - name: Install evaluator dependencies run: | pip3 install pandas scipy tabulate @@ -174,7 +164,7 @@ jobs: - name: Generate the results run: | - python3 llvm-test-suite/utils/vast_compare.py --columns vast-hl,vast-llvm,vast-bin --files resultshl.json,resultsllvm.json,results.json + python3 llvm-test-suite/utils/vast_compare.py --columns vast-hl,vast-llvm,vast-bin --files results-hl.json,results-llvm.json,results-bin.json - name: Post results as artifacts uses: actions/upload-artifact@v4 From d62643413e84c7f2438645fa0b461cce141b083d Mon Sep 17 00:00:00 2001 From: xlauko Date: Wed, 28 Feb 2024 11:14:22 +0100 Subject: [PATCH 7/7] www: Add benchmarks directory. --- CMakePresets.json | 26 ++++++++++++++++++++++++++ www/mkdocs.yml | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CMakePresets.json b/CMakePresets.json index f3b83049db..990578a75e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -107,5 +107,31 @@ "displayName": "Test cxx-common-osx-deb", "configuration": "Debug" } + ], + "packagePresets": [ + { + "name": "default", + "configurePreset": "ninja-multi-default", + "generators": [ "TGZ" ] + } + ], + "workflowPresets": [ + { + "name": "release-package", + "steps": [ + { + "type": "configure", + "name": "ninja-multi-default" + }, + { + "type": "build", + "name": "ninja-rel" + }, + { + "type": "package", + "name": "default" + } + ] + } ] } diff --git a/www/mkdocs.yml b/www/mkdocs.yml index d7d33f5a99..2c1edf12e7 100644 --- a/www/mkdocs.yml +++ b/www/mkdocs.yml @@ -24,7 +24,7 @@ nav: - Query: Tools/vast-query.md - REPL: Tools/vast-repl.md - Benchmarks: - - LLVM Test Suite SingleSource: results.md + - LLVM Test Suite SingleSource: benchmarks/results.md - About: - 'License': 'statement.md'