Integrate llvm test suite

.github/workflows/llvm-ts-run.yml

@@ -0,0 +1,175 @@
+#
+# Copyright (c) 2021-present, Trail of Bits, Inc.
+# All rights reserved.
+#
+# This source code is licensed in accordance with the terms specified in
+# the LICENSE file found in the root directory of this source tree.
+#
+
+name: "LLVM test suite"
+
+on:
+  workflow_run:
+    workflows: [Build]
+    types:
+      - completed
+    branches:
+      - "master"
+
+jobs:
+  build:
+    name: "Build VAST"
+    strategy:
+      matrix:
+        llvm-version: [17]
+        image-version: [22.04]
+
+    runs-on: ubuntu-${{ matrix.image-version }}
+    timeout-minutes: 60
+    container:
+      image:
+        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest
+
+    env:
+        CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/"
+        TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake
+        LLVM_EXTERNAL_LIT: "/usr/local/bin/lit"
+
+    steps:
+      - name: Clone the VAST repository
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Configure build
+        run: |
+            cmake --preset ninja-multi-default --toolchain ${TOOLCHAIN} \
+              -DCMAKE_VERBOSE_MAKEFILE=True \
+              -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} \
+              -DLLVM_EXTERNAL_LIT=${LLVM_EXTERNAL_LIT}
+
+      - name: Build release
+        run: |
+            cmake --build --preset ninja-rel -j $(nproc)
+            cpack -G TXZ --config ./builds/ninja-multi-default/CPackConfig.cmake
+
+      - name: Upload VAST build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: VAST
+          path: ./builds/ninja-multi-default/package/*
+          retention-days: 1
+
+  test:
+    name: "Run LLVM test suite"
+    needs: build
+    strategy:
+      matrix:
+        llvm-version: [17]
+        image-version: [22.04]
+        vast-target: ['hl', 'llvm', 'bin']
+
+
+    runs-on: ubuntu-${{ matrix.image-version }}
+    timeout-minutes: 60
+    container:
+      image:
+        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest
+
+    steps:
+     - name: Fetch VAST artifact
+       uses: actions/download-artifact@v4
+       with:
+         name: VAST
+
+     - name: Unpack VAST
+       run: |
+          mkdir vast && tar -xf VAST-* -C vast --strip-components=1
+
+     - name: Export vast binaries
+       run: |
+          echo "${PWD}/vast/bin/" >> $GITHUB_PATH
+
+     - name: Install test suite dependencies
+       run: |
+          apt-get update
+          apt-get -y install clang
+          pip3 install lit psutil
+
+     - name: Clone test suite repository
+       uses: actions/checkout@v4
+       with:
+        repository: trailofbits/vast-llvm-test-suite
+        ref: main
+        path: llvm-test-suite
+        fetch-depth: 1
+
+     - name: Run test-suite
+       continue-on-error: true
+       run: |
+        mkdir llvm-test-suite/build && cd llvm-test-suite/build/
+        cmake ../ -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_C_COMPILER=$(readlink -f ../tools/compiler) \
+          -DCMAKE_CXX_COMPILER=$(readlink -f ../tools/compiler++) \
+          -C=../cmake/caches/O3.cmake \
+          -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \
+          -DTEST_SUITE_SUBDIRS=SingleSource \
+          -DTEST_SUITE_VAST_ENABLE=ON \
+          -DTEST_SUITE_VAST_TARGET=${{ matrix.vast-target }}
+        make -j $(nproc) -k
+
+     - name: Collect run data (mlir)
+       if: matrix.vast-target != ''
+       working-directory: ./llvm-test-suite/build
+       run: python3 ../utils/mlir_compile_json_gen.py >> ../../results-${{ matrix.vast-target }}.json
+
+     - name: Collect run data (bin)
+       continue-on-error: true
+       if: matrix.vast-target == ''
+       working-directory: ./llvm-test-suite/build
+       run: lit --timeout=60 -v -o ../../results.json .
+
+     - name: Upload results
+       uses: actions/upload-artifact@v4
+       with:
+        path: ./results-${{ matrix.vast-target }}.json
+        name: results-${{ matrix.vast-target }}
+
+  eval:
+    name: "Eval test results"
+    needs: test
+    strategy:
+      matrix:
+        image-version: [22.04]
+    runs-on: ubuntu-${{ matrix.image-version }}
+    steps:
+      - name: Fetch result artifacts
+        uses: actions/download-artifact@v4
+
+      - name: Display structure of downloaded files
+        run: ls -R
+      - name: Install evaluator dependencies
+        run: |
+            pip3 install pandas scipy tabulate
+
+      - name: Clone test suite repository
+        uses: actions/checkout@v4
+        with:
+          repository: trailofbits/vast-llvm-test-suite
+          sparse-checkout: utils/vast_compare.py
+          ref: main
+          path: llvm-test-suite
+          fetch-depth: 1
+
+      - name: Generate the results
+        run: |
+          python3 llvm-test-suite/utils/vast_compare.py --columns vast-hl,vast-llvm,vast-bin --files results-hl.json,results-llvm.json,results-bin.json
+
+      - name: Post results as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: final-result
+          path: |
+            ./results.csv
+            ./results.md

.github/workflows/prerelease.yml

@@ -10,7 +10,7 @@ name: "Pre Release"
 
 on:
   workflow_run:
-    workflows: ["Build"]
+    workflows: ["LLVM test suite"]
     types:
       - completed
     branches:
@@ -69,6 +69,13 @@ jobs:
 
       - name: Build VAST Doc
         run: cmake --build --preset ninja-rel --target vast-doc
+
+      - name: Fetch LLVM test suite results
+        uses: actions/download-artifact@v4
+        with:
+          name: final-result
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
 
       - name: Build Pages
         run: |

www/mkdocs.yml

@@ -23,6 +23,8 @@ nav:
     - Optimizer: Tools/vast-opt.md
     - Query: Tools/vast-query.md
     - REPL: Tools/vast-repl.md
+  - Benchmarks:
+    - LLVM Test Suite SingleSource: results.md
   - About:
     - 'License': 'statement.md'
 

www/setup.sh

@@ -40,6 +40,9 @@ cp -rv $(pwd)/CONTRIBUTING.md $dst/docs
 # Setup auto-generated docs
 cp -rv $build/docs $dst/docs/dialects
 
+# Setup benchmark results
+cp -rv $(pwd)/results.md $dst/docs
+
 # Setup site assets
 cp -rv $(pwd)/www/assets $dst
 cp -rv $(pwd)/www/mkdocs.yml $dst