WIP: build multiplier only using vcpkg

.github/workflows/ci.yml

@@ -15,9 +15,10 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  LLVM_PASTA_VER: 4a2f2bf
   LLVM_VER: 17
   PYTHON_VER: 3.11
+  VCPKG_REGISTRY_VERSION: v1.0.0
+  VCPKG_COMMIT: fecdd89f28ed883b26a48c2b30085f6384070bb8
 
 jobs:
   build:
@@ -30,11 +31,17 @@ jobs:
       contents: write
       packages: write
 
+    strategy:
+      matrix:
+        include:
+          - name: lto
+            additional_flags: -flto
+          - name: nolto
+            additional_flags:
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
       - run: git config --global --add safe.directory '*'
 
       - name: Install prereqs
@@ -45,7 +52,24 @@ jobs:
           curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
           echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VER} main" | tee -a /etc/apt/sources.list > /dev/null
           echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VER} main" | tee -a /etc/apt/sources.list > /dev/null
-          apt-get update; apt-get install gh clang-${LLVM_VER} lld-${LLVM_VER} python${PYTHON_VER} python${PYTHON_VER}-dev -y
+          apt-get update; apt-get install gh clang-${LLVM_VER} lld-${LLVM_VER} python${PYTHON_VER} python${PYTHON_VER}-dev openssh-client -y
+
+      - name: Fetch submodules
+        run: |
+          mkdir -p ~/.ssh
+          eval $(ssh-agent -s)
+          ssh-keyscan github.com >> ~/.ssh/known_hosts
+          chmod 600 ~/.ssh/known_hosts
+          echo "$MX_REGISTRY_KEY" | ssh-add - 
+          git submodule update --init --recursive -- vcpkg-registry
+
+          cd vcpkg-registry
+          git fetch
+          head_hash="$(git rev-parse HEAD)"
+          tag_hash="$(git rev-parse $VCPKG_REGISTRY_VERSION)"
+          [ "$head_hash" = "$tag_hash" ] || { echo "The \$VCPKG_REGISTRY_VERSION specified in CI ($VCPKG_REGISTRY_VERSION) and the submodule commit don\'t match. You probably updated \$VCPKG_REGISTRY_VERSION without updating the submodule version as well." ; exit 1 ; }
+        env:
+            MX_REGISTRY_KEY: ${{ secrets.MX_REGISTRY_KEY }}
 
       - name: Setup cmake
         uses: trail-of-forks/actions-setup-cmake@master
@@ -58,52 +82,103 @@ jobs:
       - name: Set up variables
         run: |
           echo "RELEASE_DIR=${RUNNER_TEMP}/release" >> $GITHUB_ENV
-          echo "LLVM_PREFIX_DIR=${RUNNER_TEMP}/llvm" >> $GITHUB_ENV
           echo "VERSION=$(git rev-parse --short=7 HEAD)" >> $GITHUB_ENV
-          echo "ARTIFACT_NAME=multiplier-$(git rev-parse --short=7 HEAD).tar.xz" >> $GITHUB_ENV
+          echo "ARTIFACT_NAME=multiplier-$(git rev-parse --short=7 HEAD)-${{ matrix.name }}.tar.xz" >> $GITHUB_ENV
+          echo "CC=$(which clang-$LLVM_VER)" >> $GITHUB_ENV
+          echo "CXX=$(which clang++-$LLVM_VER)" >> $GITHUB_ENV
+          echo "CFLAGS=-g0 --ld-path=$(which ld.lld-$LLVM_VER) ${{ matrix.additional_flags }}" >> $GITHUB_ENV
+          echo "CXXFLAGS=-g0 --ld-path=$(which ld.lld-$LLVM_VER) ${{ matrix.additional_flags }}" >> $GITHUB_ENV
+          echo "VCPKG_BINARY_SOURCES=clear;files,$GITHUB_WORKSPACE/vcpkg-binary-cache,readwrite" >> $GITHUB_ENV
+
+      - name: Setup vcpkg
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/vcpkg-binary-cache
+          git clone https://github.com/microsoft/vcpkg $HOME/vcpkg
+          git -C $HOME/vcpkg checkout $VCPKG_COMMIT
+          $HOME/vcpkg/bootstrap-vcpkg.sh -disableMetrics
 
-      - name: Install llvm-pasta
+      - name: Preseed vcpkg cache
         run: |
-          mkdir -p ${RELEASE_DIR}
-          mkdir -p ${LLVM_PREFIX_DIR}
-          gh release download ${LLVM_PASTA_VER} --repo trail-of-forks/llvm-project --pattern 'llvm-*'
-          tar -xvJf llvm-pasta-*.tar.xz -C ${LLVM_PREFIX_DIR}
+          gh release download $VCPKG_REGISTRY_VERSION \
+            --pattern vcpkg-binary-cache-${{ matrix.name }}.tar.bz2 \
+            --repo trailofbits/multiplier-vcpkg-registry
+          tar xjf vcpkg-binary-cache-${{ matrix.name }}.tar.bz2
+          rm vcpkg-binary-cache-${{ matrix.name }}.tar.bz2
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.MX_REGISTRY_RELEASES_TOKEN }}
+
+      - name: Cache vcpkg binaries
+        id: cache-vcpkg
+        uses: actions/cache@v4
+        with:
+          path: vcpkg-binary-cache
+          key: ${{ runner.os }}-${{ matrix.name }}-${{ hashFiles('vcpkg-registry/ports/**/*') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.name }}-${{ hashFiles('vcpkg-registry/ports/**/*') }}
+            ${{ runner.os }}-
 
       - name: Build
         run: |
-
-          CXXFLAGS="-g0" \
-          CCFLAGS="-g0" \
           cmake \
+            -DVCPKG_TARGET_TRIPLET=x64-linux-rel \
+            --toolchain "$HOME/vcpkg/scripts/buildsystems/vcpkg.cmake" \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_PREFIX=${RELEASE_DIR} \
-            -DCMAKE_EXE_LINKER_FLAGS="--ld-path=$(which ld.lld-${LLVM_VER})" \
-            -DCMAKE_MODULE_LINKER_FLAGS="--ld-path=$(which ld.lld-${LLVM_VER})" \
-            -DCMAKE_SHARED_LINKER_FLAGS="--ld-path=$(which ld.lld-${LLVM_VER})" \
-            -DCMAKE_C_COMPILER="$(which clang-${LLVM_VER})" \
-            -DCMAKE_CXX_COMPILER="$(which clang++-${LLVM_VER})" \
-            -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=TRUE \
-            -DClang_DIR=${LLVM_PREFIX_DIR}/lib/cmake/clang \
-            -DLLVM_DIR=${LLVM_PREFIX_DIR}/lib/cmake/llvm \
-            -DMLIR_DIR=${LLVM_PREFIX_DIR}/lib/cmake/mlir \
-            -DMX_USE_VENDORED_CLANG=OFF \
-            -DMX_USE_VENDORED_PASTA=OFF \
-            -DLLVM_ENABLE_LLD:BOOL=TRUE \
             -GNinja \
-            ./
+            -S . \
+            -B build
 
-          ninja install
+          cmake --build build
+          cmake --install build --prefix ${RELEASE_DIR}
 
       # Release on merge to main
       - name: Package Release
         if: github.ref == 'refs/heads/main'
         working-directory: ${{ env.RELEASE_DIR }}
         run: tar --use-compress-program pixz -cf "${ARTIFACT_NAME}" *
 
-      - name: Create Release
+      - name: Upload cache artifact
         if: github.ref == 'refs/heads/main'
-        run: gh release create ${VERSION} --generate-notes ${{ env.RELEASE_DIR }}/${{ env.ARTIFACT_NAME }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: multiplier-${{ matrix.name }}
+          path: ${{ env.RELEASE_DIR }}/${{ env.ARTIFACT_NAME }}
+
+  create-release:
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: github.ref == 'refs/heads/main'
+    name: Create release
+
+    permissions:
+      contents: write
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          run-id: build
+
+      - name: Install gh and extract
+        run: |
+          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
+          sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+
+          sudo apt-get update
+          sudo apt-get install gh
+          echo "VERSION=$(git rev-parse --short=7 HEAD)" >> $GITHUB_ENV
+          echo "ARTIFACT_PREFIX=multiplier-$(git rev-parse --short=7 HEAD)" >> $GITHUB_ENV
+
+      - name: Create release
+        run: |
+          gh release create "${{ env.VERSION }}" --generate-notes \
+            multiplier-lto/${{ env.ARTIFACT_PREFIX }}-lto.tar.xz \
+            multiplier-nolto/${{ env.ARTIFACT_PREFIX }}-nolto.tar.xz \
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitmodules

@@ -1,53 +1,3 @@
-[submodule "vendor/concurrentqueue/src"]
-	path = vendor/concurrentqueue/src
-	url = https://github.com/cameron314/concurrentqueue.git
-	shallow = true
-[submodule "vendor/reproc/src"]
-	path = vendor/reproc/src
-	url = https://github.com/DaanDeMeyer/reproc.git
-	shallow = true
-[submodule "vendor/capnproto/src"]
-	path = vendor/capnproto/src
-	url = https://github.com/capnproto/capnproto.git
-	shallow = true
-[submodule "vendor/glog/src"]
-	path = vendor/glog/src
-	url = https://github.com/google/glog.git
-	shallow = true
-[submodule "vendor/gflags/src"]
-	path = vendor/gflags/src
-	url = https://github.com/gflags/gflags.git
-	shallow = true
-[submodule "vendor/zstd/src"]
-	path = vendor/zstd/src
-	url = https://github.com/facebook/zstd.git
-	shallow = true
-[submodule "vendor/gap/src"]
-	path = vendor/gap/src
-	url = https://github.com/lifting-bits/gap.git
-	shallow = true
-[submodule "vendor/pasta/src"]
-	path = vendor/pasta/src
-	url = https://github.com/trailofbits/pasta.git
-	branch = master
-[submodule "vendor/llvm-project/src"]
-	path = vendor/llvm-project/src
-	url = https://github.com/trail-of-forks/llvm-project.git
-	branch = pasta-llvmorg-17.0.4
-[submodule "vendor/abseil/src"]
-	path = vendor/abseil/src
-	url = https://github.com/abseil/abseil-cpp.git
-	shallow = true
-[submodule "vendor/re2/src"]
-	path = vendor/re2/src
-	url = https://github.com/google/re2.git
-	shallow = true
-[submodule "vendor/rocksdb/src"]
-	path = vendor/rocksdb/src
-	url = https://github.com/facebook/rocksdb.git
-[submodule "vendor/xxhash/src"]
-	path = vendor/xxhash/src
-	url = https://github.com/Cyan4973/xxHash.git
-[submodule "vendor/vast/src"]
-	path = vendor/vast/src
-	url = https://github.com/trailofbits/vast.git
+[submodule "vcpkg-registry"]
+	path = vcpkg-registry
+	url = [email protected]:trailofbits/multiplier-vcpkg-registry.git

CMakeLists.txt

@@ -25,8 +25,7 @@ endif()
 list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules")
 
 include("cmake/system.cmake")
-include("cmake/llvm.cmake")
-include("cmake/linker.cmake")
+#include("cmake/llvm.cmake")
 include("cmake/git.cmake")
 
 if(MX_ENABLE_INSTALL)

bin/Examples/CMakeLists.txt

@@ -54,7 +54,7 @@ function(define_example exe_name source_file)
         "${PROJECT_NAME}Targets"
       RUNTIME
         DESTINATION
-          "${CMAKE_INSTALL_BINDIR}"
+          "${MX_INSTALL_TOOLS_DIR}"
     )
   endif()
 endfunction(define_example)

bin/Index/CMakeLists.txt

@@ -73,6 +73,8 @@ add_executable("${exe_name}"
   "${MX_BOOTSTRAP_INCLUDE_VISITOR_INC_H}"
 )
 
+find_package(concurrentqueue MODULE REQUIRED)
+
 target_link_libraries("${exe_name}"
   PRIVATE
     "mx-api"
@@ -118,7 +120,7 @@ set_target_properties("${exe_name}"
       "${PROJECT_BINARY_DIR}/${MX_INSTALL_BIN_DIR}"
 )
 
-find_and_link_llvm_dependencies("${exe_name}")
+#find_and_link_llvm_dependencies("${exe_name}")
 
 if(MX_ENABLE_INSTALL AND NOT MX_ENABLE_BOOTSTRAP)
   install(
@@ -128,6 +130,6 @@ if(MX_ENABLE_INSTALL AND NOT MX_ENABLE_BOOTSTRAP)
       "${PROJECT_NAME}Targets"
     RUNTIME
       DESTINATION
-        "${CMAKE_INSTALL_BINDIR}"
+        "${MX_INSTALL_TOOLS_DIR}"
   )
 endif()

bindings/Python/CMakeLists.txt

@@ -117,7 +117,7 @@ if(MX_ENABLE_INSTALL AND NOT MX_ENABLE_BOOTSTRAP)
     DIRECTORY
       "${CMAKE_CURRENT_LIST_DIR}/${lower_project_name}-stubs"
     DESTINATION
-      "${CMAKE_INSTALL_PREFIX}/${MX_INSTALL_LIB_DIR}/${site_packages_dir}"
+      "${MX_INSTALL_LIB_DIR}/${site_packages_dir}"
   )
 
   # Install the binary into the target site-packages directory.

cmake/modules/Findconcurrentqueue.cmake

@@ -0,0 +1,16 @@
+find_path(concurrentqueue_INCLUDE_DIR concurrentqueue.h PATH_SUFFIXES concurrentqueue)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(concurrentqueue
+    REQUIRED_VARS concurrentqueue_INCLUDE_DIR)
+
+# Create the imported target
+if(concurrentqueue_FOUND)
+    set(concurrentqueue_INCLUDE_DIRS ${concurrentqueue_INCLUDE_DIR})
+
+    if(NOT TARGET concurrentqueue)
+        add_library(concurrentqueue INTERFACE IMPORTED)
+        set_target_properties(concurrentqueue PROPERTIES
+            INTERFACE_INCLUDE_DIRECTORIES "${concurrentqueue_INCLUDE_DIRS}")
+    endif()
+endif()

cmake/options.cmake

@@ -12,24 +12,9 @@ set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard version")
 
 set(MX_INSTALL_LIB_DIR "${CMAKE_INSTALL_LIBDIR}" CACHE PATH "Directory in which Multiplier libraries will be installed")
 set(MX_INSTALL_BIN_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH "Directory in which Multiplier binaries will be installed")
+set(MX_INSTALL_TOOLS_DIR "${CMAKE_INSTALL_BINDIR}" CACHE PATH "Directory in which Multiplier tools will be installed")
 set(MX_INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}" CACHE PATH "Directory in which Multiplier headers will be installed")
 set(MX_INSTALL_SHARE_DIR "${CMAKE_INSTALL_DATADIR}" CACHE PATH "Directory in which Multiplier CMake files will be installed")
-set(MX_VENDOR_INSTALL_DIR "${PROJECT_BINARY_DIR}/vendor/install" CACHE PATH "Directory in which multiplier's vendored dependencies are installed")
-
-option(MX_USE_VENDORED_CAPNPROTO "Set to OFF to disable default building of Cap'n Proto as a vendored library." ON)
-option(MX_USE_VENDORED_GLOG "Set to OFF to disable default building of Google glog as a vendored library." ON)
-option(MX_USE_VENDORED_GFLAGS "Set to OFF to disable default building of gflags as a vendored library." ON)
-option(MX_USE_VENDORED_ZSTD "Set to OFF to disable default building of ZStandard as a vendored library." ON)
-option(MX_USE_VENDORED_REPROC "Set to OFF to disable default building of reproc/reproc++ as a vendored library." ON)
-option(MX_USE_VENDORED_GAP "Set to OFF to disable default building of gap as a vendored library." ON)
-option(MX_USE_VENDORED_CLANG "Set to OFF to disable default building of Clang/LLVM as a vendored library." ON)
-option(MX_USE_VENDORED_PASTA "Set to OFF to disable default building of PASTA as a vendored library." ON)
-option(MX_USE_VENDORED_ABSEIL "Set to OFF to disable default building of Abseil as a vendored library." ON)
-option(MX_USE_VENDORED_RE2 "Set to OFF to disable default building of RE2 as a vendored library." ON)
-option(MX_USE_VENDORED_ROCKSDB "Set to OFF to disable default building of RocksDB as a vendored library." ON)
-option(MX_USE_VENDORED_XXHASH "Set to OFF to disable default building of xxHash as a vendored library." ON)
-option(MX_USE_VENDORED_NANOBIND "Set to OFF to disable default building of nanobind as a vendored library." ON)
-option(MX_USE_VENDORED_VAST "Set to OFF to disable default building of VAST as a vendored library." ON)
 
 option(MX_ENABLE_BOOTSTRAP "Set to ON to enable re-bootstrapping of some auto-generated files." OFF)
 option(MX_ENABLE_INSTALL "Set to ON to enable the install target" ON)

lib/CMakeLists.txt

@@ -185,6 +185,9 @@ add_dependencies("mx-api"
   "copy_headers"
 )
 
+find_package(concurrentqueue MODULE REQUIRED)
+find_package(SQLite3 REQUIRED)
+
 target_link_libraries("mx-api"
   PUBLIC
     "gap::gap"
@@ -193,7 +196,7 @@ target_link_libraries("mx-api"
     "std::coroutines"
   PRIVATE
     "mx-serialize"
-    "mx-sqlite"
+    SQLite::SQLite3
     "zstd::libzstd_static"
     "concurrentqueue"
     ${MLIR_LIBS}

vcpkg-configuration.json

@@ -0,0 +1,9 @@
+{
+    "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg-configuration.schema.json",
+    "overlay-ports": [
+        "./vcpkg-registry/ports"
+    ],
+    "overlay-triplets": [
+        "./vcpkg-registry/triplets"
+    ]
+}