diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 70340ce11..e75ac95fb 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -114,11 +114,6 @@ jobs:
         cd build
         source ${GITHUB_WORKSPACE}/.github/ci_setup.sh ${{ matrix.PLATFORM.NAME }}
         ctest --output-on-failure
-    # Run the Benchmarks(s) only for Release build type
-    - name: Benchmark
-      if: "matrix.BUILD_TYPE == 'Release'"
-      run: |
-        source ${GITHUB_WORKSPACE}/tests/scripts/run_benchmarks.sh
 
   # Containerised build jobs.
   device-container:
@@ -132,11 +127,11 @@ jobs:
           - NAME: "CUDA"
             CXX_STANDARD: "20"
             CONTAINER: "ghcr.io/acts-project/ubuntu2404_cuda:56"
-            OPTIONS: -DDETRAY_EIGEN_PLUGIN=ON -DDETRAY_SMATRIX_PLUGIN=ON -DDETRAY_VC_PLUGIN=ON
+            OPTIONS: -DDETRAY_EIGEN_PLUGIN=ON -DDETRAY_SMATRIX_PLUGIN=OFF -DDETRAY_VC_PLUGIN=OFF
           - NAME: "SYCL"
             CXX_STANDARD: "20"
             CONTAINER: "ghcr.io/acts-project/ubuntu2404_oneapi:56"
-            OPTIONS: -DDETRAY_BUILD_CUDA=OFF -DDETRAY_BUILD_SYCL=ON -DDETRAY_EIGEN_PLUGIN=ON -DDETRAY_SMATRIX_PLUGIN=ON -DDETRAY_VC_PLUGIN=ON
+            OPTIONS: -DDETRAY_BUILD_CUDA=OFF -DDETRAY_BUILD_SYCL=ON -DDETRAY_EIGEN_PLUGIN=ON -DDETRAY_SMATRIX_PLUGIN=OFF -DDETRAY_VC_PLUGIN=OFF
 
     # The system to run on.
     runs-on: ubuntu-latest
@@ -164,6 +159,7 @@ jobs:
           -DDETRAY_BUILD_INTEGRATIONTESTS=ON \
           -DDETRAY_BUILD_BENCHMARKS=ON \
           -DDETRAY_BUILD_TUTORIALS=ON \
+          -DDETRAY_BUILD_HOST=OFF \
           -DDETRAY_FAIL_ON_WARNINGS=ON \
           -DCMAKE_BUILD_TYPE=${{ matrix.BUILD_TYPE }} \
           -DDETRAY_CUSTOM_SCALARTYPE=${{ matrix.SCALAR_TYPE }} \
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ef7c496ae..ee50e7d4b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,7 +12,7 @@ stages:
 build_cuda:
   tags: [docker]
   stage: build
-  image: ghcr.io/acts-project/ubuntu2204_cuda:56
+  image: gitlab-registry.cern.ch/acts/machines/ubuntu2204_cuda:latest
   artifacts:
     paths:
       - build
@@ -22,11 +22,15 @@ build_cuda:
       - >
         cmake --preset cuda -S src -B build
         -DCMAKE_BUILD_TYPE=Release
+        -DDETRAY_EIGEN_PLUGIN=ON
+        -DDETRAY_VC_PLUGIN=OFF
+        -DDETRAY_SMATRIX_PLUGIN=OFF
         -DBUILD_TESTING=ON
         -DDETRAY_BUILD_UNITTESTS=ON
         -DDETRAY_BUILD_INTEGRATIONTESTS=ON
         -DDETRAY_BUILD_BENCHMARKS=ON
         -DDETRAY_BUILD_TUTORIALS=ON
+        -DDETRAY_BUILD_HOST=OFF
         -DDETRAY_FAIL_ON_WARNINGS=ON
       - cmake --build build
 
@@ -64,11 +68,15 @@ build_sycl:
       - >
         cmake --preset sycl -S src -B build
         -DCMAKE_BUILD_TYPE=Release
+        -DDETRAY_EIGEN_PLUGIN=ON
+        -DDETRAY_VC_PLUGIN=OFF
+        -DDETRAY_SMATRIX_PLUGIN=OFF
         -DBUILD_TESTING=ON
         -DDETRAY_BUILD_UNITTESTS=OFF
         -DDETRAY_BUILD_INTEGRATIONTESTS=ON
         -DDETRAY_BUILD_BENCHMARKS=OFF
         -DDETRAY_BUILD_TUTORIALS=OFF
+        -DDETRAY_BUILD_HOST=OFF
         -DDETRAY_FAIL_ON_WARNINGS=ON
       - cmake --build build
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d7374426f..adb7916eb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -60,6 +60,7 @@ option( DETRAY_SVG_DISPLAY "Build ActSVG display module" OFF)
 option( DETRAY_BUILD_SYCL "Build the SYCL sources included in detray" OFF )
 option( DETRAY_BUILD_CUDA "Build the CUDA sources included in detray"
    ${DETRAY_BUILD_CUDA_DEFAULT} )
+cmake_dependent_option(DETRAY_BUILD_HOST "Build the host sources included in detray" ON "DETRAY_BUILD_CUDA OR DETRAY_BUILD_SYCL" ON)
 option( DETRAY_BUILD_TEST_UTILS "Build the test utility library of Detray" OFF )
 option( DETRAY_BUILD_UNITTESTS "Build the unit tests of Detray" OFF )
 option( DETRAY_BUILD_INTEGRATIONTESTS "Build the integration tests of Detray"
@@ -128,7 +129,7 @@ endif()
 
 # Set up ACTSVG for displaying
 option( DETRAY_SETUP_ACTSVG
-   "Set up the actsvg target(s) explicitly" TRUE )
+   "Set up the actsvg target(s) explicitly" ${DETRAY_SVG_DISPLAY} )
 option( DETRAY_USE_SYSTEM_ACTSVG
    "Pick up an existing installation of actsvg from the build environment"
    ${DETRAY_USE_SYSTEM_LIBS} )
@@ -170,7 +171,7 @@ endif()
 
 # Set up GoogleTest.
 option( DETRAY_SETUP_GOOGLETEST
-   "Set up the GoogleTest target(s) explicitly" TRUE )
+   "Set up the GoogleTest target(s) explicitly" ${DETRAY_BUILD_TESTING} )
 option( DETRAY_USE_SYSTEM_GOOGLETEST
    "Pick up an existing installation of GoogleTest from the build environment"
    ${DETRAY_USE_SYSTEM_LIBS} )
@@ -184,7 +185,7 @@ endif()
 
 # Set up Google Benchmark.
 option( DETRAY_SETUP_BENCHMARK
-   "Set up the Google Benchmark target(s) explicitly" TRUE )
+   "Set up the Google Benchmark target(s) explicitly" ${DETRAY_BUILD_BENCHMARKS} )
 option( DETRAY_USE_SYSTEM_BENCHMARK
    "Pick up an existing installation of Google Benchmark from the build environment"
    ${DETRAY_USE_SYSTEM_LIBS} )
diff --git a/extern/algebra-plugins/CMakeLists.txt b/extern/algebra-plugins/CMakeLists.txt
index c3ddec154..2c09f0d2a 100644
--- a/extern/algebra-plugins/CMakeLists.txt
+++ b/extern/algebra-plugins/CMakeLists.txt
@@ -35,9 +35,9 @@ FetchContent_Declare(AlgebraPlugins ${DETRAY_ALGEBRA_PLUGINS_SOURCE_FULL})
 # Options used in the build of Algebra Plugins.
 set(ALGEBRA_PLUGINS_BUILD_TESTING FALSE CACHE BOOL
    "Turn off the build of the Algebra Plugins unit tests")
-set(ALGEBRA_PLUGINS_INCLUDE_EIGEN TRUE CACHE BOOL
+set(ALGEBRA_PLUGINS_INCLUDE_EIGEN ${DETRAY_EIGEN_PLUGIN} CACHE BOOL
    "Turn on the build of algebra::eigen")
-set(ALGEBRA_PLUGINS_INCLUDE_VC TRUE CACHE BOOL
+set(ALGEBRA_PLUGINS_INCLUDE_VC ${DETRAY_VC_PLUGIN} CACHE BOOL
    "Turn on the build of algebra::vc_array")
 set(ALGEBRA_PLUGINS_INCLUDE_VECMEM TRUE CACHE BOOL
    "Turn on the build of algebra::vecmem_array")
@@ -47,11 +47,11 @@ set(ALGEBRA_PLUGINS_INCLUDE_VECMEM TRUE CACHE BOOL
 set(ALGEBRA_PLUGINS_INCLUDE_SMATRIX ${DETRAY_SMATRIX_PLUGIN}
    CACHE BOOL "Turn on/off the build of algebra::smatrix")
 
-set(ALGEBRA_PLUGINS_SETUP_EIGEN3 TRUE CACHE BOOL
+set(ALGEBRA_PLUGINS_SETUP_EIGEN3 ${DETRAY_EIGEN_PLUGIN}  CACHE BOOL
    "Have Algebra Plugins set up Eigen3 for itself")
 set(ALGEBRA_PLUGINS_USE_SYSTEM_EIGEN3 FALSE CACHE BOOL
    "Have Algebra Plugins pick up Eigen3 from the system")
-set(ALGEBRA_PLUGINS_SETUP_VC TRUE CACHE BOOL
+set(ALGEBRA_PLUGINS_SETUP_VC ${DETRAY_VC_PLUGIN} CACHE BOOL
    "Have Algebra Plugins set up Vc for itself")
 set(ALGEBRA_PLUGINS_USE_SYSTEM_VC FALSE CACHE BOOL
    "Have Algebra Plugins build Vc itself")
diff --git a/tests/benchmarks/CMakeLists.txt b/tests/benchmarks/CMakeLists.txt
index a0859f163..12c7bb3fa 100644
--- a/tests/benchmarks/CMakeLists.txt
+++ b/tests/benchmarks/CMakeLists.txt
@@ -5,7 +5,9 @@
 # Mozilla Public License Version 2.0
 
 # Set up the host/cpu benchmarks.
-add_subdirectory( cpu )
+if( DETRAY_BUILD_HOST )
+   add_subdirectory( cpu )
+endif()
 
 # Set up all of the "device" benchmarks.
 if( DETRAY_BUILD_CUDA )
diff --git a/tests/integration_tests/CMakeLists.txt b/tests/integration_tests/CMakeLists.txt
index 2ca28bd8b..d7a809cd1 100644
--- a/tests/integration_tests/CMakeLists.txt
+++ b/tests/integration_tests/CMakeLists.txt
@@ -4,11 +4,13 @@
 #
 # Mozilla Public License Version 2.0
 
-message(STATUS "Building detray host integration tests")
-
 # Set up all of the "host" tests.
-add_subdirectory( cpu )
-add_subdirectory( io )
+if( DETRAY_BUILD_HOST )
+   message(STATUS "Building detray host integration tests")
+
+   add_subdirectory( cpu )
+   add_subdirectory( io )
+endif()
 
 # Set up all of the "device" tests.
 if( DETRAY_BUILD_CUDA OR DETRAY_BUILD_SYCL )
diff --git a/tests/scripts/analyze_benchmarks.py b/tests/scripts/analyze_benchmarks.py
deleted file mode 100644
index d161e7d1a..000000000
--- a/tests/scripts/analyze_benchmarks.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import sys
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-
-benchmarks = pd.read_csv(sys.argv[1]).groupby(['name'])
-for bench in benchmarks :
-    # Skip groups that contain "/"
-    if "/" in bench[0]:
-        continue
-    gbenchmarks = bench[1].groupby(['group']);        
-    maxl = 0
-    for gbench in gbenchmarks :                
-        maxl = max(maxl, len(gbench[1]))
-    for gbench in gbenchmarks :
-        curl = len(gbench[1])
-        if gbench[0] == 'core' : # core tests are discontinued
-            crange = range(0, curl)
-        else :
-            crange = range(maxl-curl, maxl)
-        plt.plot(range(maxl-curl, maxl), gbench[1]['cpu_time'], marker="o", label=gbench[0])
-    plt.legend()
-    plt.savefig(bench[0] + '.png', format='png')
-    plt.cla()
diff --git a/tests/scripts/run_benchmarks.sh b/tests/scripts/run_benchmarks.sh
deleted file mode 100755
index be402ffd7..000000000
--- a/tests/scripts/run_benchmarks.sh
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/bin/bash
-#
-# Detray library, part of the ACTS project (R&D line)
-#
-# (c) 2021-2023 CERN for the benefit of the ACTS project
-#
-# Mozilla Public License Version 2.0
-
-echo "===> CI Benchmark running script for detray"
-
-# Set the script path
-BASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# Set workspace directory
-if [ -z "${GITHUB_WORKSPACE}" ]; then
-    WORKSPACE=${BASEDIR}/../../
-else
-    WORKSPACE=${GITHUB_WORKSPACE}
-fi
-
-git config --global --add safe.directory $(pwd)
-
-# Generate a CSV file in data directory
-export LASTCOMMIT=$(git log -n1 | head -n1 | cut -b 8-14)
-export DETRAY_TEST_DATA_DIR=${WORKSPACE}/data/
-
-touch ${WORKSPACE}/benchmark_${LASTCOMMIT}.csv
-
-echo "===> Benchmark pipeline for commit ${LASTCOMMIT}"
-
-# Run the benchmarks
-for group in eigen array ; do
-    echo "===> Running ${group}.benchmarks ..."
-    ${WORKSPACE}/build/bin/detray_benchmark_cpu_${group} --benchmark_format=csv --benchmark_filter=-propagation > ${group}_benchmarks.csv
-
-    echo "===> Formatting benchmark results ..."
-    sed -i -e "1d" ${group}_benchmarks.csv
-    sed -i -e 's/"BM_/'$LASTCOMMIT',"'$group'","BM_/g' ${group}_benchmarks.csv
-    cat ${group}_benchmarks.csv >> benchmark_${LASTCOMMIT}.csv
-done
-
-cat benchmark_${LASTCOMMIT}.csv
-
-echo "===> Install components for benchmark analysis ..."
-
-if python3 -c 'import sys; sys.exit(not (sys.version_info.major > 3 or (sys.version_info.major == 3 and sys.version_info.minor >= 9)))' ; then
-    pip3 install matplotlib==3.7.1 numpy==1.26.4 pandas==1.5.3
-else
-    pip3 install matplotlib==3.7.1 numpy==1.24.2 pandas==1.5.3
-fi
-
-echo "===> Download benchmark history ..."
-
-# Configure Github user
-git config --local user.email "action@github.com"
-git config --local user.name "GitHub Action"
-
-# Checkout to benchmark branch
-git fetch origin
-git checkout -b gh-pages origin/gh-pages
-
-# Bring analysis script
-git checkout ${LASTCOMMIT} -- ${WORKSPACE}/tests/scripts/analyze_benchmarks.py
-
-# Record the benchmark result into benchmarks_history
-BENCHMARK_HISTORY_FILE=${WORKSPACE}/archive/benchmarks/benchmarks_history.csv
-cat ${WORKSPACE}/benchmark_${LASTCOMMIT}.csv >> ${BENCHMARK_HISTORY_FILE}
-
-echo "===> Running benchmark analysis ..."
-
-cd ${WORKSPACE}
-python3 ${BASEDIR}/analyze_benchmarks.py ${BENCHMARK_HISTORY_FILE}
-
-echo "===> Commiting the results ..."
-
-cp ${WORKSPACE}/*.png  ${WORKSPACE}/figures/.
-
-git add ${BENCHMARK_HISTORY_FILE}
-git add ${WORKSPACE}/figures/*.png
-
-git commit -m "updating benchmark data for commit ${LASTCOMMIT}" -a
-
-if [ -z "${GITHUB_WORKSPACE}" ]; then
-    echo "===> Exit"
-else
-    echo "===> Uploading the results"
-    # CI stuck here...
-    #git push origin gh-pages
-    echo "===> Exit"
-fi
diff --git a/tests/tools/CMakeLists.txt b/tests/tools/CMakeLists.txt
index f9a2b9daa..816c7e7a3 100644
--- a/tests/tools/CMakeLists.txt
+++ b/tests/tools/CMakeLists.txt
@@ -4,8 +4,6 @@
 #
 # Mozilla Public License Version 2.0
 
-message(STATUS "Building detray command line tools")
-
 include( CMakeFindDependencyMacro )
 
 find_dependency( Boost COMPONENTS program_options )
diff --git a/tests/tools/src/CMakeLists.txt b/tests/tools/src/CMakeLists.txt
index 25caf4059..2e0027941 100644
--- a/tests/tools/src/CMakeLists.txt
+++ b/tests/tools/src/CMakeLists.txt
@@ -4,7 +4,9 @@
 #
 # Mozilla Public License Version 2.0
 
-add_subdirectory( cpu )
+if( DETRAY_BUILD_HOST )
+    add_subdirectory( cpu )
+endif()
 
 if( DETRAY_BUILD_CUDA )
     add_subdirectory( cuda )
diff --git a/tests/tools/src/cpu/CMakeLists.txt b/tests/tools/src/cpu/CMakeLists.txt
index 665d2946b..2e320c4a7 100644
--- a/tests/tools/src/cpu/CMakeLists.txt
+++ b/tests/tools/src/cpu/CMakeLists.txt
@@ -4,6 +4,8 @@
 #
 #Mozilla Public License Version 2.0
 
+message(STATUS "Building detray host command line tools")
+
 include( CMakeFindDependencyMacro )
 
 find_dependency( Boost COMPONENTS program_options )
diff --git a/tests/tools/src/cuda/CMakeLists.txt b/tests/tools/src/cuda/CMakeLists.txt
index 0dc2e71c1..ec6851c1b 100644
--- a/tests/tools/src/cuda/CMakeLists.txt
+++ b/tests/tools/src/cuda/CMakeLists.txt
@@ -4,6 +4,8 @@
 #
 #Mozilla Public License Version 2.0
 
+message(STATUS "Building detray device command line tools")
+
 # Enable CUDA as a language.
 enable_language(CUDA)
 
diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt
index fa64b9cdf..14f49f891 100644
--- a/tests/unit_tests/CMakeLists.txt
+++ b/tests/unit_tests/CMakeLists.txt
@@ -4,13 +4,15 @@
 #
 # Mozilla Public License Version 2.0
 
-message(STATUS "Building detray host unit tests")
-
 # Set up all of the "host" tests.
-add_subdirectory( covfie )
-add_subdirectory( cpu )
-add_subdirectory( io )
-add_subdirectory( svgtools )
+if( DETRAY_BUILD_HOST )
+   message(STATUS "Building detray host unit tests")
+
+   add_subdirectory( covfie )
+   add_subdirectory( cpu )
+   add_subdirectory( io )
+   add_subdirectory( svgtools )
+endif()
 
 # Set up all of the "device" tests.
 if( DETRAY_BUILD_CUDA OR DETRAY_BUILD_SYCL )
diff --git a/tutorials/src/CMakeLists.txt b/tutorials/src/CMakeLists.txt
index fe13e9359..b0cb158cb 100644
--- a/tutorials/src/CMakeLists.txt
+++ b/tutorials/src/CMakeLists.txt
@@ -1,11 +1,13 @@
 # Detray library, part of the ACTS project (R&D line)
 #
-# (c) 2023 CERN for the benefit of the ACTS project
+# (c) 2023-2024 CERN for the benefit of the ACTS project
 #
 # Mozilla Public License Version 2.0
 
 # Set up all of the "host" tutorials.
-add_subdirectory( cpu )
+if( DETRAY_BUILD_HOST )
+   add_subdirectory( cpu )
+endif()
 
 # Set up all of the "device" tutorials.
 if( DETRAY_BUILD_CUDA OR DETRAY_BUILD_SYCL )
diff --git a/tutorials/src/cpu/CMakeLists.txt b/tutorials/src/cpu/CMakeLists.txt
index 6973bec4e..2eb0acaa3 100644
--- a/tutorials/src/cpu/CMakeLists.txt
+++ b/tutorials/src/cpu/CMakeLists.txt
@@ -4,6 +4,8 @@
 #
 # Mozilla Public License Version 2.0
 
+message(STATUS "Building detray host tutorials")
+
 # Set up the tutorial(s) that are algebra agnostic.
 detray_add_tutorial( define_an_actor
    "propagation/define_an_actor.cpp"
diff --git a/tutorials/src/device/cuda/CMakeLists.txt b/tutorials/src/device/cuda/CMakeLists.txt
index 3a59bf45c..b967333c4 100644
--- a/tutorials/src/device/cuda/CMakeLists.txt
+++ b/tutorials/src/device/cuda/CMakeLists.txt
@@ -4,6 +4,8 @@
 #
 # Mozilla Public License Version 2.0
 
+message(STATUS "Building detray CUDA tutorials")
+
 # C++17 support for CUDA requires CMake 3.18.
 cmake_minimum_required(VERSION 3.18)