From 1c21658fc7ffa1e5770eb8324639faa13b832f10 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 16:59:48 +0100 Subject: [PATCH 001/122] chore: Update .gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c0a676728ea..dba5bbf393b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ genConf x86_64-slc6-gcc48-opt x86_64-slc6-gcc49-opt .* -build* +/build* .asetup.save #json output @@ -30,7 +30,7 @@ build* # dont ignore hidden configs !.clang-format -!.github/** +!.github !.gitignore !.gitlab-ci.yml !.clang-tidy From 224e13ae9a26c5dcdbd5d2d0c1bb5948c42c4a3d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 16:50:32 +0100 Subject: [PATCH 002/122] remove some workflows temporarily --- .github/workflows/analysis.yml | 145 --------------------------------- .github/workflows/checks.yml | 105 ------------------------ .github/workflows/docs.yml | 61 -------------- 3 files changed, 311 deletions(-) delete mode 100644 .github/workflows/analysis.yml delete mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml deleted file mode 100644 index f33bb927f40..00000000000 --- a/.github/workflows/analysis.yml +++ /dev/null @@ -1,145 +0,0 @@ -name: Analysis - -on: - push: - branches: - - main - pull_request: - branches: - - main - - 'release/**' - - 'develop/**' - paths-ignore: - - "docs/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - CTEST_OUTPUT_ON_FAILURE: 1 - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 1.25G - CCACHE_KEY_SUFFIX: r2 - ACTS_LOG_FAILURE_THRESHOLD: WARNING - DEPENDENCY_URL: https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.v5.tar.zst - -# NOTE this only builds core unittests to reduce the output size. if we -# found a way to have Github actions not fail regularly with this job -# all unit tests should be reactivated. -jobs: - build_debug: - runs-on: ubuntu-latest - container: ghcr.io/acts-project/ubuntu2404:63 - steps: - - - uses: actions/checkout@v4 - - - name: Install dependencies - run: CI/dependencies.sh - - - name: Cache build - uses: actions/cache@v4 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - - name: Configure - run: > - ccache -z && - cmake -B build -S . - --preset=github-ci - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_FLAGS="-Werror --coverage -g -gz -g1" - -DPython_EXECUTABLE=$(which python3) - -DACTS_BUILD_ODD=OFF - - name: Build - run: cmake --build build - - name: ccache stats - run: ccache -s - - name: Unit tests - run: ctest --test-dir build -j$(nproc) - - - name: Remove .o files - run: > - du -sh build - && find build -name *.o -delete - && du -sh build - - name: Coverage - run: > - pip3 install gcovr==7.2 - && cd build - && /usr/bin/python3 ../CI/test_coverage.py - - - uses: actions/upload-artifact@v4 - with: - name: coverage-build - path: build - - - name: Save PR number to file - if: github.event_name == 'pull_request' - run: echo ${{ github.event.number }} > PR_NUMBER.txt - - - name: Persist PR number - if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v4 - with: - name: PR_NUMBER - path: PR_NUMBER.txt - - build_performance: - runs-on: ubuntu-latest - container: ghcr.io/acts-project/ubuntu2404:63 - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v4 - - - name: Install dependencies - run: CI/dependencies.sh - - - name: Install dependencies - run: pip3 install git+https://github.com/paulgessinger/cmakeperf.git@2a409b5 - - name: Configure - run: > - cmake -B build -S . - --preset=github-ci - -DCMAKE_BUILD_TYPE=RelWithDebInfo - -DCMAKE_CXX_FLAGS="-Werror" - -DPython_EXECUTABLE=$(which python3) - -DACTS_BUILD_ODD=OFF - - name: Measure - run: cmakeperf collect build/compile_commands.json -o perf.csv - - name: Results - run: cmakeperf print perf.csv - - uses: actions/upload-artifact@v4 - with: - name: cmakeperf - path: perf.csv - - # metric_tracking: - # runs-on: ubuntu-latest - # needs: build_performance - # if: github.ref == 'refs/heads/main' - # steps: - # - uses: actions/checkout@v4 - # - name: Install dependencies - # run: pip3 install git+https://github.com/paulgessinger/headwind.git@eeeaa80 - # - uses: actions/download-artifact@v4 - # with: - # name: cmakeperf - # - name: Run collection - # env: - # SSH_AUTH_SOCK: /tmp/ssh_agent.sock - # run: | - # ssh-agent -a $SSH_AUTH_SOCK > /dev/null - # ssh-add - <<< "${{ secrets.METRIC_DEPLOY_SSH_KEY }}" - # git config --global user.email "action@github.com" - # git config --global user.name "GitHub Action" - # git clone git@github.com:acts-project/metrics.git - # hdw collect CI/headwind.yml --commit $(git log --pretty=format:'%H' -1) - # cd metrics - # git add -A - # git commit -m"update metrics" - # git push diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml deleted file mode 100644 index 59ba76bb00a..00000000000 --- a/.github/workflows/checks.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Checks - -on: - push: - pull_request: - branches: - - main - - 'release/**' - - 'develop/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - env: - PRE_COMMIT_HOME: '/tmp/pre-commit' - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - uses: actions/cache@v4 - with: - path: | - ${{ env.PRE_COMMIT_HOME }} - key: ${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} - - - name: Install pre-commit - run: pip install pre-commit - - - name: Run pre-commit - run: pre-commit run --all-files --show-diff-on-failure - - smearing_config: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Check - run: > - CI/check_smearing_config.py . - - missing_includes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install clang - run: > - sudo apt-get install -y clang libeigen3-dev libboost-dev - - name: Check - run: > - CI/missing_include_check.sh - - fpe_masks: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install dependencies - run: > - pip install -r CI/fpe_masks/requirements.txt - - name: Check - run: > - CI/check_fpe_masks.py --token ${{ secrets.GITHUB_TOKEN }} - - unused_files: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Check - run: > - CI/check_unused_files.py - - codegen: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install dependencies - run: > - pip install -r codegen/requirements.txt - - name: Check - run: > - CI/check_codegen - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: changed - path: changed diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index dfa5b303ee2..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Docs - -on: - push: - pull_request: - branches: - - main - - 'release/**' - - 'develop/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - docs: - runs-on: ubuntu-latest - env: - DOXYGEN_WARN_AS_ERROR: FAIL_ON_WARNINGS - DOXYGEN_VERSION: 1.9.8 - steps: - - uses: actions/checkout@v4 - - - name: Cache doxygen - id: cache-doxygen - uses: actions/cache@v4 - with: - path: /usr/local/bin/doxygen - key: doxygen_${{ env.DOXYGEN_VERSION }} - - - name: Install doxygen - if: steps.cache-doxygen.outputs.cache-hit != 'true' - run: > - curl -SL https://sourceforge.net/projects/doxygen/files/rel-${{ env.DOXYGEN_VERSION }}/doxygen-${{ env.DOXYGEN_VERSION }}.linux.bin.tar.gz/download | tar -xzC . - && mv doxygen-${{ env.DOXYGEN_VERSION }}/bin/doxygen /usr/local/bin/doxygen - - - name: Install dependencies - run: > - pip3 install --upgrade pip - && pip install -r docs/requirements.txt - - - name: Build documentation - run: > - cd docs - && sphinx-build - -b html - -d _build/doctrees/ - -j auto - -W - --keep-going - -t run_doxygen - -t lazy_autodoc - -t white_papers - -b linkcheck - . _build/html/ - - - - uses: actions/upload-artifact@v4 - with: - name: acts-docs - path: docs/_build/html/ From 209298a962ed2cd711212e941cac6b7d160b61b1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 16:59:21 +0100 Subject: [PATCH 003/122] test --- .github/actions/dependencies/action.yml | 33 ++ .github/actions/dependencies/opengl.sh | 33 ++ .github/workflows/builds.yml | 750 ++++++++++++------------ 3 files changed, 442 insertions(+), 374 deletions(-) create mode 100644 .github/actions/dependencies/action.yml create mode 100755 .github/actions/dependencies/opengl.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml new file mode 100644 index 00000000000..44e45828e51 --- /dev/null +++ b/.github/actions/dependencies/action.yml @@ -0,0 +1,33 @@ +name: "Dependencies action" +description: "Reusable action that sets up dependencies for the ACTS CI build" +# inputs: +# example_input: +# description: "An example input" +# required: true +# default: "Hello, World!" +# outputs: +# example_output: +# description: "An example output" +runs: + using: "composite" + steps: + - name: Set up Spack + uses: spack/setup-spack@v2 + + - name: Apply spack patch + shell: bash + working-directory: spack + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + + - name: Locate OpenGL + shell: bash + run: ./opengl.sh + + # - name: Setup Xcode version + # if: startsWith(inputs.os, 'macos') + # uses: maxim-lobanov/setup-xcode@v1 + # with: + # xcode-version: "${{ inputs.xcode }}" diff --git a/.github/actions/dependencies/opengl.sh b/.github/actions/dependencies/opengl.sh new file mode 100755 index 00000000000..d3943c4d114 --- /dev/null +++ b/.github/actions/dependencies/opengl.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -u +set -e + +source detect_os.sh + +packages_file=${GITHUB_WORKSPACE}/spack/etc/spack/packages.yaml + +if [ "$os" == "ubuntu" ]; then + sudo apt-get update + sudo apt-get install -y libgl1-mesa-dev +cat < "$packages_file" +packages: + opengl: + buildable: false + externals: + - prefix: /usr/ + spec: opengl@4.5 +EOF +cat "$packages_file" +elif [ "$os" == "almalinux" ]; then + dnf install -y mesa-libGLU +cat < "$packages_file" +packages: + opengl: + buildable: false + externals: + - prefix: /usr/ + spec: opengl@4.6 +EOF +cat "$packages_file" +fi diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 2d565795d9f..fb70b81dd3e 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -36,377 +36,379 @@ jobs: submodules: true lfs: true - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: CI/dependencies.sh - - - name: Restore ccache - uses: actions/cache/restore@v4 - id: ccache-restore - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - - name: Configure - # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # dd4hep CMake configuration that gets triggered on recent CMake - # versions - # Need to set git user & email for patching to work (GeoModel plugin) - run: > - git config --global user.name 'CI' && - git config --global user.email '<>' && - ccache -z && - cmake -B build -S . - --preset=github-ci - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) - -DACTS_BUILD_PLUGIN_ONNX=ON - - - name: Build - run: cmake --build build - - - name: ccache stats - run: ccache -s - - - name: Save ccache - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ github.workspace }}/ccache - key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - - - name: Unit tests - run: ctest --test-dir build -j$(nproc) - - - name: Integration tests - run: cmake --build build --target integrationtests - - - name: Install - run: cmake --build build --target install - - - name: Package build - run: tar czf build.tar.gz -C build --exclude "*.o" --exclude "bin/ActsUnitTest*" --exclude "bin/ActsIntegrationTest*" . - - - uses: actions/upload-artifact@v4 - with: - name: acts-linux-ubuntu - path: build.tar.gz - - - name: Downstream configure - run: > - cmake -B build-downstream -S Tests/DownstreamProject - -GNinja - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=20 - -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - - - name: Downstream build - run: cmake --build build-downstream - - - name: Downstream run - run: ./build-downstream/bin/ShowActsVersion - - linux_examples_test: - runs-on: ubuntu-latest - container: ghcr.io/acts-project/ubuntu2404:63 - needs: [linux_ubuntu] - env: - ACTS_SEQUENCER_DISABLE_FPEMON: true - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: CI/dependencies.sh - - - uses: actions/download-artifact@v4 - with: - name: acts-linux-ubuntu - - - name: Unpack build - run: mkdir build && tar xf build.tar.gz -C build - - - name: Python level tests - shell: bash - env: - PYTEST_MD_REPORT: true - PYTEST_MD_REPORT_VERBOSE: 0 - PYTEST_MD_REPORT_OUTPUT: pytest.md - run: > - geant4-config --install-datasets - && source build/this_acts_withdeps.sh - && python3 -m pip install -r Examples/Python/tests/requirements.txt - && python3 -m pip install pytest-md-report - && pytest -rFsv -k "not exatrkx" -v - && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY - - linux_physmon: - runs-on: ubuntu-latest - container: ghcr.io/acts-project/ubuntu2404:63 - needs: [linux_ubuntu] - env: - ACTS_SEQUENCER_DISABLE_FPEMON: true - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - run: apt-get update && apt-get install -y time - - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: CI/dependencies.sh - - - uses: actions/download-artifact@v4 - with: - name: acts-linux-ubuntu - - - name: Unpack build - run: mkdir build && tar xf build.tar.gz -C build - - - name: Save PR number - if: ${{ github.event_name == 'pull_request' }} - run: | - mkdir -p physmon - echo ${{ github.event.number }} > physmon/pr_number - echo ${{ github.event.pull_request.head.sha }} > physmon/sha - - - name: Physics performance checks - shell: bash - run: > - echo "::group::Dependencies" - && git config --global safe.directory "$GITHUB_WORKSPACE" - && python3 -m pip install histcmp==0.6.8 matplotlib - && python3 -m pip install -r Examples/Scripts/requirements.txt - && geant4-config --install-datasets - && venv_python=$(which python3) - && echo $venv_python - && source build/this_acts_withdeps.sh - && export PATH=$(dirname $venv_python):$PATH - && echo $PATH - && which python3 - && echo "::endgroup::" - && CI/physmon/phys_perf_mon.sh all physmon - - - name: Post step summary - if: always() - run: cat physmon/summary.md >> $GITHUB_STEP_SUMMARY - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: physmon - path: physmon - - linux_physmon_perf_report: - needs: [linux_physmon] - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - steps: - - name: Install dependencies - run: pip3 install spyral-cli==1.1.2 - - - uses: actions/download-artifact@v4 - with: - name: physmon - path: physmon - - - name: Store metrics - env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - run: | - ssh-agent -a $SSH_AUTH_SOCK > /dev/null - ssh-add - <<< "${{ secrets.RUNTIME_METRIC_DEPLOY_SSH_KEY }}" - git config --global user.email "action@github.com" - git config --global user.name "GitHub Action" - git clone git@github.com:acts-project/runtime_metrics.git - spyral maxima runtime_metrics/metrics.csv physmon/memory/*.csv -e $(date +%Y-%m-%dT%H-%M-%S) -e ${GITHUB_REF_NAME} -e ${GITHUB_REF} -e ${GITHUB_SHA} - cd runtime_metrics - git add -A - git commit -m"update metrics" - git push - - linux_ubuntu_extra: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - image: ubuntu2204 - std: 20 - - image: ubuntu2204_clang - std: 20 - container: ghcr.io/acts-project/${{ matrix.image }}:63 - env: - INSTALL_DIR: ${{ github.workspace }}/install - ACTS_LOG_FAILURE_THRESHOLD: WARNING - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: CI/dependencies.sh - - - name: Restore ccache - uses: actions/cache/restore@v4 - id: ccache-restore - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - - - name: Configure - # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # dd4hep CMake configuration that gets triggered on recent CMake - # versions - run: > - ccache -z && - cmake -B build -S . - --preset=github-ci - -DCMAKE_CXX_STANDARD=${{ matrix.std }} - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) - - - name: Build - run: cmake --build build - - - name: ccache stats - run: ccache -s - - - name: Save ccache - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ github.workspace }}/ccache - key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - - - name: Unit tests - run: cmake --build build --target test - - - name: Integration tests - run: cmake --build build --target integrationtests - - - name: Install - run: cmake --build build --target install - - - name: Downstream configure - run: > - cmake -B build-downstream -S Tests/DownstreamProject - -GNinja - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=${{ matrix.std }} - -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - - - name: Downstream build - run: cmake --build build-downstream - - - name: Downstream run - run: ./build-downstream/bin/ShowActsVersion - - macos: - runs-on: macos-14 - env: - INSTALL_DIR: ${{ github.workspace }}/install_acts - ACTS_LOG_FAILURE_THRESHOLD: WARNING - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - lfs: true - - - name: Print architecture - run: uname -p - - - name: Set dependencies URL - run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/macos-14/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - - - name: Install dependencies - run: > - brew install cmake ninja ccache xerces-c - && CI/dependencies.sh - - - name: Restore ccache - uses: actions/cache/restore@v4 - id: ccache-restore - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - - name: Configure - run: > - ccache -z - && cmake -B build -S . - --preset=github-ci - -DCMAKE_PREFIX_PATH="${{ env.DEPENDENCY_DIR }}" - -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" - -DPython_EXECUTABLE=$(which python3) - - - name: Build - run: cmake --build build - - - name: ccache stats - run: ccache -s - - - name: Save ccache - uses: actions/cache/save@v4 - if: always() - with: - path: ${{ github.workspace }}/ccache - key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - - - name: Unit tests - run: cmake --build build --target test - - - name: Integration tests - run: cmake --build build --target integrationtests - - - name: Install - run: cmake --build build --target install - - - uses: actions/upload-artifact@v4 - with: - name: acts-macos - path: ${{ env.INSTALL_DIR }} - - - name: Downstream configure - run: > - cmake -B build-downstream -S Tests/DownstreamProject - -GNinja - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=20 - -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - - - name: Downstream build - run: cmake --build build-downstream - - - name: Downstream run - run: > - PATH="${{ env.DEPENDENCY_DIR }}/bin:$PATH" - && ./build-downstream/bin/ShowActsVersion + - uses: ./.github/actions/dependencies + + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: CI/dependencies.sh + # + # - name: Restore ccache + # uses: actions/cache/restore@v4 + # id: ccache-restore + # with: + # path: ${{ env.CCACHE_DIR }} + # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + # restore-keys: | + # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + # + # - name: Configure + # # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the + # # dd4hep CMake configuration that gets triggered on recent CMake + # # versions + # # Need to set git user & email for patching to work (GeoModel plugin) + # run: > + # git config --global user.name 'CI' && + # git config --global user.email '<>' && + # ccache -z && + # cmake -B build -S . + # --preset=github-ci + # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + # -DPython_EXECUTABLE=$(which python3) + # -DACTS_BUILD_PLUGIN_ONNX=ON + # + # - name: Build + # run: cmake --build build + # + # - name: ccache stats + # run: ccache -s + # + # - name: Save ccache + # uses: actions/cache/save@v4 + # if: always() + # with: + # path: ${{ github.workspace }}/ccache + # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + # + # - name: Unit tests + # run: ctest --test-dir build -j$(nproc) + # + # - name: Integration tests + # run: cmake --build build --target integrationtests + # + # - name: Install + # run: cmake --build build --target install + # + # - name: Package build + # run: tar czf build.tar.gz -C build --exclude "*.o" --exclude "bin/ActsUnitTest*" --exclude "bin/ActsIntegrationTest*" . + # + # - uses: actions/upload-artifact@v4 + # with: + # name: acts-linux-ubuntu + # path: build.tar.gz + # + # - name: Downstream configure + # run: > + # cmake -B build-downstream -S Tests/DownstreamProject + # -GNinja + # -DCMAKE_BUILD_TYPE=Release + # -DCMAKE_CXX_FLAGS=-Werror + # -DCMAKE_CXX_STANDARD=20 + # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + # + # - name: Downstream build + # run: cmake --build build-downstream + # + # - name: Downstream run + # run: ./build-downstream/bin/ShowActsVersion + + # linux_examples_test: + # runs-on: ubuntu-latest + # container: ghcr.io/acts-project/ubuntu2404:63 + # needs: [linux_ubuntu] + # env: + # ACTS_SEQUENCER_DISABLE_FPEMON: true + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: CI/dependencies.sh + # + # - uses: actions/download-artifact@v4 + # with: + # name: acts-linux-ubuntu + # + # - name: Unpack build + # run: mkdir build && tar xf build.tar.gz -C build + # + # - name: Python level tests + # shell: bash + # env: + # PYTEST_MD_REPORT: true + # PYTEST_MD_REPORT_VERBOSE: 0 + # PYTEST_MD_REPORT_OUTPUT: pytest.md + # run: > + # geant4-config --install-datasets + # && source build/this_acts_withdeps.sh + # && python3 -m pip install -r Examples/Python/tests/requirements.txt + # && python3 -m pip install pytest-md-report + # && pytest -rFsv -k "not exatrkx" -v + # && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY + # + # linux_physmon: + # runs-on: ubuntu-latest + # container: ghcr.io/acts-project/ubuntu2404:63 + # needs: [linux_ubuntu] + # env: + # ACTS_SEQUENCER_DISABLE_FPEMON: true + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - run: apt-get update && apt-get install -y time + # + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: CI/dependencies.sh + # + # - uses: actions/download-artifact@v4 + # with: + # name: acts-linux-ubuntu + # + # - name: Unpack build + # run: mkdir build && tar xf build.tar.gz -C build + # + # - name: Save PR number + # if: ${{ github.event_name == 'pull_request' }} + # run: | + # mkdir -p physmon + # echo ${{ github.event.number }} > physmon/pr_number + # echo ${{ github.event.pull_request.head.sha }} > physmon/sha + # + # - name: Physics performance checks + # shell: bash + # run: > + # echo "::group::Dependencies" + # && git config --global safe.directory "$GITHUB_WORKSPACE" + # && python3 -m pip install histcmp==0.6.8 matplotlib + # && python3 -m pip install -r Examples/Scripts/requirements.txt + # && geant4-config --install-datasets + # && venv_python=$(which python3) + # && echo $venv_python + # && source build/this_acts_withdeps.sh + # && export PATH=$(dirname $venv_python):$PATH + # && echo $PATH + # && which python3 + # && echo "::endgroup::" + # && CI/physmon/phys_perf_mon.sh all physmon + # + # - name: Post step summary + # if: always() + # run: cat physmon/summary.md >> $GITHUB_STEP_SUMMARY + # + # - uses: actions/upload-artifact@v4 + # if: always() + # with: + # name: physmon + # path: physmon + # + # linux_physmon_perf_report: + # needs: [linux_physmon] + # runs-on: ubuntu-latest + # if: github.ref == 'refs/heads/main' + # steps: + # - name: Install dependencies + # run: pip3 install spyral-cli==1.1.2 + # + # - uses: actions/download-artifact@v4 + # with: + # name: physmon + # path: physmon + # + # - name: Store metrics + # env: + # SSH_AUTH_SOCK: /tmp/ssh_agent.sock + # run: | + # ssh-agent -a $SSH_AUTH_SOCK > /dev/null + # ssh-add - <<< "${{ secrets.RUNTIME_METRIC_DEPLOY_SSH_KEY }}" + # git config --global user.email "action@github.com" + # git config --global user.name "GitHub Action" + # git clone git@github.com:acts-project/runtime_metrics.git + # spyral maxima runtime_metrics/metrics.csv physmon/memory/*.csv -e $(date +%Y-%m-%dT%H-%M-%S) -e ${GITHUB_REF_NAME} -e ${GITHUB_REF} -e ${GITHUB_SHA} + # cd runtime_metrics + # git add -A + # git commit -m"update metrics" + # git push + # + # linux_ubuntu_extra: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # include: + # - image: ubuntu2204 + # std: 20 + # - image: ubuntu2204_clang + # std: 20 + # container: ghcr.io/acts-project/${{ matrix.image }}:63 + # env: + # INSTALL_DIR: ${{ github.workspace }}/install + # ACTS_LOG_FAILURE_THRESHOLD: WARNING + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: CI/dependencies.sh + # + # - name: Restore ccache + # uses: actions/cache/restore@v4 + # id: ccache-restore + # with: + # path: ${{ env.CCACHE_DIR }} + # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + # restore-keys: | + # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + # + # + # - name: Configure + # # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the + # # dd4hep CMake configuration that gets triggered on recent CMake + # # versions + # run: > + # ccache -z && + # cmake -B build -S . + # --preset=github-ci + # -DCMAKE_CXX_STANDARD=${{ matrix.std }} + # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + # -DPython_EXECUTABLE=$(which python3) + # + # - name: Build + # run: cmake --build build + # + # - name: ccache stats + # run: ccache -s + # + # - name: Save ccache + # uses: actions/cache/save@v4 + # if: always() + # with: + # path: ${{ github.workspace }}/ccache + # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + # + # - name: Unit tests + # run: cmake --build build --target test + # + # - name: Integration tests + # run: cmake --build build --target integrationtests + # + # - name: Install + # run: cmake --build build --target install + # + # - name: Downstream configure + # run: > + # cmake -B build-downstream -S Tests/DownstreamProject + # -GNinja + # -DCMAKE_BUILD_TYPE=Release + # -DCMAKE_CXX_FLAGS=-Werror + # -DCMAKE_CXX_STANDARD=${{ matrix.std }} + # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + # + # - name: Downstream build + # run: cmake --build build-downstream + # + # - name: Downstream run + # run: ./build-downstream/bin/ShowActsVersion + # + # macos: + # runs-on: macos-14 + # env: + # INSTALL_DIR: ${{ github.workspace }}/install_acts + # ACTS_LOG_FAILURE_THRESHOLD: WARNING + # + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # lfs: true + # + # - name: Print architecture + # run: uname -p + # + # - name: Set dependencies URL + # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/macos-14/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV + # + # - name: Install dependencies + # run: > + # brew install cmake ninja ccache xerces-c + # && CI/dependencies.sh + # + # - name: Restore ccache + # uses: actions/cache/restore@v4 + # id: ccache-restore + # with: + # path: ${{ env.CCACHE_DIR }} + # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + # restore-keys: | + # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + # + # - name: Configure + # run: > + # ccache -z + # && cmake -B build -S . + # --preset=github-ci + # -DCMAKE_PREFIX_PATH="${{ env.DEPENDENCY_DIR }}" + # -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" + # -DPython_EXECUTABLE=$(which python3) + # + # - name: Build + # run: cmake --build build + # + # - name: ccache stats + # run: ccache -s + # + # - name: Save ccache + # uses: actions/cache/save@v4 + # if: always() + # with: + # path: ${{ github.workspace }}/ccache + # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + # + # - name: Unit tests + # run: cmake --build build --target test + # + # - name: Integration tests + # run: cmake --build build --target integrationtests + # + # - name: Install + # run: cmake --build build --target install + # + # - uses: actions/upload-artifact@v4 + # with: + # name: acts-macos + # path: ${{ env.INSTALL_DIR }} + # + # - name: Downstream configure + # run: > + # cmake -B build-downstream -S Tests/DownstreamProject + # -GNinja + # -DCMAKE_BUILD_TYPE=Release + # -DCMAKE_CXX_FLAGS=-Werror + # -DCMAKE_CXX_STANDARD=20 + # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + # + # - name: Downstream build + # run: cmake --build build-downstream + # + # - name: Downstream run + # run: > + # PATH="${{ env.DEPENDENCY_DIR }}/bin:$PATH" + # && ./build-downstream/bin/ShowActsVersion From b1ca7e44c6ac4b4308c850bf07997da3d32367ad Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:05:24 +0100 Subject: [PATCH 004/122] use correct folder --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 44e45828e51..c947ecdef74 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -24,7 +24,7 @@ runs: - name: Locate OpenGL shell: bash - run: ./opengl.sh + run: "${GITHUB_ACTION_PATH}/opengl.sh" # - name: Setup Xcode version # if: startsWith(inputs.os, 'macos') From e35480b0824a71a98768c0355a30d8ca10478fd1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:17:24 +0100 Subject: [PATCH 005/122] prog --- .github/actions/dependencies/action.yml | 22 +++++++++++++++++----- .github/workflows/builds.yml | 3 ++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index c947ecdef74..edf892be396 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -1,10 +1,10 @@ name: "Dependencies action" description: "Reusable action that sets up dependencies for the ACTS CI build" -# inputs: -# example_input: -# description: "An example input" -# required: true -# default: "Hello, World!" +inputs: + xcode-version: + description: "Which Xcode version to use (only for macOS)" + required: true + default: "" # outputs: # example_output: # description: "An example output" @@ -26,6 +26,18 @@ runs: shell: bash run: "${GITHUB_ACTION_PATH}/opengl.sh" + - name: Get spack lock file name + shell: bash + run: | + arch=$(spack arch --family) + if [[ $arch == darwin* ]]; then + lock_file="spack-${arch}-${{ inputs.xcode }}.lock" + else + lock_file="spack-${arch}.lock" + fi + echo "Lock file: ${lock_file}" + echo "SPACK_LOCK_FILE=${lock_file}" >> $GITHUB_ENV + # - name: Setup Xcode version # if: startsWith(inputs.os, 'macos') # uses: maxim-lobanov/setup-xcode@v1 diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index fb70b81dd3e..b90566eda12 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -36,7 +36,8 @@ jobs: submodules: true lfs: true - - uses: ./.github/actions/dependencies + - name: Install dependencies + uses: ./.github/actions/dependencies # - name: Set dependencies URL # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV From f1a81405641afbb9a98d801660f5147c8c95d76b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:18:44 +0100 Subject: [PATCH 006/122] add missing file --- .github/actions/dependencies/detect_os.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/actions/dependencies/detect_os.sh diff --git a/.github/actions/dependencies/detect_os.sh b/.github/actions/dependencies/detect_os.sh new file mode 100644 index 00000000000..b943539cbe1 --- /dev/null +++ b/.github/actions/dependencies/detect_os.sh @@ -0,0 +1,17 @@ +if [ $(uname) == "Linux" ]; then + os_name=$(cat /etc/os-release | grep -e "^PRETTY_NAME=" | sed 's/PRETTY_NAME="\(.*\)"/\1/g') + if [[ $os_name == *"Ubuntu"* ]]; then + os="ubuntu" + elif [[ $os_name == *"AlmaLinux"* ]]; then + os="almalinux" + fi +elif [ $(uname) == "Darwin" ]; then + os_name="$(sw_vers -productName) $(sw_vers -productVersion)" + os="macos" +else + echo "Only Ubuntu, AlmaLinux and macOS are supported. Exiting." + exit 1 +fi + +export os +export os_name From cacea4fa46de38eb8531d9d2c7daf8d53e9ccefb Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:20:10 +0100 Subject: [PATCH 007/122] script_dir --- .github/actions/dependencies/opengl.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/opengl.sh b/.github/actions/dependencies/opengl.sh index d3943c4d114..4c7240a60c5 100755 --- a/.github/actions/dependencies/opengl.sh +++ b/.github/actions/dependencies/opengl.sh @@ -3,7 +3,9 @@ set -u set -e -source detect_os.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source ${SCRIPT_DIR}/detect_os.sh packages_file=${GITHUB_WORKSPACE}/spack/etc/spack/packages.yaml From b3a618dd9740a80abf0d8a5e7a6ad84a24ddb2d1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:22:45 +0100 Subject: [PATCH 008/122] maybe sudo --- .github/actions/dependencies/opengl.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/actions/dependencies/opengl.sh b/.github/actions/dependencies/opengl.sh index 4c7240a60c5..bb6f68dc747 100755 --- a/.github/actions/dependencies/opengl.sh +++ b/.github/actions/dependencies/opengl.sh @@ -9,9 +9,17 @@ source ${SCRIPT_DIR}/detect_os.sh packages_file=${GITHUB_WORKSPACE}/spack/etc/spack/packages.yaml +if ! command -v sudo &> /dev/null +then + SUDO="" +else + SUDO="sudo" +fi + + if [ "$os" == "ubuntu" ]; then - sudo apt-get update - sudo apt-get install -y libgl1-mesa-dev + ${SUDO} apt-get update + ${SUDO} apt-get install -y libgl1-mesa-dev cat < "$packages_file" packages: opengl: @@ -22,7 +30,7 @@ packages: EOF cat "$packages_file" elif [ "$os" == "almalinux" ]; then - dnf install -y mesa-libGLU + ${SUDO} dnf install -y mesa-libGLU cat < "$packages_file" packages: opengl: From 33b98fdc35053febf3cc10aaa6d169c2963c4301 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:27:38 +0100 Subject: [PATCH 009/122] download lock file --- .github/actions/dependencies/action.yml | 26 +++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index edf892be396..61646ed6252 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -5,9 +5,10 @@ inputs: description: "Which Xcode version to use (only for macOS)" required: true default: "" -# outputs: -# example_output: -# description: "An example output" + dependency-tag: + description: "Which dependency tag to use" + required: true + default: "v6-test" runs: using: "composite" steps: @@ -38,8 +39,17 @@ runs: echo "Lock file: ${lock_file}" echo "SPACK_LOCK_FILE=${lock_file}" >> $GITHUB_ENV - # - name: Setup Xcode version - # if: startsWith(inputs.os, 'macos') - # uses: maxim-lobanov/setup-xcode@v1 - # with: - # xcode-version: "${{ inputs.xcode }}" + - name: Setup Xcode version + if: startsWith(runner.os, 'macos') + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: "${{ inputs.xcode-version }}" + + - name: Get spack lock file from tag + shell: bash + run: | + url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE}" + echo "URL: ${url}" + curl -L -o spack.lock $url + + ls -al From 9bf83d591b608e2403c990c0aae2af70608157eb Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:30:30 +0100 Subject: [PATCH 010/122] run the install --- .github/actions/dependencies/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 61646ed6252..9c306eea78a 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -52,4 +52,9 @@ runs: echo "URL: ${url}" curl -L -o spack.lock $url - ls -al + - name: Create spack environment + shell: bash + run: | + spack env create -d spack_env spack.lock + spack env activate -d spack-env + spack install --use-buildcache only From 3b2073b4e9a4361aae4b0c70794b0fb278105d7e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:32:58 +0100 Subject: [PATCH 011/122] don't activate --- .github/actions/dependencies/action.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 9c306eea78a..4374861559a 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -55,6 +55,5 @@ runs: - name: Create spack environment shell: bash run: | - spack env create -d spack_env spack.lock - spack env activate -d spack-env - spack install --use-buildcache only + spack env create -d . spack.lock + spack -e . install --use-buildcache only From 44d4ae61479b7bcaa75e10ff647ecb2b6074d7c8 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:35:23 +0100 Subject: [PATCH 012/122] named --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4374861559a..8ad7eeb7ed0 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -55,5 +55,5 @@ runs: - name: Create spack environment shell: bash run: | - spack env create -d . spack.lock - spack -e . install --use-buildcache only + spack env create ci spack.lock + spack -e ci install --use-buildcache only From af175e751f4141c014916c66e61c3b357382fa08 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:39:41 +0100 Subject: [PATCH 013/122] add the build cache --- .github/actions/dependencies/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 8ad7eeb7ed0..e714a493cfb 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -23,6 +23,12 @@ runs: git config user.email 41898282+github-actions[bot]@users.noreply.github.com curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + + - name: Add spack buildcache mirror + shell: bash + run: | + spack mirror add acts-spack-buildcache oci://ghcr.io/acts-project/spack-buildcache --unsigned + - name: Locate OpenGL shell: bash run: "${GITHUB_ACTION_PATH}/opengl.sh" From 0310289501eb53d0d51da9ae7e4c836a473cf7e4 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:44:35 +0100 Subject: [PATCH 014/122] add some other jobs --- .github/workflows/builds.yml | 94 ++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 53 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index b90566eda12..fd7689f512a 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,12 +39,6 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: CI/dependencies.sh - # # - name: Restore ccache # uses: actions/cache/restore@v4 # id: ccache-restore @@ -244,31 +238,28 @@ jobs: # git commit -m"update metrics" # git push # - # linux_ubuntu_extra: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # include: - # - image: ubuntu2204 - # std: 20 - # - image: ubuntu2204_clang - # std: 20 - # container: ghcr.io/acts-project/${{ matrix.image }}:63 - # env: - # INSTALL_DIR: ${{ github.workspace }}/install - # ACTS_LOG_FAILURE_THRESHOLD: WARNING - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-22.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: CI/dependencies.sh + linux_ubuntu_extra: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - image: ubuntu2204 + std: 20 + - image: ubuntu2204_clang + std: 20 + container: ghcr.io/acts-project/${{ matrix.image }}:63 + env: + INSTALL_DIR: ${{ github.workspace }}/install + ACTS_LOG_FAILURE_THRESHOLD: WARNING + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + - name: Install dependencies + uses: ./.github/actions/dependencies # # - name: Restore ccache # uses: actions/cache/restore@v4 @@ -329,28 +320,25 @@ jobs: # - name: Downstream run # run: ./build-downstream/bin/ShowActsVersion # - # macos: - # runs-on: macos-14 - # env: - # INSTALL_DIR: ${{ github.workspace }}/install_acts - # ACTS_LOG_FAILURE_THRESHOLD: WARNING - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - name: Print architecture - # run: uname -p - # - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/macos-14/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: > - # brew install cmake ninja ccache xerces-c - # && CI/dependencies.sh + macos: + runs-on: macos-14 + env: + INSTALL_DIR: ${{ github.workspace }}/install_acts + ACTS_LOG_FAILURE_THRESHOLD: WARNING + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + # - name: Install dependencies + # run: > + # brew install cmake ninja ccache xerces-c + # && CI/dependencies.sh + + - name: Install dependencies + uses: ./.github/actions/dependencies # # - name: Restore ccache # uses: actions/cache/restore@v4 From 2722f2bfbcf973074fe8fb84117fee4ae14a31f5 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:46:32 +0100 Subject: [PATCH 015/122] correct xcode version --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index e714a493cfb..a4b19ca93b0 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -38,7 +38,7 @@ runs: run: | arch=$(spack arch --family) if [[ $arch == darwin* ]]; then - lock_file="spack-${arch}-${{ inputs.xcode }}.lock" + lock_file="spack-${arch}-${{ inputs.xcode-version }}.lock" else lock_file="spack-${arch}.lock" fi From 701b72eab2a9967fab0e63a41a9e9a9dec73d765 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:48:39 +0100 Subject: [PATCH 016/122] lock to xcode 16.1.0 --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index a4b19ca93b0..4a38b333f97 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -4,7 +4,7 @@ inputs: xcode-version: description: "Which Xcode version to use (only for macOS)" required: true - default: "" + default: "16.0.0" dependency-tag: description: "Which dependency tag to use" required: true From 3976a2d61c199aa9a01788a953519d8f94140776 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 11 Dec 2024 17:50:24 +0100 Subject: [PATCH 017/122] another error --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4a38b333f97..8a852c2433e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -38,7 +38,7 @@ runs: run: | arch=$(spack arch --family) if [[ $arch == darwin* ]]; then - lock_file="spack-${arch}-${{ inputs.xcode-version }}.lock" + lock_file="spack-${arch}-xcode${{ inputs.xcode-version }}.lock" else lock_file="spack-${arch}.lock" fi From d7e703860a9eeaec83dbe9199eb2871b7dc1d816 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 09:46:58 +0100 Subject: [PATCH 018/122] print folder, create env --- .github/actions/dependencies/action.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 8a852c2433e..f7b707326cb 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -62,4 +62,11 @@ runs: shell: bash run: | spack env create ci spack.lock + spack -e ci spec + spack -e ci find spack -e ci install --use-buildcache only + pwd + spack -e view enable dependencies + ls -al + + # - name: Install Geant4 data From 9363828efa119b2e0a7b0fadc6e7ea2721bf15b0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 09:57:03 +0100 Subject: [PATCH 019/122] add view creation --- .github/actions/dependencies/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index f7b707326cb..72b17e4de9e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -61,12 +61,10 @@ runs: - name: Create spack environment shell: bash run: | - spack env create ci spack.lock + spack env create ci spack.lock --with-view $PWD/dependencies spack -e ci spec spack -e ci find spack -e ci install --use-buildcache only - pwd - spack -e view enable dependencies ls -al # - name: Install Geant4 data From 2bebf9af00ce7f293a88253cd9e995136ac86e27 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 11:03:20 +0100 Subject: [PATCH 020/122] get g4 data --- .github/actions/dependencies/action.yml | 7 ++++++- .github/actions/dependencies/with_spack_env.sh | 11 +++++++++++ .github/workflows/builds.yml | 3 +++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100755 .github/actions/dependencies/with_spack_env.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 72b17e4de9e..545565b6c83 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -67,4 +67,9 @@ runs: spack -e ci install --use-buildcache only ls -al - # - name: Install Geant4 data + - name: Install Geant4 data + shell: bash + run: | + spack -e ci install --add geant4-data + "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 | awk '{print "export " $0}' > geant4_data.sh + source geant4_data.sh diff --git a/.github/actions/dependencies/with_spack_env.sh b/.github/actions/dependencies/with_spack_env.sh new file mode 100755 index 00000000000..13cefd50977 --- /dev/null +++ b/.github/actions/dependencies/with_spack_env.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +set -u + +source "${SPACK_ROOT}/share/spack/setup-env.sh" + +env=$1 +shift +spack env activate "$env" + +"$@" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index fd7689f512a..29c42aa3ee7 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,6 +39,9 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies + - run: | + env | grep G4 + # - name: Restore ccache # uses: actions/cache/restore@v4 # id: ccache-restore From ce5c656789da732fcd75ef34a3cf61a871c09765 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 11:15:15 +0100 Subject: [PATCH 021/122] different root detection mechanism --- .github/actions/dependencies/with_spack_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/with_spack_env.sh b/.github/actions/dependencies/with_spack_env.sh index 13cefd50977..ae0de83d4b4 100755 --- a/.github/actions/dependencies/with_spack_env.sh +++ b/.github/actions/dependencies/with_spack_env.sh @@ -2,7 +2,7 @@ set -e set -u -source "${SPACK_ROOT}/share/spack/setup-env.sh" +source "$(spack location -r)/share/spack/setup-env.sh" env=$1 shift From d668d2d2c567deab9ee9190e3f92007e310e1746 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 11:25:50 +0100 Subject: [PATCH 022/122] other env mechanism --- .github/actions/dependencies/action.yml | 5 ++--- .github/workflows/builds.yml | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 545565b6c83..806c5080ad7 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -43,7 +43,7 @@ runs: lock_file="spack-${arch}.lock" fi echo "Lock file: ${lock_file}" - echo "SPACK_LOCK_FILE=${lock_file}" >> $GITHUB_ENV + echo "SPACK_LOCK_FILE=${lock_file}" >> "$GITHUB_ENV" - name: Setup Xcode version if: startsWith(runner.os, 'macos') @@ -71,5 +71,4 @@ runs: shell: bash run: | spack -e ci install --add geant4-data - "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 | awk '{print "export " $0}' > geant4_data.sh - source geant4_data.sh + "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 29c42aa3ee7..81cf0d799c0 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,7 +40,10 @@ jobs: uses: ./.github/actions/dependencies - run: | - env | grep G4 + env + ls -al + source geant4_data.sh + env # - name: Restore ccache # uses: actions/cache/restore@v4 From cf25749bcd493fb10e8d5edd2158b60cf0412cf3 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 11:45:22 +0100 Subject: [PATCH 023/122] get python going --- .github/actions/dependencies/action.yml | 1 + .github/workflows/builds.yml | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 806c5080ad7..7688184d7a4 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -66,6 +66,7 @@ runs: spack -e ci find spack -e ci install --use-buildcache only ls -al + echo "PATH=$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" - name: Install Geant4 data shell: bash diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 81cf0d799c0..892d4ee0c7e 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,11 +39,8 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies - - run: | - env - ls -al - source geant4_data.sh - env + - run: env + - run: python3 -m pip install jinja2 pyyaml # - name: Restore ccache # uses: actions/cache/restore@v4 From 2997210246d12dc097e15148a2dbb30a4420ea79 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 12:45:56 +0100 Subject: [PATCH 024/122] paths --- .github/actions/dependencies/action.yml | 4 +++- .github/workflows/builds.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 7688184d7a4..f02b3a12fc4 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -66,7 +66,9 @@ runs: spack -e ci find spack -e ci install --use-buildcache only ls -al - echo "PATH=$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + venv_dir="$PWD/dependencies/venv" + "$PWD/dependencies/bin/python3" -m venv "$venv_dir" + echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" - name: Install Geant4 data shell: bash diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 892d4ee0c7e..74506668703 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,7 +40,9 @@ jobs: uses: ./.github/actions/dependencies - run: env - - run: python3 -m pip install jinja2 pyyaml + - run: | + which python3 + python3 -m pip install jinja2 pyyaml # - name: Restore ccache # uses: actions/cache/restore@v4 From bed013fce79bb9a3b7259994a0bed725efd6ecad Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 13:02:01 +0100 Subject: [PATCH 025/122] debug --- .github/actions/dependencies/action.yml | 1 + .github/workflows/builds.yml | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index f02b3a12fc4..6acaf0c8985 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -69,6 +69,7 @@ runs: venv_dir="$PWD/dependencies/venv" "$PWD/dependencies/bin/python3" -m venv "$venv_dir" echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + ls -al "${venv_dir}/bin" - name: Install Geant4 data shell: bash diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 74506668703..17b6dbb006b 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,7 +40,11 @@ jobs: uses: ./.github/actions/dependencies - run: env - - run: | + - run: | + ls -al + ls -al dependencies + ls -al dependencies/venv + ls -al dependencies/venv/bin which python3 python3 -m pip install jinja2 pyyaml From 4b6950cd44af1b036904aa65c190b27937923168 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 13:13:57 +0100 Subject: [PATCH 026/122] make python stuff later --- .github/actions/dependencies/action.yml | 14 +++++++++----- .github/workflows/builds.yml | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 6acaf0c8985..0442b97dbf0 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -65,14 +65,18 @@ runs: spack -e ci spec spack -e ci find spack -e ci install --use-buildcache only - ls -al - venv_dir="$PWD/dependencies/venv" - "$PWD/dependencies/bin/python3" -m venv "$venv_dir" - echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" - ls -al "${venv_dir}/bin" - name: Install Geant4 data shell: bash run: | spack -e ci install --add geant4-data "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" + + - name: Prepare python environment + shell: bash + run: | + ls -al + venv_dir="$PWD/dependencies/venv" + "$PWD/dependencies/bin/python3" -m venv "$venv_dir" + echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + ls -al "${venv_dir}/bin" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 17b6dbb006b..96ee9febb6d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -42,7 +42,7 @@ jobs: - run: env - run: | ls -al - ls -al dependencies + ls -al dependencies/ ls -al dependencies/venv ls -al dependencies/venv/bin which python3 From dd74b87793fd9e778dfd17988a2969369f5e5ece Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 13:54:49 +0100 Subject: [PATCH 027/122] more debug --- .github/actions/dependencies/action.yml | 2 +- .github/workflows/builds.yml | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 0442b97dbf0..9161989f456 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -78,5 +78,5 @@ runs: ls -al venv_dir="$PWD/dependencies/venv" "$PWD/dependencies/bin/python3" -m venv "$venv_dir" - echo "PATH=$PWD/dependencies/venv/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + echo "PATH=${venv_dir}/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" ls -al "${venv_dir}/bin" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 96ee9febb6d..4ae0462fe6c 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,7 +40,12 @@ jobs: uses: ./.github/actions/dependencies - run: env + - run: | + pwd + ls -al + ls -al /__w/acts/acts/dependencies/venv/bin/ - run: | + echo $PATH ls -al ls -al dependencies/ ls -al dependencies/venv @@ -49,8 +54,8 @@ jobs: python3 -m pip install jinja2 pyyaml # - name: Restore ccache + # id: ccache-restore # uses: actions/cache/restore@v4 - # id: ccache-restore # with: # path: ${{ env.CCACHE_DIR }} # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} From c00bfd6b9007e7620f2d90da7354518c1dc4b5b2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 15:18:28 +0100 Subject: [PATCH 028/122] different way of setting PATH --- .github/actions/dependencies/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 9161989f456..016292a4842 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -65,6 +65,7 @@ runs: spack -e ci spec spack -e ci find spack -e ci install --use-buildcache only + echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" - name: Install Geant4 data shell: bash @@ -77,6 +78,6 @@ runs: run: | ls -al venv_dir="$PWD/dependencies/venv" - "$PWD/dependencies/bin/python3" -m venv "$venv_dir" - echo "PATH=${venv_dir}/bin/:$PWD/dependencies/bin:$PATH" >> "$GITHUB_ENV" + python3 -m venv "$venv_dir" + echo "${venv_dir}/bin/" >> "$GITHUB_PATH" ls -al "${venv_dir}/bin" From b3906effa7c7e8339430114f807b67548ae7c12b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 15:27:05 +0100 Subject: [PATCH 029/122] enable build --- .github/workflows/builds.yml | 136 +++++++++++++++++------------------ 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 4ae0462fe6c..b2fd3d9d0dd 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -53,74 +53,74 @@ jobs: which python3 python3 -m pip install jinja2 pyyaml - # - name: Restore ccache - # id: ccache-restore - # uses: actions/cache/restore@v4 - # with: - # path: ${{ env.CCACHE_DIR }} - # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - # restore-keys: | - # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - # - name: Configure - # # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # # dd4hep CMake configuration that gets triggered on recent CMake - # # versions - # # Need to set git user & email for patching to work (GeoModel plugin) - # run: > - # git config --global user.name 'CI' && - # git config --global user.email '<>' && - # ccache -z && - # cmake -B build -S . - # --preset=github-ci - # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - # -DPython_EXECUTABLE=$(which python3) - # -DACTS_BUILD_PLUGIN_ONNX=ON - # - # - name: Build - # run: cmake --build build - # - # - name: ccache stats - # run: ccache -s - # - # - name: Save ccache - # uses: actions/cache/save@v4 - # if: always() - # with: - # path: ${{ github.workspace }}/ccache - # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - # - # - name: Unit tests - # run: ctest --test-dir build -j$(nproc) - # - # - name: Integration tests - # run: cmake --build build --target integrationtests - # - # - name: Install - # run: cmake --build build --target install - # - # - name: Package build - # run: tar czf build.tar.gz -C build --exclude "*.o" --exclude "bin/ActsUnitTest*" --exclude "bin/ActsIntegrationTest*" . - # - # - uses: actions/upload-artifact@v4 - # with: - # name: acts-linux-ubuntu - # path: build.tar.gz - # - # - name: Downstream configure - # run: > - # cmake -B build-downstream -S Tests/DownstreamProject - # -GNinja - # -DCMAKE_BUILD_TYPE=Release - # -DCMAKE_CXX_FLAGS=-Werror - # -DCMAKE_CXX_STANDARD=20 - # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - # - # - name: Downstream build - # run: cmake --build build-downstream - # - # - name: Downstream run - # run: ./build-downstream/bin/ShowActsVersion + - name: Restore ccache + id: ccache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + + - name: Configure + # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the + # dd4hep CMake configuration that gets triggered on recent CMake + # versions + # Need to set git user & email for patching to work (GeoModel plugin) + run: > + git config --global user.name 'CI' && + git config --global user.email '<>' && + ccache -z && + cmake -B build -S . + --preset=github-ci + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -DPython_EXECUTABLE=$(which python3) + -DACTS_BUILD_PLUGIN_ONNX=ON + + - name: Build + run: cmake --build build + + - name: ccache stats + run: ccache -s + + - name: Save ccache + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + + - name: Unit tests + run: ctest --test-dir build -j$(nproc) + + - name: Integration tests + run: cmake --build build --target integrationtests + + - name: Install + run: cmake --build build --target install + + - name: Package build + run: tar czf build.tar.gz -C build --exclude "*.o" --exclude "bin/ActsUnitTest*" --exclude "bin/ActsIntegrationTest*" . + + - uses: actions/upload-artifact@v4 + with: + name: acts-linux-ubuntu + path: build.tar.gz + + - name: Downstream configure + run: > + cmake -B build-downstream -S Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_STANDARD=20 + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + - name: Downstream build + run: cmake --build build-downstream + + - name: Downstream run + run: ./build-downstream/bin/ShowActsVersion # linux_examples_test: # runs-on: ubuntu-latest From cfe63f77cd7c88f1c11cef2bd772b4bc334a8bf7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 16:07:08 +0100 Subject: [PATCH 030/122] set CMake prefix path --- .github/actions/dependencies/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 016292a4842..2c096b79be4 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -66,6 +66,7 @@ runs: spack -e ci find spack -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" + echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" - name: Install Geant4 data shell: bash @@ -79,5 +80,5 @@ runs: ls -al venv_dir="$PWD/dependencies/venv" python3 -m venv "$venv_dir" - echo "${venv_dir}/bin/" >> "$GITHUB_PATH" + echo "${venv_dir}/bin" >> "$GITHUB_PATH" ls -al "${venv_dir}/bin" From 801f2bce23a5e87fdc09f78a774464d9b64b2608 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 16:08:09 +0100 Subject: [PATCH 031/122] additional variables --- .github/actions/dependencies/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 2c096b79be4..eb429b1f682 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -67,6 +67,12 @@ runs: spack -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" + echo "ROOT_INCLUDE_PATH=${destination}/include" >> "$GITHUB_ENV" + # # Geant4 puts CLHEP in a subdirectory + # set_env ROOT_INCLUDE_PATH "${destination}/include/Geant4" + # Pythia8 looks for settings in this directory + echo "PYTHIA8DATA=${destination}/share/Pythia8/xmldoc" >> "$GITHUB_ENV" - name: Install Geant4 data shell: bash From 34df3ac7f0c399825f2402788decf21b8783df6c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 18:05:49 +0100 Subject: [PATCH 032/122] use more CPUs to install --- .github/actions/dependencies/action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index eb429b1f682..f60a161b477 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -61,10 +61,16 @@ runs: - name: Create spack environment shell: bash run: | + if [[ $(spack arch --platform) == darwin ]]; then + NCPUS=$(sysctl -n hw.ncpu) + else + NCPUS=$(nproc) + fi + spack env create ci spack.lock --with-view $PWD/dependencies spack -e ci spec spack -e ci find - spack -e ci install --use-buildcache only + spack -n$NCPUS -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" From bc656cc91d3654a2ad85b3e6a86c4ada6e3416da Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 12 Dec 2024 18:11:40 +0100 Subject: [PATCH 033/122] add timing --- .github/actions/dependencies/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index f60a161b477..4e23c16cf41 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -67,10 +67,10 @@ runs: NCPUS=$(nproc) fi - spack env create ci spack.lock --with-view $PWD/dependencies - spack -e ci spec - spack -e ci find - spack -n$NCPUS -e ci install --use-buildcache only + time spack env create ci spack.lock --with-view $PWD/dependencies + time spack -e ci spec + time spack -e ci find + time spack -n$NCPUS -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" @@ -83,7 +83,7 @@ runs: - name: Install Geant4 data shell: bash run: | - spack -e ci install --add geant4-data + time spack -e ci install --add geant4-data "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" - name: Prepare python environment From 23483817ded9d39939e36d03fb8bd72a77bfb4b0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 09:40:13 +0100 Subject: [PATCH 034/122] try better parallel --- .github/actions/dependencies/action.yml | 5 +++-- .github/actions/dependencies/parallel.sh | 28 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 .github/actions/dependencies/parallel.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4e23c16cf41..abfd84fdf4a 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -70,7 +70,7 @@ runs: time spack env create ci spack.lock --with-view $PWD/dependencies time spack -e ci spec time spack -e ci find - time spack -n$NCPUS -e ci install --use-buildcache only + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" @@ -83,7 +83,8 @@ runs: - name: Install Geant4 data shell: bash run: | - time spack -e ci install --add geant4-data + time spack -e ci add geant4-data + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" - name: Prepare python environment diff --git a/.github/actions/dependencies/parallel.sh b/.github/actions/dependencies/parallel.sh new file mode 100644 index 00000000000..4ec4ece0f1f --- /dev/null +++ b/.github/actions/dependencies/parallel.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +N=$1 +shift +declare -a pids=() + +# Start all processes in the background +for i in $(seq 1 "$N"); do + # Replace `sleep 10` with the actual command you want to run. + # For demonstration, we are using a command that sleeps for 10 seconds. + # Make sure it runs in the background with '&'. + "$@" & + pids+=($!) +done + +# Wait for all processes to finish, if any fails, kill them all +for pid in "${pids[@]}"; do + if ! wait "$pid"; then + echo "Process $pid failed. Terminating all remaining processes..." + # Kill all started processes + kill "${pids[@]}" 2>/dev/null || true + exit 1 + fi +done + +echo "All processes completed successfully." +exit 0 From 24a4542ec1cf912bd562cbe4118584372bee83bf Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 09:40:45 +0100 Subject: [PATCH 035/122] number --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index abfd84fdf4a..8dde3859934 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -84,7 +84,7 @@ runs: shell: bash run: | time spack -e ci add geant4-data - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install + time ${GITHUB_ACTION_PATH}/parallel.sh 10 spack -e ci install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" - name: Prepare python environment From 3f8dfa348a4fd2be4b543ede5ee86e2736dfcc31 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 09:55:35 +0100 Subject: [PATCH 036/122] make parallel script executable --- .github/actions/dependencies/parallel.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/actions/dependencies/parallel.sh diff --git a/.github/actions/dependencies/parallel.sh b/.github/actions/dependencies/parallel.sh old mode 100644 new mode 100755 From a8a703d9b4d3544ed385835d3383406b049e1110 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 10:04:39 +0100 Subject: [PATCH 037/122] higher count on install from buildcache --- .github/actions/dependencies/action.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 8dde3859934..73d83d933f8 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -61,11 +61,12 @@ runs: - name: Create spack environment shell: bash run: | - if [[ $(spack arch --platform) == darwin ]]; then - NCPUS=$(sysctl -n hw.ncpu) - else - NCPUS=$(nproc) - fi + # if [[ $(spack arch --platform) == darwin ]]; then + # NCPUS=$(sysctl -n hw.ncpu) + # else + # NCPUS=$(nproc) + # fi + NCPUS=16 # we're only downloading time spack env create ci spack.lock --with-view $PWD/dependencies time spack -e ci spec From 649f487bd8187adeb990c1e363a12c208ebf2de2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 11:17:41 +0100 Subject: [PATCH 038/122] separate concretize --- .github/actions/dependencies/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 73d83d933f8..b30f4da5288 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -85,6 +85,7 @@ runs: shell: bash run: | time spack -e ci add geant4-data + time spack -e ci concretize time ${GITHUB_ACTION_PATH}/parallel.sh 10 spack -e ci install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" From 5324c33f69275b2a0890084e8e48299f4710dd51 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 14:43:14 +0100 Subject: [PATCH 039/122] install combo --- .github/actions/dependencies/action.yml | 38 ++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index b30f4da5288..c1747b40278 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -39,11 +39,16 @@ runs: arch=$(spack arch --family) if [[ $arch == darwin* ]]; then lock_file="spack-${arch}-xcode${{ inputs.xcode-version }}.lock" + lock_file_g4="spack-${arch}-xcode${{ inputs.xcode-version }}-geant4-data.lock" else lock_file="spack-${arch}.lock" + lock_file_g4="spack-${arch}-geant4-data.lock" fi + echo "Lock file: ${lock_file}" + echo "Lock file G4: ${lock_file_g4}" echo "SPACK_LOCK_FILE=${lock_file}" >> "$GITHUB_ENV" + echo "SPACK_LOCK_FILE_G4=${lock_file_g4}" >> "$GITHUB_ENV" - name: Setup Xcode version if: startsWith(runner.os, 'macos') @@ -51,27 +56,36 @@ runs: with: xcode-version: "${{ inputs.xcode-version }}" - - name: Get spack lock file from tag + - name: Get spack lock files from tag shell: bash run: | url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE}" echo "URL: ${url}" curl -L -o spack.lock $url + g4url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE_G4}" + echo "URL: ${g4url}" + curl -L -o spack-g4.lock $g4url - name: Create spack environment shell: bash run: | - # if [[ $(spack arch --platform) == darwin ]]; then - # NCPUS=$(sysctl -n hw.ncpu) - # else - # NCPUS=$(nproc) - # fi - NCPUS=16 # we're only downloading + NCPUS=32 # we're only downloading + + time spack env create ci spack.lock --with-view $PWD/dependencies & + time spack env create ci-g4 spack-g4.lock & + wait - time spack env create ci spack.lock --with-view $PWD/dependencies time spack -e ci spec time spack -e ci find + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install + + "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" + + - name: Configure downstream environment + shell: bash + run: | echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" @@ -81,14 +95,6 @@ runs: # Pythia8 looks for settings in this directory echo "PYTHIA8DATA=${destination}/share/Pythia8/xmldoc" >> "$GITHUB_ENV" - - name: Install Geant4 data - shell: bash - run: | - time spack -e ci add geant4-data - time spack -e ci concretize - time ${GITHUB_ACTION_PATH}/parallel.sh 10 spack -e ci install - "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci env | grep G4 >> "$GITHUB_ENV" - - name: Prepare python environment shell: bash run: | From fb2dbd8081009e922266b075088c945e2d685538 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 14:47:41 +0100 Subject: [PATCH 040/122] correct tag --- .github/actions/dependencies/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index c1747b40278..cac415bd90e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -8,7 +8,7 @@ inputs: dependency-tag: description: "Which dependency tag to use" required: true - default: "v6-test" + default: "v6-test3" runs: using: "composite" steps: @@ -78,8 +78,9 @@ runs: time spack -e ci spec time spack -e ci find - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only & + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install & + wait "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" From 642796f5645ea47608462fa4a24f335bd770282d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 14:56:34 +0100 Subject: [PATCH 041/122] don't overlap the install --- .github/actions/dependencies/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index cac415bd90e..557162049f3 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -72,15 +72,17 @@ runs: NCPUS=32 # we're only downloading time spack env create ci spack.lock --with-view $PWD/dependencies & + pid1=$! time spack env create ci-g4 spack-g4.lock & - wait + pid2=$! + wait $pid1 + wait $pid2 time spack -e ci spec time spack -e ci find - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only & - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install & - wait + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" From e079196b978706cf94ece58d5a41e17d89a13889 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 15:02:43 +0100 Subject: [PATCH 042/122] i don't understand the spec --- .github/actions/dependencies/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 557162049f3..faf6a2bb6e6 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -82,6 +82,10 @@ runs: time spack -e ci find time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only + + time spack -e ci-g4 spec + time spack -e ci-g4 find + time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" From 1573b51c7e04064d45be30ef0328a4e1b3ac6421 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:17:14 +0100 Subject: [PATCH 043/122] move all to a script --- .github/actions/dependencies/action.yml | 112 -------------- .github/actions/dependencies/detect_os.sh | 17 --- .../actions/dependencies/with_spack_env.sh | 11 -- .github/workflows/builds.yml | 6 +- CI/dependencies.sh | 103 ------------- .../actions => CI}/dependencies/opengl.sh | 11 +- .../actions => CI}/dependencies/parallel.sh | 0 CI/dependencies/setup.sh | 137 ++++++++++++++++++ 8 files changed, 144 insertions(+), 253 deletions(-) delete mode 100644 .github/actions/dependencies/action.yml delete mode 100644 .github/actions/dependencies/detect_os.sh delete mode 100755 .github/actions/dependencies/with_spack_env.sh delete mode 100755 CI/dependencies.sh rename {.github/actions => CI}/dependencies/opengl.sh (68%) rename {.github/actions => CI}/dependencies/parallel.sh (100%) create mode 100755 CI/dependencies/setup.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml deleted file mode 100644 index faf6a2bb6e6..00000000000 --- a/.github/actions/dependencies/action.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: "Dependencies action" -description: "Reusable action that sets up dependencies for the ACTS CI build" -inputs: - xcode-version: - description: "Which Xcode version to use (only for macOS)" - required: true - default: "16.0.0" - dependency-tag: - description: "Which dependency tag to use" - required: true - default: "v6-test3" -runs: - using: "composite" - steps: - - name: Set up Spack - uses: spack/setup-spack@v2 - - - name: Apply spack patch - shell: bash - working-directory: spack - run: | - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - - - - name: Add spack buildcache mirror - shell: bash - run: | - spack mirror add acts-spack-buildcache oci://ghcr.io/acts-project/spack-buildcache --unsigned - - - name: Locate OpenGL - shell: bash - run: "${GITHUB_ACTION_PATH}/opengl.sh" - - - name: Get spack lock file name - shell: bash - run: | - arch=$(spack arch --family) - if [[ $arch == darwin* ]]; then - lock_file="spack-${arch}-xcode${{ inputs.xcode-version }}.lock" - lock_file_g4="spack-${arch}-xcode${{ inputs.xcode-version }}-geant4-data.lock" - else - lock_file="spack-${arch}.lock" - lock_file_g4="spack-${arch}-geant4-data.lock" - fi - - echo "Lock file: ${lock_file}" - echo "Lock file G4: ${lock_file_g4}" - echo "SPACK_LOCK_FILE=${lock_file}" >> "$GITHUB_ENV" - echo "SPACK_LOCK_FILE_G4=${lock_file_g4}" >> "$GITHUB_ENV" - - - name: Setup Xcode version - if: startsWith(runner.os, 'macos') - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: "${{ inputs.xcode-version }}" - - - name: Get spack lock files from tag - shell: bash - run: | - url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE}" - echo "URL: ${url}" - curl -L -o spack.lock $url - g4url="https://github.com/acts-project/ci-dependencies/releases/download/${{ inputs.dependency-tag }}/${SPACK_LOCK_FILE_G4}" - echo "URL: ${g4url}" - curl -L -o spack-g4.lock $g4url - - - name: Create spack environment - shell: bash - run: | - NCPUS=32 # we're only downloading - - time spack env create ci spack.lock --with-view $PWD/dependencies & - pid1=$! - time spack env create ci-g4 spack-g4.lock & - pid2=$! - wait $pid1 - wait $pid2 - - time spack -e ci spec - time spack -e ci find - - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci install --use-buildcache only - - time spack -e ci-g4 spec - time spack -e ci-g4 find - - time ${GITHUB_ACTION_PATH}/parallel.sh $NCPUS spack -e ci-g4 install - - "${GITHUB_ACTION_PATH}/with_spack_env.sh" ci-g4 env | grep G4 >> "$GITHUB_ENV" - - - name: Configure downstream environment - shell: bash - run: | - echo "$PWD/dependencies/bin" >> "$GITHUB_PATH" - echo "CMAKE_PREFIX_PATH=$PWD/dependencies" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${destination}/lib" >> "$GITHUB_ENV" - echo "ROOT_INCLUDE_PATH=${destination}/include" >> "$GITHUB_ENV" - # # Geant4 puts CLHEP in a subdirectory - # set_env ROOT_INCLUDE_PATH "${destination}/include/Geant4" - # Pythia8 looks for settings in this directory - echo "PYTHIA8DATA=${destination}/share/Pythia8/xmldoc" >> "$GITHUB_ENV" - - - name: Prepare python environment - shell: bash - run: | - ls -al - venv_dir="$PWD/dependencies/venv" - python3 -m venv "$venv_dir" - echo "${venv_dir}/bin" >> "$GITHUB_PATH" - ls -al "${venv_dir}/bin" diff --git a/.github/actions/dependencies/detect_os.sh b/.github/actions/dependencies/detect_os.sh deleted file mode 100644 index b943539cbe1..00000000000 --- a/.github/actions/dependencies/detect_os.sh +++ /dev/null @@ -1,17 +0,0 @@ -if [ $(uname) == "Linux" ]; then - os_name=$(cat /etc/os-release | grep -e "^PRETTY_NAME=" | sed 's/PRETTY_NAME="\(.*\)"/\1/g') - if [[ $os_name == *"Ubuntu"* ]]; then - os="ubuntu" - elif [[ $os_name == *"AlmaLinux"* ]]; then - os="almalinux" - fi -elif [ $(uname) == "Darwin" ]; then - os_name="$(sw_vers -productName) $(sw_vers -productVersion)" - os="macos" -else - echo "Only Ubuntu, AlmaLinux and macOS are supported. Exiting." - exit 1 -fi - -export os -export os_name diff --git a/.github/actions/dependencies/with_spack_env.sh b/.github/actions/dependencies/with_spack_env.sh deleted file mode 100755 index ae0de83d4b4..00000000000 --- a/.github/actions/dependencies/with_spack_env.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e -set -u - -source "$(spack location -r)/share/spack/setup-env.sh" - -env=$1 -shift -spack env activate "$env" - -"$@" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index b2fd3d9d0dd..06fd96ba9ad 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -37,7 +37,7 @@ jobs: lfs: true - name: Install dependencies - uses: ./.github/actions/dependencies + uses: CI/dependencies/setup.sh - run: env - run: | @@ -273,7 +273,7 @@ jobs: lfs: true - name: Install dependencies - uses: ./.github/actions/dependencies + uses: CI/dependencies/setup.sh # # - name: Restore ccache # uses: actions/cache/restore@v4 @@ -352,7 +352,7 @@ jobs: # && CI/dependencies.sh - name: Install dependencies - uses: ./.github/actions/dependencies + uses: CI/dependencies/setup.sh # # - name: Restore ccache # uses: actions/cache/restore@v4 diff --git a/CI/dependencies.sh b/CI/dependencies.sh deleted file mode 100755 index 7c8879b499e..00000000000 --- a/CI/dependencies.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -function run() { - set -x - "$@" - { set +x; } 2> /dev/null -} - -function set_env { - key="$1" - value="$2" - - echo "=> ${key}=${value}" - - if [ -n "${GITHUB_ACTIONS:-}" ]; then - echo "${key}=${value}" >> $GITHUB_ENV - else - export ${key}=${value} - fi -} - -url=${1:-${DEPENDENCY_URL:-}} - -if [ -n "${GITHUB_ACTIONS:-}" ]; then - destination="${GITHUB_WORKSPACE}/dependencies" -elif [ -n "${GITLAB_CI:-}" ];then - destination="${CI_PROJECT_DIR}/dependencies" -else - destination=${2} -fi - -set_env DEPENDENCY_DIR "${destination}" - -if [ -z "${url}" ]; then - echo "url is not set" - exit 1 -fi - -echo "URL: $url" -echo "DESTINATION: $destination" - -# check curl location -CURL=$(command -v curl) -if [ -z "$CURL" ]; then - echo "curl is not available" - exit 1 -fi - -UNZSTD=$(command -v unzstd) -if [ -z "$UNZSTD" ]; then - echo "unzstd is not available" - exit 1 -fi - -TAR=$(command -v tar) -if [ -z "$TAR" ]; then - echo "tar is not available" - exit 1 -fi - -run mkdir -p "${destination}" - -run $CURL \ - --retry 5 \ - --connect-timeout 2 \ - --location $url \ - | unzstd \ - | tar \ - -x \ - --strip-components=1 \ - --directory "${destination}" - -# Patch up geant4-config data install script -out=$(${destination}/bin/geant4-config --datasets) -line=$(echo "$out" | head -n1) -orig_share=$(echo "$line" | perl -pe 's|.*?(\/.*)\/share.*|\1|') -orig_share_escaped=$(echo $orig_share|perl -pe 's|/|\\/|g') -destination_escaped=$(echo "$destination"|perl -pe 's|/|\\/|g') -perl -pi.bak -e "s/$orig_share_escaped/$destination_escaped/g" ${destination}/bin/geant4-config - -if [ -n "${GITHUB_ACTIONS:-}" ]; then - echo "Running in GitHub Actions" - venv="${GITHUB_WORKSPACE}/venv" -fi - -if [ -n "${GITLAB_CI:-}" ];then - echo "Running in GitLab CI" - venv="${CI_PROJECT_DIR}/venv" -fi - -if [ -n "${CI:-}" ];then - run "${destination}/bin/python3" -m venv "${venv}" - run "${venv}/bin/python3" -m pip install pyyaml jinja2 - set_env PATH "${venv}/bin:${destination}/bin/:${PATH}" -fi - -set_env CMAKE_PREFIX_PATH "${destination}" -set_env LD_LIBRARY_PATH "${destination}/lib" -set_env ROOT_INCLUDE_PATH "${destination}/include" -# Geant4 puts CLHEP in a subdirectory -set_env ROOT_INCLUDE_PATH "${destination}/include/Geant4" -# Pythia8 looks for settings in this directory -set_env PYTHIA8DATA "${destination}/share/Pythia8/xmldoc" diff --git a/.github/actions/dependencies/opengl.sh b/CI/dependencies/opengl.sh similarity index 68% rename from .github/actions/dependencies/opengl.sh rename to CI/dependencies/opengl.sh index bb6f68dc747..10e3eda6544 100755 --- a/.github/actions/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -3,11 +3,7 @@ set -u set -e -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -source ${SCRIPT_DIR}/detect_os.sh - -packages_file=${GITHUB_WORKSPACE}/spack/etc/spack/packages.yaml +packages_file=$(spack location -r)/spack/etc/spack/packages.yaml if ! command -v sudo &> /dev/null then @@ -16,8 +12,9 @@ else SUDO="sudo" fi +os=$(spack arch --platform) -if [ "$os" == "ubuntu" ]; then +if [ "$os" == ubuntu* ]; then ${SUDO} apt-get update ${SUDO} apt-get install -y libgl1-mesa-dev cat < "$packages_file" @@ -29,7 +26,7 @@ packages: spec: opengl@4.5 EOF cat "$packages_file" -elif [ "$os" == "almalinux" ]; then +elif [ "$os" == almalinux* ]; then ${SUDO} dnf install -y mesa-libGLU cat < "$packages_file" packages: diff --git a/.github/actions/dependencies/parallel.sh b/CI/dependencies/parallel.sh similarity index 100% rename from .github/actions/dependencies/parallel.sh rename to CI/dependencies/parallel.sh diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh new file mode 100755 index 00000000000..f0f0a14b5f6 --- /dev/null +++ b/CI/dependencies/setup.sh @@ -0,0 +1,137 @@ +#!/bin/bash +set -e +set -u + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +export SPACK_COLOR=always + +function set_env { + key="$1" + value="$2" + + echo "=> ${key}=${value}" + + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "${key}=${value}" >> "$GITHUB_ENV" + else + export "${key}"="${value}" + fi +} + +function start_section() { + local section_name="$1" + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::group::${section_name}" + else + echo "${section_name}" + fi +} + +function end_section() { + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::endgroup::" + fi +} + +tag=${1:-${DEPENDENCY_TAG}} +if [ -n "${GITHUB_ACTIONS:-}" ]; then + destination="${GITHUB_WORKSPACE}/dependencies" +elif [ -n "${GITLAB_CI:-}" ];then + destination="${CI_PROJECT_DIR}/dependencies" +else + destination=${2} +fi + +echo "Install tag: $tag" +echo "Install destination: $destination" + +mkdir -p ${destination} + + +start_section "Install spack if not already installed" +if ! command -v spack &> /dev/null; then + git clone -c feature.manyFiles=true https://github.com/spack/spack.git + pushd spack > /dev/null + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # Apply patch for spack improvements + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + popd > /dev/null +fi +end_section + +if [ -n "${CI:-}" ]; then +start_section "Add buildcache mirror" +mirror_name="acts-spack-buildcache" +mirror_url="oci://ghcr.io/acts-project/spack-buildcache" +spack mirror add ${mirror_name} ${mirror_url} --unsigned +end_section + +start_section "Locate OpenGL" +"${SCRIPT_DIR}/opengl.sh" +end_section +fi + +start_section "Get spack lock file name" +arch=$(spack arch --family) +lock_file="spack-${arch}.lock" +end_section + + +url="https://github.com/acts-project/ci-dependencies/releases/download/${tag}/${lock_file}" +echo "URL: $url" + +env_dir="${destination}/env" +view_dir="${destination}/view" +mkdir -p ${env_dir} +lock_file_path="${destination}/spack.lock" +start_section "Get spack lock files from tag" +curl -fL -o $lock_file_path $url +end_section + + + +start_section "Create spack environment" +time spack env create -d ${env_dir} ${lock_file_path} --with-view $view_dir +time spack -e ${env_dir} spec +time spack -e ${env_dir} find +end_section + +start_section "Install spack packages" +NCPUS=32 +time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only | tee install.log +end_section + +start_section "Patch up Geant4 data directory" +# ${SCRIPT_DIR}/with_spack_env.sh ${env_dir} geant4-config --install-datasets +geant4_dir=$(spack -e ${env_dir} location -i geant4) +# Prepare the folder for G4 data, and symlink it to where G4 will look for it +mkdir -p ${geant4_dir}/share/Geant4 +ln -s ${geant4_dir}/share/Geant4/data ${view_dir}/share/Geant4/data +end_section + +start_section "Set environment variables" +set_env CMAKE_PREFIX_PATH "${view_dir}" +set_env LD_LIBRARY_PATH "${view_dir}/lib" +set_env ROOT_INCLUDE_PATH "${view_dir}/include" +# Geant4 puts CLHEP in a subdirectory +set_env ROOT_INCLUDE_PATH "${view_dir}/include/Geant4" +end_section + +start_section "Prepare python environment" +ls -al +venv_dir="${view_dir}/venv" +python3 -m venv "$venv_dir" + +if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "${venv_dir}/bin" >> "$GITHUB_PATH" + echo "${view_dir}/bin" >> "$GITHUB_PATH" +fi +set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}" +end_section + + + +# Pythia8 looks for settings in this directory +# set_env PYTHIA8DATA "${destination}/share/Pythia8/xmldoc" From 3b66c790196cf61891b78e4acc422dc184b8d7b6 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:18:44 +0100 Subject: [PATCH 044/122] run --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 06fd96ba9ad..debe1878670 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -37,7 +37,7 @@ jobs: lfs: true - name: Install dependencies - uses: CI/dependencies/setup.sh + run: CI/dependencies/setup.sh - run: env - run: | From 727793c89a8173a50c721008da0ba451488b9d80 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:19:26 +0100 Subject: [PATCH 045/122] all --- .github/workflows/builds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index debe1878670..acabc98dacf 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -273,7 +273,7 @@ jobs: lfs: true - name: Install dependencies - uses: CI/dependencies/setup.sh + run: CI/dependencies/setup.sh # # - name: Restore ccache # uses: actions/cache/restore@v4 @@ -352,7 +352,7 @@ jobs: # && CI/dependencies.sh - name: Install dependencies - uses: CI/dependencies/setup.sh + run: CI/dependencies/setup.sh # # - name: Restore ccache # uses: actions/cache/restore@v4 From 40bd37844beb7d8b5ea406c09f31d069b43cf3e3 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:21:52 +0100 Subject: [PATCH 046/122] set it up if we just installed it --- CI/dependencies/setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index f0f0a14b5f6..65e825a3ddf 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -57,6 +57,7 @@ if ! command -v spack &> /dev/null; then git config user.email 41898282+github-actions[bot]@users.noreply.github.com # Apply patch for spack improvements curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + source "$(pwd)/share/spack/setup-env.sh" popd > /dev/null fi end_section From 1a1e6865eeece363efb8b74285c26f9a9d8efb7e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:23:48 +0100 Subject: [PATCH 047/122] correct tag --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index acabc98dacf..42104415706 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v5 + DEPENDENCY_TAG: v6-test3 jobs: linux_ubuntu: From ca296d0d650d695f965183d7fc542a1d1fc7a435 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:28:02 +0100 Subject: [PATCH 048/122] diagnostics --- .github/workflows/builds.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 42104415706..630f0004b57 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -48,8 +48,9 @@ jobs: echo $PATH ls -al ls -al dependencies/ - ls -al dependencies/venv - ls -al dependencies/venv/bin + ls -al dependencies/view + ls -al dependencies/view/venv + ls -al dependencies/view/venv/bin which python3 python3 -m pip install jinja2 pyyaml From d4a60247d5db6a90cdfb2193e97b7b1967bb05dd Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:28:42 +0100 Subject: [PATCH 049/122] use the right python --- CI/dependencies/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 65e825a3ddf..1bc4016d671 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -123,7 +123,7 @@ end_section start_section "Prepare python environment" ls -al venv_dir="${view_dir}/venv" -python3 -m venv "$venv_dir" +${view_dir}/bin/python3 -m venv "$venv_dir" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${venv_dir}/bin" >> "$GITHUB_PATH" From 9bea0f091bef2724379620f490ca7d3602ef6b89 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:35:48 +0100 Subject: [PATCH 050/122] diag --- .github/workflows/builds.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 630f0004b57..766cdf71502 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -40,10 +40,6 @@ jobs: run: CI/dependencies/setup.sh - run: env - - run: | - pwd - ls -al - ls -al /__w/acts/acts/dependencies/venv/bin/ - run: | echo $PATH ls -al From d7b977e6fbcddac0404e34b6792aa20c8c543836 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:40:54 +0100 Subject: [PATCH 051/122] output filter --- CI/dependencies/setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 1bc4016d671..fd106289b7e 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -101,7 +101,9 @@ end_section start_section "Install spack packages" NCPUS=32 -time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only | tee install.log +time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only \ + | tee install.log \ + | grep -v "^Waiting\|^\[+\]" end_section start_section "Patch up Geant4 data directory" From 040bd656e730a71f645cbff45b6228e2d4e3289b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 16:55:23 +0100 Subject: [PATCH 052/122] limit parallel --- CI/dependencies/setup.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index fd106289b7e..787ef237018 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -100,7 +100,11 @@ time spack -e ${env_dir} find end_section start_section "Install spack packages" -NCPUS=32 +if [ "$(uname)" = "Darwin" ]; then + NCPUS=$(sysctl -n hw.ncpu) +else + NCPUS=$(nproc) +fi time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only \ | tee install.log \ | grep -v "^Waiting\|^\[+\]" From f49d92772ab9844059b1d21f5adb9aa29cf288d7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 17:10:37 +0100 Subject: [PATCH 053/122] update tag --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 766cdf71502..38b990013f2 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v6-test3 + DEPENDENCY_TAG: v7_b1 jobs: linux_ubuntu: From aa7a3052379485dcf984bca689d7e04555b17892 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 18:02:34 +0100 Subject: [PATCH 054/122] quotes --- CI/dependencies/setup.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 787ef237018..c389ce5f8c2 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -94,9 +94,9 @@ end_section start_section "Create spack environment" -time spack env create -d ${env_dir} ${lock_file_path} --with-view $view_dir -time spack -e ${env_dir} spec -time spack -e ${env_dir} find +time spack env create -d "${env_dir}" "${lock_file_path}" --with-view "$view_dir" +time spack -e "${env_dir}" spec +time spack -e "${env_dir}" find end_section start_section "Install spack packages" @@ -105,17 +105,17 @@ if [ "$(uname)" = "Darwin" ]; then else NCPUS=$(nproc) fi -time ${SCRIPT_DIR}/parallel.sh $NCPUS spack -e ${env_dir} install --use-buildcache only \ +time "${SCRIPT_DIR}"/parallel.sh "$NCPUS" spack -e "${env_dir}" install --use-buildcache only \ | tee install.log \ | grep -v "^Waiting\|^\[+\]" end_section start_section "Patch up Geant4 data directory" # ${SCRIPT_DIR}/with_spack_env.sh ${env_dir} geant4-config --install-datasets -geant4_dir=$(spack -e ${env_dir} location -i geant4) +geant4_dir=$(spack -e "${env_dir}" location -i geant4) # Prepare the folder for G4 data, and symlink it to where G4 will look for it -mkdir -p ${geant4_dir}/share/Geant4 -ln -s ${geant4_dir}/share/Geant4/data ${view_dir}/share/Geant4/data +mkdir -p "${geant4_dir}"/share/Geant4 +ln -s "${geant4_dir}"/share/Geant4/data ${view_dir}/share/Geant4/data end_section start_section "Set environment variables" @@ -129,7 +129,7 @@ end_section start_section "Prepare python environment" ls -al venv_dir="${view_dir}/venv" -${view_dir}/bin/python3 -m venv "$venv_dir" +"${view_dir}"/bin/python3 -m venv "$venv_dir" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${venv_dir}/bin" >> "$GITHUB_PATH" From f2f0363ecdf9b2920ac1061c310c706103d4fa3d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 18:02:41 +0100 Subject: [PATCH 055/122] long spec output --- CI/dependencies/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index c389ce5f8c2..372d1e687dd 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -95,7 +95,7 @@ end_section start_section "Create spack environment" time spack env create -d "${env_dir}" "${lock_file_path}" --with-view "$view_dir" -time spack -e "${env_dir}" spec +time spack -e "${env_dir}" spec -l time spack -e "${env_dir}" find end_section From 73bbd8c6bd9c442a2778e39267669f435e9968b3 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 18:04:45 +0100 Subject: [PATCH 056/122] see if the other builds work --- .github/workflows/builds.yml | 263 +++++++++++++++++------------------ 1 file changed, 124 insertions(+), 139 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 38b990013f2..98365cbeedb 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -39,17 +39,6 @@ jobs: - name: Install dependencies run: CI/dependencies/setup.sh - - run: env - - run: | - echo $PATH - ls -al - ls -al dependencies/ - ls -al dependencies/view - ls -al dependencies/view/venv - ls -al dependencies/view/venv/bin - which python3 - python3 -m pip install jinja2 pyyaml - - name: Restore ccache id: ccache-restore uses: actions/cache/restore@v4 @@ -271,66 +260,66 @@ jobs: - name: Install dependencies run: CI/dependencies/setup.sh - # - # - name: Restore ccache - # uses: actions/cache/restore@v4 - # id: ccache-restore - # with: - # path: ${{ env.CCACHE_DIR }} - # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - # restore-keys: | - # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - # - # - name: Configure - # # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # # dd4hep CMake configuration that gets triggered on recent CMake - # # versions - # run: > - # ccache -z && - # cmake -B build -S . - # --preset=github-ci - # -DCMAKE_CXX_STANDARD=${{ matrix.std }} - # -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - # -DPython_EXECUTABLE=$(which python3) - # - # - name: Build - # run: cmake --build build - # - # - name: ccache stats - # run: ccache -s - # - # - name: Save ccache - # uses: actions/cache/save@v4 - # if: always() - # with: - # path: ${{ github.workspace }}/ccache - # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - # - # - name: Unit tests - # run: cmake --build build --target test - # - # - name: Integration tests - # run: cmake --build build --target integrationtests - # - # - name: Install - # run: cmake --build build --target install - # - # - name: Downstream configure - # run: > - # cmake -B build-downstream -S Tests/DownstreamProject - # -GNinja - # -DCMAKE_BUILD_TYPE=Release - # -DCMAKE_CXX_FLAGS=-Werror - # -DCMAKE_CXX_STANDARD=${{ matrix.std }} - # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - # - # - name: Downstream build - # run: cmake --build build-downstream - # - # - name: Downstream run - # run: ./build-downstream/bin/ShowActsVersion - # + + - name: Restore ccache + uses: actions/cache/restore@v4 + id: ccache-restore + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + + + - name: Configure + # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the + # dd4hep CMake configuration that gets triggered on recent CMake + # versions + run: > + ccache -z && + cmake -B build -S . + --preset=github-ci + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -DPython_EXECUTABLE=$(which python3) + + - name: Build + run: cmake --build build + + - name: ccache stats + run: ccache -s + + - name: Save ccache + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + + - name: Unit tests + run: cmake --build build --target test + + - name: Integration tests + run: cmake --build build --target integrationtests + + - name: Install + run: cmake --build build --target install + + - name: Downstream configure + run: > + cmake -B build-downstream -S Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + - name: Downstream build + run: cmake --build build-downstream + + - name: Downstream run + run: ./build-downstream/bin/ShowActsVersion + macos: runs-on: macos-14 env: @@ -343,72 +332,68 @@ jobs: submodules: true lfs: true - # - name: Install dependencies - # run: > - # brew install cmake ninja ccache xerces-c - # && CI/dependencies.sh - - name: Install dependencies - run: CI/dependencies/setup.sh - # - # - name: Restore ccache - # uses: actions/cache/restore@v4 - # id: ccache-restore - # with: - # path: ${{ env.CCACHE_DIR }} - # key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} - # restore-keys: | - # ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - # - name: Configure - # run: > - # ccache -z - # && cmake -B build -S . - # --preset=github-ci - # -DCMAKE_PREFIX_PATH="${{ env.DEPENDENCY_DIR }}" - # -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" - # -DPython_EXECUTABLE=$(which python3) - # - # - name: Build - # run: cmake --build build - # - # - name: ccache stats - # run: ccache -s - # - # - name: Save ccache - # uses: actions/cache/save@v4 - # if: always() - # with: - # path: ${{ github.workspace }}/ccache - # key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - # - # - name: Unit tests - # run: cmake --build build --target test - # - # - name: Integration tests - # run: cmake --build build --target integrationtests - # - # - name: Install - # run: cmake --build build --target install - # - # - uses: actions/upload-artifact@v4 - # with: - # name: acts-macos - # path: ${{ env.INSTALL_DIR }} - # - # - name: Downstream configure - # run: > - # cmake -B build-downstream -S Tests/DownstreamProject - # -GNinja - # -DCMAKE_BUILD_TYPE=Release - # -DCMAKE_CXX_FLAGS=-Werror - # -DCMAKE_CXX_STANDARD=20 - # -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - # - # - name: Downstream build - # run: cmake --build build-downstream - # - # - name: Downstream run - # run: > - # PATH="${{ env.DEPENDENCY_DIR }}/bin:$PATH" - # && ./build-downstream/bin/ShowActsVersion + run: > + brew install ninja ccache + && CI/dependencies/setup.sh + + - name: Restore ccache + uses: actions/cache/restore@v4 + id: ccache-restore + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + + - name: Configure + run: > + ccache -z + && cmake -B build -S . + --preset=github-ci + -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" + -DPython_EXECUTABLE=$(which python3) + + - name: Build + run: cmake --build build + + - name: ccache stats + run: ccache -s + + - name: Save ccache + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ github.workspace }}/ccache + key: ${{ steps.ccache-restore.outputs.cache-primary-key }} + + - name: Unit tests + run: cmake --build build --target test + + - name: Integration tests + run: cmake --build build --target integrationtests + + - name: Install + run: cmake --build build --target install + + - uses: actions/upload-artifact@v4 + with: + name: acts-macos + path: ${{ env.INSTALL_DIR }} + + - name: Downstream configure + run: > + cmake -B build-downstream -S Tests/DownstreamProject + -GNinja + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_FLAGS=-Werror + -DCMAKE_CXX_STANDARD=20 + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" + + - name: Downstream build + run: cmake --build build-downstream + + - name: Downstream run + run: > + PATH="${{ env.DEPENDENCY_DIR }}/bin:$PATH" + && ./build-downstream/bin/ShowActsVersion From 197c57de389c623aba42c9f4960500b332904825 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 18:49:12 +0100 Subject: [PATCH 057/122] fix: Use toUnderlying to avoid cross-enum comparison --- Core/include/Acts/EventData/TrackStateProxy.hpp | 3 ++- Core/include/Acts/EventData/VectorMultiTrajectory.hpp | 3 ++- .../include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Core/include/Acts/EventData/TrackStateProxy.hpp b/Core/include/Acts/EventData/TrackStateProxy.hpp index 4338c434f77..9e7f8f57392 100644 --- a/Core/include/Acts/EventData/TrackStateProxy.hpp +++ b/Core/include/Acts/EventData/TrackStateProxy.hpp @@ -783,7 +783,8 @@ class TrackStateProxy { void allocateCalibrated(const Eigen::DenseBase& val, const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && + Eigen::PlainObjectBase::RowsAtCompileTime <= + toUnderlying(eBoundSize) && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == diff --git a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp index e13498cea2c..e7d1e77a711 100644 --- a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp +++ b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp @@ -497,7 +497,8 @@ class VectorMultiTrajectory final const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && + Eigen::PlainObjectBase::RowsAtCompileTime <= + toUnderlying(eBoundSize) && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == diff --git a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp index 53eebf0e9b4..0f66b9c1c4b 100644 --- a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp +++ b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp @@ -621,7 +621,8 @@ class MutablePodioTrackStateContainer final const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && + Eigen::PlainObjectBase::RowsAtCompileTime <= + toUnderlying(eBoundSize) && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == From 926e53d46c8fac9ca519cfefff88f871ece76da6 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 20:01:20 +0100 Subject: [PATCH 058/122] Revert "fix: Use toUnderlying to avoid cross-enum comparison" This reverts commit 197c57de389c623aba42c9f4960500b332904825. --- Core/include/Acts/EventData/TrackStateProxy.hpp | 3 +-- Core/include/Acts/EventData/VectorMultiTrajectory.hpp | 3 +-- .../include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Core/include/Acts/EventData/TrackStateProxy.hpp b/Core/include/Acts/EventData/TrackStateProxy.hpp index 9e7f8f57392..4338c434f77 100644 --- a/Core/include/Acts/EventData/TrackStateProxy.hpp +++ b/Core/include/Acts/EventData/TrackStateProxy.hpp @@ -783,8 +783,7 @@ class TrackStateProxy { void allocateCalibrated(const Eigen::DenseBase& val, const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= - toUnderlying(eBoundSize) && + Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == diff --git a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp index e7d1e77a711..e13498cea2c 100644 --- a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp +++ b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp @@ -497,8 +497,7 @@ class VectorMultiTrajectory final const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= - toUnderlying(eBoundSize) && + Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == diff --git a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp index 0f66b9c1c4b..53eebf0e9b4 100644 --- a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp +++ b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp @@ -621,8 +621,7 @@ class MutablePodioTrackStateContainer final const Eigen::DenseBase& cov) requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= - toUnderlying(eBoundSize) && + Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && Eigen::PlainObjectBase::RowsAtCompileTime == Eigen::PlainObjectBase::RowsAtCompileTime && Eigen::PlainObjectBase::RowsAtCompileTime == From f6e9869d19245b9f901950c9834fd418c3ab50a2 Mon Sep 17 00:00:00 2001 From: Stephen Nicholas Swatman Date: Sat, 7 Dec 2024 09:47:47 +0100 Subject: [PATCH 059/122] refactor: Add new concepts for Eigen types This commit adds several new concepts for dealing with Eigen base types. The motivation for this is superficially to simplify some of the requirements in the track state proxy and the multi-trajectory, but it also allows me to more easily deal with some clang compiler warnings seen in #3949, as clang won't let me compare `A::ColsAtCompileTime` and `B::ColsAtCompileTime` if `A` and `B` are different types; this allows me to hide the conversion to integers inside a concept. --- .../Acts/EventData/TrackStateProxy.hpp | 12 ++-- .../Acts/EventData/VectorMultiTrajectory.hpp | 13 ++--- Core/include/Acts/Utilities/EigenConcepts.hpp | 56 +++++++++++++++++++ 3 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 Core/include/Acts/Utilities/EigenConcepts.hpp diff --git a/Core/include/Acts/EventData/TrackStateProxy.hpp b/Core/include/Acts/EventData/TrackStateProxy.hpp index 4338c434f77..3d91547c725 100644 --- a/Core/include/Acts/EventData/TrackStateProxy.hpp +++ b/Core/include/Acts/EventData/TrackStateProxy.hpp @@ -16,6 +16,7 @@ #include "Acts/EventData/TrackStateType.hpp" #include "Acts/EventData/Types.hpp" #include "Acts/Surfaces/Surface.hpp" +#include "Acts/Utilities/EigenConcepts.hpp" #include "Acts/Utilities/HashedString.hpp" #include "Acts/Utilities/Helpers.hpp" @@ -782,12 +783,11 @@ class TrackStateProxy { template void allocateCalibrated(const Eigen::DenseBase& val, const Eigen::DenseBase& cov) - requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && - Eigen::PlainObjectBase::RowsAtCompileTime == - Eigen::PlainObjectBase::RowsAtCompileTime && - Eigen::PlainObjectBase::RowsAtCompileTime == - Eigen::PlainObjectBase::ColsAtCompileTime) + requires(Concepts::eigen_base_is_fixed_size && + Concepts::eigen_bases_have_same_num_rows && + Concepts::eigen_base_is_square && + Eigen::PlainObjectBase::RowsAtCompileTime <= + static_cast>(eBoundSize)) { m_traj->template allocateCalibrated< Eigen::PlainObjectBase::RowsAtCompileTime>(m_istate, val, cov); diff --git a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp index e13498cea2c..8f416dd6969 100644 --- a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp +++ b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp @@ -16,6 +16,7 @@ #include "Acts/EventData/Types.hpp" #include "Acts/EventData/detail/DynamicColumn.hpp" #include "Acts/EventData/detail/DynamicKeyIterator.hpp" +#include "Acts/Utilities/EigenConcepts.hpp" #include "Acts/Utilities/HashedString.hpp" #include "Acts/Utilities/Helpers.hpp" #include "Acts/Utilities/ThrowAssert.hpp" @@ -495,13 +496,11 @@ class VectorMultiTrajectory final void allocateCalibrated_impl(IndexType istate, const Eigen::DenseBase& val, const Eigen::DenseBase& cov) - - requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && - Eigen::PlainObjectBase::RowsAtCompileTime == - Eigen::PlainObjectBase::RowsAtCompileTime && - Eigen::PlainObjectBase::RowsAtCompileTime == - Eigen::PlainObjectBase::ColsAtCompileTime) + requires(Concepts::eigen_base_is_fixed_size && + Concepts::eigen_bases_have_same_num_rows && + Concepts::eigen_base_is_square && + Eigen::PlainObjectBase::RowsAtCompileTime <= + static_cast>(eBoundSize)) { constexpr std::size_t measdim = val_t::RowsAtCompileTime; diff --git a/Core/include/Acts/Utilities/EigenConcepts.hpp b/Core/include/Acts/Utilities/EigenConcepts.hpp new file mode 100644 index 00000000000..4151dd6e237 --- /dev/null +++ b/Core/include/Acts/Utilities/EigenConcepts.hpp @@ -0,0 +1,56 @@ +// This file is part of the ACTS project. +// +// Copyright (C) 2016 CERN for the benefit of the ACTS project +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#pragma once + +#include + +namespace Acts::Concepts { +/// @brief Concept that is true iff T is a valid Eigen dense base. +template +concept is_eigen_base = requires { + { T::RowsAtCompileTime }; + { T::ColsAtCompileTime }; +}; + +/// @brief Concept that is true iff T is a valid Eigen dense base with fixed +/// size. +template +concept eigen_base_is_fixed_size = + is_eigen_base && Eigen::PlainObjectBase::RowsAtCompileTime > 0 && + Eigen::PlainObjectBase::ColsAtCompileTime > 0; + +/// @brief Concept that is true iff T is a valid Eigen dense base with fixed, +/// square size. +template +concept eigen_base_is_square = eigen_base_is_fixed_size && + Eigen::PlainObjectBase::RowsAtCompileTime == + Eigen::PlainObjectBase::ColsAtCompileTime; + +/// @brief Concept that is true iff T1 and T2 have the same, known at compile +/// time, number of rows. +template +concept eigen_bases_have_same_num_rows = + eigen_base_is_fixed_size && eigen_base_is_fixed_size && + static_cast(Eigen::PlainObjectBase::RowsAtCompileTime) == + static_cast(Eigen::PlainObjectBase::RowsAtCompileTime); + +/// @brief Concept that is true iff T1 and T2 have the same, known at compile +/// time, number of columns. +template +concept eigen_bases_have_same_num_cols = + eigen_base_is_fixed_size && eigen_base_is_fixed_size && + static_cast(Eigen::PlainObjectBase::ColsAtCompileTime) == + static_cast(Eigen::PlainObjectBase::ColsAtCompileTime); + +/// @brief Concept that is true iff T1 and T2 have the same, known at compile +/// time, size. +template +concept eigen_bases_have_same_size = eigen_bases_have_same_num_rows && + eigen_bases_have_same_num_cols; +} // namespace Acts::Concepts From 8dcc12e4c03b9549bc1fea0834febcceb96564e7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 13 Dec 2024 20:03:14 +0100 Subject: [PATCH 060/122] update concepts with toUnderlying --- Core/include/Acts/Utilities/EigenConcepts.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/include/Acts/Utilities/EigenConcepts.hpp b/Core/include/Acts/Utilities/EigenConcepts.hpp index 4151dd6e237..77718dda92f 100644 --- a/Core/include/Acts/Utilities/EigenConcepts.hpp +++ b/Core/include/Acts/Utilities/EigenConcepts.hpp @@ -37,16 +37,16 @@ concept eigen_base_is_square = eigen_base_is_fixed_size && template concept eigen_bases_have_same_num_rows = eigen_base_is_fixed_size && eigen_base_is_fixed_size && - static_cast(Eigen::PlainObjectBase::RowsAtCompileTime) == - static_cast(Eigen::PlainObjectBase::RowsAtCompileTime); + toUnderlying(Eigen::PlainObjectBase::RowsAtCompileTime) == + toUnderlying(Eigen::PlainObjectBase::RowsAtCompileTime); /// @brief Concept that is true iff T1 and T2 have the same, known at compile /// time, number of columns. template concept eigen_bases_have_same_num_cols = eigen_base_is_fixed_size && eigen_base_is_fixed_size && - static_cast(Eigen::PlainObjectBase::ColsAtCompileTime) == - static_cast(Eigen::PlainObjectBase::ColsAtCompileTime); + toUnderlying(Eigen::PlainObjectBase::ColsAtCompileTime) == + toUnderlying(Eigen::PlainObjectBase::ColsAtCompileTime); /// @brief Concept that is true iff T1 and T2 have the same, known at compile /// time, size. From 5b22555abf27b7b1d06e6b9d991ddaa436edc7c8 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sat, 14 Dec 2024 12:29:58 +0100 Subject: [PATCH 061/122] another compile fix --- Core/include/Acts/EventData/VectorMultiTrajectory.hpp | 2 +- .../Acts/Plugins/Podio/PodioTrackStateContainer.hpp | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp index 8f416dd6969..6c9fe454aea 100644 --- a/Core/include/Acts/EventData/VectorMultiTrajectory.hpp +++ b/Core/include/Acts/EventData/VectorMultiTrajectory.hpp @@ -500,7 +500,7 @@ class VectorMultiTrajectory final Concepts::eigen_bases_have_same_num_rows && Concepts::eigen_base_is_square && Eigen::PlainObjectBase::RowsAtCompileTime <= - static_cast>(eBoundSize)) + toUnderlying(eBoundSize)) { constexpr std::size_t measdim = val_t::RowsAtCompileTime; diff --git a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp index 53eebf0e9b4..af1a2a88412 100644 --- a/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp +++ b/Plugins/Podio/include/Acts/Plugins/Podio/PodioTrackStateContainer.hpp @@ -620,12 +620,11 @@ class MutablePodioTrackStateContainer final const Eigen::DenseBase& val, const Eigen::DenseBase& cov) - requires(Eigen::PlainObjectBase::RowsAtCompileTime > 0 && - Eigen::PlainObjectBase::RowsAtCompileTime <= eBoundSize && - Eigen::PlainObjectBase::RowsAtCompileTime == - Eigen::PlainObjectBase::RowsAtCompileTime && - Eigen::PlainObjectBase::RowsAtCompileTime == - Eigen::PlainObjectBase::ColsAtCompileTime) + requires(Concepts::eigen_base_is_fixed_size && + Concepts::eigen_bases_have_same_num_rows && + Concepts::eigen_base_is_square && + Eigen::PlainObjectBase::RowsAtCompileTime <= + toUnderlying(eBoundSize)) { constexpr std::size_t measdim = val_t::RowsAtCompileTime; From 9a9bb8622573f95434930282050acf0ae0646e12 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 15 Dec 2024 14:44:54 +0100 Subject: [PATCH 062/122] be smarter about selecting a compiler --- .github/workflows/builds.yml | 16 ++- CI/dependencies/select_lockfile.py | 175 +++++++++++++++++++++++++++++ CI/dependencies/setup.sh | 99 +++++++++++++--- 3 files changed, 272 insertions(+), 18 deletions(-) create mode 100755 CI/dependencies/select_lockfile.py diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 98365cbeedb..c66730b3ff0 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v7_b1 + DEPENDENCY_TAG: v7_b4 jobs: linux_ubuntu: @@ -37,6 +37,8 @@ jobs: lfs: true - name: Install dependencies + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: CI/dependencies/setup.sh - name: Restore ccache @@ -245,9 +247,11 @@ jobs: include: - image: ubuntu2204 std: 20 - - image: ubuntu2204_clang + compiler: g++ + - image: ubuntu2204 std: 20 - container: ghcr.io/acts-project/${{ matrix.image }}:63 + compiler: clang++ + container: ghcr.io/acts-project/${{ matrix.image }}:71 env: INSTALL_DIR: ${{ github.workspace }}/install ACTS_LOG_FAILURE_THRESHOLD: WARNING @@ -259,7 +263,9 @@ jobs: lfs: true - name: Install dependencies - run: CI/dependencies/setup.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: CI/dependencies/setup.sh -c ${{ matrix.compiler }} - name: Restore ccache uses: actions/cache/restore@v4 @@ -333,6 +339,8 @@ jobs: lfs: true - name: Install dependencies + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: > brew install ninja ccache && CI/dependencies/setup.sh diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py new file mode 100755 index 00000000000..e23df66f360 --- /dev/null +++ b/CI/dependencies/select_lockfile.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 + +import os +import argparse +import json +import urllib.request +import urllib.error +import re +import subprocess +from typing import Tuple, Dict + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--tag", type=str, required=True, help="Tag to use") + parser.add_argument("--arch", type=str, required=True, help="Architecture to use") + parser.add_argument( + "--compiler-binary", + type=str, + default=os.environ.get("CXX"), + help="Compiler to use (defaults to CXX environment variable if set)", + ) + parser.add_argument( + "--compiler", + type=str, + default=None, + help="Compiler to use (defaults to compiler binary if set)", + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Output file to write lockfile to", + ) + args = parser.parse_args() + + print("Fetching lockfiles for tag:", args.tag) + print("Architecture:", args.arch) + if args.compiler_binary is not None: + compiler = determine_compiler_version(args.compiler_binary) + print("Compiler:", args.compiler_binary, f"{compiler}") + elif args.compiler is not None: + if not re.match(r"^([\w-]+)@(\d+\.\d+\.\d+)$", args.compiler): + print(f"Invalid compiler format: {args.compiler}") + exit(1) + compiler = args.compiler + print("Compiler:", f"{compiler}") + else: + print("No compiler specified") + exit(1) + + base_url = f"https://api.github.com/repos/acts-project/ci-dependencies/releases/tags/{args.tag}" + + data = json.loads(fetch_github(base_url)) + + lockfiles = parse_assets(data) + + print("Available lockfiles:") + for arch, compilers in lockfiles.items(): + print(f"> {arch}:") + for c, (n, _) in compilers.items(): + print(f" - {c}: {n}") + + if args.arch not in lockfiles: + print(f"No lockfile found for architecture {args.arch}") + exit(1) + + lockfile = select_lockfile(lockfiles, args.arch, compiler) + + print("Selected lockfile:", lockfile) + + if args.output: + with open(args.output, "wb") as f: + f.write(fetch_github(lockfile)) + + +def parse_assets(data: Dict) -> Dict[str, Dict[str, Tuple[str, str]]]: + lockfiles: Dict[str, Dict[str, Tuple[str, str]]] = {} + + for asset in data["assets"]: + url = asset["browser_download_url"] + + name = asset["name"] + if not name.endswith(".lock") or not name.startswith("spack_"): + continue + + m = re.match(r"spack_(.*(?:aarch64|x86_64))(?:_(.*))?\.lock", name) + if m is None: + continue + + arch, compiler = m.groups() + compiler = compiler if compiler else "default" + lockfiles.setdefault(arch, {})[compiler] = (name, url) + + return lockfiles + + +def select_lockfile( + lockfiles: Dict[str, Dict[str, Tuple[str, str]]], arch: str, compiler: str +): + # Default to the default lockfile + _, lockfile = lockfiles[arch]["default"] + + # Extract compiler family and version + compiler_family = compiler.split("@")[0] + + # Find all matching compiler families + matching_compilers = { + comp: ver + for comp, ver in lockfiles[arch].items() + if comp != "default" and comp.split("@")[0] == compiler_family + } + + if matching_compilers: + if compiler in matching_compilers: + # Exact match found + _, lockfile = matching_compilers[compiler] + else: + # Find highest version of same compiler family + highest_version = max( + matching_compilers.keys(), + key=lambda x: [int(v) for v in x.split("@")[1].split(".")], + ) + _, lockfile = matching_compilers[highest_version] + + return lockfile + + +def determine_compiler_version(binary: str): + try: + result = subprocess.run([binary, "--version"], capture_output=True, text=True) + + line = result.stdout.split("\n", 1)[0] + print(line) + if "clang" in line: + compiler = "clang" + if "Apple" in line: + compiler = "apple-clang" + elif "gcc" in line or "GCC" in line: + compiler = "gcc" + else: + print(f"Unknown compiler: {binary}") + exit(1) + + m = re.search(r"(\d+\.\d+\.\d+)", line) + if m is None: + print(f"Failed to determine version for compiler: {binary}") + exit(1) + (version,) = m.groups() + return f"{compiler}@{version}" + + except (subprocess.SubprocessError, FileNotFoundError): + print(f"Failed to determine version for compiler: {binary}") + exit(1) + + +def fetch_github(base_url): + headers = {} + if "GITHUB_TOKEN" in os.environ: + headers["Authorization"] = f"token {os.environ['GITHUB_TOKEN']}" + + try: + req = urllib.request.Request(base_url, headers=headers) + with urllib.request.urlopen(req) as response: + return response.read() + except urllib.error.URLError as e: + print(f"Failed to fetch from {base_url}: {e}") + exit(1) + except json.JSONDecodeError as e: + print(f"Failed to parse JSON response: {e}") + exit(1) + + +if __name__ == "__main__": + main() diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 372d1e687dd..18e979d00ef 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -34,15 +34,88 @@ function end_section() { fi } -tag=${1:-${DEPENDENCY_TAG}} -if [ -n "${GITHUB_ACTIONS:-}" ]; then +# if [ -n "${1:-${DEPENDENCY_TAG:-}}" ]; then +# tag=${1:-${DEPENDENCY_TAG}} +# else +# echo "Usage: $0 " +# exit 1 +# fi + +# if [ -n "${GITHUB_ACTIONS:-}" ]; then +# destination="${GITHUB_WORKSPACE}/dependencies" +# elif [ -n "${GITLAB_CI:-}" ];then +# destination="${CI_PROJECT_DIR}/dependencies" +# else +# if [ -n "${2:-}" ]; then +# destination=${2} +# else +# echo "Usage: $0 " +# exit 1 +# fi +# fi + +# Parse command line arguments +while getopts "c:t:d:h" opt; do + case ${opt} in + c ) + compiler=$OPTARG + ;; + t ) + tag=$OPTARG + ;; + d ) + destination=$OPTARG + ;; + h ) + echo "Usage: $0 [-c compiler] [-t tag] [-d destination]" + echo "Options:" + echo " -c Specify compiler (defaults to CXX env var)" + echo " -t Specify dependency tag (defaults to DEPENDENCY_TAG env var)" + echo " -d Specify install destination (defaults based on CI environment)" + echo " -h Show this help message" + exit 0 + ;; + \? ) + echo "Invalid option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Option -$OPTARG requires an argument" 1>&2 + exit 1 + ;; + esac +done + +# Set defaults if not specified +if [ -z "${compiler:-}" ]; then + compiler="${CXX:-}" + if [ -z "${compiler:-}" ]; then + echo "No compiler specified via -c or CXX environment variable" + exit 1 + fi +fi + +if [ -z "${tag:-}" ]; then + tag="${DEPENDENCY_TAG:-}" + if [ -z "${tag:-}" ]; then + echo "No tag specified via -t or DEPENDENCY_TAG environment variable" + exit 1 + fi +fi + +if [ -z "${destination:-}" ]; then + if [ -n "${GITHUB_ACTIONS:-}" ]; then destination="${GITHUB_WORKSPACE}/dependencies" -elif [ -n "${GITLAB_CI:-}" ];then + elif [ -n "${GITLAB_CI:-}" ]; then destination="${CI_PROJECT_DIR}/dependencies" -else - destination=${2} + else + echo "No destination specified via -d and not running in CI" + exit 1 + fi fi + + echo "Install tag: $tag" echo "Install destination: $destination" @@ -74,21 +147,19 @@ start_section "Locate OpenGL" end_section fi -start_section "Get spack lock file name" +start_section "Get spack lock file" arch=$(spack arch --family) -lock_file="spack-${arch}.lock" -end_section - - -url="https://github.com/acts-project/ci-dependencies/releases/download/${tag}/${lock_file}" -echo "URL: $url" env_dir="${destination}/env" view_dir="${destination}/view" mkdir -p ${env_dir} + lock_file_path="${destination}/spack.lock" -start_section "Get spack lock files from tag" -curl -fL -o $lock_file_path $url +"${SCRIPT_DIR}/select_lockfile.py" \ + --tag "${tag}" \ + --arch "${arch}"\ + --output "${lock_file_path}" \ + --compiler-binary "${compiler}" end_section From f3460157bc146088d58dce38c5046baafd1a2a4b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 15 Dec 2024 14:45:45 +0100 Subject: [PATCH 063/122] use the compiler in cmake --- .github/workflows/builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index c66730b3ff0..7244039ad35 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -286,6 +286,7 @@ jobs: cmake -B build -S . --preset=github-ci -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DPython_EXECUTABLE=$(which python3) From 3fa02073603ff14d5dc4d7284a6eba4d4089273b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 15 Dec 2024 14:51:43 +0100 Subject: [PATCH 064/122] handle empty argument --- CI/dependencies/select_lockfile.py | 10 ++++++---- CI/dependencies/setup.sh | 24 ++++++++++++++---------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index e23df66f360..cc44207e878 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -7,7 +7,7 @@ import urllib.error import re import subprocess -from typing import Tuple, Dict +from typing import Tuple, Dict, Optional def main(): @@ -46,8 +46,7 @@ def main(): compiler = args.compiler print("Compiler:", f"{compiler}") else: - print("No compiler specified") - exit(1) + compiler = None base_url = f"https://api.github.com/repos/acts-project/ci-dependencies/releases/tags/{args.tag}" @@ -96,11 +95,14 @@ def parse_assets(data: Dict) -> Dict[str, Dict[str, Tuple[str, str]]]: def select_lockfile( - lockfiles: Dict[str, Dict[str, Tuple[str, str]]], arch: str, compiler: str + lockfiles: Dict[str, Dict[str, Tuple[str, str]]], arch: str, compiler: Optional[str] ): # Default to the default lockfile _, lockfile = lockfiles[arch]["default"] + if compiler is None: + return lockfile + # Extract compiler family and version compiler_family = compiler.split("@")[0] diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 18e979d00ef..75bb1bf054d 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -88,11 +88,7 @@ done # Set defaults if not specified if [ -z "${compiler:-}" ]; then - compiler="${CXX:-}" - if [ -z "${compiler:-}" ]; then - echo "No compiler specified via -c or CXX environment variable" - exit 1 - fi + compiler="${CXX:-default}" fi if [ -z "${tag:-}" ]; then @@ -155,11 +151,19 @@ view_dir="${destination}/view" mkdir -p ${env_dir} lock_file_path="${destination}/spack.lock" -"${SCRIPT_DIR}/select_lockfile.py" \ - --tag "${tag}" \ - --arch "${arch}"\ - --output "${lock_file_path}" \ - --compiler-binary "${compiler}" +cmd=( + "${SCRIPT_DIR}/select_lockfile.py" + "--tag" "${tag}" + "--arch" "${arch}" + "--output" "${lock_file_path}" +) + +if [ "${compiler}" != "default" ]; then + cmd+=("--compiler-binary" "${compiler}") +fi + +"${cmd[@]}" + end_section From bebf290e708950946827f091fcb280bc095b9172 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Sun, 15 Dec 2024 14:54:51 +0100 Subject: [PATCH 065/122] detect g++ --- CI/dependencies/select_lockfile.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index cc44207e878..732b482f307 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -36,17 +36,6 @@ def main(): print("Fetching lockfiles for tag:", args.tag) print("Architecture:", args.arch) - if args.compiler_binary is not None: - compiler = determine_compiler_version(args.compiler_binary) - print("Compiler:", args.compiler_binary, f"{compiler}") - elif args.compiler is not None: - if not re.match(r"^([\w-]+)@(\d+\.\d+\.\d+)$", args.compiler): - print(f"Invalid compiler format: {args.compiler}") - exit(1) - compiler = args.compiler - print("Compiler:", f"{compiler}") - else: - compiler = None base_url = f"https://api.github.com/repos/acts-project/ci-dependencies/releases/tags/{args.tag}" @@ -64,6 +53,18 @@ def main(): print(f"No lockfile found for architecture {args.arch}") exit(1) + if args.compiler_binary is not None: + compiler = determine_compiler_version(args.compiler_binary) + print("Compiler:", args.compiler_binary, f"{compiler}") + elif args.compiler is not None: + if not re.match(r"^([\w-]+)@(\d+\.\d+\.\d+)$", args.compiler): + print(f"Invalid compiler format: {args.compiler}") + exit(1) + compiler = args.compiler + print("Compiler:", f"{compiler}") + else: + compiler = None + lockfile = select_lockfile(lockfiles, args.arch, compiler) print("Selected lockfile:", lockfile) @@ -138,7 +139,7 @@ def determine_compiler_version(binary: str): compiler = "clang" if "Apple" in line: compiler = "apple-clang" - elif "gcc" in line or "GCC" in line: + elif "gcc" in line or "GCC" in line or "g++" in line: compiler = "gcc" else: print(f"Unknown compiler: {binary}") From 9060ae7cdcb8f0b0d933cc0314162db87c915031 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 09:24:14 +0100 Subject: [PATCH 066/122] b5 --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 7244039ad35..5966d803827 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v7_b4 + DEPENDENCY_TAG: v7_b5 jobs: linux_ubuntu: From 67f1685943ac6c3fc58f8e3f25d3aa5c367ad7a1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 09:29:09 +0100 Subject: [PATCH 067/122] fix opengl detection --- CI/dependencies/opengl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index 10e3eda6544..4cec4cda4e4 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -14,7 +14,7 @@ fi os=$(spack arch --platform) -if [ "$os" == ubuntu* ]; then +if [[ "$os" == ubuntu* ]]; then ${SUDO} apt-get update ${SUDO} apt-get install -y libgl1-mesa-dev cat < "$packages_file" @@ -26,7 +26,7 @@ packages: spec: opengl@4.5 EOF cat "$packages_file" -elif [ "$os" == almalinux* ]; then +elif [[ "$os" == almalinux* ]]; then ${SUDO} dnf install -y mesa-libGLU cat < "$packages_file" packages: From 25e8e410d457f7a301b190395c167057da8bc452 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 09:34:37 +0100 Subject: [PATCH 068/122] fix glob --- CI/dependencies/opengl.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index 4cec4cda4e4..fe8c94e0120 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -14,7 +14,7 @@ fi os=$(spack arch --platform) -if [[ "$os" == ubuntu* ]]; then +if [[ "$os" == *ubuntu* ]]; then ${SUDO} apt-get update ${SUDO} apt-get install -y libgl1-mesa-dev cat < "$packages_file" @@ -26,7 +26,7 @@ packages: spec: opengl@4.5 EOF cat "$packages_file" -elif [[ "$os" == almalinux* ]]; then +elif [[ "$os" == *almalinux* ]]; then ${SUDO} dnf install -y mesa-libGLU cat < "$packages_file" packages: @@ -37,4 +37,6 @@ packages: spec: opengl@4.6 EOF cat "$packages_file" +else [[ "$os" == *darwin* ]] + echo "Nothing to do on Darwin" fi From 0323fc83726422367724444d85eca2a6c3fd61fd Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 09:43:22 +0100 Subject: [PATCH 069/122] family --- CI/dependencies/opengl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index fe8c94e0120..5a1c83c4293 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -12,7 +12,7 @@ else SUDO="sudo" fi -os=$(spack arch --platform) +os=$(spack arch --family) if [[ "$os" == *ubuntu* ]]; then ${SUDO} apt-get update From f0f2572d30c8f42ceea26db0b54c80c023732271 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 11:32:59 +0100 Subject: [PATCH 070/122] opengl --- CI/dependencies/opengl.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/opengl.sh b/CI/dependencies/opengl.sh index 5a1c83c4293..b7db00bbcda 100755 --- a/CI/dependencies/opengl.sh +++ b/CI/dependencies/opengl.sh @@ -3,7 +3,9 @@ set -u set -e -packages_file=$(spack location -r)/spack/etc/spack/packages.yaml +packages_file=$(spack location -r)/etc/spack/packages.yaml +echo "Packages file: $packages_file" +stat "$packages_file" || true if ! command -v sudo &> /dev/null then @@ -14,16 +16,28 @@ fi os=$(spack arch --family) +echo "OS: $os" + if [[ "$os" == *ubuntu* ]]; then ${SUDO} apt-get update ${SUDO} apt-get install -y libgl1-mesa-dev + +if [[ "$os" == *ubuntu24* ]]; then + version="4.6" +elif [[ "$os" == *ubuntu20* ]]; then + version="4.5" +else + echo "Unknown OS version, default OpenGL version" + version="4.5" +fi + cat < "$packages_file" packages: opengl: buildable: false externals: - prefix: /usr/ - spec: opengl@4.5 + spec: opengl@${version} EOF cat "$packages_file" elif [[ "$os" == *almalinux* ]]; then From 8ee96b4d60db0273cd5cb1e2a101ad4eb72ecd8b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 11:39:37 +0100 Subject: [PATCH 071/122] wrap script in action --- .github/actions/dependencies/action.yml | 28 +++++++++++++++++++++++++ .github/workflows/builds.yml | 16 +++++--------- 2 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 .github/actions/dependencies/action.yml diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml new file mode 100644 index 00000000000..286925f4245 --- /dev/null +++ b/.github/actions/dependencies/action.yml @@ -0,0 +1,28 @@ +name: 'Dependencies' +description: 'Install dependencies from lockfile' + +inputs: + compiler: + description: 'Compiler to use' + required: false + default: '' + +runs: + using: 'composite' + steps: + - name: Prerequisites + if: startsWith(inputs.os, 'macos') + shell: bash + run: | + brew install ninja ccache + + - name: Install dependencies + shell: run + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ -n "${{ inputs.compiler }}" ]; then + CI/dependencies/setup.sh -c "${{ inputs.compiler }}" + else + CI/dependencies/setup.sh + fi diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 5966d803827..5c428f1cfb5 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -37,9 +37,7 @@ jobs: lfs: true - name: Install dependencies - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: CI/dependencies/setup.sh + uses: ./.github/actions/dependencies - name: Restore ccache id: ccache-restore @@ -263,9 +261,9 @@ jobs: lfs: true - name: Install dependencies - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: CI/dependencies/setup.sh -c ${{ matrix.compiler }} + uses: ./.github/actions/dependencies + with: + compiler: ${{ matrix.compiler }} - name: Restore ccache uses: actions/cache/restore@v4 @@ -340,11 +338,7 @@ jobs: lfs: true - name: Install dependencies - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: > - brew install ninja ccache - && CI/dependencies/setup.sh + uses: ./.github/actions/dependencies - name: Restore ccache uses: actions/cache/restore@v4 From b70b0f72cf0989cc804f4f7ac9b2578c6a4a2101 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 11:55:41 +0100 Subject: [PATCH 072/122] explicit token --- .github/actions/dependencies/action.yml | 6 +++++- .github/workflows/builds.yml | 5 +++++ CI/dependencies/select_lockfile.py | 5 +++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 286925f4245..2507f9f8181 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -6,6 +6,10 @@ inputs: description: 'Compiler to use' required: false default: '' + GITHUB_TOKEN: + description: 'Secrets to pass to the action' + required: false + default: '' runs: using: 'composite' @@ -19,7 +23,7 @@ runs: - name: Install dependencies shell: run env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} run: | if [ -n "${{ inputs.compiler }}" ]; then CI/dependencies/setup.sh -c "${{ inputs.compiler }}" diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 5c428f1cfb5..24e157c0667 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -38,6 +38,8 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Restore ccache id: ccache-restore @@ -264,6 +266,7 @@ jobs: uses: ./.github/actions/dependencies with: compiler: ${{ matrix.compiler }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Restore ccache uses: actions/cache/restore@v4 @@ -339,6 +342,8 @@ jobs: - name: Install dependencies uses: ./.github/actions/dependencies + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Restore ccache uses: actions/cache/restore@v4 diff --git a/CI/dependencies/select_lockfile.py b/CI/dependencies/select_lockfile.py index 732b482f307..5bf3b4aa568 100755 --- a/CI/dependencies/select_lockfile.py +++ b/CI/dependencies/select_lockfile.py @@ -159,8 +159,9 @@ def determine_compiler_version(binary: str): def fetch_github(base_url): headers = {} - if "GITHUB_TOKEN" in os.environ: - headers["Authorization"] = f"token {os.environ['GITHUB_TOKEN']}" + token = os.environ.get("GITHUB_TOKEN") + if token is not None and token != "": + headers["Authorization"] = f"token {token}" try: req = urllib.request.Request(base_url, headers=headers) From 0144c90e3d415a4e6e769e16a035a3c3113bbca6 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 14:08:38 +0100 Subject: [PATCH 073/122] action --- .github/actions/dependencies/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 2507f9f8181..2690caf309e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -25,8 +25,11 @@ runs: env: GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} run: | + pwd if [ -n "${{ inputs.compiler }}" ]; then + echo "With compiler" CI/dependencies/setup.sh -c "${{ inputs.compiler }}" else + echo "Without compiler" CI/dependencies/setup.sh fi From bea93d4bcba7c6828f98f74b30c3adc7ed2e9659 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 14:11:35 +0100 Subject: [PATCH 074/122] shell --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 2690caf309e..05aa72a905c 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -21,7 +21,7 @@ runs: brew install ninja ccache - name: Install dependencies - shell: run + shell: bash env: GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} run: | From 8771d43a527f36a8fd7b0c04f7a779c84a5d3511 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 16 Dec 2024 14:42:23 +0100 Subject: [PATCH 075/122] use runner os --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 05aa72a905c..17787e7035b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -15,7 +15,7 @@ runs: using: 'composite' steps: - name: Prerequisites - if: startsWith(inputs.os, 'macos') + if: startsWith(runner.os, 'macos') shell: bash run: | brew install ninja ccache From 463e14f95fe9cfefc6030f2084f1adb80fa68617 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 15:44:01 +0100 Subject: [PATCH 076/122] update tag --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 24e157c0667..c57a7209c7b 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -20,7 +20,7 @@ env: CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 500M CCACHE_KEY_SUFFIX: r2 - DEPENDENCY_TAG: v7_b5 + DEPENDENCY_TAG: v7_b6 jobs: linux_ubuntu: From f2c5601be269a2157daab54d8011e7b8623c8ac6 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:09:21 +0100 Subject: [PATCH 077/122] dedupe matrix cache keys --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index c57a7209c7b..7925acdd1a8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -273,7 +273,7 @@ jobs: id: ccache-restore with: path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- From cee6d44ab4ae293149ddc750f1b1b9f8ac003cff Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:09:41 +0100 Subject: [PATCH 078/122] remove remnant --- .github/workflows/builds.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 7925acdd1a8..4f5c8e9d8b8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -279,9 +279,6 @@ jobs: - name: Configure - # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the - # dd4hep CMake configuration that gets triggered on recent CMake - # versions run: > ccache -z && cmake -B build -S . From 6f3ce37f3efb0f13ab383f8360776e2d7da15084 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:10:15 +0100 Subject: [PATCH 079/122] see if we still need to supply the python interpreter --- .github/workflows/builds.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 4f5c8e9d8b8..9d36cfc85d3 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -62,7 +62,6 @@ jobs: cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) -DACTS_BUILD_PLUGIN_ONNX=ON - name: Build @@ -101,7 +100,6 @@ jobs: -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=20 -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - name: Downstream build @@ -286,7 +284,6 @@ jobs: -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DPython_EXECUTABLE=$(which python3) - name: Build run: cmake --build build @@ -357,7 +354,6 @@ jobs: && cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" - -DPython_EXECUTABLE=$(which python3) - name: Build run: cmake --build build From 8cf4c481faf9cfd2711be4b3f852993dfa2b0655 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:14:47 +0100 Subject: [PATCH 080/122] change path ordering --- CI/dependencies/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 75bb1bf054d..c6174ea0661 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -207,8 +207,8 @@ venv_dir="${view_dir}/venv" "${view_dir}"/bin/python3 -m venv "$venv_dir" if [ -n "${GITHUB_ACTIONS:-}" ]; then - echo "${venv_dir}/bin" >> "$GITHUB_PATH" echo "${view_dir}/bin" >> "$GITHUB_PATH" + echo "${venv_dir}/bin" >> "$GITHUB_PATH" fi set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}" end_section From 7f89ef82dc71c5b0ecda15ab80daa78d0a6bc5d9 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:16:58 +0100 Subject: [PATCH 081/122] we need it in macos at least --- .github/workflows/builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 9d36cfc85d3..468446d4325 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -354,6 +354,7 @@ jobs: && cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" + -DPython_EXECUTABLE=$(which python3) - name: Build run: cmake --build build From 0d01f2f7e95b8cb45faa46539b72c9d61774c033 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:26:17 +0100 Subject: [PATCH 082/122] let's try to cache the spack checkout --- .github/actions/dependencies/action.yml | 29 ++++++++++++++++ CI/dependencies/setup.sh | 45 ++++++++++--------------- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 17787e7035b..12366030395 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -20,6 +20,22 @@ runs: run: | brew install ninja ccache + - name: Restore spack cache + id: cache-spack-restore + uses: actions/cache/restore@v4 + with: + path: spack.tar.zst + key: ${{ env.CACHE_KEY }}-spack + + - name: Extract spack cache if found + if: steps.cache-spack-restore.outputs.cache-hit == 'true' + shell: bash + run: | + echo "Cache hit, extracting spack..." + zstd -d spack.tar.zst -o spack.tar + tar xf spack.tar + rm spack.tar + - name: Install dependencies shell: bash env: @@ -33,3 +49,16 @@ runs: echo "Without compiler" CI/dependencies/setup.sh fi + + - name: Compress spack directory for caching + shell: bash + run: | + # Compress spack directory for caching + echo "Compressing spack directory..." + tar cf spack.tar spack/ + zstd -T0 -10 spack.tar -o spack.tar.zst + + uses: actions/cache/save@v4 + with: + path: spack.tar.zst + key: ${{ env.CACHE_KEY }}-spack diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index c6174ea0661..b0690919d56 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -34,26 +34,6 @@ function end_section() { fi } -# if [ -n "${1:-${DEPENDENCY_TAG:-}}" ]; then -# tag=${1:-${DEPENDENCY_TAG}} -# else -# echo "Usage: $0 " -# exit 1 -# fi - -# if [ -n "${GITHUB_ACTIONS:-}" ]; then -# destination="${GITHUB_WORKSPACE}/dependencies" -# elif [ -n "${GITLAB_CI:-}" ];then -# destination="${CI_PROJECT_DIR}/dependencies" -# else -# if [ -n "${2:-}" ]; then -# destination=${2} -# else -# echo "Usage: $0 " -# exit 1 -# fi -# fi - # Parse command line arguments while getopts "c:t:d:h" opt; do case ${opt} in @@ -120,14 +100,23 @@ mkdir -p ${destination} start_section "Install spack if not already installed" if ! command -v spack &> /dev/null; then - git clone -c feature.manyFiles=true https://github.com/spack/spack.git - pushd spack > /dev/null - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - # Apply patch for spack improvements - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - source "$(pwd)/share/spack/setup-env.sh" - popd > /dev/null + if [ ! -d "spack" ]; then + echo "Cloning spack" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git + pushd spack > /dev/null + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # Apply patch for spack improvements + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + popd > /dev/null + else + echo "Updating spack" + pushd spack > /dev/null + git pull --rebase + popd > /dev/null + fi + + source "$(pwd)/spack/share/spack/setup-env.sh" fi end_section From b9f4c6492cc40f600e8320cbd75fb8d9dab46988 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:27:20 +0100 Subject: [PATCH 083/122] yaml error --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 12366030395..13559991df7 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -58,7 +58,7 @@ runs: tar cf spack.tar spack/ zstd -T0 -10 spack.tar -o spack.tar.zst - uses: actions/cache/save@v4 + - uses: actions/cache/save@v4 with: path: spack.tar.zst key: ${{ env.CACHE_KEY }}-spack From 04c42358fd37d05026614ea6e134de8d461e8e9a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:33:22 +0100 Subject: [PATCH 084/122] compress before installing, rebase --- .github/actions/dependencies/action.yml | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 13559991df7..6625c0d8e68 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -35,6 +35,19 @@ runs: zstd -d spack.tar.zst -o spack.tar tar xf spack.tar rm spack.tar + git -C spack pull --rebase + + - name: Compress spack directory for caching if we cloned + if: steps.cache-spack-restore.outputs.cache-hit == 'false' + shell: bash + run: | + tar cf spack.tar spack/ + zstd -T0 -10 spack.tar -o spack.tar.zst + + - uses: actions/cache/save@v4 + with: + path: spack.tar.zst + key: ${{ env.CACHE_KEY }}-spack - name: Install dependencies shell: bash @@ -49,16 +62,3 @@ runs: echo "Without compiler" CI/dependencies/setup.sh fi - - - name: Compress spack directory for caching - shell: bash - run: | - # Compress spack directory for caching - echo "Compressing spack directory..." - tar cf spack.tar spack/ - zstd -T0 -10 spack.tar -o spack.tar.zst - - - uses: actions/cache/save@v4 - with: - path: spack.tar.zst - key: ${{ env.CACHE_KEY }}-spack From ebd27cbf231182bbea4337bcaceccfe09d889f96 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:34:07 +0100 Subject: [PATCH 085/122] better compression --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 6625c0d8e68..84cc0085615 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -42,7 +42,7 @@ runs: shell: bash run: | tar cf spack.tar spack/ - zstd -T0 -10 spack.tar -o spack.tar.zst + zstd -T0 -19 spack.tar -o spack.tar.zst - uses: actions/cache/save@v4 with: From 88621b6cecbc0454b34524e359377dba241b6265 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:36:15 +0100 Subject: [PATCH 086/122] repack? --- .github/actions/dependencies/action.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 84cc0085615..36481206a38 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -25,7 +25,7 @@ runs: uses: actions/cache/restore@v4 with: path: spack.tar.zst - key: ${{ env.CACHE_KEY }}-spack + key: spack-clone - name: Extract spack cache if found if: steps.cache-spack-restore.outputs.cache-hit == 'true' @@ -37,8 +37,7 @@ runs: rm spack.tar git -C spack pull --rebase - - name: Compress spack directory for caching if we cloned - if: steps.cache-spack-restore.outputs.cache-hit == 'false' + - name: Compress spack directory for caching shell: bash run: | tar cf spack.tar spack/ @@ -47,7 +46,7 @@ runs: - uses: actions/cache/save@v4 with: path: spack.tar.zst - key: ${{ env.CACHE_KEY }}-spack + key: spack-clone - name: Install dependencies shell: bash From 14c520f4ad3442d26d16868ad2f2022fe3b77b72 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:38:42 +0100 Subject: [PATCH 087/122] actually, set up with update --- .github/actions/dependencies/action.yml | 7 +++++-- CI/dependencies/setup.sh | 20 +------------------- CI/dependencies/setup_spack.sh | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+), 21 deletions(-) create mode 100644 CI/dependencies/setup_spack.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 36481206a38..4b99eb2b341 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -31,11 +31,14 @@ runs: if: steps.cache-spack-restore.outputs.cache-hit == 'true' shell: bash run: | - echo "Cache hit, extracting spack..." zstd -d spack.tar.zst -o spack.tar tar xf spack.tar rm spack.tar - git -C spack pull --rebase + + - name: Install spack if not found + shell: bash + run: | + CI/dependencies/setup_spack.sh - name: Compress spack directory for caching shell: bash diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index b0690919d56..9cef587dd8c 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -99,25 +99,7 @@ mkdir -p ${destination} start_section "Install spack if not already installed" -if ! command -v spack &> /dev/null; then - if [ ! -d "spack" ]; then - echo "Cloning spack" - git clone -c feature.manyFiles=true https://github.com/spack/spack.git - pushd spack > /dev/null - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - # Apply patch for spack improvements - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - popd > /dev/null - else - echo "Updating spack" - pushd spack > /dev/null - git pull --rebase - popd > /dev/null - fi - - source "$(pwd)/spack/share/spack/setup-env.sh" -fi +source "${SCRIPT_DIR}/install_spack.sh" end_section if [ -n "${CI:-}" ]; then diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh new file mode 100644 index 00000000000..1ebc3a896d8 --- /dev/null +++ b/CI/dependencies/setup_spack.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e +set -u + +if ! command -v spack &> /dev/null; then + if [ ! -d "spack" ]; then + echo "Cloning spack" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git + pushd spack > /dev/null + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # Apply patch for spack improvements + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + popd > /dev/null + else + echo "Updating spack" + pushd spack > /dev/null + git pull --rebase + popd > /dev/null + fi + + source "$(pwd)/spack/share/spack/setup-env.sh" +fi From 6ae61f236d8eb74d591a71305f8407c244baf3a2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:40:29 +0100 Subject: [PATCH 088/122] shuffle --- .github/actions/dependencies/action.yml | 11 ++++++----- CI/dependencies/setup_spack.sh | 0 2 files changed, 6 insertions(+), 5 deletions(-) mode change 100644 => 100755 CI/dependencies/setup_spack.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4b99eb2b341..fd92196d53b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -14,11 +14,6 @@ inputs: runs: using: 'composite' steps: - - name: Prerequisites - if: startsWith(runner.os, 'macos') - shell: bash - run: | - brew install ninja ccache - name: Restore spack cache id: cache-spack-restore @@ -51,6 +46,12 @@ runs: path: spack.tar.zst key: spack-clone + - name: Prerequisites + if: startsWith(runner.os, 'macos') + shell: bash + run: | + brew install ninja ccache + - name: Install dependencies shell: bash env: diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh old mode 100644 new mode 100755 From 53daaef70f3c147e1a68f2269cd6601467234a15 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:43:51 +0100 Subject: [PATCH 089/122] need to do it after we did the environment --- .github/actions/dependencies/action.yml | 22 +++++++++++----------- CI/dependencies/setup.sh | 20 +++++++++++++++++++- CI/dependencies/setup_spack.sh | 23 ----------------------- 3 files changed, 30 insertions(+), 35 deletions(-) delete mode 100755 CI/dependencies/setup_spack.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index fd92196d53b..928c4d78499 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -35,17 +35,6 @@ runs: run: | CI/dependencies/setup_spack.sh - - name: Compress spack directory for caching - shell: bash - run: | - tar cf spack.tar spack/ - zstd -T0 -19 spack.tar -o spack.tar.zst - - - uses: actions/cache/save@v4 - with: - path: spack.tar.zst - key: spack-clone - - name: Prerequisites if: startsWith(runner.os, 'macos') shell: bash @@ -65,3 +54,14 @@ runs: echo "Without compiler" CI/dependencies/setup.sh fi + + - name: Compress spack directory for caching + shell: bash + run: | + tar --exclude='opt' cf spack.tar spack/ + zstd -T0 -19 spack.tar -o spack.tar.zst + + - uses: actions/cache/save@v4 + with: + path: spack.tar.zst + key: spack-clone diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 9cef587dd8c..b0690919d56 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -99,7 +99,25 @@ mkdir -p ${destination} start_section "Install spack if not already installed" -source "${SCRIPT_DIR}/install_spack.sh" +if ! command -v spack &> /dev/null; then + if [ ! -d "spack" ]; then + echo "Cloning spack" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git + pushd spack > /dev/null + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + # Apply patch for spack improvements + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + popd > /dev/null + else + echo "Updating spack" + pushd spack > /dev/null + git pull --rebase + popd > /dev/null + fi + + source "$(pwd)/spack/share/spack/setup-env.sh" +fi end_section if [ -n "${CI:-}" ]; then diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh deleted file mode 100755 index 1ebc3a896d8..00000000000 --- a/CI/dependencies/setup_spack.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -e -set -u - -if ! command -v spack &> /dev/null; then - if [ ! -d "spack" ]; then - echo "Cloning spack" - git clone -c feature.manyFiles=true https://github.com/spack/spack.git - pushd spack > /dev/null - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - # Apply patch for spack improvements - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - popd > /dev/null - else - echo "Updating spack" - pushd spack > /dev/null - git pull --rebase - popd > /dev/null - fi - - source "$(pwd)/spack/share/spack/setup-env.sh" -fi From 57d9856688b00c12bba3b3bbe82125af3e0eb3f0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:46:03 +0100 Subject: [PATCH 090/122] exclude tarball --- .github/actions/dependencies/action.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 928c4d78499..a0e87891c36 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -58,7 +58,11 @@ runs: - name: Compress spack directory for caching shell: bash run: | - tar --exclude='opt' cf spack.tar spack/ + tar \ + --exclude='opt' \ + --exclude='var/spack/cache' \ + --exclude='var/spack/environments' \ + cf spack.tar spack/ zstd -T0 -19 spack.tar -o spack.tar.zst - uses: actions/cache/save@v4 From 07f45ab386854fedba8dff03726afb27cb88a9bc Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Thu, 19 Dec 2024 17:55:10 +0100 Subject: [PATCH 091/122] drop the caching again --- .github/actions/dependencies/action.yml | 36 ------------------------- 1 file changed, 36 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index a0e87891c36..17787e7035b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -14,27 +14,6 @@ inputs: runs: using: 'composite' steps: - - - name: Restore spack cache - id: cache-spack-restore - uses: actions/cache/restore@v4 - with: - path: spack.tar.zst - key: spack-clone - - - name: Extract spack cache if found - if: steps.cache-spack-restore.outputs.cache-hit == 'true' - shell: bash - run: | - zstd -d spack.tar.zst -o spack.tar - tar xf spack.tar - rm spack.tar - - - name: Install spack if not found - shell: bash - run: | - CI/dependencies/setup_spack.sh - - name: Prerequisites if: startsWith(runner.os, 'macos') shell: bash @@ -54,18 +33,3 @@ runs: echo "Without compiler" CI/dependencies/setup.sh fi - - - name: Compress spack directory for caching - shell: bash - run: | - tar \ - --exclude='opt' \ - --exclude='var/spack/cache' \ - --exclude='var/spack/environments' \ - cf spack.tar spack/ - zstd -T0 -19 spack.tar -o spack.tar.zst - - - uses: actions/cache/save@v4 - with: - path: spack.tar.zst - key: spack-clone From afe5e73d3f1c00ca78ae80201b6e6b5fc0481864 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:18:47 +0100 Subject: [PATCH 092/122] path python macos --- .github/workflows/builds.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 468446d4325..31181743721 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -349,9 +349,11 @@ jobs: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - name: Configure - run: > + run: | ccache -z - && cmake -B build -S . + echo $PATH + which python3 + cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" -DPython_EXECUTABLE=$(which python3) From 5e76eb48923ab2a703ca06394b83ddf96b8ea5e2 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:19:57 +0100 Subject: [PATCH 093/122] tmate --- .github/workflows/builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 31181743721..4cefc0be04d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -348,6 +348,9 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + - uses: mxschmitt/action-tmate@v3 + # if: failure() + - name: Configure run: | ccache -z From dad7b0a9dcda43bf8f631bb55d9b19aa3b231715 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:20:14 +0100 Subject: [PATCH 094/122] other tmate --- .github/workflows/builds.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 4cefc0be04d..f3351c4be16 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -50,6 +50,9 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + - uses: mxschmitt/action-tmate@v3 + # if: failure() + - name: Configure # setting CMAKE_CXX_STANDARD=20 is a workaround for a bug in the # dd4hep CMake configuration that gets triggered on recent CMake @@ -348,8 +351,6 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - uses: mxschmitt/action-tmate@v3 - # if: failure() - name: Configure run: | From 713b425089565a0e85183f8247d5d2be5f7d6b8a Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:33:06 +0100 Subject: [PATCH 095/122] point cmakeprefixpath at venv dir --- CI/dependencies/setup.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index b0690919d56..8c64bf86a6c 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -178,17 +178,10 @@ start_section "Patch up Geant4 data directory" # ${SCRIPT_DIR}/with_spack_env.sh ${env_dir} geant4-config --install-datasets geant4_dir=$(spack -e "${env_dir}" location -i geant4) # Prepare the folder for G4 data, and symlink it to where G4 will look for it -mkdir -p "${geant4_dir}"/share/Geant4 -ln -s "${geant4_dir}"/share/Geant4/data ${view_dir}/share/Geant4/data +mkdir -p "${geant4_dir}/share/Geant4" +ln -s "${geant4_dir}/share/Geant4/data" "${view_dir}/share/Geant4/data" end_section -start_section "Set environment variables" -set_env CMAKE_PREFIX_PATH "${view_dir}" -set_env LD_LIBRARY_PATH "${view_dir}/lib" -set_env ROOT_INCLUDE_PATH "${view_dir}/include" -# Geant4 puts CLHEP in a subdirectory -set_env ROOT_INCLUDE_PATH "${view_dir}/include/Geant4" -end_section start_section "Prepare python environment" ls -al @@ -202,7 +195,13 @@ fi set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}" end_section - +start_section "Set environment variables" +set_env CMAKE_PREFIX_PATH "${venv_dir}:${view_dir}" +set_env LD_LIBRARY_PATH "${view_dir}/lib" +set_env ROOT_INCLUDE_PATH "${view_dir}/include" +# Geant4 puts CLHEP in a subdirectory +set_env ROOT_INCLUDE_PATH "${view_dir}/include/Geant4" +end_section # Pythia8 looks for settings in this directory # set_env PYTHIA8DATA "${destination}/share/Pythia8/xmldoc" From f3083251c9904558bc0161369ca639f019a2efe0 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:33:12 +0100 Subject: [PATCH 096/122] chain updates --- .github/workflows/builds.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index f3351c4be16..51356421ff3 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -50,7 +50,7 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - uses: mxschmitt/action-tmate@v3 + # - uses: mxschmitt/action-tmate@v3 # if: failure() - name: Configure @@ -59,8 +59,6 @@ jobs: # versions # Need to set git user & email for patching to work (GeoModel plugin) run: > - git config --global user.name 'CI' && - git config --global user.email '<>' && ccache -z && cmake -B build -S . --preset=github-ci @@ -353,14 +351,11 @@ jobs: - name: Configure - run: | - ccache -z - echo $PATH - which python3 + run: > + ccache -z && cmake -B build -S . --preset=github-ci -DCMAKE_INSTALL_PREFIX="${{ env.INSTALL_DIR }}" - -DPython_EXECUTABLE=$(which python3) - name: Build run: cmake --build build From cf22ce309523ffb581a9ff228c479f0be1a23bbd Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:39:55 +0100 Subject: [PATCH 097/122] system site packages? --- CI/dependencies/setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 8c64bf86a6c..8cb8768319d 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -186,7 +186,9 @@ end_section start_section "Prepare python environment" ls -al venv_dir="${view_dir}/venv" -"${view_dir}"/bin/python3 -m venv "$venv_dir" +"${view_dir}"/bin/python3 -m venv \ + --system-site-packages \ + "$venv_dir" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${view_dir}/bin" >> "$GITHUB_PATH" From d36b9d516b8922b8b1f848fb79d276f533fc4e08 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 08:50:55 +0100 Subject: [PATCH 098/122] install base python packages --- CI/dependencies/setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 8cb8768319d..a90765625c0 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -190,6 +190,8 @@ venv_dir="${view_dir}/venv" --system-site-packages \ "$venv_dir" +"${venv_dir}/bin/python3" -m pip install pyyaml jinja2 + if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${view_dir}/bin" >> "$GITHUB_PATH" echo "${venv_dir}/bin" >> "$GITHUB_PATH" From 31ff1e3fc42c80045bc84271344f6fb6ecb5e4f7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 09:56:46 +0100 Subject: [PATCH 099/122] tmae macos --- .github/workflows/builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 51356421ff3..c20c53413b6 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -349,6 +349,9 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + - uses: mxschmitt/action-tmate@v3 + # if: failure() + - name: Configure run: > From d249cff84583ba3de0e050f53e7a2d3201bb0c3c Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:18:03 +0100 Subject: [PATCH 100/122] add ptyhon find framework directive --- CMakePresets.json | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakePresets.json b/CMakePresets.json index 2aef0f3fe94..2c781e1c807 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -47,6 +47,7 @@ "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "CMAKE_COMPILE_WARNING_AS_ERROR": "ON", + "Python_FIND_FRAMEWORK": "LAST", "ACTS_FORCE_ASSERTIONS": "ON", "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON", "ACTS_BUILD_BENCHMARKS": "ON", From 7fa9c3619a4906e0ac231cb785671aa770401980 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:18:52 +0100 Subject: [PATCH 101/122] no tmate --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index c20c53413b6..917f6570042 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -349,7 +349,7 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - - uses: mxschmitt/action-tmate@v3 + # - uses: mxschmitt/action-tmate@v3 # if: failure() From 0a79be8b976f16ed124eb9d0b8a81930fd919fc1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:24:36 +0100 Subject: [PATCH 102/122] location strategy --- CMakePresets.json | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakePresets.json b/CMakePresets.json index 2c781e1c807..b6cb70bfa91 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -48,6 +48,7 @@ "CMAKE_BUILD_TYPE": "Release", "CMAKE_COMPILE_WARNING_AS_ERROR": "ON", "Python_FIND_FRAMEWORK": "LAST", + "Python_FIND_STRATEGY": "LOCATION", "ACTS_FORCE_ASSERTIONS": "ON", "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON", "ACTS_BUILD_BENCHMARKS": "ON", From 79670bf88822b4c96afc1f9e1bcb2447c1c98246 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:31:24 +0100 Subject: [PATCH 103/122] see if policy also works --- CMakeLists.txt | 2 ++ CMakePresets.json | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04fb58d6825..b8a53a90c4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,6 +258,8 @@ if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() +cmake_policy(SET CMP0094 NEW) + # required packages if(ACTS_SETUP_BOOST) if(POLICY CMP0167) diff --git a/CMakePresets.json b/CMakePresets.json index b6cb70bfa91..2c781e1c807 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -48,7 +48,6 @@ "CMAKE_BUILD_TYPE": "Release", "CMAKE_COMPILE_WARNING_AS_ERROR": "ON", "Python_FIND_FRAMEWORK": "LAST", - "Python_FIND_STRATEGY": "LOCATION", "ACTS_FORCE_ASSERTIONS": "ON", "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON", "ACTS_BUILD_BENCHMARKS": "ON", From 2d89ca12487129305fd3ba4ae953f5440acf099d Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:46:19 +0100 Subject: [PATCH 104/122] more general CMAKE_FIND_FRAMEWORK --- CMakePresets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakePresets.json b/CMakePresets.json index 2c781e1c807..fc7275cbd4e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -47,7 +47,7 @@ "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "CMAKE_COMPILE_WARNING_AS_ERROR": "ON", - "Python_FIND_FRAMEWORK": "LAST", + "CMAKE_FIND_FRAMEWORK": "LAST", "ACTS_FORCE_ASSERTIONS": "ON", "ACTS_ENABLE_LOG_FAILURE_THRESHOLD": "ON", "ACTS_BUILD_BENCHMARKS": "ON", From 730415f5d67a9c881c89287c323148f3091259dc Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 10:52:44 +0100 Subject: [PATCH 105/122] collect all policy settings in one place --- CMakeLists.txt | 52 +++++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8a53a90c4b..de68185d7a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,36 @@ include(ActsRetrieveVersion) project(Acts VERSION ${_acts_version} LANGUAGES CXX) +# policy settings + +# Steers how project() handles the VERSION option +cmake_policy(SET CMP0048 NEW) + +# the `_VERSION` variables set by `setup(... VERSION ...)` have +# only local scope, i.e. they are not accessible her for dependencies added +# via `add_subdirectory`. this overrides the `project(...)` function for +# sub-projects such that the resulting `_VERSION` has +# global scope and is accessible within the main project later on. +macro(project) + _project(${ARGN}) + set(${PROJECT_NAME}_VERSION "${${PROJECT_NAME}_VERSION}" CACHE INTERNAL "") +endmacro() + +# Controls the way python is located +if(POLICY CMP0094) + cmake_policy(SET CMP0094 NEW) +endif() + +# Controls behavior of DOWNLOAD_EXTRACT_TIMESTAMP +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() + +# Use boost's cmake config files +if(POLICY CMP0167) + cmake_policy(SET CMP0167 NEW) +endif() + # build options # all options and compile-time parameters must be defined here for clear @@ -253,19 +283,8 @@ set(_acts_boost_recommended_version 1.78.0) # Include the sources for the external dependencies. include(ActsExternSources) -# Controls behavior of DOWNLOAD_EXTRACT_TIMESTAMP -if(POLICY CMP0135) - cmake_policy(SET CMP0135 NEW) -endif() - -cmake_policy(SET CMP0094 NEW) - # required packages if(ACTS_SETUP_BOOST) - if(POLICY CMP0167) - cmake_policy(SET CMP0167 NEW) - endif() - # Enable both program_options and unit_test_framework to reduce complexity # Also Cuda tests seem to use program_options if( @@ -313,17 +332,6 @@ endif() find_package(Filesystem REQUIRED) -# the `_VERSION` variables set by `setup(... VERSION ...)` have -# only local scope, i.e. they are not accessible her for dependencies added -# via `add_subdirectory`. this overrides the `project(...)` function for -# sub-projects such that the resulting `_VERSION` has -# global scope and is accessible within the main project later on. -cmake_policy(SET CMP0048 NEW) -macro(project) - _project(${ARGN}) - set(${PROJECT_NAME}_VERSION "${${PROJECT_NAME}_VERSION}" CACHE INTERNAL "") -endmacro() - # CUDA settings are collected here in a macro, so that they can be reused by different plugins macro(enable_cuda) enable_language(CUDA) From 994c585849227c3d0898c39969aeee4357f04739 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 11:47:30 +0100 Subject: [PATCH 106/122] run downstream jobs --- .github/workflows/builds.yml | 207 +++++++++++++++++------------------ 1 file changed, 100 insertions(+), 107 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 917f6570042..047d431599f 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -109,107 +109,105 @@ jobs: - name: Downstream run run: ./build-downstream/bin/ShowActsVersion - # linux_examples_test: - # runs-on: ubuntu-latest - # container: ghcr.io/acts-project/ubuntu2404:63 - # needs: [linux_ubuntu] - # env: - # ACTS_SEQUENCER_DISABLE_FPEMON: true - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: CI/dependencies.sh - # - # - uses: actions/download-artifact@v4 - # with: - # name: acts-linux-ubuntu - # - # - name: Unpack build - # run: mkdir build && tar xf build.tar.gz -C build - # - # - name: Python level tests - # shell: bash - # env: - # PYTEST_MD_REPORT: true - # PYTEST_MD_REPORT_VERBOSE: 0 - # PYTEST_MD_REPORT_OUTPUT: pytest.md - # run: > - # geant4-config --install-datasets - # && source build/this_acts_withdeps.sh - # && python3 -m pip install -r Examples/Python/tests/requirements.txt - # && python3 -m pip install pytest-md-report - # && pytest -rFsv -k "not exatrkx" -v - # && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY - # - # linux_physmon: - # runs-on: ubuntu-latest - # container: ghcr.io/acts-project/ubuntu2404:63 - # needs: [linux_ubuntu] - # env: - # ACTS_SEQUENCER_DISABLE_FPEMON: true - # - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # lfs: true - # - # - run: apt-get update && apt-get install -y time - # - # - name: Set dependencies URL - # run: echo "DEPENDENCY_URL=https://acts.web.cern.ch/ACTS/ci/ubuntu-24.04/deps.${DEPENDENCY_TAG}.tar.zst" >> $GITHUB_ENV - # - # - name: Install dependencies - # run: CI/dependencies.sh - # - # - uses: actions/download-artifact@v4 - # with: - # name: acts-linux-ubuntu - # - # - name: Unpack build - # run: mkdir build && tar xf build.tar.gz -C build - # - # - name: Save PR number - # if: ${{ github.event_name == 'pull_request' }} - # run: | - # mkdir -p physmon - # echo ${{ github.event.number }} > physmon/pr_number - # echo ${{ github.event.pull_request.head.sha }} > physmon/sha - # - # - name: Physics performance checks - # shell: bash - # run: > - # echo "::group::Dependencies" - # && git config --global safe.directory "$GITHUB_WORKSPACE" - # && python3 -m pip install histcmp==0.6.8 matplotlib - # && python3 -m pip install -r Examples/Scripts/requirements.txt - # && geant4-config --install-datasets - # && venv_python=$(which python3) - # && echo $venv_python - # && source build/this_acts_withdeps.sh - # && export PATH=$(dirname $venv_python):$PATH - # && echo $PATH - # && which python3 - # && echo "::endgroup::" - # && CI/physmon/phys_perf_mon.sh all physmon - # - # - name: Post step summary - # if: always() - # run: cat physmon/summary.md >> $GITHUB_STEP_SUMMARY - # - # - uses: actions/upload-artifact@v4 - # if: always() - # with: - # name: physmon - # path: physmon + linux_examples_test: + runs-on: ubuntu-latest + container: ghcr.io/acts-project/ubuntu2404:63 + needs: [linux_ubuntu] + env: + ACTS_SEQUENCER_DISABLE_FPEMON: true + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + - name: Install dependencies + uses: ./.github/actions/dependencies + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/download-artifact@v4 + with: + name: acts-linux-ubuntu + + - name: Unpack build + run: mkdir build && tar xf build.tar.gz -C build + + - name: Python level tests + shell: bash + env: + PYTEST_MD_REPORT: true + PYTEST_MD_REPORT_VERBOSE: 0 + PYTEST_MD_REPORT_OUTPUT: pytest.md + run: > + geant4-config --install-datasets + && source build/this_acts_withdeps.sh + && python3 -m pip install -r Examples/Python/tests/requirements.txt + && python3 -m pip install pytest-md-report + && pytest -rFsv -k "not exatrkx" -v + && cat ${PYTEST_MD_REPORT_OUTPUT} >> $GITHUB_STEP_SUMMARY + + linux_physmon: + runs-on: ubuntu-latest + container: ghcr.io/acts-project/ubuntu2404:63 + needs: [linux_ubuntu] + env: + ACTS_SEQUENCER_DISABLE_FPEMON: true + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + lfs: true + + - run: apt-get update && apt-get install -y time + + - name: Install dependencies + uses: ./.github/actions/dependencies + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/download-artifact@v4 + with: + name: acts-linux-ubuntu + + - name: Unpack build + run: mkdir build && tar xf build.tar.gz -C build + + - name: Save PR number + if: ${{ github.event_name == 'pull_request' }} + run: | + mkdir -p physmon + echo ${{ github.event.number }} > physmon/pr_number + echo ${{ github.event.pull_request.head.sha }} > physmon/sha + + - name: Physics performance checks + shell: bash + run: > + echo "::group::Dependencies" + && git config --global safe.directory "$GITHUB_WORKSPACE" + && python3 -m pip install histcmp==0.6.8 matplotlib + && python3 -m pip install -r Examples/Scripts/requirements.txt + && geant4-config --install-datasets + && venv_python=$(which python3) + && echo $venv_python + && source build/this_acts_withdeps.sh + && export PATH=$(dirname $venv_python):$PATH + && echo $PATH + && which python3 + && echo "::endgroup::" + && CI/physmon/phys_perf_mon.sh all physmon + + - name: Post step summary + if: always() + run: cat physmon/summary.md >> $GITHUB_STEP_SUMMARY + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: physmon + path: physmon # # linux_physmon_perf_report: # needs: [linux_physmon] @@ -272,7 +270,7 @@ jobs: id: ccache-restore with: path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ github.job }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ github.sha }} + key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ github.sha }} restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- @@ -349,10 +347,6 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- - # - uses: mxschmitt/action-tmate@v3 - # if: failure() - - - name: Configure run: > ccache -z && @@ -393,7 +387,6 @@ jobs: -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-Werror - -DCMAKE_CXX_STANDARD=20 -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" - name: Downstream build From e8dea2558f6b6f3335cc6412c72fa2a4b9e61014 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 12:58:36 +0100 Subject: [PATCH 107/122] update hashes due to G4 version change --- Examples/Python/tests/root_file_hashes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/Python/tests/root_file_hashes.txt b/Examples/Python/tests/root_file_hashes.txt index 07d42c66ad1..3b16b9e63a0 100644 --- a/Examples/Python/tests/root_file_hashes.txt +++ b/Examples/Python/tests/root_file_hashes.txt @@ -1,8 +1,8 @@ test_pythia8__particles.root: 125182a9647ef3cec71afbc1b9e1676e40c13826c8333d6704345dd5133d5e91 test_fatras__particles_simulation.root: bc970873fef0c2efd86ed5413623802353d2cd04abea72de14e8cdfc0e40076f test_fatras__hits.root: 6e4beb045fa1712c4d14c280ba33c3fa13e4aff9de88d55c3e32f62ad226f724 -test_geant4__particles_simulation.root: 49926c71a9b54e13aa1cc7596d3302baf3c87d8e2c1d0267cb4523f6abdc0ac2 -test_geant4__hits.root: 4c9e704a75f47ed2e61652679a1d6f18fa4d9cf53faa8f8f5bbf7995634207aa +test_geant4__particles_simulation.root: fd10da8000715f7f515001670e713efd8df21501b32e9b2635f0152d63eecaf4 +test_geant4__hits.root: d8efffea763de487454c23c48bde26548df59f2342807b3b486eb98bdc6a1559 test_seeding__estimatedparams.root: 6759004f945cabe03098c94b3eea7e3323acd9f37edfa71641797007336643c8 test_seeding__performance_seeding.root: 992f9c611d30dde0d3f3ab676bab19ada61ab6a4442828e27b65ec5e5b7a2880 test_seeding__particles.root: c423bc666df3674f1a1140dec68ea13f44173232b8057e8a02572aee4f3e7d5b From 655f0be8e6ffebae12715ecc91dfadf0c69a624b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:06:17 +0100 Subject: [PATCH 108/122] include more in the cache key --- .github/workflows/builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 047d431599f..5bc26b27618 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -272,7 +272,7 @@ jobs: path: ${{ env.CCACHE_DIR }} key: ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}-${{ github.sha }} restore-keys: | - ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}- + ccache-${{ runner.os }}-${{ github.job }}-${{ env.CCACHE_KEY_SUFFIX }}-${{ matrix.image }}-${{ matrix.std }}-${{ matrix.compiler }}- - name: Configure From 6f9827aad87d3761a2c4fd97c94a122a79a6d262 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:25:30 +0100 Subject: [PATCH 109/122] try spack caching again --- .github/actions/dependencies/action.yml | 38 ++++++++++++++++++++++++- CI/dependencies/setup.sh | 20 ++----------- CI/dependencies/setup_spack.sh | 20 +++++++++++++ 3 files changed, 60 insertions(+), 18 deletions(-) create mode 100755 CI/dependencies/setup_spack.sh diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 17787e7035b..d9702eed9bf 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -10,6 +10,10 @@ inputs: description: 'Secrets to pass to the action' required: false default: '' + spack_version: + description: 'Version of Spack to use' + required: false + default: '2024-11-13' runs: using: 'composite' @@ -18,7 +22,39 @@ runs: if: startsWith(runner.os, 'macos') shell: bash run: | - brew install ninja ccache + brew install ninja ccache zstd + + - name: Restore cache + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: spack.tar.zst + key: spack-${{ runner.os }}-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + + - name: Extract cached Spack + if: steps.cache-restore.outputs.cache-hit == 'true' + shell: bash + run: | + zstd -d spack.tar.zst -f + tar xf spack.tar + rm spack.tar + + - name: Setup Spack from scratch + if: steps.cache-restore.outputs.cache-hit != 'true' + shell: bash + env: + SPACK_VERSION: ${{ inputs.spack_version }} + run: | + CI/dependencies/setup_spack.sh + tar cf spack.tar spack/ + zstd -19 --rm spack.tar + + - name: Save cache + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: spack.tar.zst + key: spack-${{ runner.os }}-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies shell: bash diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index a90765625c0..80d439b5296 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -100,26 +100,12 @@ mkdir -p ${destination} start_section "Install spack if not already installed" if ! command -v spack &> /dev/null; then - if [ ! -d "spack" ]; then - echo "Cloning spack" - git clone -c feature.manyFiles=true https://github.com/spack/spack.git - pushd spack > /dev/null - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - # Apply patch for spack improvements - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - popd > /dev/null - else - echo "Updating spack" - pushd spack > /dev/null - git pull --rebase - popd > /dev/null - fi - - source "$(pwd)/spack/share/spack/setup-env.sh" + "${SCRIPT_DIR}/setup_spack.sh" fi end_section +source "$(pwd)/spack/share/spack/setup-env.sh" + if [ -n "${CI:-}" ]; then start_section "Add buildcache mirror" mirror_name="acts-spack-buildcache" diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh new file mode 100755 index 00000000000..3fc5a644f34 --- /dev/null +++ b/CI/dependencies/setup_spack.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +_spack_version=${SPACK_VERSION:-develop} + +if [ ! -d "spack" ]; then + echo "Cloning spack" + git clone -c feature.manyFiles=true https://github.com/spack/spack.git -b ${_spack_version} + pushd spack > /dev/null + git config user.name CI + git config user.email <> + echo "Applying patch for spack improvements" + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am + curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/48236.patch | git am + + rm -rf .git + + echo "Populating the repository index" + bin/spack list > /dev/null + popd > /dev/null +fi From f7ffaa4d15de0a396e2cb9e5269398d2f9be49db Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:27:51 +0100 Subject: [PATCH 110/122] syntax --- CI/dependencies/setup_spack.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index 3fc5a644f34..268d718bc03 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -6,8 +6,8 @@ if [ ! -d "spack" ]; then echo "Cloning spack" git clone -c feature.manyFiles=true https://github.com/spack/spack.git -b ${_spack_version} pushd spack > /dev/null - git config user.name CI - git config user.email <> + git config user.name 'CI' + git config user.email '<>' echo "Applying patch for spack improvements" curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/48236.patch | git am From 1eae13d239dcdb292b9483e78f8a2340afd72b22 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:33:14 +0100 Subject: [PATCH 111/122] tag fix --- .github/actions/dependencies/action.yml | 2 +- CI/dependencies/setup_spack.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index d9702eed9bf..7c5f23571ad 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -13,7 +13,7 @@ inputs: spack_version: description: 'Version of Spack to use' required: false - default: '2024-11-13' + default: 'develop-2024-12-15' runs: using: 'composite' diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index 268d718bc03..58e9abca3bc 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -1,10 +1,12 @@ #!/bin/bash +set -e +set -u _spack_version=${SPACK_VERSION:-develop} if [ ! -d "spack" ]; then echo "Cloning spack" - git clone -c feature.manyFiles=true https://github.com/spack/spack.git -b ${_spack_version} + git clone --branch ${_spack_version} -c feature.manyFiles=true https://github.com/spack/spack.git pushd spack > /dev/null git config user.name 'CI' git config user.email '<>' From ba00215ad1bba1aab560f253d1fe9e2e28d4acd9 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:40:24 +0100 Subject: [PATCH 112/122] switch repo, no patches --- CI/dependencies/setup_spack.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index 58e9abca3bc..dba3939d282 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -6,13 +6,10 @@ _spack_version=${SPACK_VERSION:-develop} if [ ! -d "spack" ]; then echo "Cloning spack" - git clone --branch ${_spack_version} -c feature.manyFiles=true https://github.com/spack/spack.git + git clone --branch ${_spack_version} -c feature.manyFiles=true https://github.com/acts-project/spack.git pushd spack > /dev/null git config user.name 'CI' git config user.email '<>' - echo "Applying patch for spack improvements" - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/47370.patch | git am - curl https://patch-diff.githubusercontent.com/raw/spack/spack/pull/48236.patch | git am rm -rf .git From 2a95037e99038927459408b6e6db3af81b9d11a7 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:42:10 +0100 Subject: [PATCH 113/122] special branch --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 7c5f23571ad..4a42c13c287 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -13,7 +13,7 @@ inputs: spack_version: description: 'Version of Spack to use' required: false - default: 'develop-2024-12-15' + default: 'develop-acts' runs: using: 'composite' From bc646c8875db094883665b6c2c7e2e63805145cc Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 13:50:14 +0100 Subject: [PATCH 114/122] single cache key --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 4a42c13c287..1195e236b26 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,7 +29,7 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-${{ runner.os }}-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' @@ -54,7 +54,7 @@ runs: uses: actions/cache/save@v3 with: path: spack.tar.zst - key: spack-${{ runner.os }}-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies shell: bash From 0fb774424811cff566c86e2b285d2580db7281ad Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:07:49 +0100 Subject: [PATCH 115/122] let's keep the .spack directory as well --- .github/actions/dependencies/action.yml | 9 +++++---- CI/dependencies/setup.sh | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 1195e236b26..106542a7904 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,15 +29,16 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' shell: bash run: | - zstd -d spack.tar.zst -f tar xf spack.tar + tar xf -C ~ spack_cache.tar rm spack.tar + rm spack_cache.tar - name: Setup Spack from scratch if: steps.cache-restore.outputs.cache-hit != 'true' @@ -47,14 +48,14 @@ runs: run: | CI/dependencies/setup_spack.sh tar cf spack.tar spack/ - zstd -19 --rm spack.tar + tar cf spack_cache.tar -C ~ .spack - name: Save cache if: steps.cache-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v3 with: path: spack.tar.zst - key: spack-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies shell: bash diff --git a/CI/dependencies/setup.sh b/CI/dependencies/setup.sh index 80d439b5296..a922c009e82 100755 --- a/CI/dependencies/setup.sh +++ b/CI/dependencies/setup.sh @@ -178,14 +178,14 @@ venv_dir="${view_dir}/venv" "${venv_dir}/bin/python3" -m pip install pyyaml jinja2 +end_section + +start_section "Set environment variables" if [ -n "${GITHUB_ACTIONS:-}" ]; then echo "${view_dir}/bin" >> "$GITHUB_PATH" echo "${venv_dir}/bin" >> "$GITHUB_PATH" fi set_env PATH "${venv_dir}/bin:${view_dir}/bin/:${PATH}" -end_section - -start_section "Set environment variables" set_env CMAKE_PREFIX_PATH "${venv_dir}:${view_dir}" set_env LD_LIBRARY_PATH "${view_dir}/lib" set_env ROOT_INCLUDE_PATH "${view_dir}/include" From d98a6c6d430455bbbebd3bb06cc549e8960c9470 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:10:27 +0100 Subject: [PATCH 116/122] namwes, shuffle --- .github/actions/dependencies/action.yml | 6 +++++- CI/dependencies/setup_spack.sh | 5 ----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 106542a7904..10ef86a8daf 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -47,6 +47,8 @@ runs: SPACK_VERSION: ${{ inputs.spack_version }} run: | CI/dependencies/setup_spack.sh + bin/spack list > /dev/null + rm -rf spack/.git tar cf spack.tar spack/ tar cf spack_cache.tar -C ~ .spack @@ -54,7 +56,9 @@ runs: if: steps.cache-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v3 with: - path: spack.tar.zst + path: | + spack.tar + spack_cache.tar key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies diff --git a/CI/dependencies/setup_spack.sh b/CI/dependencies/setup_spack.sh index dba3939d282..4cb0772e01e 100755 --- a/CI/dependencies/setup_spack.sh +++ b/CI/dependencies/setup_spack.sh @@ -10,10 +10,5 @@ if [ ! -d "spack" ]; then pushd spack > /dev/null git config user.name 'CI' git config user.email '<>' - - rm -rf .git - - echo "Populating the repository index" - bin/spack list > /dev/null popd > /dev/null fi From a3862601391115b0132034f005eec18214805aa1 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:12:02 +0100 Subject: [PATCH 117/122] binary path --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 10ef86a8daf..68fa9e50887 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -47,7 +47,7 @@ runs: SPACK_VERSION: ${{ inputs.spack_version }} run: | CI/dependencies/setup_spack.sh - bin/spack list > /dev/null + spack/bin/spack list > /dev/null rm -rf spack/.git tar cf spack.tar spack/ tar cf spack_cache.tar -C ~ .spack From eff06a33878030ea2d4f8cd2fa508852c9a22d44 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:29:03 +0100 Subject: [PATCH 118/122] no zstd install --- .github/actions/dependencies/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 68fa9e50887..8664480cd6e 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -22,7 +22,7 @@ runs: if: startsWith(runner.os, 'macos') shell: bash run: | - brew install ninja ccache zstd + brew install ninja ccache - name: Restore cache id: cache-restore From a1135c7d41ee1557b2cb522b61d69b0de9e77999 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 14:30:26 +0100 Subject: [PATCH 119/122] key issue? --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 8664480cd6e..83c72aeefcf 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,7 +29,7 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' @@ -59,7 +59,7 @@ runs: path: | spack.tar spack_cache.tar - key: spack-v2-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Install dependencies shell: bash From 6ea6ae30ad7d2e5813455bd86312eef353c4c0b6 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 15:01:36 +0100 Subject: [PATCH 120/122] trigger From b477ff3d6dbcb4cec2b56949c1672a7a04f0ebf9 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 15:08:29 +0100 Subject: [PATCH 121/122] restore key? --- .github/actions/dependencies/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index 83c72aeefcf..a3c305e1126 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -30,6 +30,8 @@ runs: with: path: spack.tar.zst key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + restore-keys: | + spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} - name: Extract cached Spack if: steps.cache-restore.outputs.cache-hit == 'true' From 10360faa3321f1529d50e384d06079e3270c0049 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Fri, 20 Dec 2024 15:15:21 +0100 Subject: [PATCH 122/122] keys? --- .github/actions/dependencies/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dependencies/action.yml b/.github/actions/dependencies/action.yml index a3c305e1126..6dc4ee1bc9b 100644 --- a/.github/actions/dependencies/action.yml +++ b/.github/actions/dependencies/action.yml @@ -29,7 +29,7 @@ runs: uses: actions/cache/restore@v3 with: path: spack.tar.zst - key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }}-${{ github.sha }} restore-keys: | spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} @@ -61,7 +61,7 @@ runs: path: | spack.tar spack_cache.tar - key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }} + key: spack-r1-${{ inputs.spack_version }}-${{ hashFiles('CI/dependencies/setup_spack.sh') }}-${{ github.sha }} - name: Install dependencies shell: bash