erf-model · asalmgren · Jun 19, 2024 · Jun 19, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -46,12 +46,24 @@ jobs:
         # Install MPI
         ${{matrix.install_deps}}
 
+    - name: Install CCache
+      run: Submodules/AMReX/.github/workflows/dependencies/dependencies_ccache.sh
+
+    - name: Set Up Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+
     - name: Configure CMake
       run: |
         cmake \
           -B${{runner.workspace}}/ERF/build-${{matrix.os}} \
           -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \
           -DCMAKE_BUILD_TYPE:STRING=Debug \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -DERF_DIM:STRING=3 \
           -DERF_ENABLE_MPI:BOOL=ON \
           -DERF_ENABLE_TESTS:BOOL=ON \
@@ -63,8 +75,16 @@ jobs:
 
     - name: Build
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=600M
+        ccache -z
+
         cmake --build ${{runner.workspace}}/ERF/build-${{matrix.os}} --parallel ${{env.NPROCS}};
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
     - name: Regression Tests
       run: |
         ctest -L regression -VV
@@ -94,3 +114,18 @@ jobs:
     #     name: build-and-test
     #     path: |
     #       ${{runner.workspace}}/ERF/regressioncov.html
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1
diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml
@@ -8,16 +8,16 @@ on:
 
 jobs:
   CleanUpCcacheCachePostPR:
-    name: Clean Up Ccahe Cache Post PR
+    name: Clean Up Ccache Cache Post PR
     runs-on: ubuntu-latest
     permissions:
       actions: write
       contents: read
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v3
-      - name: Clean up ccahe
+      - uses: actions/checkout@v4
+      - name: Clean up ccache
         run: |
           gh extension install actions/gh-actions-cache
 
@@ -31,7 +31,10 @@ jobs:
           set +e
 
           keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1)
+          # $keys might contain spaces. Thus we set IFS to \n.
+          IFS=$'\n'
           for k in $keys
           do
-            gh actions-cache delete $k -R $REPO -B $BRANCH --confirm
+            gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
           done
+          unset IFS
diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml
@@ -2,22 +2,22 @@ name: CleanUpCache
 
 on:
   workflow_run:
-    workflows: [GCC,CUDA,HIP,SYCL]
+    workflows: [ERF CI, ERF CUDA CI, ERF CI (hip), ERF CI (sycl), Linux GCC, MacOS]
     types:
       - completed
 
 jobs:
   CleanUpCcacheCache:
-    name: Clean Up Ccahe Cache for ${{ github.event.workflow_run.name }}
+    name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }}
     runs-on: ubuntu-latest
     permissions:
       actions: write
       contents: read
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v3
-      - name: Clean up ccahe
+      - uses: actions/checkout@v4
+      - name: Clean up ccache
         run: |
           gh extension install actions/gh-actions-cache
 
@@ -27,7 +27,7 @@ jobs:
           EVENT=${{ github.event.workflow_run.event }}
 
           # Triggering workflow run name (e.g., LinuxClang)
-          WORKFLOW_NAME=${{ github.event.workflow_run.name }}
+          WORKFLOW_NAME="${{ github.event.workflow_run.name }}"
 
           if [[ $EVENT == "pull_request" ]]; then
             gh run download ${{ github.event.workflow_run.id }} -n pr_number
@@ -45,16 +45,19 @@ jobs:
           # The goal is to keep the last used key of each job and delete all others.
 
           # something like ccache-LinuxClang-
-          keyprefix=ccache-${WORKFLOW_NAME}-
+          keyprefix="ccache-${WORKFLOW_NAME}-"
 
-          cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key $keyprefix | awk -F '-git-' '{print $1}' | sort | uniq)
+          cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq)
 
           # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d".
+          # It might also contain spaces. Thus we set IFS to \n.
+          IFS=$'\n'
           for j in $cached_jobs
           do
-            old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key ${j}-git- --sort last-used | cut -f 1 | tail -n +2)
+            old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2)
             for k in $old_keys
             do
-              gh actions-cache delete $k -R $REPO -B $BRANCH --confirm
+              gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
             done
           done
+          unset IFS
diff --git a/.github/workflows/cuda-ci.yml b/.github/workflows/cuda-ci.yml
@@ -35,17 +35,51 @@ jobs:
           submodules: true
       - name: Prepare CUDA environment
         run: Submodules/AMReX/.github/workflows/dependencies/dependencies_nvcc.sh ${{matrix.cuda_ver}}
+      - name: Install CCache
+        run: Submodules/AMReX/.github/workflows/dependencies/dependencies_ccache.sh
+      - name: Set Up Cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ccache
+          key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+          restore-keys: |
+               ccache-${{ github.workflow }}-${{ github.job }}-git-
       - name: Configure and build
         run: |
+          export CCACHE_COMPRESS=1
+          export CCACHE_COMPRESSLEVEL=10
+          export CCACHE_MAXSIZE=600M
+          ccache -z
+
           export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
           export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
           which nvcc || echo "nvcc not in PATH!"
           cmake -Bbuild-${{matrix.cuda_pkg}} \
             -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
             -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
             -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON           \
             -DERF_DIM:STRING=3 \
             -DERF_ENABLE_MPI:BOOL=ON \
             -DERF_ENABLE_CUDA:BOOL=ON \
             -DERF_ENABLE_REGRESSION_TESTS_ONLY:BOOL=ON .
           cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc)
+
+          ccache -s
+          du -hs ~/.cache/ccache
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1
diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml
@@ -19,12 +19,24 @@ jobs:
     - name: Install Dependencies
       run: Submodules/AMReX/.github/workflows/dependencies/dependencies.sh
 
+    - name: Install CCache
+      run: Submodules/AMReX/.github/workflows/dependencies/dependencies_ccache.sh
+
+    - name: Set Up Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+
     - name: Configure Project and Generate Build System
       run: |
         cmake \
           -B${{runner.workspace}}/ERF/build \
           -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \
           -DCMAKE_BUILD_TYPE:STRING=Debug \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -DERF_DIM:STRING=3 \
           -DERF_ENABLE_MPI:BOOL=ON \
           -DERF_ENABLE_TESTS:BOOL=ON \
@@ -34,9 +46,32 @@ jobs:
 
     - name: Compile and Link
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=300M
+        ccache -z
+
         cmake --build ${{runner.workspace}}/ERF/build --parallel 2 --verbose
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
     - name: CMake Tests # see file ERF/Tests/CTestList.cmake
       run: |
         ctest -L regression -VV
       working-directory: ${{runner.workspace}}/ERF/build
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1
diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml
@@ -42,8 +42,24 @@ jobs:
     - name: Dependencies
       run: Submodules/AMReX/.github/workflows/dependencies/dependencies_hip.sh
 
+    - name: Install CCache
+      run: Submodules/AMReX/.github/workflows/dependencies/dependencies_ccache.sh
+
+    - name: Set Up Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+
     - name: Build & Install
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=300M
+        ccache -z
+
         source /etc/profile.d/rocm.sh
         hipcc --version
         which clang
@@ -59,6 +75,7 @@ jobs:
           -DCMAKE_VERBOSE_MAKEFILE=ON                   \
           -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \
           -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -DERF_DIM:STRING=3 \
           -DERF_ENABLE_MPI:BOOL=ON \
           -DERF_ENABLE_HIP:BOOL=ON \
@@ -81,6 +98,9 @@ jobs:
         # make -j ${{env.NPROCS}};
         make -j 2;
 
+        ccache -s
+        du -hs ~/.cache/ccache
+
     # - name: Regression Tests
     #   run: |
     #     ctest -L regression -VV
@@ -119,3 +139,18 @@ jobs:
         name: build-and-test
         path: |
           ${{runner.workspace}}/ERF/regressioncov.xml
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -19,13 +19,22 @@ jobs:
     - name: Install Dependencies
       run: Submodules/AMReX/.github/workflows/dependencies/dependencies_mac.sh
 
+    - name: Set Up Cache
+      uses: actions/cache@v4
+      with:
+        path: /Users/runner/Library/Caches/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+
     - name: Configure Project and Generate Build System
       run: |
         cmake \
           -B${{runner.workspace}}/ERF/build \
           -DBUILD_SHARED_LIBS:BOOL=TRUE \
           -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
           -DCMAKE_BUILD_TYPE:STRING=Debug \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -DERF_DIM:STRING=3 \
           -DERF_ENABLE_MPI:BOOL=TRUE \
           -DERF_ENABLE_TESTS:BOOL=TRUE \
@@ -35,9 +44,31 @@ jobs:
 
     - name: Compile and Link
       run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=250M
+        ccache -z
+
         cmake --build ${{runner.workspace}}/ERF/build --parallel 2 --verbose
 
+        ccache -s
+
     - name: CMake Tests # see file ERF/Tests/CTestList.cmake
       run: |
         ctest -L regression -VV
       working-directory: ${{runner.workspace}}/ERF/build
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1