diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1397dbc1..05a988b47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,11 +137,27 @@ jobs: echo "IGNDELAY_WORKING_DIRECTORY=${{github.workspace}}/PelePhysics-${{matrix.comp}}/Testing/Exec/IgnitionDelay" >> $GITHUB_ENV echo "JAC_WORKING_DIRECTORY=${{github.workspace}}/PelePhysics-${{matrix.comp}}/Testing/Exec/Jacobian" >> $GITHUB_ENV echo "NPROCS=$(nproc)" >> $GITHUB_ENV + echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=5" >> $GITHUB_ENV + echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV if [ "${{matrix.comp}}" == 'cuda' ]; then \ echo "CUDA_HOME=/usr/local/cuda-11.2" >> $GITHUB_ENV; \ echo "LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV; \ echo "/usr/local/cuda-11.2/bin" >> $GITHUB_PATH; \ fi + - name: Install Ccache + run: | + wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz + tar xvf ccache-4.8-linux-x86_64.tar.xz + sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/ + - name: Set Up Ccache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.comp}}-git-${{github.sha}} + restore-keys: | + ccache-${{github.workflow}}-${{github.job}}-${{matrix.comp}}-git- - name: Dependencies working-directory: ${{env.TRANSPORT_WORKING_DIRECTORY}} run: | @@ -174,39 +190,52 @@ jobs: echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json" if [ "${{matrix.comp}}" == 'hip' ]; then source /etc/profile.d/rocm.sh; fi; if [ "${{matrix.comp}}" == 'sycl' ]; then source /opt/intel/oneapi/setvars.sh || true; fi; + ccache -z for TYPE in Constant Simple Sutherland; do \ printf "\n-------- ${TYPE} --------\n"; \ - make -j ${{env.NPROCS}} Transport_Model=${TYPE} TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}; \ + make -j ${{env.NPROCS}} Transport_Model=${TYPE} TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}; \ if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \ ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.2d_${TYPE}; \ fi; \ make realclean; \ if [ $? -ne 0 ]; then exit 1; fi; \ done + - name: Transport ccache report + working-directory: ${{env.TRANSPORT_WORKING_DIRECTORY}} + run: | + ccache -s + du -hs ${HOME}/.cache/ccache - name: Test EOS working-directory: ${{env.EOS_WORKING_DIRECTORY}} run: | echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json" if [ "${{matrix.comp}}" == 'hip' ]; then source /etc/profile.d/rocm.sh; fi; if [ "${{matrix.comp}}" == 'sycl' ]; then source /opt/intel/oneapi/setvars.sh || true; fi; + ccache -z for TYPE in Fuego GammaLaw Soave-Redlich-Kwong; do \ if [ "${TYPE}" == 'Fuego' ] || [ "${TYPE}" == 'Soave-Redlich-Kwong' ]; then CHEMISTRY=LiDryer; else CHEMISTRY=Null; fi; \ if [ "${TYPE}" == 'Soave-Redlich-Kwong' ]; then TRANSPORT=Simple; else TRANSPORT=Constant; fi; \ printf "\n-------- ${TYPE} --------\n"; \ - make -j ${{env.NPROCS}} Eos_Model=${TYPE} Chemistry_Model=${CHEMISTRY} Transport_Model=${TRANSPORT} TINY_PROFILE=TRUE ${{matrix.amrex_build_args}}; \ + make -j ${{env.NPROCS}} Eos_Model=${TYPE} Chemistry_Model=${CHEMISTRY} Transport_Model=${TRANSPORT} TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}}; \ if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \ ./Pele2d.${{matrix.comp}}.TPROF.ex inputs.2d; \ fi; \ make realclean; \ if [ $? -ne 0 ]; then exit 1; fi; \ done + - name: EOS ccache report + working-directory: ${{env.EOS_WORKING_DIRECTORY}} + run: | + ccache -s + du -hs ${HOME}/.cache/ccache - name: Test Integration working-directory: ${{env.REACT_WORKING_DIRECTORY}} run: | echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json" if [ "${{matrix.comp}}" == 'hip' ]; then source /etc/profile.d/rocm.sh; fi; if [ "${{matrix.comp}}" == 'sycl' ]; then source /opt/intel/oneapi/setvars.sh || true; fi; - make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=drm19 TINY_PROFILE=TRUE ${{matrix.amrex_build_args}} + ccache -z + make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=drm19 TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}} if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \ for TYPE in Cvode Arkode RK64; do \ printf "\n-------- ${TYPE} --------\n"; \ @@ -216,7 +245,7 @@ jobs: done \ fi; make realclean - make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=LiDryer TINY_PROFILE=TRUE ${{matrix.amrex_build_args}} + make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=LiDryer TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}} if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \ for TYPE in Cvode Arkode RK64; do \ printf "\n-------- ${TYPE} --------\n"; \ @@ -226,6 +255,11 @@ jobs: done \ fi make realclean + - name: Integration ccache report + working-directory: ${{env.REACT_WORKING_DIRECTORY}} + run: | + ccache -s + du -hs ${HOME}/.cache/ccache - name: Test Ignition delay working-directory: ${{env.IGNDELAY_WORKING_DIRECTORY}} run: | @@ -233,23 +267,35 @@ jobs: if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \ python -m pip install --upgrade pip pip install numpy - make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu TINY_PROFILE=TRUE ${{matrix.amrex_build_args}} + ccache -z + make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}} bash exec_ignDelay.sh python check_ignDelay.py if [ $? -ne 0 ]; then exit 1; fi; \ fi; make realclean + - name: Ignition delay ccache report + working-directory: ${{env.IGNDELAY_WORKING_DIRECTORY}} + run: | + ccache -s + du -hs ${HOME}/.cache/ccache - name: Test Jacobian working-directory: ${{env.JAC_WORKING_DIRECTORY}} run: | echo "::add-matcher::${{github.workspace}}/PelePhysics-${{matrix.comp}}/.github/problem-matchers/gcc.json" if [ "${{matrix.comp}}" == 'gnu' ] || [ "${{matrix.comp}}" == 'llvm' ]; then \ + ccache -z cd ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/ceptr poetry run qssa -f ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/Mechanism/Models/dodecane_lu_qss/skeletal.yaml -n ${{github.workspace}}/Support/Mechanism/Models/dodecane_lu_qss/non_qssa_list.yaml poetry run convert -f ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/Mechanism/Models/dodecane_lu_qss/qssa.yaml --qss_format_input ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Support/Mechanism/Models/dodecane_lu_qss/qssa_input_non_optimized.toml --qss_symbolic_jacobian cd ${{env.JAC_WORKING_DIRECTORY}} - make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu_qss DEBUG=TRUE TINY_PROFILE=TRUE ${{matrix.amrex_build_args}} + make -j ${{env.NPROCS}} Eos_Model=Fuego Chemistry_Model=dodecane_lu_qss DEBUG=TRUE TINY_PROFILE=TRUE USE_CCACHE=TRUE ${{matrix.amrex_build_args}} ./Pele3d.${{matrix.comp}}.DEBUG.TPROF.ex; \ if [ $? -ne 0 ]; then exit 1; fi; \ fi; make realclean + - name: Jacobian ccache report + working-directory: ${{env.JAC_WORKING_DIRECTORY}} + run: | + ccache -s + du -hs ${HOME}/.cache/ccache diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml new file mode 100644 index 000000000..a43b89ee8 --- /dev/null +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -0,0 +1,37 @@ +name: CleanUpCachePostPR + +on: + workflow_run: + workflows: [PostPR] + types: + - completed + +jobs: + CleanUpCcacheCachePostPR: + name: Clean Up Ccahe Cache Post PR + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Clean up ccahe + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) + for k in $keys + do + gh actions-cache delete $k -R $REPO -B $BRANCH --confirm + done diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml new file mode 100644 index 000000000..1fe423c35 --- /dev/null +++ b/.github/workflows/cleanup-cache.yml @@ -0,0 +1,60 @@ +name: CleanUpCache + +on: + workflow_run: + workflows: [PelePhysics-CI] + types: + - completed + +jobs: + CleanUpCcacheCache: + name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Clean up ccahe + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + # push or pull_request or schedule or ... + EVENT=${{ github.event.workflow_run.event }} + + # Triggering workflow run name (e.g., LinuxClang) + WORKFLOW_NAME=${{ github.event.workflow_run.name }} + + if [[ $EVENT == "pull_request" ]]; then + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + else + BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} + fi + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + # In our cache keys, substring after `-git-` is git hash, substring + # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`). + # The goal is to keep the last used key of each job and delete all others. + + # something like ccache-LinuxClang- + keyprefix=ccache-${WORKFLOW_NAME}- + + cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key $keyprefix | awk -F '-git-' '{print $1}' | sort | uniq) + + # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". + for j in $cached_jobs + do + old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key ${j}-git- --sort last-used | cut -f 1 | tail -n +2) + for k in $old_keys + do + gh actions-cache delete $k -R $REPO -B $BRANCH --confirm + done + done diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml new file mode 100644 index 000000000..f5b914033 --- /dev/null +++ b/.github/workflows/post-pr.yml @@ -0,0 +1,20 @@ +name: PostPR +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1