diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47958f42e..0ff6de6c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ concurrency: jobs: Formatting: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -26,7 +26,7 @@ jobs: clangFormatVersion: 18 CPU-GNUmake: needs: Formatting - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Clone uses: actions/checkout@v4 @@ -72,7 +72,7 @@ jobs: - name: PMF debug report working-directory: ./Exec/RegTests/PMF run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt @@ -97,7 +97,7 @@ jobs: - name: EB debug report working-directory: ./Exec/RegTests/EB-C10 run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt @@ -122,7 +122,7 @@ jobs: - name: EB OMP debug report working-directory: ./Exec/RegTests/EB-C10 run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt @@ -147,7 +147,7 @@ jobs: - name: TG debug single precision report working-directory: ./Exec/RegTests/TG run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt @@ -169,9 +169,9 @@ jobs: - name: Soot debug report working-directory: ./Exec/RegTests/Soot-Flame run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ - | egrep -v "Submodules/PelePhysics/Mechanisms" | sort | uniq \ + | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt cat warnings.txt export return=$(tail -n 1 warnings.txt | awk '{print $2}') @@ -191,9 +191,9 @@ jobs: - name: Spray debug report working-directory: ./Exec/RegTests/Spray-Conv run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ - | egrep -v "Submodules/PelePhysics/Mechanisms" | sort | uniq \ + | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt cat warnings.txt export return=$(tail -n 1 warnings.txt | awk '{print $2}') @@ -203,7 +203,7 @@ jobs: runs-on: ${{matrix.os}} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-22.04, macos-latest] build_type: [Release, Debug] include: - os: macos-latest @@ -211,7 +211,7 @@ jobs: comp: llvm procs: $(sysctl -n hw.ncpu) ccache_cache: /Users/runner/Library/Caches/ccache - - os: ubuntu-latest + - os: ubuntu-22.04 install_deps: sudo apt-get update && sudo apt-get install mpich libmpich-dev libtool-bin comp: gnu procs: $(nproc) @@ -320,7 +320,7 @@ jobs: - name: Report run: | echo "::add-matcher::.github/problem-matchers/gcc.json" - egrep "warning:|error:" ${{runner.workspace}}/build-output.txt \ + egrep "Warning:|Error:|warning:|error:" ${{runner.workspace}}/build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials" \ | egrep -v "ld: warning:" | egrep -v "lto-wrapper: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > ${{runner.workspace}}/build-output-warnings.txt @@ -338,13 +338,7 @@ jobs: GPU-Nvidia: name: GPU-CUDA needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-20.04 - strategy: - matrix: - cuda_ver: [11.4] - include: - - cuda_ver: "11.4" - cuda_pkg: 11-4 + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -360,16 +354,15 @@ jobs: - name: Dependencies run: | ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh - curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb - sudo dpkg -i cuda-keyring_1.0-1_all.deb + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo apt-get update - sudo apt-get install -y \ - cuda-command-line-tools-${{matrix.cuda_pkg}} \ - cuda-compiler-${{matrix.cuda_pkg}} cuda-minimal-build-${{matrix.cuda_pkg}} \ - cuda-nvml-dev-${{matrix.cuda_pkg}} cuda-nvtx-${{matrix.cuda_pkg}} \ - libcurand-dev-${{matrix.cuda_pkg}} cuda-cupti-dev-${{matrix.cuda_pkg}} \ - libcusolver-dev-${{matrix.cuda_pkg}} libcusparse-dev-${{matrix.cuda_pkg}} \ - libcublas-dev-${{matrix.cuda_pkg}} + sudo apt-get install -y cuda-command-line-tools-12-6 \ + cuda-compiler-12-6 cuda-minimal-build-12-6 \ + cuda-nvml-dev-12-6 cuda-nvtx-12-6 \ + libcurand-dev-12-6 cuda-cupti-dev-12-6 \ + libcusolver-dev-12-6 libcusparse-dev-12-6 \ + libcublas-dev-12-6 libcurand-dev-12-6 libnvjitlink-12-6 - name: Install Ccache run: | wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz @@ -384,23 +377,23 @@ jobs: ccache-${{github.workflow}}-${{github.job}}-git- - name: Configure run: | - export PATH=/usr/local/nvidia/bin:/usr/local/cuda-${{matrix.cuda_ver}}/bin:${PATH} - export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-${{matrix.cuda_ver}}/lib:${LD_LIBRARY_PATH} - cmake -Bbuild-${{matrix.cuda_pkg}} \ + export PATH=/usr/local/nvidia/bin:/usr/local/cuda-12.6/bin:${PATH} + export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-12.6/lib:${LD_LIBRARY_PATH} + cmake -Bbuild-cuda \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DPELE_ENABLE_MPI:BOOL=OFF \ -DPELE_ENABLE_CUDA:BOOL=ON \ -DPELE_EXCLUDE_BUILD_IN_CI:BOOL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS:BOOL=ON \ - -DCMAKE_CUDA_ARCHITECTURES:STRING=70 \ + -DCMAKE_CUDA_ARCHITECTURES:STRING=80 \ -DCMAKE_CUDA_COMPILER_LAUNCHER:STRING=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \ ${{github.workspace}} - name: Build run: | ccache -z - cmake --build build-${{matrix.cuda_pkg}} --parallel ${{env.NPROCS}} + cmake --build build-cuda --parallel ${{env.NPROCS}} - name: Ccache Report run: | ccache -s @@ -408,7 +401,9 @@ jobs: GPU-AMD: name: GPU-HIP needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-20.04 + # AMReX uses std::clamp in EB and it breaks newer ROCm or ubuntu releases due to + # error: reference to __host__ function '__glibcxx_assert_fail' in __host__ __device__ function + runs-on: ubuntu-22.04 steps: - name: Clone uses: actions/checkout@v4 @@ -424,14 +419,13 @@ jobs: - name: Dependencies run: | ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh - curl -O https://repo.radeon.com/rocm/rocm.gpg.key - sudo apt-key add rocm.gpg.key - echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/5.6.1 ubuntu main' \ - | sudo tee /etc/apt/sources.list.d/rocm.list - echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \ - | sudo tee -a /etc/profile.d/rocm.sh + sudo mkdir --parents --mode=0755 /etc/apt/keyrings + wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.2.4 jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list + echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600 + echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh sudo apt-get update - sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev + sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev hiprand-dev - name: Install Ccache run: | wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz @@ -448,13 +442,15 @@ jobs: run: | source /etc/profile.d/rocm.sh which clang + which clang++ which hipcc + hipcc --version + hipconfig --full cmake -B${{runner.workspace}}/build-hip \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ - -DCMAKE_CXX_COMPILER:STRING=$(which hipcc) \ + -DCMAKE_CXX_COMPILER:STRING=$(which clang++) \ -DCMAKE_C_COMPILER:STRING=$(which clang) \ - -DCMAKE_CXX_STANDARD:STRING=17 \ -DPELE_ENABLE_MPI:BOOL=OFF \ -DPELE_ENABLE_HIP:BOOL=ON \ -DPELE_EXCLUDE_BUILD_IN_CI:BOOL=ON \ @@ -480,7 +476,7 @@ jobs: GPU-Intel: name: GPU-SYCL needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -499,8 +495,7 @@ jobs: ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - echo "deb https://apt.repos.intel.com/oneapi all main" \ - | sudo tee /etc/apt/sources.list.d/oneAPI.list + echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 intel-oneapi-mkl-devel=2024.2.2-15 - name: Install Ccache @@ -581,7 +576,7 @@ jobs: # exit ${return} Lint-clang-tidy: needs: Formatting - runs-on: macos-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -589,25 +584,30 @@ jobs: submodules: 'recursive' - name: Setup run: | - echo "NPROCS=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV + echo "NPROCS=$(nproc)" >> $GITHUB_ENV echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=500M" >> $GITHUB_ENV - - name: Dependencies + echo "CCACHE_EXTRAFILES=${{github.workspace}}/.clang-tidy" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=50M" >> $GITHUB_ENV + echo "CTCACHE_DIR=~/.cache/ctcache" >> $GITHUB_ENV + - name: Install Ccache run: | - brew install ccache - brew install llvm + wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz + sudo curl https://raw.githubusercontent.com/matus-chochlik/ctcache/7fd516e91c17779cbc6fc18bd119313d9532dd90/clang-tidy-cache -Lo /usr/bin/clang-tidy-cache + tar xvf ccache-4.8-linux-x86_64.tar.xz + sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/ + sudo chmod +x /usr/bin/clang-tidy-cache + mkdir -p ~/.cache/ctcache - name: Set Up Ccache uses: actions/cache@v4 with: - path: /Users/runner/Library/Caches/ccache + path: ~/.cache key: ccache-${{github.workflow}}-${{github.job}}-git-${{github.sha}} restore-keys: | ccache-${{github.workflow}}-${{github.job}}-git- - name: Configure run: | - export PATH=$(brew --prefix llvm)/bin:${PATH} cmake -B${{runner.workspace}}/build-clang-tidy \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ @@ -623,16 +623,19 @@ jobs: - name: Check working-directory: ${{runner.workspace}}/build-clang-tidy run: | - export PATH=$(brew --prefix llvm)/bin:${PATH} cmake --build . --parallel ${{env.NPROCS}} 2>&1 | tee -a clang-tidy-full-report.txt - egrep "warning:|error:" clang-tidy-full-report.txt \ + egrep "Warning:|Error:|warning:|error:" clang-tidy-full-report.txt \ | egrep -v "Submodules/amrex|Submodules/sundials" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > clang-tidy-warnings.txt - name: Ccache Report run: | + ls ~/.cache + ls ~/.cache/ccache + du -hs ~/.cache/ccache + ls ~/.cache/ctcache + du -hs ~/.cache/ctcache ccache -s - du -hs /Users/runner/Library/Caches/ccache - name: Full report working-directory: ${{runner.workspace}}/build-clang-tidy run: cat clang-tidy-full-report.txt @@ -645,7 +648,7 @@ jobs: exit ${return} Lint-codeql: needs: Formatting - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -693,7 +696,7 @@ jobs: sarif_file: sarif-results/cpp.sarif Lint-codespell: needs: Formatting - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -705,14 +708,13 @@ jobs: python-version: '3.12' - name: Dependencies run: | - # Install Python packages python -m pip install --upgrade pip pip install codespell - name: Run codespell run: codespell Save-PR-Number: if: github.event_name == 'pull_request' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Save PR number env: diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index 5dc32a685..6a30a57d9 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -9,7 +9,7 @@ on: jobs: CleanUpCcacheCachePostPR: name: Clean Up Ccahe Cache Post PR - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: actions: write contents: read diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index eca78d4cb..952d95097 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -9,7 +9,7 @@ on: jobs: CleanUpCcacheCache: name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: actions: write contents: read diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6d0dcff45..01162d1cc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,7 +16,7 @@ on: jobs: Docs: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml index 2768ef376..8dbeb26f1 100644 --- a/.github/workflows/post-pr.yml +++ b/.github/workflows/post-pr.yml @@ -6,7 +6,7 @@ on: jobs: cleanup: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Save PR number env: diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 0106d18e3..769f6b00a 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -56,7 +56,13 @@ macro(setup_test) file(COPY ${TEST_FILES} DESTINATION "${CURRENT_TEST_BINARY_DIR}/") # Set some default runtime options for all tests - set(RUNTIME_OPTIONS "amr.plot_file=plt amr.checkpoint_files_output=0 amr.plot_files_output=1 amrex.the_arena_is_managed=0 amrex.abort_on_unused_inputs=1") + set(RUNTIME_OPTIONS "amr.checkpoint_files_output=0 amrex.the_arena_is_managed=0") + if(PELE_ENABLE_FCOMPARE_FOR_TESTS) + set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=1 amr.plot_int=10 amr.plot_file=plt") + else() + set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=0") + endif() + if(PELE_ENABLE_FPE_TRAP_FOR_TESTS AND (NOT APPLE)) set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amrex.signal_handling=1 amrex.fpe_trap_invalid=1 amrex.fpe_trap_zero=1 amrex.fpe_trap_overflow=1") else() @@ -64,7 +70,7 @@ macro(setup_test) endif() if(PELE_ENABLE_MPI) if(PELE_ENABLE_CUDA) - set(PELE_NP 2) # 1 rank per GPU on Eagle + set(PELE_NP 2) else() set(PELE_NP 4) endif()