Skip to content

Commit

Permalink
Merge branch 'development' into xyuan/rad_develop
Browse files Browse the repository at this point in the history
  • Loading branch information
xyuan authored Jan 4, 2024
2 parents ba8ae60 + aa250d0 commit 092344e
Show file tree
Hide file tree
Showing 300 changed files with 11,440 additions and 6,691 deletions.
43 changes: 0 additions & 43 deletions .github/workflows/cuda-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,46 +56,3 @@ jobs:
-DERF_ENABLE_MPI:BOOL=ON \
-DERF_ENABLE_CUDA:BOOL=ON .
cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc)
cuda-moisture-build:
runs-on: ubuntu-20.04
name: (MOISTURE ON) CUDA v${{matrix.cuda_ver}}
strategy:
matrix:
cuda_pkg: [11-0]
include:
- cuda_ver: "11.0"
cuda_pkg: 11-0
cuda_extra: libcurand-dev-11-0 cuda-cupti-dev-11-0 libcusolver-dev-11-0 libcublas-dev-11-0 libcusparse-dev-11-0
steps:
- name: Cancel previous runs
uses: styfle/[email protected]
with:
access_token: ${{github.token}}
- uses: actions/checkout@v3
with:
submodules: true
- name: Prepare CUDA environment
run: |
export DEBIAN_FRONTEND=noninteractive
sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64/7fa2af80.pub
echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" | sudo tee /etc/apt/sources.list.d/cuda.list
echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64 /" | sudo tee /etc/apt/sources.list.d/nvidia-ml.list
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libopenmpi-dev cuda-command-line-tools-${{matrix.cuda_pkg}} cuda-compiler-${{matrix.cuda_pkg}} cuda-minimal-build-${{matrix.cuda_pkg}} cuda-nvml-dev-${{matrix.cuda_pkg}} cuda-nvtx-${{matrix.cuda_pkg}} ${{matrix.cuda_extra}}
- name: Configure and build
run: |
export PATH=/usr/local/nvidia/bin:/usr/local/cuda-${{matrix.cuda_ver}}/bin:${PATH}
export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-${{matrix.cuda_ver}}/lib:${LD_LIBRARY_PATH}
cmake -Bbuild-${{matrix.cuda_pkg}} \
-DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
-DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
-DAMReX_CUDA_ERROR_CAPTURE_THIS=ON \
-DERF_DIM:STRING=3 \
-DERF_ENABLE_MPI:BOOL=OFF \
-DERF_ENABLE_MOISTURE:BOOL=ON \
-DERF_ENABLE_CUDA:BOOL=ON .
cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc)
3 changes: 2 additions & 1 deletion .github/workflows/dependencies/dependencies_hip.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ sudo apt-get install -y --no-install-recommends \
rocprim-dev \
rocfft \
rocprim \
rocrand
rocrand \
hiprand-dev

# activate
#
Expand Down
102 changes: 0 additions & 102 deletions .github/workflows/hip.yml
Original file line number Diff line number Diff line change
Expand Up @@ -119,105 +119,3 @@ jobs:
name: build-and-test
path: |
${{runner.workspace}}/ERF/regressioncov.xml
Build-And-Test-HIP-Moisture:
name: (MOISTURE ON) HIP ROCm [email protected] C++17 [tests]
runs-on: ubuntu-20.04
# Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
# Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
# amrex/Src/Base/AMReX_GpuLaunchGlobal.H:15:5: error: must specify at least one argument for '...' parameter of variadic macro [-Werror,-Wgnu-zero-variadic-macro-arguments]
# __launch_bounds__(amrex_launch_bounds_max_threads)
# ^
# /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:178:71: note: expanded from macro '__launch_bounds__'
# select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__)
# ^
# /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:176:9: note: macro 'select_impl_' defined here
# #define select_impl_(_1, _2, impl_, ...) impl_
# NOTE: -Werror was removed because ERF specifically had a lot of warnings. It will be a separate task to go through and fix them all...
env: {CXXFLAGS: "-Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments -Wno-pass-failed"}
steps:
- uses: actions/checkout@v3
with:
submodules: true

- name: Dependencies
run: .github/workflows/dependencies/dependencies_hip.sh

- name: Build & Install
run: |
source /etc/profile.d/rocm.sh
hipcc --version
which clang
which clang++
# "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3
# https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
# https://github.com/open-mpi/ompi/issues/9317
export LDFLAGS="-lopen-pal"
cmake \
-B${{runner.workspace}}/ERF/build-${{matrix.os}} \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \
-DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
-DERF_DIM:STRING=3 \
-DERF_ENABLE_MPI:BOOL=ON \
-DERF_ENABLE_HIP:BOOL=ON \
-DAMReX_AMD_ARCH=gfx908 \
-DERF_ENABLE_TESTS:BOOL=ON \
-DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
-DERF_ENABLE_FCOMPARE:BOOL=ON \
-DERF_ENABLE_MOISTURE:BOOL=ON \
-DCODECOV:BOOL=ON \
-DCMAKE_C_COMPILER=$(which clang) \
-DCMAKE_CXX_COMPILER=$(which clang++) \
-DCMAKE_CXX_STANDARD=17 \
${{github.workspace}};
# ${{matrix.mpipreflags}} \
# for some reason this cmake command fails to build the code,
# and we need to use the make command that follows instead ...
# cmake --build ${{runner.workspace}}/ERF/build-${{matrix.os}} --parallel ${{env.NPROCS}};
pushd ${{runner.workspace}}/ERF/build-${{matrix.os}};
# make -j ${{env.NPROCS}};
make -j 2;
# - name: Regression Tests
# run: |
# ctest -L regression -VV
# working-directory: ${{runner.workspace}}/ERF/build-${{matrix.os}}

# - name: Generate coverage report
# working-directory: ${{runner.workspace}}/ERF/build-${{matrix.os}}
# run: |
# find . -type f -name '*.gcno' -path "**Source**" -exec gcov -pb {} +
# cd ..
# gcovr -g -k -r . --xml regressioncov.xml # -v

# - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v3
# with:
# dry_run: false
# # token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
# files: ./regressioncov.xml # optional
# flags: regtests # optional
# # name: codecov-umbrella # optional
# fail_ci_if_error: true # optional (default = false)
# verbose: true # optional (default = false)
# directory: ${{runner.workspace}}/ERF

- name: Success artifacts
uses: actions/upload-artifact@v3
if: success()
with:
name: build-and-test
path: |
${{runner.workspace}}/ERF/regressioncov.xml
- name: Failing test artifacts
uses: actions/upload-artifact@v3
if: failure()
with:
name: build-and-test
path: |
${{runner.workspace}}/ERF/regressioncov.xml
18 changes: 0 additions & 18 deletions Build/cmake_with_moisture.sh

This file was deleted.

18 changes: 0 additions & 18 deletions Build/cmake_with_warm_no_precip.sh

This file was deleted.

44 changes: 34 additions & 10 deletions CHANGES
Original file line number Diff line number Diff line change
@@ -1,18 +1,42 @@
#23.11
# 24.01
-- AMReX submodule set to 24.01 release hash (a068330)

-- Addition of more moisture vars (#1350)

-- AMReX submodule set to 23.11 release hash (ae7b64b)
-- Make PBL model terrain aware (#1348)

-- Bug fix with vertical grid stretching and MOST (#1275)
-- Moisture is no longer a compile time option (#1334)

-- Bug fix with negative K_turb below surface and MYNN2.5 PBL model (#1271)
-- Radiation cleanup (#1318)

-- Correction to equation of state with moisture (#1263)
-- Enable multiple moisture models (#1303)

# 23.12
-- AMReX submodule set to 23.12 release hash (9b733ec)

-- Generalized multi-level sponge zones. This allows for nudging of fine solution
towards coarse solution with complex box arrays. (#1233)
-- Inclusion of Kessler microphysics option (#1324)

# 23.10
-- Fix for particles with redistribute call (#1314)

-- Inclusion of radiation (#1311)

-- Inclusion of refluxing for two-way coupling (#1289)

-- Metgrid initialization with SST and LandMask (#1280)

# 23.11
-- AMReX submodule set to 23.11 release hash (ae7b64b)

-- First release since JOSS paper. Too many key ERF PRs to summarize.
-- Bug fix with vertical grid stretching and MOST (#1275)

-- Bug fix with negative K_turb below surface and MYNN2.5 PBL model (#1271)

-- Correction to equation of state with moisture (#1263)

-- Generalized multi-level sponge zones. This allows for nudging of fine solution
towards coarse solution with complex box arrays. (#1233)

# 23.10
-- First release since JOSS paper. Too many key ERF PRs to summarize.

-- AMReX submodule set to 23.10 release hash (388738d)
-- AMReX submodule set to 23.10 release hash (388738d)
43 changes: 21 additions & 22 deletions CMake/BuildERFExe.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@ function(build_erf_lib erf_lib_name)
include(${CMAKE_SOURCE_DIR}/CMake/SetERFCompileFlags.cmake)
set_erf_compile_flags(${erf_lib_name})

if(ERF_ENABLE_MOISTURE)
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MOISTURE)
endif()
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MOISTURE)

if(ERF_ENABLE_MULTIBLOCK)
target_sources(${erf_lib_name} PRIVATE
Expand Down Expand Up @@ -57,18 +55,6 @@ function(build_erf_lib erf_lib_name)
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_NETCDF)
endif()

if(ERF_ENABLE_MOISTURE)
target_sources(${erf_lib_name} PRIVATE
${SRC_DIR}/Microphysics/SAM/Init.cpp
${SRC_DIR}/Microphysics/SAM/Cloud.cpp
${SRC_DIR}/Microphysics/SAM/IceFall.cpp
${SRC_DIR}/Microphysics/SAM/Precip.cpp
${SRC_DIR}/Microphysics/SAM/PrecipFall.cpp
${SRC_DIR}/Microphysics/SAM/Diagnose.cpp
${SRC_DIR}/Microphysics/SAM/Update.cpp)
target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MOISTURE)
endif()

if(ERF_ENABLE_RRTMGP)
target_sources(${erf_lib_name} PRIVATE
${SRC_DIR}/Radiation/Init_rrtmgp.cpp
Expand Down Expand Up @@ -150,7 +136,6 @@ function(build_erf_lib erf_lib_name)
${SRC_DIR}/TimeIntegration/ERF_advance_microphysics.cpp
${SRC_DIR}/TimeIntegration/ERF_advance_radiation.cpp
${SRC_DIR}/TimeIntegration/ERF_make_buoyancy.cpp
${SRC_DIR}/TimeIntegration/ERF_make_condensation_source.cpp
${SRC_DIR}/TimeIntegration/ERF_make_fast_coeffs.cpp
${SRC_DIR}/TimeIntegration/ERF_slow_rhs_pre.cpp
${SRC_DIR}/TimeIntegration/ERF_ApplySpongeZoneBCs.cpp
Expand All @@ -162,6 +147,21 @@ function(build_erf_lib erf_lib_name)
${SRC_DIR}/Utils/TerrainMetrics.cpp
${SRC_DIR}/Utils/VelocityToMomentum.cpp
${SRC_DIR}/Utils/InteriorGhostCells.cpp
${SRC_DIR}/Microphysics/SAM/Init_SAM.cpp
${SRC_DIR}/Microphysics/SAM/Cloud_SAM.cpp
${SRC_DIR}/Microphysics/SAM/IceFall.cpp
${SRC_DIR}/Microphysics/SAM/Precip.cpp
${SRC_DIR}/Microphysics/SAM/PrecipFall.cpp
${SRC_DIR}/Microphysics/SAM/Diagnose_SAM.cpp
${SRC_DIR}/Microphysics/SAM/Update_SAM.cpp
${SRC_DIR}/Microphysics/Kessler/Init_Kessler.cpp
${SRC_DIR}/Microphysics/Kessler/Kessler.cpp
${SRC_DIR}/Microphysics/Kessler/Diagnose_Kessler.cpp
${SRC_DIR}/Microphysics/Kessler/Update_Kessler.cpp
${SRC_DIR}/Microphysics/FastEddy/Init_FE.cpp
${SRC_DIR}/Microphysics/FastEddy/FastEddy.cpp
${SRC_DIR}/Microphysics/FastEddy/Diagnose_FE.cpp
${SRC_DIR}/Microphysics/FastEddy/Update_FE.cpp
)

if(NOT "${erf_exe_name}" STREQUAL "erf_unit_tests")
Expand All @@ -183,12 +183,6 @@ function(build_erf_lib erf_lib_name)
endif()
endif()

if(ERF_ENABLE_MOISTURE)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Null)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/SAM)
endif()

if(ERF_ENABLE_RRTMGP)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Radiation)
endif()
Expand All @@ -208,6 +202,11 @@ function(build_erf_lib erf_lib_name)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/TimeIntegration)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Utils)
target_include_directories(${erf_lib_name} PUBLIC ${CMAKE_BINARY_DIR})
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Null)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/SAM)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Kessler)
target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/FastEddy)

if(ERF_ENABLE_RRTMGP)
target_link_libraries(${erf_lib_name} PUBLIC yakl)
Expand Down
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ option(ERF_ENABLE_HDF5 "Enable HDF5 IO" ${ERF_ENABLE_NETCDF})
option(ERF_ENABLE_FCOMPARE "Enable building fcompare when not testing" OFF)
set(ERF_PRECISION "DOUBLE" CACHE STRING "Floating point precision SINGLE or DOUBLE")

option(ERF_ENABLE_MOISTURE "Enable Full Moisture" OFF)
option(ERF_ENABLE_MOISTURE "Enable Full Moisture" ON)
option(ERF_ENABLE_WARM_NO_PRECIP "Enable Warm Moisture" OFF)
option(ERF_ENABLE_RRTMGP "Enable RTE-RRTMGP Radiation" OFF)

Expand Down
Loading

0 comments on commit 092344e

Please sign in to comment.