Move away from moisture being a compile-time option ... we no longer …

.github/workflows/cuda-ci.yml

@@ -56,46 +56,3 @@ jobs:
             -DERF_ENABLE_MPI:BOOL=ON \
             -DERF_ENABLE_CUDA:BOOL=ON .
           cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc)
-
-  cuda-moisture-build:
-    runs-on: ubuntu-20.04
-    name: (MOISTURE ON) CUDA v${{matrix.cuda_ver}}
-    strategy:
-      matrix:
-        cuda_pkg: [11-0]
-        include:
-          - cuda_ver: "11.0"
-            cuda_pkg: 11-0
-            cuda_extra: libcurand-dev-11-0 cuda-cupti-dev-11-0 libcusolver-dev-11-0 libcublas-dev-11-0 libcusparse-dev-11-0
-    steps:
-      - name: Cancel previous runs
-        uses: styfle/[email protected]
-        with:
-          access_token: ${{github.token}}
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Prepare CUDA environment
-        run: |
-          export DEBIAN_FRONTEND=noninteractive
-          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
-          sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64/7fa2af80.pub
-          echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" | sudo tee /etc/apt/sources.list.d/cuda.list
-          echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64 /" | sudo tee /etc/apt/sources.list.d/nvidia-ml.list
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-              libopenmpi-dev cuda-command-line-tools-${{matrix.cuda_pkg}} cuda-compiler-${{matrix.cuda_pkg}} cuda-minimal-build-${{matrix.cuda_pkg}} cuda-nvml-dev-${{matrix.cuda_pkg}} cuda-nvtx-${{matrix.cuda_pkg}} ${{matrix.cuda_extra}}
-      - name: Configure and build
-        run: |
-          export PATH=/usr/local/nvidia/bin:/usr/local/cuda-${{matrix.cuda_ver}}/bin:${PATH}
-          export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-${{matrix.cuda_ver}}/lib:${LD_LIBRARY_PATH}
-          cmake -Bbuild-${{matrix.cuda_pkg}} \
-            -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
-            -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
-            -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON           \
-            -DERF_DIM:STRING=3 \
-            -DERF_ENABLE_MPI:BOOL=OFF \
-            -DERF_ENABLE_MOISTURE:BOOL=ON \
-            -DERF_ENABLE_CUDA:BOOL=ON .
-          cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc)
-

.github/workflows/hip.yml

@@ -119,105 +119,3 @@ jobs:
         name: build-and-test
         path: |
           ${{runner.workspace}}/ERF/regressioncov.xml
-
-  Build-And-Test-HIP-Moisture:
-    name: (MOISTURE ON) HIP ROCm [email protected] C++17 [tests]
-    runs-on: ubuntu-20.04
-    # Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
-    # Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
-    #    amrex/Src/Base/AMReX_GpuLaunchGlobal.H:15:5: error: must specify at least one argument for '...' parameter of variadic macro [-Werror,-Wgnu-zero-variadic-macro-arguments]
-    #        __launch_bounds__(amrex_launch_bounds_max_threads)
-    #        ^
-    #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:178:71: note: expanded from macro '__launch_bounds__'
-    #        select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__)
-    #                                                                          ^
-    #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:176:9: note: macro 'select_impl_' defined here
-    #    #define select_impl_(_1, _2, impl_, ...) impl_
-    # NOTE: -Werror was removed because ERF specifically had a lot of warnings. It will be a separate task to go through and fix them all...
-    env: {CXXFLAGS: "-Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments -Wno-pass-failed"}
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-
-    - name: Dependencies
-      run: .github/workflows/dependencies/dependencies_hip.sh
-
-    - name: Build & Install
-      run: |
-        source /etc/profile.d/rocm.sh
-        hipcc --version
-        which clang
-        which clang++
-
-        # "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3
-        #   https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
-        #   https://github.com/open-mpi/ompi/issues/9317
-        export LDFLAGS="-lopen-pal"
-
-        cmake \
-          -B${{runner.workspace}}/ERF/build-${{matrix.os}} \
-          -DCMAKE_VERBOSE_MAKEFILE=ON                   \
-          -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \
-          -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
-          -DERF_DIM:STRING=3 \
-          -DERF_ENABLE_MPI:BOOL=ON \
-          -DERF_ENABLE_HIP:BOOL=ON \
-          -DAMReX_AMD_ARCH=gfx908 \
-          -DERF_ENABLE_TESTS:BOOL=ON \
-          -DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
-          -DERF_ENABLE_FCOMPARE:BOOL=ON \
-          -DERF_ENABLE_MOISTURE:BOOL=ON \
-          -DCODECOV:BOOL=ON \
-          -DCMAKE_C_COMPILER=$(which clang)             \
-          -DCMAKE_CXX_COMPILER=$(which clang++)         \
-          -DCMAKE_CXX_STANDARD=17 \
-          ${{github.workspace}};
-        # ${{matrix.mpipreflags}} \
-
-        # for some reason this cmake command fails to build the code,
-        # and we need to use the make command that follows instead ...
-        # cmake --build ${{runner.workspace}}/ERF/build-${{matrix.os}} --parallel ${{env.NPROCS}};
-
-        pushd ${{runner.workspace}}/ERF/build-${{matrix.os}};
-        # make -j ${{env.NPROCS}};
-        make -j 2;
-
-    # - name: Regression Tests
-    #   run: |
-    #     ctest -L regression -VV
-    #   working-directory: ${{runner.workspace}}/ERF/build-${{matrix.os}}
-
-    # - name: Generate coverage report
-    #   working-directory: ${{runner.workspace}}/ERF/build-${{matrix.os}}
-    #   run: |
-    #     find . -type f -name '*.gcno' -path "**Source**" -exec gcov -pb {} +
-    #     cd ..
-    #     gcovr -g -k -r . --xml regressioncov.xml  # -v
-
-    # - name: Upload coverage to Codecov
-    #   uses: codecov/codecov-action@v3
-    #   with:
-    #     dry_run: false
-    #     # token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-    #     files: ./regressioncov.xml # optional
-    #     flags: regtests # optional
-    #     # name: codecov-umbrella # optional
-    #     fail_ci_if_error: true # optional (default = false)
-    #     verbose: true # optional (default = false)
-    #     directory: ${{runner.workspace}}/ERF
-
-    - name: Success artifacts
-      uses: actions/upload-artifact@v3
-      if: success()
-      with:
-        name: build-and-test
-        path: |
-          ${{runner.workspace}}/ERF/regressioncov.xml
-    - name: Failing test artifacts
-      uses: actions/upload-artifact@v3
-      if: failure()
-      with:
-        name: build-and-test
-        path: |
-          ${{runner.workspace}}/ERF/regressioncov.xml

CMake/BuildERFExe.cmake

@@ -15,9 +15,7 @@ function(build_erf_lib erf_lib_name)
   include(${CMAKE_SOURCE_DIR}/CMake/SetERFCompileFlags.cmake)
   set_erf_compile_flags(${erf_lib_name})
 
-  if(ERF_ENABLE_MOISTURE)
-    target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MOISTURE)
-  endif()
+  target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MOISTURE)
 
   if(ERF_ENABLE_MULTIBLOCK)
     target_sources(${erf_lib_name} PRIVATE
@@ -57,22 +55,6 @@ function(build_erf_lib erf_lib_name)
     target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_NETCDF)
   endif()
 
-  if(ERF_ENABLE_MOISTURE)
-    target_sources(${erf_lib_name} PRIVATE
-       ${SRC_DIR}/Microphysics/SAM/Init.cpp
-       ${SRC_DIR}/Microphysics/SAM/Cloud.cpp
-       ${SRC_DIR}/Microphysics/SAM/IceFall.cpp
-       ${SRC_DIR}/Microphysics/SAM/Precip.cpp
-       ${SRC_DIR}/Microphysics/SAM/PrecipFall.cpp
-       ${SRC_DIR}/Microphysics/SAM/Diagnose.cpp
-       ${SRC_DIR}/Microphysics/SAM/Update.cpp
-       ${SRC_DIR}/Microphysics/Kessler/Init_Kessler.cpp
-       ${SRC_DIR}/Microphysics/Kessler/Kessler.cpp
-       ${SRC_DIR}/Microphysics/Kessler/Diagnose_Kessler.cpp
-       ${SRC_DIR}/Microphysics/Kessler/Update_Kessler.cpp)
-    target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_MOISTURE)
-  endif()
-
   if(ERF_ENABLE_RRTMGP)
     target_sources(${erf_lib_name} PRIVATE
                    ${SRC_DIR}/Radiation/Init_rrtmgp.cpp
@@ -166,6 +148,17 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/Utils/TerrainMetrics.cpp
        ${SRC_DIR}/Utils/VelocityToMomentum.cpp
        ${SRC_DIR}/Utils/InteriorGhostCells.cpp 
+       ${SRC_DIR}/Microphysics/SAM/Init_SAM.cpp
+       ${SRC_DIR}/Microphysics/SAM/Cloud_SAM.cpp
+       ${SRC_DIR}/Microphysics/SAM/IceFall.cpp
+       ${SRC_DIR}/Microphysics/SAM/Precip.cpp
+       ${SRC_DIR}/Microphysics/SAM/PrecipFall.cpp
+       ${SRC_DIR}/Microphysics/SAM/Diagnose_SAM.cpp
+       ${SRC_DIR}/Microphysics/SAM/Update_SAM.cpp
+       ${SRC_DIR}/Microphysics/Kessler/Init_Kessler.cpp
+       ${SRC_DIR}/Microphysics/Kessler/Kessler.cpp
+       ${SRC_DIR}/Microphysics/Kessler/Diagnose_Kessler.cpp
+       ${SRC_DIR}/Microphysics/Kessler/Update_Kessler.cpp
   )
 
   if(NOT "${erf_exe_name}" STREQUAL "erf_unit_tests")
@@ -187,13 +180,6 @@ function(build_erf_lib erf_lib_name)
     endif()
   endif()
 
-  if(ERF_ENABLE_MOISTURE)
-    target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics)
-    target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Null)
-    target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/SAM)
-    target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Kessler)
-  endif()
-
   if(ERF_ENABLE_RRTMGP)
     target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Radiation)
   endif()
@@ -213,6 +199,10 @@ function(build_erf_lib erf_lib_name)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/TimeIntegration)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Utils)
   target_include_directories(${erf_lib_name} PUBLIC ${CMAKE_BINARY_DIR})
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics)
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Null)
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/SAM)
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Kessler)
 
   if(ERF_ENABLE_RRTMGP)
      target_link_libraries(${erf_lib_name} PUBLIC yakl)

CMakeLists.txt

@@ -17,7 +17,7 @@ option(ERF_ENABLE_HDF5 "Enable HDF5 IO" ${ERF_ENABLE_NETCDF})
 option(ERF_ENABLE_FCOMPARE "Enable building fcompare when not testing" OFF)
 set(ERF_PRECISION "DOUBLE" CACHE STRING "Floating point precision SINGLE or DOUBLE")
 
-option(ERF_ENABLE_MOISTURE "Enable Full Moisture" OFF)
+option(ERF_ENABLE_MOISTURE "Enable Full Moisture" ON)
 option(ERF_ENABLE_WARM_NO_PRECIP "Enable Warm Moisture" OFF)
 option(ERF_ENABLE_RRTMGP "Enable RTE-RRTMGP Radiation" OFF)
 

Docs/sphinx_doc/Inputs.rst

@@ -883,14 +883,8 @@ Examples of Usage
 Moisture
 ========
 
-ERF has two different moisture models -- one that includes only water vapor and cloud water,
-and a more complete model that includes water vapor, cloud water, cloud ice, rain, snow and graupel.
-
-If ERF is compiled with ERF_USE_WARM_NO_PRECIP defined, then the first model is used and no
-further inputs are required.
-
-If ERF is compiled with ERF_USE_MOISTURE defined, then the following run-time options control how
-the full moisture model is used.
+ERF has several different moisture models.
+The following run-time options control how the full moisture model is used.
 
 List of Parameters
 ------------------

Docs/sphinx_doc/RegressionTests.rst

@@ -24,7 +24,7 @@ The following problems are currently tested in the CI:
 +-------------------------------+----------+----------+----------+------------+-------+-----------------------+
 | Test                          | nx ny nz | xbc      | ybc      | zbc        | Ext   | Other                 |
 +===============================+==========+==========+==========+============+=======+=======================+
-| Bubble_Density_Current        | 256 4 64 | Symmetry | Periodic | SlipWall   | None  | USE_MOISTURE=TRUE     |
+| Bubble_Density_Current        | 256 4 64 | Symmetry | Periodic | SlipWall   | None  | moist bubble          |
 |                               |          | Outflow  |          | SlipWall   |       |                       |
 +-------------------------------+----------+----------+----------+------------+-------+-----------------------+
 | CouetteFlow                   | 32 4  16 | Periodic | Periodic | SlipWall   | None  | inhomogeneous         |

Docs/sphinx_doc/building.rst

@@ -85,8 +85,6 @@ or if using tcsh,
    +--------------------+------------------------------+------------------+-------------+
    | USE_HDF5           | Whether to enable HDF5       | TRUE / FALSE     | FALSE       |
    +--------------------+------------------------------+------------------+-------------+
-   | USE_MOISTURE       | Whether to enable moisture   | TRUE / FALSE     | FALSE       |
-   +--------------------+------------------------------+------------------+-------------+
    | USE_WARM_NO_PRECIP | Whether to use warm moisture | TRUE / FALSE     | FALSE       |
    +--------------------+------------------------------+------------------+-------------+
    | USE_MULTIBLOCK     | Whether to enable multiblock | TRUE / FALSE     | FALSE       |
@@ -176,8 +174,6 @@ Analogous to GNU Make, the list of cmake directives is as follows:
    +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_HDF5           | Whether to enable HDF5       | TRUE / FALSE     | FALSE       |
    +---------------------------+------------------------------+------------------+-------------+
-   | ERF_ENABLE_MOISTURE       | Whether to enable moisture   | TRUE / FALSE     | FALSE       |
-   +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_WARM_NO_PRECIP | Whether to use warm moisture | TRUE / FALSE     | FALSE       |
    +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_MULTIBLOCK     | Whether to enable multiblock | TRUE / FALSE     | FALSE       |

Exec/ABL/GNUmakefile

@@ -18,9 +18,6 @@ USE_CUDA = FALSE
 USE_HIP  = FALSE
 USE_SYCL = FALSE
 
-# Physics
-USE_MOISTURE = FALSE
-
 # Debugging
 DEBUG = FALSE
 

Exec/ABL/prob.H

@@ -62,14 +62,9 @@ public:
         amrex::Array4<amrex::Real      > const& p_hse,
         amrex::Array4<amrex::Real const> const& z_nd,
         amrex::Array4<amrex::Real const> const& z_cc,
-#if defined(ERF_USE_MOISTURE)
         amrex::Array4<amrex::Real      > const& qv,
         amrex::Array4<amrex::Real      > const& qc,
         amrex::Array4<amrex::Real      > const& qi,
-#elif defined(ERF_USE_WARM_NO_PRECIP)
-        amrex::Array4<amrex::Real      > const& qv,
-        amrex::Array4<amrex::Real      > const& qc,
-#endif
         amrex::GeometryData const& geomdata,
         amrex::Array4<amrex::Real const> const& mf_m,
         amrex::Array4<amrex::Real const> const& mf_u,

Exec/ABL/prob.cpp

@@ -56,14 +56,9 @@ Problem::init_custom_pert(
     amrex::Array4<amrex::Real      > const& /*p_hse*/,
     amrex::Array4<amrex::Real const> const& /*z_nd*/,
     amrex::Array4<amrex::Real const> const& /*z_cc*/,
-#if defined(ERF_USE_MOISTURE)
     amrex::Array4<amrex::Real      > const& /*qv*/,
     amrex::Array4<amrex::Real      > const& /*qc*/,
     amrex::Array4<amrex::Real      > const& /*qi*/,
-#elif defined(ERF_USE_WARM_NO_PRECIP)
-    amrex::Array4<amrex::Real      > const& /*qv*/,
-    amrex::Array4<amrex::Real      > const& /*qc*/,
-#endif
     amrex::GeometryData const& geomdata,
     amrex::Array4<amrex::Real const> const& /*mf_m*/,
     amrex::Array4<amrex::Real const> const& /*mf_u*/,
@@ -98,13 +93,8 @@ Problem::init_custom_pert(
     // Set an initial value for QKE
     state(i, j, k, RhoQKE_comp) = parms.QKE_0;
 
-#if defined(ERF_USE_MOISTURE)
-    state(i, j, k, RhoQt_comp) = 0.0;
-    state(i, j, k, RhoQp_comp) = 0.0;
-#elif defined(ERF_USE_WARM_NO_PRECIP)
-    state(i, j, k, RhoQv_comp) = 0.0;
-    state(i, j, k, RhoQc_comp) = 0.0;
-#endif
+    state(i, j, k, RhoQ1_comp) = 0.0;
+    state(i, j, k, RhoQ2_comp) = 0.0;
   });
 
   // Set the x-velocity

Exec/ABL_input_sounding/prob.H

@@ -61,14 +61,9 @@ public:
         amrex::Array4<amrex::Real      > const& p_hse,
         amrex::Array4<amrex::Real const> const& z_nd,
         amrex::Array4<amrex::Real const> const& z_cc,
-#if defined(ERF_USE_MOISTURE)
         amrex::Array4<amrex::Real      > const& qv,
         amrex::Array4<amrex::Real      > const& qc,
         amrex::Array4<amrex::Real      > const& qi,
-#elif defined(ERF_USE_WARM_NO_PRECIP)
-        amrex::Array4<amrex::Real      > const& qv,
-        amrex::Array4<amrex::Real      > const& qc,
-#endif
         amrex::GeometryData const& geomdata,
         amrex::Array4<amrex::Real const> const& mf_m,
         amrex::Array4<amrex::Real const> const& mf_u,