Merge branch 'development' into metgrid_interpolator

erf-model · Nov 7, 2024 · 747bf6a · 747bf6a
2 parents ef46453 + 91dcdd7
commit 747bf6a
Show file tree

Hide file tree

Showing 56 changed files with 3,307 additions and 319 deletions.
diff --git a/Build/cmake_with_poisson.sh → Build/cmake_with_eb.sh b/Build/cmake_with_poisson.sh → Build/cmake_with_eb.sh
@@ -8,11 +8,12 @@ cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \
       -DCMAKE_Fortran_COMPILER:STRING=mpifort \
       -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
       -DCMAKE_BUILD_TYPE:STRING=Release \
+      -DAMREX_EB:BOOL=ON \
       -DERF_DIM:STRING=3 \
       -DERF_ENABLE_MPI:BOOL=ON \
       -DERF_ENABLE_TESTS:BOOL=ON \
       -DERF_ENABLE_FCOMPARE:BOOL=ON \
       -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \
-      -DERF_ENABLE_POISSON_SOLVE:BOOL=ON \
+      -DERF_ENABLE_EB:BOOL=ON \
       -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
       .. && make -j8
diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake
@@ -25,13 +25,6 @@ function(build_erf_lib erf_lib_name)
     target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_WARM_NO_PRECIP)
   endif()
 
-  if(ERF_ENABLE_POISSON_SOLVE)
-    target_sources(${erf_lib_name} PRIVATE
-                   ${SRC_DIR}/Utils/ERF_PoissonSolve.cpp
-                   ${SRC_DIR}/Utils/ERF_PoissonSolve_tb.cpp)
-    target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_POISSON_SOLVE)
-  endif()
-
   if(ERF_ENABLE_PARTICLES)
     target_sources(${erf_lib_name} PRIVATE
                    ${SRC_DIR}/Particles/ERFPCEvolve.cpp
@@ -190,6 +183,8 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/Utils/ERF_AverageDown.cpp
        ${SRC_DIR}/Utils/ERF_ChopGrids.cpp
        ${SRC_DIR}/Utils/ERF_MomentumToVelocity.cpp
+       ${SRC_DIR}/Utils/ERF_PoissonSolve.cpp
+       ${SRC_DIR}/Utils/ERF_PoissonSolve_tb.cpp
        ${SRC_DIR}/Utils/ERF_TerrainMetrics.cpp
        ${SRC_DIR}/Utils/ERF_VelocityToMomentum.cpp
        ${SRC_DIR}/Utils/ERF_InteriorGhostCells.cpp

diff --git a/CMake/SetAmrexOptions.cmake b/CMake/SetAmrexOptions.cmake
@@ -27,12 +27,9 @@ set(AMReX_ACC OFF)
 set(AMReX_PLOTFILE_TOOLS ${ERF_ENABLE_FCOMPARE})
 set(AMReX_FORTRAN OFF)
 
-set(AMReX_LINEAR_SOLVERS OFF)
-if(ERF_ENABLE_POISSON_SOLVE)
-  set(AMReX_LINEAR_SOLVERS ON)
-  set(AMReX_LINEAR_SOLVERS_EM OFF)
-  set(AMReX_LINEAR_SOLVERS_INCFLO OFF)
-endif()
+set(AMReX_LINEAR_SOLVERS ON)
+set(AMReX_LINEAR_SOLVERS_EM OFF)
+set(AMReX_LINEAR_SOLVERS_INCFLO OFF)
 
 set(AMReX_PARTICLES OFF)
 if(ERF_ENABLE_PARTICLES)

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -24,8 +24,6 @@ option(ERF_ENABLE_MOISTURE "Enable Full Moisture" ON)
 option(ERF_ENABLE_WARM_NO_PRECIP "Enable Warm Moisture" OFF)
 option(ERF_ENABLE_RRTMGP "Enable RTE-RRTMGP Radiation" OFF)
 
-option(ERF_ENABLE_POISSON_SOLVE "Enable Poisson solve for anelastic/incompressible flow" OFF)
-
 #Options for performance
 option(ERF_ENABLE_MPI "Enable MPI" OFF)
 option(ERF_ENABLE_OPENMP "Enable OpenMP" OFF)

diff --git a/Docs/sphinx_doc/Inputs.rst b/Docs/sphinx_doc/Inputs.rst
@@ -34,10 +34,8 @@ Governing Equations
 
 .. note::
 
-   To solve the anelastic equations, you must set USE_POISSON_SOLVE = TRUE if using
-   gmake or ERF_ENABLE_POISSON_SOLVE if using cmake.  This will enable use of the
-   AMReX-based Poisson solver.   To optionally use the FFT solver, you must additionally
-   set USE_FFT = TRUE if using gmake.
+   To use the FFT solver if running with anelastic, you must set USE_FFT = TRUE if using gmake
+   or ERF_ENABLE_FFT if using cmake.
 
 Problem Geometry
 ================
@@ -1162,9 +1160,7 @@ This file has the same format as used by ``ideal.exe`` executable in WRF.
 Using this option for initialization, running ``ideal.exe`` and reading from the resulting ``wrfinput_d01`` file are not needed.
 This option is used for initializing ERF domain to a horizontally homogeneous mesoscale state and does not include terrain or map scale factors.
 
-In addition, there is a run-time option to project the initial velocity field to make it divergence-free.  To take
-advantage of this option, the code must be built with ``USE_POISSON_SOLVE = TRUE`` in the GNUmakefile if using gmake, or with
-``-DERF_ENABLE_POISSON_SOLVE:BOOL=ON`` in the cmake.sh file if using cmake.
+In addition, there is a run-time option to project the initial velocity field to make it divergence-free.
 
 List of Parameters
 ------------------
@@ -1341,9 +1337,6 @@ integrating the hydrostatic equation from the surface.
 
 If **erf.init_type = custom** or **erf.init_type = input_sounding**, ``erf.nc_init_file`` and ``erf.nc_bdy_file`` do not need to be set.
 
-Setting **erf.project_initial_velocity = 1** will have no effect if the code is not built with **ERF_USE_POISSON_SOLVE** defined
-if using cmake or with **USE_POISSON_SOLVE = TRUE** if using gmake.
-
 Map Scale Factors
 =================
 

diff --git a/Docs/sphinx_doc/building.rst b/Docs/sphinx_doc/building.rst
@@ -96,8 +96,6 @@ or if using tcsh,
    +--------------------+------------------------------+------------------+-------------+
    | USE_PARTICLES      | Whether to enable particles  | TRUE / FALSE     | FALSE       |
    +--------------------+------------------------------+------------------+-------------+
-   | USE_POISSON_SOLVE  | Whether to enable anelastic  | TRUE / FALSE     | FALSE       |
-   +--------------------+------------------------------+------------------+-------------+
    | USE_WARM_NO_PRECIP | Whether to use warm moisture | TRUE / FALSE     | FALSE       |
    +--------------------+------------------------------+------------------+-------------+
    | USE_MULTIBLOCK     | Whether to enable multiblock | TRUE / FALSE     | FALSE       |
@@ -113,9 +111,6 @@ or if using tcsh,
    | TRACE_PROFILE      | Include trace profiling info | TRUE / FALSE     | FALSE       |
    +--------------------+------------------------------+------------------+-------------+
 
-   .. note::
-      **To run with the anelastic option, USE_POISSON_SOLVE must be set to TRUE.**
-
    .. note::
       **At most one of USE_OMP, USE_CUDA, USE_HIP, USE_SYCL should be set to true.**
 
@@ -192,8 +187,6 @@ Analogous to GNU Make, the list of cmake directives is as follows:
    +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_PARTICLES      | Whether to enable particles  | TRUE / FALSE     | FALSE       |
    +---------------------------+------------------------------+------------------+-------------+
-   | ERF_ENABLE_POISSON_SOVLE  | Whether to enable anelastic  | TRUE / FALSE     | FALSE       |
-   +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_WARM_NO_PRECIP | Whether to use warm moisture | TRUE / FALSE     | FALSE       |
    +---------------------------+------------------------------+------------------+-------------+
    | ERF_ENABLE_MULTIBLOCK     | Whether to enable multiblock | TRUE / FALSE     | FALSE       |
@@ -205,9 +198,6 @@ Analogous to GNU Make, the list of cmake directives is as follows:
    | ERF_ENABLE_FCOMPARE       | Whether to enable fcompare   | TRUE / FALSE     | FALSE       |
    +---------------------------+------------------------------+------------------+-------------+
 
-   .. note::
-      **To run with the anelastic option, ERF_ENABLE_POISSON_SOLVE must be set to TRUE.**
-
    .. note::
       **At most one of ERF_ENABLE_OMP, ERF_ENABLE_CUDA, ERF_ENABLE_HIP and ERF_ENABLE_SYCL should be set to true.**
 

diff --git a/Exec/ABL/GNUmakefile b/Exec/ABL/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/ABL_input_sounding/GNUmakefile b/Exec/ABL_input_sounding/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-USE_POISSON_SOLVE = FALSE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DevTests/FlowInABox/GNUmakefile b/Exec/DevTests/FlowInABox/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DevTests/TemperatureSource/GNUmakefile b/Exec/DevTests/TemperatureSource/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DryRegTests/DensityCurrent/GNUmakefile b/Exec/DryRegTests/DensityCurrent/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/DryRegTests/TurbulentInflow/GNUmakefile b/Exec/DryRegTests/TurbulentInflow/GNUmakefile
@@ -24,9 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-# Incompressible/Anelastic Solver
-USE_POISSON_SOLVE = FALSE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/Make.ERF b/Exec/Make.ERF
@@ -111,11 +111,9 @@ ifeq ($(USE_EB),TRUE)
 AMReXdirs             += EB
 endif
 
-ifeq ($(USE_POISSON_SOLVE),TRUE)
 USE_LINEAR_SOLVERS_INCFLO = FALSE
 USE_LINEAR_SOLVERS_EM     = FALSE
 AMReXdirs                += LinearSolvers
-endif
 
 ifeq ($(USE_HDF5),TRUE)
 AMReXdirs             += Extern/HDF5
@@ -234,11 +232,6 @@ ifeq ($(COMPUTE_ERROR), TRUE)
   DEFINES += -DERF_COMPUTE_ERROR
 endif
 
-ifeq ($(USE_POISSON_SOLVE), TRUE)
-  DEFINES += -DERF_USE_POISSON_SOLVE
-  USERSuffix += .INC
-endif
-
 ifeq ($(USE_PARTICLES), TRUE)
   DEFINES += -DERF_USE_PARTICLES
 endif

diff --git a/Exec/MoistRegTests/Bomex/GNUmakefile b/Exec/MoistRegTests/Bomex/GNUmakefile
@@ -24,8 +24,6 @@ DEBUG = FALSE
 TEST = TRUE
 USE_ASSERTION = TRUE
 
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/MoistRegTests/SquallLine_2D/CMakeLists.txt b/Exec/MoistRegTests/SquallLine_2D/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(erf_exe_name erf_squallline_2d)
+set(erf_exe_name erf_squallline)
 
 add_executable(${erf_exe_name} "")
 target_sources(${erf_exe_name}

diff --git a/Exec/MoistRegTests/SquallLine_2D/inputs_moisture_Gabersek b/Exec/MoistRegTests/SquallLine_2D/inputs_moisture_Gabersek
@@ -62,13 +62,6 @@ erf.alpha_C = 200.0
 erf.moisture_model = "Kessler"
 erf.use_moist_background = true
 
-erf.dycore_horiz_adv_type    = "Centered_2nd"
-erf.dycore_vert_adv_type     = "Centered_2nd"
-erf.dryscal_horiz_adv_type   = "Centered_2nd"
-erf.dryscal_vert_adv_type    = "Centered_2nd"
-erf.moistscal_horiz_adv_type = "Centered_2nd"
-erf.moistscal_vert_adv_type  = "Centered_2nd"
-
 # PROBLEM PARAMETERS (optional)
 prob.z_tr = 12000.0
 prob.height = 1200.0

diff --git a/Exec/MoistRegTests/SuperCell_3D/CMakeLists.txt b/Exec/MoistRegTests/SuperCell_3D/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(erf_exe_name erf_supercell_3d)
+set(erf_exe_name erf_supercell)
 
 add_executable(${erf_exe_name} "")
 target_sources(${erf_exe_name}

diff --git a/...RegTests/SuperCell_3D/inputs_Supercell_3D → ...sts/SuperCell_3D/inputs_moisture_Tissaoui b/...RegTests/SuperCell_3D/inputs_Supercell_3D → ...sts/SuperCell_3D/inputs_moisture_Tissaoui
@@ -1,6 +1,6 @@
 # ------------------  INPUTS TO MAIN PROGRAM  -------------------
-max_step = 40000
-stop_time = 10000.0
+max_step = 28800
+stop_time = 14400.0
 
 amrex.fpe_trap_invalid = 1
 
@@ -22,8 +22,8 @@ zlo.type = "SlipWall"
 zhi.type = "HO_Outflow"
 
 # TIME STEP CONTROL
-erf.fixed_dt       = 0.25      # fixed time step [s] -- Straka et al 1993
-erf.fixed_fast_dt  = 0.125     # fixed time step [s] -- Straka et al 1993
+erf.fixed_dt       = 0.5      # fixed time step [s] -- Straka et al 1993
+erf.fixed_fast_dt  = 0.25     # fixed time step [s] -- Straka et al 1993
 
 # DIAGNOSTICS & VERBOSITY
 erf.sum_interval   = 1       # timesteps between computing mass
@@ -40,7 +40,7 @@ amr.check_int       = 1000       # number of timesteps between checkpoints
 
 # PLOTFILES
 erf.plot_file_1         = plt        # root name of plotfile
-erf.plot_int_1          = 240         # number of timesteps between plotfiles
+erf.plot_int_1          = 120         # number of timesteps between plotfiles
 erf.plot_vars_1         = density rhotheta rhoQ1 rhoQ2 rhoQ3 x_velocity y_velocity z_velocity pressure theta temp qv qc qrain rain_accum pert_dens
 
 # SOLVER CHOICE
@@ -59,27 +59,13 @@ erf.les_type = "None"
 erf.molec_diff_type = "ConstantAlpha"
 #erf.molec_diff_type = "Constant"
 erf.rho0_trans = 1.0 # [kg/m^3], used to convert input diffusivities
-erf.dynamicViscosity = 66.67 # [kg/(m-s)] ==> nu = 75.0 m^2/s
-erf.alpha_T = 66.67 # [m^2/s]
-erf.alpha_C = 66.67
+erf.dynamicViscosity = 33.33 # [kg/(m-s)] ==> nu = 75.0 m^2/s
+erf.alpha_T = 33.33 # [m^2/s]
+erf.alpha_C = 33.33
 
 erf.moisture_model = "Kessler"
 erf.use_moist_background = true
 
-#erf.dycore_horiz_adv_type    = Upwind_3rd
-#erf.dycore_vert_adv_type     = Upwind_3rd
-#erf.dryscal_horiz_adv_type   = WENOZ5
-#erf.dryscal_vert_adv_type    = WENOZ5
-#erf.moistscal_horiz_adv_type = WENOZ5
-#erf.moistscal_vert_adv_type  = WENOZ5
-
-erf.dycore_horiz_adv_type    = Centered_4th
-erf.dycore_vert_adv_type     = Centered_4th
-erf.dryscal_horiz_adv_type   = Centered_4th
-erf.dryscal_vert_adv_type    = Centered_4th
-erf.moistscal_horiz_adv_type = Centered_4th
-erf.moistscal_vert_adv_type  = Centered_4th
-
 # PROBLEM PARAMETERS (optional)
 prob.z_tr = 12000.0
 prob.height = 1200.0

diff --git a/Exec/WindFarmTests/AWAKEN/GNUmakefile b/Exec/WindFarmTests/AWAKEN/GNUmakefile
@@ -26,9 +26,6 @@ USE_ASSERTION = TRUE
 
 USE_WINDFARM = TRUE
 
-
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/WindFarmTests/EWP/GNUmakefile b/Exec/WindFarmTests/EWP/GNUmakefile
@@ -26,9 +26,6 @@ USE_ASSERTION = TRUE
 
 USE_WINDFARM = TRUE
 
-
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Exec/WindFarmTests/SimpleActuatorDisk/GNUmakefile b/Exec/WindFarmTests/SimpleActuatorDisk/GNUmakefile
@@ -26,9 +26,6 @@ USE_ASSERTION = TRUE
 
 USE_WINDFARM = TRUE
 
-
-#USE_POISSON_SOLVE = TRUE
-
 # GNU Make
 Bpack := ./Make.package
 Blocs := .

diff --git a/Source/DataStructs/ERF_DataStruct.H b/Source/DataStructs/ERF_DataStruct.H
@@ -198,7 +198,6 @@ struct SolverChoice {
 
         // *******************************************************************************
 
-#ifdef ERF_USE_POISSON_SOLVE
         bool any_anelastic = false;
         for (int i = 0; i <= max_level; ++i) {
             if (anelastic[i] == 1) any_anelastic = true;
@@ -225,14 +224,6 @@ struct SolverChoice {
                 substepping_type[lev] = SubsteppingType::None;
             }
         }
-#else
-        for (int i = 0; i <= max_level; ++i) {
-            if (anelastic[i] != 0) amrex::Abort("To run with anelastic you must compile with USE_POISSON_SOLVE = TRUE");
-        }
-        if (project_initial_velocity) {
-            amrex::Abort("To project the initial velocity you must compile with USE_POISSON_SOLVE = TRUE");
-        }
-#endif
 
         pp.query("force_stage1_single_substep", force_stage1_single_substep);
 

diff --git a/Source/ERF.cpp b/Source/ERF.cpp
@@ -856,7 +856,6 @@ ERF::InitData_post ()
         }
     }
 
-#ifdef ERF_USE_POISSON_SOLVE
     if (restart_chkfile == "")
     {
         // Note -- this projection is only defined for no terrain
@@ -870,7 +869,6 @@ ERF::InitData_post ()
             }
         }
     }
-#endif
 
     // Copy from new into old just in case
     for (int lev = 0; lev <= finest_level; ++lev)

diff --git a/Source/Microphysics/Kessler/ERF_Kessler.H b/Source/Microphysics/Kessler/ERF_Kessler.H
@@ -143,6 +143,9 @@ private:
     // Number of qstate variables
     int m_qstate_size = 3;
 
+    // CFL MAX for vertical advection
+    static constexpr amrex::Real CFL_MAX = 0.5;
+
     // MicVar map (Qmoist indices -> MicVar enum)
     amrex::Vector<int> MicVarMap;