Merge remote-tracking branch 'upstream/development' into AWAKEN_Refactor

Build/cmake_with_poisson.sh → Build/cmake_with_eb.sh

@@ -8,11 +8,12 @@ cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \
       -DCMAKE_Fortran_COMPILER:STRING=mpifort \
       -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \
       -DCMAKE_BUILD_TYPE:STRING=Release \
+      -DAMREX_EB:BOOL=ON \
       -DERF_DIM:STRING=3 \
       -DERF_ENABLE_MPI:BOOL=ON \
       -DERF_ENABLE_TESTS:BOOL=ON \
       -DERF_ENABLE_FCOMPARE:BOOL=ON \
       -DERF_ENABLE_DOCUMENTATION:BOOL=OFF \
-      -DERF_ENABLE_POISSON_SOLVE:BOOL=ON \
+      -DERF_ENABLE_EB:BOOL=ON \
       -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
       .. && make -j8

CMake/BuildERFExe.cmake

@@ -25,13 +25,6 @@ function(build_erf_lib erf_lib_name)
     target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_WARM_NO_PRECIP)
   endif()
 
-  if(ERF_ENABLE_POISSON_SOLVE)
-    target_sources(${erf_lib_name} PRIVATE
-                   ${SRC_DIR}/Utils/ERF_PoissonSolve.cpp
-                   ${SRC_DIR}/Utils/ERF_PoissonSolve_tb.cpp)
-    target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_POISSON_SOLVE)
-  endif()
-
   if(ERF_ENABLE_PARTICLES)
     target_sources(${erf_lib_name} PRIVATE
                    ${SRC_DIR}/Particles/ERFPCEvolve.cpp
@@ -190,6 +183,8 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/Utils/ERF_AverageDown.cpp
        ${SRC_DIR}/Utils/ERF_ChopGrids.cpp
        ${SRC_DIR}/Utils/ERF_MomentumToVelocity.cpp
+       ${SRC_DIR}/Utils/ERF_PoissonSolve.cpp
+       ${SRC_DIR}/Utils/ERF_PoissonSolve_tb.cpp
        ${SRC_DIR}/Utils/ERF_TerrainMetrics.cpp
        ${SRC_DIR}/Utils/ERF_VelocityToMomentum.cpp
        ${SRC_DIR}/Utils/ERF_InteriorGhostCells.cpp

CMake/SetAmrexOptions.cmake

@@ -27,12 +27,9 @@ set(AMReX_ACC OFF)
 set(AMReX_PLOTFILE_TOOLS ${ERF_ENABLE_FCOMPARE})
 set(AMReX_FORTRAN OFF)
 
-set(AMReX_LINEAR_SOLVERS OFF)
-if(ERF_ENABLE_POISSON_SOLVE)
-  set(AMReX_LINEAR_SOLVERS ON)
-  set(AMReX_LINEAR_SOLVERS_EM OFF)
-  set(AMReX_LINEAR_SOLVERS_INCFLO OFF)
-endif()
+set(AMReX_LINEAR_SOLVERS ON)
+set(AMReX_LINEAR_SOLVERS_EM OFF)
+set(AMReX_LINEAR_SOLVERS_INCFLO OFF)
 
 set(AMReX_PARTICLES OFF)
 if(ERF_ENABLE_PARTICLES)

CMakeLists.txt

@@ -24,8 +24,6 @@ option(ERF_ENABLE_MOISTURE "Enable Full Moisture" ON)
 option(ERF_ENABLE_WARM_NO_PRECIP "Enable Warm Moisture" OFF)
 option(ERF_ENABLE_RRTMGP "Enable RTE-RRTMGP Radiation" OFF)
 
-option(ERF_ENABLE_POISSON_SOLVE "Enable Poisson solve for anelastic/incompressible flow" OFF)
-
 #Options for performance
 option(ERF_ENABLE_MPI "Enable MPI" OFF)
 option(ERF_ENABLE_OPENMP "Enable OpenMP" OFF)
@@ -247,7 +245,7 @@ target_link_libraries(erf_api PUBLIC erf_srclib)
 add_library(${PROJECT_NAME}::erf_api ALIAS erf_srclib)
 
 # Collect all headers and make them installable with the target
-set(ERF_INCLUDES "Source/ERF.H;Source/ERF_Constants.H;Source/WindFarmParametrization/SimpleActuatorDisk/ERF_SimpleAD.H;Source/WindFarmParametrization/EWP/ERF_EWP.H;Source/WindFarmParametrization/Null/ERF_NullWindFarm.H;Source/WindFarmParametrization/ERF_WindFarm.H;Source/WindFarmParametrization/Fitch/ERF_Fitch.H;Source/BoundaryConditions/ERF_PhysBCFunct.H;Source/BoundaryConditions/ERF_MOSTAverage.H;Source/BoundaryConditions/ERF_MOSTRoughness.H;Source/BoundaryConditions/ERF_ABLMost.H;Source/BoundaryConditions/ERF_FillPatcher.H;Source/BoundaryConditions/ERF_MOSTStress.H;Source/BoundaryConditions/ERF_TimeInterpolatedData.H;Source/Utils/ERF_Interpolation.H;Source/Utils/ERF_TileNoZ.H;Source/Utils/ERF_PlaneAverage.H;Source/Utils/ERF_Interpolation_WENO.H;Source/Utils/ERF_DirectionSelector.H;Source/Utils/ERF_ParFunctions.H;Source/Utils/ERF_Wstar.H;Source/Utils/ERF_Microphysics_Utils.H;Source/Utils/ERF_Sat_methods.H;Source/Utils/ERF_Interpolation_1D.H;Source/Utils/ERF_Interpolation_UPW.H;Source/Utils/ERF_TerrainMetrics.H;Source/Utils/ERF_Interpolation_WENO_Z.H;Source/Utils/ERF_Thetav.H;Source/Utils/ERF_Water_vapor_saturation.H;Source/Utils/ERF_Utils.H;Source/Utils/ERF_Orbit.H;Source/Utils/ERF_EOS.H;Source/Utils/ERF_HSE_utils.H;Source/EB/ERF_TerrainIF.H;Source/EB/ERF_FlowerIF.H;Source/EB/ERF_eb_if.H;Source/Particles/ERFPC.H;Source/Particles/ERF_ParticleData.H;Source/Prob/ERF_init_density_hse_dry.H;Source/Prob/ERF_init_rayleigh_damping.H;Source/Prob/ERF_init_constant_density_hse.H;Source/ERF_prob_common.H;Source/ERF_Derive.H;Source/Radiation/ERF_Mam4_constituents.H;Source/Radiation/ERF_Mam4_aero.H;Source/Radiation/ERF_Optics.H;Source/Radiation/ERF_Modal_aero_wateruptake.H;Source/Radiation/ERF_Cloud_rad_props.H;Source/Radiation/ERF_Phys_prop.H;Source/Radiation/ERF_Radiation.H;Source/Radiation/ERF_Albedo.H;Source/Radiation/ERF_Parameterizations.H;Source/Radiation/ERF_Rad_constants.H;Source/Radiation/ERF_Aero_rad_props.H;Source/Radiation/ERF_m2005_effradius.H;Source/Radiation/ERF_Linear_interpolate.H;Source/Radiation/ERF_Slingo.H;Source/Radiation/ERF_Rrtmgp.H;Source/Radiation/ERF_Ebert_curry.H;Source/SourceTerms/ERF_NumericalDiffusion.H;Source/SourceTerms/ERF_Src_headers.H;Source/IO/ERF_NCInterface.H;Source/IO/ERF_NCWpsFile.H;Source/IO/ERF_NCPlotFile.H;Source/IO/ERF_ReadBndryPlanes.H;Source/IO/ERF_WriteBndryPlanes.H;Source/PBL/ERF_MYNNStruct.H;Source/PBL/ERF_PBLModels.H;Source/PBL/ERF_PBLHeight.H;Source/TimeIntegration/ERF_TI_substep_fun.H;Source/TimeIntegration/ERF_TI_slow_headers.H;Source/TimeIntegration/ERF_TI_slow_rhs_fun.H;Source/TimeIntegration/ERF_TI_fast_headers.H;Source/TimeIntegration/ERF_TI_utils.H;Source/TimeIntegration/ERF_MRI.H;Source/TimeIntegration/ERF_TI_no_substep_fun.H;Source/LandSurfaceModel/Null/ERF_NullSurf.H;Source/LandSurfaceModel/ERF_LandSurface.H;Source/LandSurfaceModel/MM5/ERF_MM5.H;Source/LandSurfaceModel/SLM/ERF_SLM.H;Source/ERF_IndexDefines.H;Source/Advection/ERF_AdvectionSrcForMom_N.H;Source/Advection/ERF_AdvectionSrcForScalars.H;Source/Advection/ERF_AdvectionSrcForMom_T.H;Source/Advection/ERF_Advection.H;Source/MultiBlock/ERF_MultiBlockContainer.H;Source/Initialization/ERF_Metgrid_utils.H;Source/Diffusion/ERF_EddyViscosity.H;Source/Diffusion/ERF_Diffusion.H;Source/Microphysics/Null/ERF_NullMoistLagrangian.H;Source/Microphysics/Null/ERF_NullMoist.H;Source/Microphysics/ERF_Microphysics.H;Source/Microphysics/ERF_LagrangianMicrophysics.H;Source/Microphysics/ERF_EulerianMicrophysics.H;Source/Microphysics/Kessler/ERF_Kessler.H;Source/Microphysics/SAM/ERF_SAM.H;Source/DataStructs/ERF_InputSpongeData.H;Source/DataStructs/ERF_TurbPertStruct.H;Source/DataStructs/ERF_SpongeStruct.H;Source/DataStructs/ERF_AdvStruct.H;Source/DataStructs/ERF_DataStruct.H;Source/DataStructs/ERF_InputSoundingData.H;Source/DataStructs/ERF_DiffStruct.H;Source/DataStructs/ERF_TurbStruct.H")
+set(ERF_INCLUDES "Source/ERF.H;Source/ERF_Constants.H;Source/WindFarmParametrization/SimpleActuatorDisk/ERF_SimpleAD.H;Source/WindFarmParametrization/EWP/ERF_EWP.H;Source/WindFarmParametrization/Null/ERF_NullWindFarm.H;Source/WindFarmParametrization/ERF_WindFarm.H;Source/WindFarmParametrization/Fitch/ERF_Fitch.H;Source/BoundaryConditions/ERF_PhysBCFunct.H;Source/BoundaryConditions/ERF_MOSTAverage.H;Source/BoundaryConditions/ERF_MOSTRoughness.H;Source/BoundaryConditions/ERF_ABLMost.H;Source/BoundaryConditions/ERF_FillPatcher.H;Source/BoundaryConditions/ERF_MOSTStress.H;Source/BoundaryConditions/ERF_TimeInterpolatedData.H;Source/Utils/ERF_Interpolation.H;Source/Utils/ERF_TileNoZ.H;Source/Utils/ERF_PlaneAverage.H;Source/Utils/ERF_Interpolation_WENO.H;Source/Utils/ERF_DirectionSelector.H;Source/Utils/ERF_ParFunctions.H;Source/Utils/ERF_Wstar.H;Source/Utils/ERF_Microphysics_Utils.H;Source/Utils/ERF_Sat_methods.H;Source/Utils/ERF_Interpolation_1D.H;Source/Utils/ERF_Interpolation_UPW.H;Source/Utils/ERF_TerrainMetrics.H;Source/Utils/ERF_Interpolation_WENO_Z.H;Source/Utils/ERF_Thetav.H;Source/Utils/ERF_Water_vapor_saturation.H;Source/Utils/ERF_Utils.H;Source/Utils/ERF_Orbit.H;Source/Utils/ERF_EOS.H;Source/Utils/ERF_HSE_utils.H;Source/EB/ERF_TerrainIF.H;Source/EB/ERF_FlowerIF.H;Source/EB/ERF_eb_if.H;Source/Particles/ERFPC.H;Source/Particles/ERF_ParticleData.H;Source/Prob/ERF_init_density_hse_dry.H;Source/Prob/ERF_init_rayleigh_damping.H;Source/Prob/ERF_init_constant_density_hse.H;Source/ERF_prob_common.H;Source/ERF_Derive.H;Source/Radiation/ERF_Mam4_constituents.H;Source/Radiation/ERF_Mam4_aero.H;Source/Radiation/ERF_Optics.H;Source/Radiation/ERF_Modal_aero_wateruptake.H;Source/Radiation/ERF_Cloud_rad_props.H;Source/Radiation/ERF_Phys_prop.H;Source/Radiation/ERF_Radiation.H;Source/Radiation/ERF_Albedo.H;Source/Radiation/ERF_Parameterizations.H;Source/Radiation/ERF_Rad_constants.H;Source/Radiation/ERF_Aero_rad_props.H;Source/Radiation/ERF_m2005_effradius.H;Source/Radiation/ERF_Linear_interpolate.H;Source/Radiation/ERF_Slingo.H;Source/Radiation/ERF_Rrtmgp.H;Source/Radiation/ERF_Ebert_curry.H;Source/SourceTerms/ERF_NumericalDiffusion.H;Source/SourceTerms/ERF_Src_headers.H;Source/IO/ERF_SampleData.H;Source/IO/ERF_NCInterface.H;Source/IO/ERF_NCWpsFile.H;Source/IO/ERF_NCPlotFile.H;Source/IO/ERF_ReadBndryPlanes.H;Source/IO/ERF_WriteBndryPlanes.H;Source/PBL/ERF_MYNNStruct.H;Source/PBL/ERF_PBLModels.H;Source/PBL/ERF_PBLHeight.H;Source/TimeIntegration/ERF_TI_substep_fun.H;Source/TimeIntegration/ERF_TI_slow_headers.H;Source/TimeIntegration/ERF_TI_slow_rhs_fun.H;Source/TimeIntegration/ERF_TI_fast_headers.H;Source/TimeIntegration/ERF_TI_utils.H;Source/TimeIntegration/ERF_MRI.H;Source/TimeIntegration/ERF_TI_no_substep_fun.H;Source/LandSurfaceModel/Null/ERF_NullSurf.H;Source/LandSurfaceModel/ERF_LandSurface.H;Source/LandSurfaceModel/MM5/ERF_MM5.H;Source/LandSurfaceModel/SLM/ERF_SLM.H;Source/ERF_IndexDefines.H;Source/Advection/ERF_AdvectionSrcForMom_N.H;Source/Advection/ERF_AdvectionSrcForScalars.H;Source/Advection/ERF_AdvectionSrcForMom_T.H;Source/Advection/ERF_Advection.H;Source/MultiBlock/ERF_MultiBlockContainer.H;Source/Initialization/ERF_Metgrid_utils.H;Source/Diffusion/ERF_EddyViscosity.H;Source/Diffusion/ERF_Diffusion.H;Source/Microphysics/Null/ERF_NullMoistLagrangian.H;Source/Microphysics/Null/ERF_NullMoist.H;Source/Microphysics/ERF_Microphysics.H;Source/Microphysics/ERF_LagrangianMicrophysics.H;Source/Microphysics/ERF_EulerianMicrophysics.H;Source/Microphysics/Kessler/ERF_Kessler.H;Source/Microphysics/SAM/ERF_SAM.H;Source/DataStructs/ERF_InputSpongeData.H;Source/DataStructs/ERF_TurbPertStruct.H;Source/DataStructs/ERF_SpongeStruct.H;Source/DataStructs/ERF_AdvStruct.H;Source/DataStructs/ERF_DataStruct.H;Source/DataStructs/ERF_InputSoundingData.H;Source/DataStructs/ERF_DiffStruct.H;Source/DataStructs/ERF_TurbStruct.H")
 set_target_properties(
   erf_srclib PROPERTIES PUBLIC_HEADER "${ERF_INCLUDES}")
 

Docs/sphinx_doc/Inputs.rst

@@ -34,10 +34,8 @@ Governing Equations
 
 .. note::
 
-   To solve the anelastic equations, you must set USE_POISSON_SOLVE = TRUE if using
-   gmake or ERF_ENABLE_POISSON_SOLVE if using cmake.  This will enable use of the
-   AMReX-based Poisson solver.   To optionally use the FFT solver, you must additionally
-   set USE_FFT = TRUE if using gmake.
+   To use the FFT solver if running with anelastic, you must set USE_FFT = TRUE if using gmake
+   or ERF_ENABLE_FFT if using cmake.
 
 Problem Geometry
 ================
@@ -1162,48 +1160,46 @@ This file has the same format as used by ``ideal.exe`` executable in WRF.
 Using this option for initialization, running ``ideal.exe`` and reading from the resulting ``wrfinput_d01`` file are not needed.
 This option is used for initializing ERF domain to a horizontally homogeneous mesoscale state and does not include terrain or map scale factors.
 
-In addition, there is a run-time option to project the initial velocity field to make it divergence-free.  To take
-advantage of this option, the code must be built with ``USE_POISSON_SOLVE = TRUE`` in the GNUmakefile if using gmake, or with
-``-DERF_ENABLE_POISSON_SOLVE:BOOL=ON`` in the cmake.sh file if using cmake.
+In addition, there is a run-time option to project the initial velocity field to make it divergence-free.
 
 List of Parameters
 ------------------
 
-+----------------------------------+-------------------+--------------------+------------------+
-| Parameter                        | Definition        | Acceptable         | Default          |
-|                                  |                   | Values             |                  |
-+==================================+===================+====================+==================+
-| **erf.init_type**                | Initialization    | “custom”,          | “*custom*”       |
-|                                  | type              | “ideal”,           |                  |
-|                                  |                   | "real",            |                  |
-|                                  |                   |"input_sounding"    |                  |
-+----------------------------------+-------------------+--------------------+------------------+
-| **erf.input_sounding_file**      | Path to WRF-style |  String            | "input_sounding" |
-|                                  | input sounding    |                    |                  |
-|                                  | file              |                    |                  |
-+----------------------------------+-------------------+--------------------+------------------+
-| **erf.init_sounding_ideal**      | Perform           |  true or false     | false            |
-|                                  | initialization    |                    |                  |
-|                                  | like WRF's        |                    |                  |
-|                                  | ideal.exe         |                    |                  |
-+----------------------------------+-------------------+--------------------+------------------+
-| **erf.use_real_bcs**             | If init_type is   | true or false      | true if          |
-|                                  | real or metgrid,  |                    | if init_type     |
-|                                  | do we want to use |                    | is real or       |
-|                                  | these bcs?        |                    | metgrid;         |
-|                                  |                   |                    | else false       |
-+----------------------------------+-------------------+--------------------+------------------+
-| **erf.nc_init_file**             | NetCDF file with  |  String            | NONE             |
-|                                  | initial mesoscale |                    |                  |
-|                                  | data              |                    |                  |
-+----------------------------------+-------------------+--------------------+------------------+
-| **erf.nc_bdy_file**              | NetCDF file with  |  String            | NONE             |
-|                                  | mesoscale data at |                    |                  |
-|                                  | lateral boundaries|                    |                  |
-+----------------------------------+-------------------+--------------------+------------------+
-| **erf.project_initial_velocity** | project initial   |  Integer           | 1                |
-|                                  | velocity?         |                    |                  |
-+----------------------------------+-------------------+--------------------+------------------+
++----------------------------------+-------------------+--------------------+-----------------------+
+| Parameter                        | Definition        | Acceptable         | Default               |
+|                                  |                   | Values             |                       |
++==================================+===================+====================+=======================+
+| **erf.init_type**                | Initialization    | “custom”,          | “*custom*”            |
+|                                  | type              | “ideal”,           |                       |
+|                                  |                   | "real",            |                       |
+|                                  |                   |"input_sounding"    |                       |
++----------------------------------+-------------------+--------------------+-----------------------+
+| **erf.input_sounding_file**      | Path to WRF-style |  String            | "input_sounding"      |
+|                                  | input sounding    |                    |                       |
+|                                  | file              |                    |                       |
++----------------------------------+-------------------+--------------------+-----------------------+
+| **erf.init_sounding_ideal**      | Perform           |  true or false     | false                 |
+|                                  | initialization    |                    |                       |
+|                                  | like WRF's        |                    |                       |
+|                                  | ideal.exe         |                    |                       |
++----------------------------------+-------------------+--------------------+-----------------------+
+| **erf.use_real_bcs**             | If init_type is   | true or false      | true if               |
+|                                  | real or metgrid,  |                    | if init_type          |
+|                                  | do we want to use |                    | is real or            |
+|                                  | these bcs?        |                    | metgrid;              |
+|                                  |                   |                    | else false            |
++----------------------------------+-------------------+--------------------+-----------------------+
+| **erf.nc_init_file**             | NetCDF file with  |  String            | NONE                  |
+|                                  | initial mesoscale |                    |                       |
+|                                  | data              |                    |                       |
++----------------------------------+-------------------+--------------------+-----------------------+
+| **erf.nc_bdy_file**              | NetCDF file with  |  String            | NONE                  |
+|                                  | mesoscale data at |                    |                       |
+|                                  | lateral boundaries|                    |                       |
++----------------------------------+-------------------+--------------------+-----------------------+
+| **erf.project_initial_velocity** | project initial   |  true or false     | true if anelastic;    |
+|                                  | velocity?         |                    | false if compressible |
++----------------------------------+-------------------+--------------------+-----------------------+
 
 Notes
 -----------------
@@ -1231,8 +1227,9 @@ integrating the hydrostatic equation from the surface.
 
 If **erf.init_type = custom** or **erf.init_type = input_sounding**, ``erf.nc_init_file`` and ``erf.nc_bdy_file`` do not need to be set.
 
-Setting **erf.project_initial_velocity = 1** will have no effect if the code is not built with **ERF_USE_POISSON_SOLVE** defined
-if using cmake or with **USE_POISSON_SOLVE = TRUE** if using gmake.
+Note that the **erf.project_initial_velocity** option is available for all **init_type** options.  If using the anelastic
+formulation this will be true regardless of the input; if using the compressible formulation the default is false but
+that can be over-written.
 
 Map Scale Factors
 =================