Framework for LSM (#1389)

* This ran 1 step in debug. * This runs but has odd behavior in relaxation zone. The smag viscosity becomes VERY large; remains stable with setting k_turb=10. * Fix copy error. * Fix issue with nudge zone and most aware SmnSmn. Issue with PBL now. * Runs 100 steps. Will not restart. * This will restart. --------- Co-authored-by: Ann Almgren <[email protected]>
erf-model · Feb 1, 2024 · fbe71ac · fbe71ac
1 parent 709d24e
commit fbe71ac
Show file tree

Hide file tree

Showing 33 changed files with 1,233 additions and 62 deletions.
diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake
@@ -136,6 +136,7 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/TimeIntegration/ERF_TimeStep.cpp
        ${SRC_DIR}/TimeIntegration/ERF_advance_dycore.cpp
        ${SRC_DIR}/TimeIntegration/ERF_advance_microphysics.cpp
+       ${SRC_DIR}/TimeIntegration/ERF_advance_lsm.cpp
        ${SRC_DIR}/TimeIntegration/ERF_advance_radiation.cpp
        ${SRC_DIR}/TimeIntegration/ERF_make_buoyancy.cpp
        ${SRC_DIR}/TimeIntegration/ERF_make_fast_coeffs.cpp
@@ -164,6 +165,8 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/Microphysics/FastEddy/FastEddy.cpp
        ${SRC_DIR}/Microphysics/FastEddy/Diagnose_FE.cpp
        ${SRC_DIR}/Microphysics/FastEddy/Update_FE.cpp
+       ${SRC_DIR}/LandSurfaceModel/SLM/SLM.cpp
+       ${SRC_DIR}/LandSurfaceModel/MM5/MM5.cpp
   )
 
   if(NOT "${erf_exe_name}" STREQUAL "erf_unit_tests")
@@ -208,8 +211,12 @@ function(build_erf_lib erf_lib_name)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Null)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/SAM)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Kessler)
-  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/FastEddy)
-
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/FastEddy) 
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/LandSurfaceModel)
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/LandSurfaceModel/Null)
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/LandSurfaceModel/SLM)
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/LandSurfaceModel/MM5)
+
   if(ERF_ENABLE_RRTMGP)
      target_link_libraries(${erf_lib_name} PUBLIC yakl)
      target_link_libraries(${erf_lib_name} PUBLIC rrtmgp)

diff --git a/Exec/CMakeLists.txt b/Exec/CMakeLists.txt
@@ -27,5 +27,6 @@ else ()
   add_subdirectory(DevTests/ParticlesOverWoA)
   add_subdirectory(DevTests/MiguelDev)
   add_subdirectory(DevTests/MetGrid)
+  add_subdirectory(DevTests/LandSurfaceModel)
   add_subdirectory(DevTests/TemperatureSource)
 endif()
diff --git a/Exec/DevTests/LandSurfaceModel/CMakeLists.txt b/Exec/DevTests/LandSurfaceModel/CMakeLists.txt
@@ -0,0 +1,15 @@
+set(erf_exe_name LandSurfaceModel)
+
+add_executable(${erf_exe_name} "")
+target_sources(${erf_exe_name}
+   PRIVATE
+     prob.cpp
+)
+
+target_include_directories(${erf_exe_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+
+include(${CMAKE_SOURCE_DIR}/CMake/BuildERFExe.cmake)
+build_erf_exe(${erf_exe_name})
+
+#find_package(AMReX)
+#target_link_libraries( ${_target}  AMReX::amrex)
diff --git a/Exec/DevTests/LandSurfaceModel/GNUmakefile b/Exec/DevTests/LandSurfaceModel/GNUmakefile
@@ -0,0 +1,35 @@
+# AMReX
+COMP = gnu
+PRECISION = DOUBLE
+
+# Profiling
+PROFILE       = FALSE
+TINY_PROFILE  = FALSE
+COMM_PROFILE  = FALSE
+TRACE_PROFILE = FALSE
+MEM_PROFILE   = FALSE
+USE_GPROF     = FALSE
+
+# Performance
+USE_MPI  = TRUE
+USE_OMP  = FALSE
+
+USE_CUDA = FALSE
+USE_HIP  = FALSE
+USE_SYCL = FALSE
+
+# Debugging
+DEBUG = FALSE
+
+TEST = TRUE
+USE_ASSERTION = TRUE
+
+USE_NETCDF = TRUE
+
+# GNU Make
+Bpack := ./Make.package
+Blocs := .
+
+ERF_HOME := ../../..
+ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/DevTests/LandSurfaceModel
+include $(ERF_HOME)/Exec/Make.ERF
diff --git a/Exec/DevTests/LandSurfaceModel/Make.package b/Exec/DevTests/LandSurfaceModel/Make.package
@@ -0,0 +1,2 @@
+CEXE_headers += prob.H
+CEXE_sources += prob.cpp
diff --git a/Exec/DevTests/LandSurfaceModel/README b/Exec/DevTests/LandSurfaceModel/README
@@ -0,0 +1 @@
+This problem setup is to test the ability of ERF to handle a land surface model.
diff --git a/Exec/DevTests/LandSurfaceModel/inputs b/Exec/DevTests/LandSurfaceModel/inputs
@@ -0,0 +1,74 @@
+# ------------------  INPUTS TO MAIN PROGRAM  -------------------
+max_step = 100
+
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1
+
+# PROBLEM SIZE & GEOMETRY
+geometry.prob_extent =  2430000 2673000 12000
+amr.n_cell           =  90      99      50
+
+geometry.is_periodic = 0 0 0
+
+xlo.type = "Outflow"
+xhi.type = "Outflow"
+
+ylo.type = "Outflow"
+yhi.type = "Outflow"
+
+zlo.type      = "Most"
+erf.most.z0   =  0.1
+erf.most.zref = 120.0
+zhi.type = "SlipWall"
+
+# TIME STEP CONTROL
+erf.fixed_dt = 0.01
+erf.fixed_mri_dt_ratio = 4
+
+# DIAGNOSTICS & VERBOSITY
+erf.sum_interval   = -1       # timesteps between computing mass
+#erf.data_log      = my_data my_1d_data
+erf.v              = 1       # verbosity in ERF.cpp
+amr.v              = 1       # verbosity in Amr.cpp
+
+# REFINEMENT / REGRIDDING
+amr.max_level       = 0       # maximum level number allowed
+
+# CHECKPOINT FILES
+erf.check_file      = chk        # root name of checkpoint file
+erf.check_int       = 100        # number of timesteps between checkpoints
+erf.restart_type    = "native"
+# PLOTFILES
+erf.plot_file_1     = plt        # prefix of plotfile name
+erf.plot_int_1      = 1          # number of timesteps between plotfiles
+erf.plot_vars_1     = density rhotheta rhoadv_0 x_velocity y_velocity z_velocity pressure theta scalar temp pres_hse dens_hse pert_pres pert_dens
+
+erf.plot_lsm = true
+
+# SOLVER CHOICE
+erf.alpha_T = 1.0
+erf.alpha_C = 1.0
+erf.use_gravity = true
+
+erf.molec_diff_type = "None"
+erf.les_type        = "Smagorinsky"
+erf.Cs              = 0.1
+#erf.pbl_type        = "MYNN2.5"
+#erf.QKE_0           = 0.5
+
+erf.use_terrain = true
+erf.terrain_smoothing = 2
+
+erf.moisture_model = "NullMoist"
+
+erf.land_surface_model = "MM5"
+
+# INITIALIZATION WITH METGRID DATA
+erf.metgrid_bdy_width = 5
+erf.metgrid_bdy_set_width = 1
+erf.init_type      = "metgrid"
+erf.nc_init_file_0 = "met_em.d01.2016-10-06_00_00_00.nc" "met_em.d01.2016-10-06_06_00_00.nc"
+
+# NO CACHE TILING
+fabarray.mfiter_tile_size = 1024 1024 1024
diff --git a/Exec/DevTests/LandSurfaceModel/prob.H b/Exec/DevTests/LandSurfaceModel/prob.H
@@ -0,0 +1,24 @@
+#ifndef _PROB_H_
+#define _PROB_H_
+
+#include <string>
+
+#include "AMReX_REAL.H"
+
+#include "prob_common.H"
+
+struct ProbParm : ProbParmDefaults {
+}; // namespace ProbParm
+class Problem : public ProblemBase
+{
+public:
+    Problem(const amrex::Real* /*problo*/, const amrex::Real* /*probhi*/);
+
+protected:
+    std::string name() override { return "LandSurfaceModel"; }
+
+private:
+    ProbParm parms;
+};
+
+#endif
diff --git a/Exec/DevTests/LandSurfaceModel/prob.cpp b/Exec/DevTests/LandSurfaceModel/prob.cpp
@@ -0,0 +1,12 @@
+#include "prob.H"
+
+using namespace amrex;
+
+std::unique_ptr<ProblemBase>
+amrex_probinit(const amrex_real* problo, const amrex_real* probhi)
+{
+    return std::make_unique<Problem>(problo, probhi);
+}
+
+Problem::Problem(const amrex::Real* /*problo*/, const amrex::Real* /*probhi*/)
+{}
diff --git a/Exec/Make.ERF b/Exec/Make.ERF
@@ -127,6 +127,26 @@ include $(ERF_MOISTURE_KESSLER_DIR)/Make.package
 VPATH_LOCATIONS   += $(ERF_MOISTURE_KESSLER_DIR)
 INCLUDE_LOCATIONS += $(ERF_MOISTURE_KESSLER_DIR)
 
+ERF_LSM_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel
+include $(ERF_LSM_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_LSM_DIR)
+INCLUDE_LOCATIONS += $(ERF_LSM_DIR)
+
+ERF_LSM_NULL_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/Null
+include $(ERF_LSM_NULL_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_LSM_NULL_DIR)
+INCLUDE_LOCATIONS += $(ERF_LSM_NULL_DIR)
+
+ERF_LSM_SLM_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/SLM
+include $(ERF_LSM_SLM_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_LSM_SLM_DIR)
+INCLUDE_LOCATIONS += $(ERF_LSM_SLM_DIR)
+
+ERF_LSM_MM5_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/MM5
+include $(ERF_LSM_MM5_DIR)/Make.package
+VPATH_LOCATIONS   += $(ERF_LSM_MM5_DIR)
+INCLUDE_LOCATIONS += $(ERF_LSM_MM5_DIR)
+
 ifeq ($(COMPUTE_ERROR), TRUE)
   DEFINES += -DERF_COMPUTE_ERROR
 endif

diff --git a/Source/BoundaryConditions/ABLMost.H b/Source/BoundaryConditions/ABLMost.H
@@ -35,6 +35,8 @@ public:
                       amrex::Vector<std::unique_ptr<amrex::MultiFab>>& z_phys_nd,
                       amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab>>>& sst_lev,
                       amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab>>>& lmask_lev,
+                      amrex::Vector<amrex::Vector<amrex::MultiFab*>> lsm_data,
+                      amrex::Vector<amrex::Vector<amrex::MultiFab*>> lsm_flux,
                       amrex::Real start_bdy_time = 0.0,
                       amrex::Real bdy_time_interval = 0.0)
     : m_start_bdy_time(start_bdy_time),
@@ -113,6 +115,19 @@ public:
             }
         }
 
+        // Get pointers to LSM data and Fluxes
+        m_lsm_data_lev.resize(nlevs);
+        m_lsm_flux_lev.resize(nlevs);
+        for (int lev(0); lev<nlevs; ++lev) {
+            int nvar = lsm_data[lev].size();
+            m_lsm_data_lev[lev].resize(nvar);
+            m_lsm_flux_lev[lev].resize(nvar);
+            for (int n(0); n<nvar; ++n) {
+                m_lsm_data_lev[lev][n] = lsm_data[lev][n];
+                m_lsm_flux_lev[lev][n] = lsm_flux[lev][n];
+            }
+        }
+
         for (int lev = 0; lev < nlevs; lev++) {
             // Attributes for MFs and FABs
             //--------------------------------------------------------
@@ -149,10 +164,12 @@ public:
 
             t_surf[lev] = std::make_unique<amrex::MultiFab>(ba2d,dm,ncomp,ng);
 
-            if (m_sst_lev[lev][0]) { // Valid SST data at t==0
+            // TODO: Do we want lsm_data to have theta at 0 index always?
+            //       Do we want an external enum struct for indexing?
+            if (m_sst_lev[lev][0] || m_lsm_data_lev[lev][0]) {
+                // Valid SST or LSM data; t_surf set before computing fluxes (avoids extended lambda capture)
                 theta_type = ThetaCalcType::SURFACE_TEMPERATURE;
-                amrex::MultiFab::Copy(*(t_surf[lev]), *(m_sst_lev[lev][0]), 0, 0, 1, ng);
-            } else if (erf_st) {     // Constant temp
+            } else if (erf_st) {
                 t_surf[lev]->setVal(surf_temp);
             } else {
                 t_surf[lev]->setVal(0.0);
@@ -187,8 +204,11 @@ public:
                       const FluxCalc& flux_comp);
 
     void
-    time_interp_tsurf(const int& lev,
-                      const amrex::Real& time);
+    time_interp_sst (const int& lev,
+                     const amrex::Real& time);
+
+    void
+    get_lsm_tsurf (const int& lev);
 
     void
     update_surf_temp (const amrex::Real& time)
@@ -265,6 +285,8 @@ private:
 
     amrex::Vector<amrex::Vector<amrex::MultiFab*>>  m_sst_lev;
     amrex::Vector<amrex::Vector<amrex::iMultiFab*>> m_lmask_lev;
+    amrex::Vector<amrex::Vector<amrex::MultiFab*>>  m_lsm_data_lev;
+    amrex::Vector<amrex::Vector<amrex::MultiFab*>>  m_lsm_flux_lev;
 };
 
 #endif /* ABLMOST_H */
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		CEXE_headers += prob.H
		CEXE_sources += prob.cpp
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		This problem setup is to test the ability of ERF to handle a land surface model.