Interface for NOAH-MP Landsurface model (#1835)

* Add inital interface for NOAH-M

* Update NOAH module and the corresponding FindNetCDF cmake recipe

* Initial interface for NOAH LSM

* Initial interface for NOAH-MP land surface model

* Add linker options

* Updates to cmake configuration to include NOAH

* Update CMakeLists and move Fortran linkage to Noah-MP:

* Update .gitmodules

* Updates to NOAH-MP interface

* Saving some interface work

* udpate gitmodules

* checkpoint changes for merge

* Update Noah-MP submodules to resolve cmake errors

* Resolve style errors

* Update gmake interface for noahmp and add a dev test

.gitmodules

@@ -9,7 +9,11 @@
 [submodule "Submodules/RRTMGP"]
 	path = Submodules/RRTMGP
 	url = https://github.com/E3SM-Project/rte-rrtmgp
-	shallow = true
+        shallow = true
+[submodule "Submodules/NOAH-MP"]
+	path = Submodules/NOAH-MP
+        url = https://github.com/AIEADA/noahmp.git
+        shallow=true
 [submodule "Submodules/WW3"]
 	path = Submodules/WW3
 	url = https://github.com/erf-model/WW3

CMake/BuildERFExe.cmake

@@ -67,6 +67,16 @@ function(build_erf_lib erf_lib_name)
     target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_NETCDF)
   endif()
 
+  if(ERF_ENABLE_NOAH)
+    target_include_directories(${erf_lib_name} PUBLIC
+                               $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Source/LandSurfaceModel/NOAH>
+                               $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/Submodules/NOAH-MP/drivers/hrldas>)
+    target_sources(${erf_lib_name} PRIVATE
+                   ${SRC_DIR}/LandSurfaceModel/NOAH/ERF_NOAH.cpp)
+    target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_NOAH)
+    target_link_libraries_system(${erf_lib_name} PUBLIC NoahMP::noahmp)
+  endif()
+
   if(ERF_ENABLE_RRTMGP)
     target_sources(${erf_lib_name} PRIVATE
                    ${SRC_DIR}/Utils/ERF_Orbit.cpp

CMake/FindNetCDF.cmake

@@ -23,10 +23,44 @@ endif (NETCDF_INCLUDES AND NETCDF_LIBRARIES)
 find_package(PkgConfig REQUIRED QUIET)
 pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET netcdf)
 
+find_path(NETCDF_INCLUDES netcdf.h
+    HINTS NETCDF_DIR/include ENV NETCDF_DIR)
+
+find_library(NETCDF_LIBRARIES_C NAMES netcdf HINTS NETCDF_DIR/lib ENV NETCDF_DIR)
+mark_as_advanced(NETCDF_LIBRARIES_C)
+
+set(NetCDF_has_interfaces "YES") # will be set to NO if we're missing any interfaces
+set(NetCDF_libs "${NETCDF_LIBRARIES_C}")
+
+get_filename_component(NetCDF_lib_dirs "${NETCDF_LIBRARIES_C}" PATH)
+
+macro(NetCDF_check_interface lang header libs)
+    if(NETCDF_${lang})
+        find_path(NETCDF_INCLUDES_${lang} NAMES ${header}
+            HINTS "${NETCDF_INCLUDES}" NO_DEFAULT_PATH)
+        find_library(NETCDF_LIBRARIES_${lang} NAMES ${libs}
+            HINTS "${NetCDF_lib_dirs}" NO_DEFAULT_PATH)
+        mark_as_advanced(NETCDF_INCLUDES_${lang} NETCDF_LIBRARIES_${lang})
+
+        if(NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang})
+            list(INSERT NetCDF_libs 0 ${NETCDF_LIBRARIES_${lang}}) # prepend so that -lnetcdf is last
+        else(NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang})
+            set(NetCDF_has_interfaces "NO")
+            message(STATUS "Failed to find NetCDF interface for ${lang}")
+        endif(NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang})
+    endif(NETCDF_${lang})
+endmacro(NetCDF_check_interface)
+
+NetCDF_check_interface(CXX netcdfcpp.h netcdf_c++)
+NetCDF_check_interface(F77 netcdf.inc netcdff)
+NetCDF_check_interface(F90 netcdf.mod netcdff)
+
+set(NETCDF_LIBRARIES "${NetCDF_libs}" CACHE STRING "All NetCDF libraries required for interface level")
+
 # handle the QUIETLY and REQUIRED arguments and set NETCDF_FOUND to TRUE if
 # all listed variables are TRUE
 include (FindPackageHandleStandardArgs)
-find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_LINK_LIBRARIES NETCDF_INCLUDE_DIRS)
+find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_LINK_LIBRARIES NETCDF_INCLUDE_DIRS NETCDF_INCLUDES NetCDF_has_interfaces)
 
 mark_as_advanced (NETCDF_LIBRARIES NETCDF_INCLUDES)
 

CMakeLists.txt

@@ -33,6 +33,9 @@ option(ERF_ENABLE_CUDA "Enable CUDA" OFF)
 option(ERF_ENABLE_HIP  "Enable HIP" OFF)
 option(ERF_ENABLE_SYCL "Enable SYCL" OFF)
 
+#Options for NOAH-MP
+option(ERF_ENABLE_NOAH "Enable Noah-MP" OFF)
+
 #Options for C++
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_EXTENSIONS OFF)
@@ -134,6 +137,18 @@ if(ERF_ENABLE_NETCDF)
   endif()
 endif()
 
+########################## NOAH-MP ##################################
+
+if(ERF_ENABLE_NOAH)
+  if(ERF_ENABLE_NETCDF)
+     set(NOAHMP_HOME ${CMAKE_SOURCE_DIR}/Submodules/NOAH-MP)
+     set(NOAHMP_BIN  ${CMAKE_BINARY_DIR}/Submodules/NOAH-MP)
+     add_subdirectory(${NOAHMP_HOME} ${NOAHMP_BIN})
+  else()
+     message(FATAL_ERROR "Noah-MP requires NetCDF be enabled")
+  endif()
+endif()
+
 ########################### RRTMGP #################################
 
 if(ERF_ENABLE_RRTMGP)

Exec/DevTests/NoahMP/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(erf_exe_name erf_noahmp)
+add_executable(${erf_exe_name} "")
+include(${CMAKE_SOURCE_DIR}/CMake/BuildERFExe.cmake)
+build_erf_exe(${erf_exe_name})

Exec/DevTests/NoahMP/GNUmakefile

@@ -0,0 +1,33 @@
+# AMReX
+COMP = gnu
+PRECISION = DOUBLE
+
+# Profiling
+PROFILE       = FALSE
+TINY_PROFILE  = FALSE
+COMM_PROFILE  = FALSE
+TRACE_PROFILE = FALSE
+MEM_PROFILE   = FALSE
+USE_GPROF     = FALSE
+
+# Performance
+USE_MPI  = TRUE
+USE_OMP  = FALSE
+
+USE_CUDA = FALSE
+USE_HIP  = FALSE
+USE_SYCL = FALSE
+
+# Debugging
+DEBUG = FALSE
+
+# Land model
+USE_NETCDF = TRUE
+USE_NOAH = TRUE
+
+# GNU Make
+Bpack := ./Make.package
+Blocs := .
+ERF_HOME := ../../..
+ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/ABL
+include $(ERF_HOME)/Exec/Make.ERF

Exec/Make.ERF

@@ -206,6 +206,25 @@ include $(ERF_LSM_SLM_DIR)/Make.package
 VPATH_LOCATIONS   += $(ERF_LSM_SLM_DIR)
 INCLUDE_LOCATIONS += $(ERF_LSM_SLM_DIR)
 
+# If using NOAH-MP model, then compile relevant source and headers
+ifeq ($(USE_NOAH), TRUE)
+  ifneq ($(USE_NETCDF), TRUE)
+    $(error USE_NETCDF must be true for using NOAH-MP interface)
+  else
+    DEFINES += -DERF_USE_NOAH
+    includes += $(shell pkg-config --cflags netcdf-fortran)
+    LIBRARIES += $(shell pkg-config --libs netcdf-fortran)
+    NOAH_HOME  ?= $(ERF_HOME)/Submodules/NOAH-MP
+    VPATH_LOCATIONS += $(NOAH_HOME)/drivers/hrldas
+    INCLUDE_LOCATIONS += $(NOAH_HOME)/drivers/hrldas
+    ERF_LSM_NOAH_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/NOAH
+    include $(ERF_LSM_NOAH_DIR)/Make.package
+    VPATH_LOCATIONS   += $(ERF_LSM_NOAH_DIR)
+    INCLUDE_LOCATIONS += $(ERF_LSM_NOAH_DIR)
+  endif
+endif
+
+
 ERF_LSM_MM5_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/MM5
 include $(ERF_LSM_MM5_DIR)/Make.package
 VPATH_LOCATIONS   += $(ERF_LSM_MM5_DIR)

Source/DataStructs/ERF_DataStruct.H

@@ -50,7 +50,7 @@ AMREX_ENUM(WindFarmLocType,
 );
 
 AMREX_ENUM(LandSurfaceType,
-    SLM, MM5, None
+    SLM, MM5, None, NOAH
 );
 
 AMREX_ENUM(PerturbationType,

Source/ERF.cpp

@@ -1557,6 +1557,11 @@ ERF::ReadParameters ()
     } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
         lsm.SetModel<MM5>();
         Print() << "MM5 land surface model!\n";
+#ifdef ERF_USE_NOAH
+    } else if (solverChoice.lsm_type == LandSurfaceType::NOAH) {
+        lsm.SetModel<NOAH>();
+        Print() << "NOAH land surface model!\n";
+#endif
     } else if (solverChoice.lsm_type == LandSurfaceType::None) {
         lsm.SetModel<NullSurf>();
         Print() << "Null land surface model!\n";

Source/LandSurfaceModel/ERF_LandSurface.H

@@ -7,6 +7,10 @@
 #include <ERF_SLM.H>
 #include <ERF_MM5.H>
 
+#if ERF_USE_NOAH
+#include <ERF_NOAH.H>
+#endif
+
 class LandSurface {
 
 public:

Source/LandSurfaceModel/NOAH/ERF_NOAH.H

@@ -0,0 +1,41 @@
+#ifndef NOAH_H
+#define NOAH_H
+
+#include <string>
+#include <vector>
+#include <memory>
+
+#include <AMReX_FArrayBox.H>
+#include <AMReX_Geometry.H>
+#include <AMReX_MultiFabUtil.H>
+
+#include <ERF_NullSurf.H>
+#include <ERF_Constants.H>
+#include <ERF_IndexDefines.H>
+#include <ERF_DataStruct.H>
+
+// External include from the noahmp library
+#include <NoahmpIO.H>
+
+class NOAH : public NullSurf {
+public:
+    // Constructor
+    NOAH () {}
+
+    // Destructor
+    virtual ~NOAH () = default;
+
+    // Initialize data structures
+    void
+    Init (const amrex::MultiFab& cons_in,
+          const amrex::Geometry& geom,
+          const amrex::Real& dt) override;
+
+
+private:
+
+   // C++ variable for NoahmpIO struct
+   NoahmpIO_struct noahmpio;
+
+};
+#endif

Source/LandSurfaceModel/NOAH/ERF_NOAH.cpp

@@ -0,0 +1,30 @@
+
+#include<iostream>
+
+#include <AMReX_Print.H>
+#include <ERF_NOAH.H>
+
+using namespace amrex;
+
+/* Initialize lsm data structures */
+void
+NOAH::Init (const MultiFab& cons_in,
+            const Geometry& geom,
+            const Real& dt)
+{
+    // Initialize Noahmp IO
+    amrex::Print() << "Initializing Noahmp IO" << std::endl;
+
+    /*
+     * noahmpio.xstart = 1;
+     * noahmpio.xend = 4;
+     * noahmpio.ystart = 1;
+     * noahmpio.yend = 2;
+     *
+     */
+
+    NoahmpIOVarInitDefault(&noahmpio);
+    NoahmpInitMain(&noahmpio);
+
+    amrex::Print() << "Noahmp IO Initialized" << std::endl;
+};

Source/LandSurfaceModel/NOAH/Make.package

@@ -0,0 +1,2 @@
+CEXE_sources += ERF_NOAH.cpp
+CEXE_headers += ERF_NOAH.H