From 4cd3dddc4347096f91cd83f0642eb17668d0925c Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 24 Oct 2024 13:40:40 -0500 Subject: [PATCH] Interface for NOAH-MP Landsurface model (#1835) * Add inital interface for NOAH-M * Update NOAH module and the corresponding FindNetCDF cmake recipe * Initial interface for NOAH LSM * Initial interface for NOAH-MP land surface model * Add linker options * Updates to cmake configuration to include NOAH * Update CMakeLists and move Fortran linkage to Noah-MP: * Update .gitmodules * Updates to NOAH-MP interface * Saving some interface work * udpate gitmodules * checkpoint changes for merge * Update Noah-MP submodules to resolve cmake errors * Resolve style errors * Update gmake interface for noahmp and add a dev test --- .gitmodules | 6 +++- CMake/BuildERFExe.cmake | 10 ++++++ CMake/FindNetCDF.cmake | 36 +++++++++++++++++++- CMakeLists.txt | 15 +++++++++ Exec/DevTests/NoahMP/CMakeLists.txt | 4 +++ Exec/DevTests/NoahMP/GNUmakefile | 33 ++++++++++++++++++ Exec/Make.ERF | 19 +++++++++++ Source/DataStructs/ERF_DataStruct.H | 2 +- Source/ERF.cpp | 5 +++ Source/LandSurfaceModel/ERF_LandSurface.H | 4 +++ Source/LandSurfaceModel/NOAH/ERF_NOAH.H | 41 +++++++++++++++++++++++ Source/LandSurfaceModel/NOAH/ERF_NOAH.cpp | 30 +++++++++++++++++ Source/LandSurfaceModel/NOAH/Make.package | 2 ++ Submodules/NOAH-MP | 1 + 14 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 Exec/DevTests/NoahMP/CMakeLists.txt create mode 100644 Exec/DevTests/NoahMP/GNUmakefile create mode 100644 Source/LandSurfaceModel/NOAH/ERF_NOAH.H create mode 100644 Source/LandSurfaceModel/NOAH/ERF_NOAH.cpp create mode 100644 Source/LandSurfaceModel/NOAH/Make.package create mode 160000 Submodules/NOAH-MP diff --git a/.gitmodules b/.gitmodules index f96430215..ba9b4da82 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,7 +9,11 @@ [submodule "Submodules/RRTMGP"] path = Submodules/RRTMGP url = https://github.com/E3SM-Project/rte-rrtmgp - shallow = true + shallow = true +[submodule "Submodules/NOAH-MP"] + path = Submodules/NOAH-MP + url = https://github.com/AIEADA/noahmp.git + shallow=true [submodule "Submodules/WW3"] path = Submodules/WW3 url = https://github.com/erf-model/WW3 diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake index 7bc51f20a..6c5d6195d 100644 --- a/CMake/BuildERFExe.cmake +++ b/CMake/BuildERFExe.cmake @@ -67,6 +67,16 @@ function(build_erf_lib erf_lib_name) target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_NETCDF) endif() + if(ERF_ENABLE_NOAH) + target_include_directories(${erf_lib_name} PUBLIC + $ + $) + target_sources(${erf_lib_name} PRIVATE + ${SRC_DIR}/LandSurfaceModel/NOAH/ERF_NOAH.cpp) + target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_NOAH) + target_link_libraries_system(${erf_lib_name} PUBLIC NoahMP::noahmp) + endif() + if(ERF_ENABLE_RRTMGP) target_sources(${erf_lib_name} PRIVATE ${SRC_DIR}/Utils/ERF_Orbit.cpp diff --git a/CMake/FindNetCDF.cmake b/CMake/FindNetCDF.cmake index 3e54ea3e8..6fac102b3 100644 --- a/CMake/FindNetCDF.cmake +++ b/CMake/FindNetCDF.cmake @@ -23,10 +23,44 @@ endif (NETCDF_INCLUDES AND NETCDF_LIBRARIES) find_package(PkgConfig REQUIRED QUIET) pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET netcdf) +find_path(NETCDF_INCLUDES netcdf.h + HINTS NETCDF_DIR/include ENV NETCDF_DIR) + +find_library(NETCDF_LIBRARIES_C NAMES netcdf HINTS NETCDF_DIR/lib ENV NETCDF_DIR) +mark_as_advanced(NETCDF_LIBRARIES_C) + +set(NetCDF_has_interfaces "YES") # will be set to NO if we're missing any interfaces +set(NetCDF_libs "${NETCDF_LIBRARIES_C}") + +get_filename_component(NetCDF_lib_dirs "${NETCDF_LIBRARIES_C}" PATH) + +macro(NetCDF_check_interface lang header libs) + if(NETCDF_${lang}) + find_path(NETCDF_INCLUDES_${lang} NAMES ${header} + HINTS "${NETCDF_INCLUDES}" NO_DEFAULT_PATH) + find_library(NETCDF_LIBRARIES_${lang} NAMES ${libs} + HINTS "${NetCDF_lib_dirs}" NO_DEFAULT_PATH) + mark_as_advanced(NETCDF_INCLUDES_${lang} NETCDF_LIBRARIES_${lang}) + + if(NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang}) + list(INSERT NetCDF_libs 0 ${NETCDF_LIBRARIES_${lang}}) # prepend so that -lnetcdf is last + else(NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang}) + set(NetCDF_has_interfaces "NO") + message(STATUS "Failed to find NetCDF interface for ${lang}") + endif(NETCDF_INCLUDES_${lang} AND NETCDF_LIBRARIES_${lang}) + endif(NETCDF_${lang}) +endmacro(NetCDF_check_interface) + +NetCDF_check_interface(CXX netcdfcpp.h netcdf_c++) +NetCDF_check_interface(F77 netcdf.inc netcdff) +NetCDF_check_interface(F90 netcdf.mod netcdff) + +set(NETCDF_LIBRARIES "${NetCDF_libs}" CACHE STRING "All NetCDF libraries required for interface level") + # handle the QUIETLY and REQUIRED arguments and set NETCDF_FOUND to TRUE if # all listed variables are TRUE include (FindPackageHandleStandardArgs) -find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_LINK_LIBRARIES NETCDF_INCLUDE_DIRS) +find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_LINK_LIBRARIES NETCDF_INCLUDE_DIRS NETCDF_INCLUDES NetCDF_has_interfaces) mark_as_advanced (NETCDF_LIBRARIES NETCDF_INCLUDES) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03fb265ad..e045d2eb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,9 @@ option(ERF_ENABLE_CUDA "Enable CUDA" OFF) option(ERF_ENABLE_HIP "Enable HIP" OFF) option(ERF_ENABLE_SYCL "Enable SYCL" OFF) +#Options for NOAH-MP +option(ERF_ENABLE_NOAH "Enable Noah-MP" OFF) + #Options for C++ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_EXTENSIONS OFF) @@ -134,6 +137,18 @@ if(ERF_ENABLE_NETCDF) endif() endif() +########################## NOAH-MP ################################## + +if(ERF_ENABLE_NOAH) + if(ERF_ENABLE_NETCDF) + set(NOAHMP_HOME ${CMAKE_SOURCE_DIR}/Submodules/NOAH-MP) + set(NOAHMP_BIN ${CMAKE_BINARY_DIR}/Submodules/NOAH-MP) + add_subdirectory(${NOAHMP_HOME} ${NOAHMP_BIN}) + else() + message(FATAL_ERROR "Noah-MP requires NetCDF be enabled") + endif() +endif() + ########################### RRTMGP ################################# if(ERF_ENABLE_RRTMGP) diff --git a/Exec/DevTests/NoahMP/CMakeLists.txt b/Exec/DevTests/NoahMP/CMakeLists.txt new file mode 100644 index 000000000..995474bb5 --- /dev/null +++ b/Exec/DevTests/NoahMP/CMakeLists.txt @@ -0,0 +1,4 @@ +set(erf_exe_name erf_noahmp) +add_executable(${erf_exe_name} "") +include(${CMAKE_SOURCE_DIR}/CMake/BuildERFExe.cmake) +build_erf_exe(${erf_exe_name}) diff --git a/Exec/DevTests/NoahMP/GNUmakefile b/Exec/DevTests/NoahMP/GNUmakefile new file mode 100644 index 000000000..04aafbb6d --- /dev/null +++ b/Exec/DevTests/NoahMP/GNUmakefile @@ -0,0 +1,33 @@ +# AMReX +COMP = gnu +PRECISION = DOUBLE + +# Profiling +PROFILE = FALSE +TINY_PROFILE = FALSE +COMM_PROFILE = FALSE +TRACE_PROFILE = FALSE +MEM_PROFILE = FALSE +USE_GPROF = FALSE + +# Performance +USE_MPI = TRUE +USE_OMP = FALSE + +USE_CUDA = FALSE +USE_HIP = FALSE +USE_SYCL = FALSE + +# Debugging +DEBUG = FALSE + +# Land model +USE_NETCDF = TRUE +USE_NOAH = TRUE + +# GNU Make +Bpack := ./Make.package +Blocs := . +ERF_HOME := ../../.. +ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/ABL +include $(ERF_HOME)/Exec/Make.ERF diff --git a/Exec/Make.ERF b/Exec/Make.ERF index 4205dafa6..7aca7d619 100644 --- a/Exec/Make.ERF +++ b/Exec/Make.ERF @@ -206,6 +206,25 @@ include $(ERF_LSM_SLM_DIR)/Make.package VPATH_LOCATIONS += $(ERF_LSM_SLM_DIR) INCLUDE_LOCATIONS += $(ERF_LSM_SLM_DIR) +# If using NOAH-MP model, then compile relevant source and headers +ifeq ($(USE_NOAH), TRUE) + ifneq ($(USE_NETCDF), TRUE) + $(error USE_NETCDF must be true for using NOAH-MP interface) + else + DEFINES += -DERF_USE_NOAH + includes += $(shell pkg-config --cflags netcdf-fortran) + LIBRARIES += $(shell pkg-config --libs netcdf-fortran) + NOAH_HOME ?= $(ERF_HOME)/Submodules/NOAH-MP + VPATH_LOCATIONS += $(NOAH_HOME)/drivers/hrldas + INCLUDE_LOCATIONS += $(NOAH_HOME)/drivers/hrldas + ERF_LSM_NOAH_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/NOAH + include $(ERF_LSM_NOAH_DIR)/Make.package + VPATH_LOCATIONS += $(ERF_LSM_NOAH_DIR) + INCLUDE_LOCATIONS += $(ERF_LSM_NOAH_DIR) + endif +endif + + ERF_LSM_MM5_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/MM5 include $(ERF_LSM_MM5_DIR)/Make.package VPATH_LOCATIONS += $(ERF_LSM_MM5_DIR) diff --git a/Source/DataStructs/ERF_DataStruct.H b/Source/DataStructs/ERF_DataStruct.H index c8f639a4a..645b5c7a5 100644 --- a/Source/DataStructs/ERF_DataStruct.H +++ b/Source/DataStructs/ERF_DataStruct.H @@ -50,7 +50,7 @@ AMREX_ENUM(WindFarmLocType, ); AMREX_ENUM(LandSurfaceType, - SLM, MM5, None + SLM, MM5, None, NOAH ); AMREX_ENUM(PerturbationType, diff --git a/Source/ERF.cpp b/Source/ERF.cpp index ef6ec5bfc..97ffabcfd 100644 --- a/Source/ERF.cpp +++ b/Source/ERF.cpp @@ -1557,6 +1557,11 @@ ERF::ReadParameters () } else if (solverChoice.lsm_type == LandSurfaceType::MM5) { lsm.SetModel(); Print() << "MM5 land surface model!\n"; +#ifdef ERF_USE_NOAH + } else if (solverChoice.lsm_type == LandSurfaceType::NOAH) { + lsm.SetModel(); + Print() << "NOAH land surface model!\n"; +#endif } else if (solverChoice.lsm_type == LandSurfaceType::None) { lsm.SetModel(); Print() << "Null land surface model!\n"; diff --git a/Source/LandSurfaceModel/ERF_LandSurface.H b/Source/LandSurfaceModel/ERF_LandSurface.H index 9db80be66..56ad3b2ea 100644 --- a/Source/LandSurfaceModel/ERF_LandSurface.H +++ b/Source/LandSurfaceModel/ERF_LandSurface.H @@ -7,6 +7,10 @@ #include #include +#if ERF_USE_NOAH +#include +#endif + class LandSurface { public: diff --git a/Source/LandSurfaceModel/NOAH/ERF_NOAH.H b/Source/LandSurfaceModel/NOAH/ERF_NOAH.H new file mode 100644 index 000000000..962c2dd2b --- /dev/null +++ b/Source/LandSurfaceModel/NOAH/ERF_NOAH.H @@ -0,0 +1,41 @@ +#ifndef NOAH_H +#define NOAH_H + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +// External include from the noahmp library +#include + +class NOAH : public NullSurf { +public: + // Constructor + NOAH () {} + + // Destructor + virtual ~NOAH () = default; + + // Initialize data structures + void + Init (const amrex::MultiFab& cons_in, + const amrex::Geometry& geom, + const amrex::Real& dt) override; + + +private: + + // C++ variable for NoahmpIO struct + NoahmpIO_struct noahmpio; + +}; +#endif diff --git a/Source/LandSurfaceModel/NOAH/ERF_NOAH.cpp b/Source/LandSurfaceModel/NOAH/ERF_NOAH.cpp new file mode 100644 index 000000000..3e4acf206 --- /dev/null +++ b/Source/LandSurfaceModel/NOAH/ERF_NOAH.cpp @@ -0,0 +1,30 @@ + +#include + +#include +#include + +using namespace amrex; + +/* Initialize lsm data structures */ +void +NOAH::Init (const MultiFab& cons_in, + const Geometry& geom, + const Real& dt) +{ + // Initialize Noahmp IO + amrex::Print() << "Initializing Noahmp IO" << std::endl; + + /* + * noahmpio.xstart = 1; + * noahmpio.xend = 4; + * noahmpio.ystart = 1; + * noahmpio.yend = 2; + * + */ + + NoahmpIOVarInitDefault(&noahmpio); + NoahmpInitMain(&noahmpio); + + amrex::Print() << "Noahmp IO Initialized" << std::endl; +}; diff --git a/Source/LandSurfaceModel/NOAH/Make.package b/Source/LandSurfaceModel/NOAH/Make.package new file mode 100644 index 000000000..4f1e6f483 --- /dev/null +++ b/Source/LandSurfaceModel/NOAH/Make.package @@ -0,0 +1,2 @@ +CEXE_sources += ERF_NOAH.cpp +CEXE_headers += ERF_NOAH.H diff --git a/Submodules/NOAH-MP b/Submodules/NOAH-MP new file mode 160000 index 000000000..482646ac5 --- /dev/null +++ b/Submodules/NOAH-MP @@ -0,0 +1 @@ +Subproject commit 482646ac5641492048973f7706db57145a3db11a