Adapt to PeleRad in PelePhysics

.github/workflows/linux.yml

@@ -85,40 +85,29 @@ jobs:
       {CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
     steps:
     - uses: actions/checkout@v3
-    - name: System Dependencies
-      run: .github/workflows/dependencies/dependencies_gcc10.sh
     - name: Repo Dependencies
       run: |
         Utils/CloneDeps.sh
-        git clone --branch main https://github.com/AMReX-Combustion/PeleRad.git Submodules/PeleRad
+    - name: System Dependencies
+      run: .github/workflows/dependencies/dependencies_gcc10.sh
     - name: Build Release
-      env:
-         PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
       working-directory: ./Exec/RegTests/SootRadTest/
       run: |
         make TPL COMP=gnu USE_MPI=TRUE TINY_PROFILE=FALSE
         make -j 2 COMP=gnu USE_MPI=TRUE TINY_PROFILE=FALSE
     - name: Run Release
-      env:
-         PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
       working-directory: ./Exec/RegTests/SootRadTest/
       run: |
-        eval DATA_PATH=$PELERAD_HOME
-        mpirun -n 2 ./PeleLMeX2d.gnu.MPI.ex first-input.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2 pelerad.kppath="$DATA_PATH/data/kpDB/"
+        mpirun -n 2 ./PeleLMeX2d.gnu.MPI.ex first-input.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2
     - name: Build Debug
-      env:
-         PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
       working-directory: ./Exec/RegTests/SootRadTest/
       run: |
         make TPL COMP=gnu USE_MPI=TRUE DEBUG=TRUE TINY_PROFILE=FALSE
         make -j 2 COMP=gnu USE_MPI=TRUE DEBUG=TRUE TINY_PROFILE=FALSE
     - name: Run Debug
-      env:
-         PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
       working-directory: ./Exec/RegTests/SootRadTest/
       run: |
-        eval DATA_PATH=$PELERAD_HOME
-        mpirun -n 2 ./PeleLMeX2d.gnu.DEBUG.MPI.ex first-input.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2 pelerad.kppath="$DATA_PATH/data/kpDB/"
+        mpirun -n 2 ./PeleLMeX2d.gnu.DEBUG.MPI.ex first-input.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2
 
   # Build and Run the EB_BackwardStepFlame RegTest with GNU9.3 and MPI support
   EBBFS:

CMake/BuildExeAndLib.cmake

@@ -1,6 +1,6 @@
 get_filename_component(DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 set(pelelmex_lib_name PelePhysics-Lib-${PELELMEX_EOS_MODEL}-${PELELMEX_CHEMISTRY_MODEL}-${PELELMEX_TRANSPORT_MODEL})
-set(pelelmex_exe_name PeleLMeX-${DIR_NAME})
+set(pelelmex_exe_name ${PROJECT_NAME}-${DIR_NAME})
 include(BuildPeleLMeXLib)
 include(BuildPeleLMeXExe)
 build_pelelmex_lib(${pelelmex_lib_name})

CMake/BuildPeleLMeXLib.cmake

@@ -2,14 +2,14 @@ function(build_pelelmex_lib pelelmex_lib_name)
   if (NOT (TARGET ${pelelmex_lib_name}))
     add_library(${pelelmex_lib_name} OBJECT)
 
-    set(PELE_PHYSICS_SRC_DIR ${CMAKE_SOURCE_DIR}/Submodules/PelePhysics)
+    set(PELE_PHYSICS_SRC_DIR "${CMAKE_SOURCE_DIR}/Submodules/PelePhysics")
     set(PELE_PHYSICS_TRANSPORT_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Transport")
     set(PELE_PHYSICS_EOS_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Eos")
     set(PELE_PHYSICS_MECHANISM_DIR "${PELE_PHYSICS_SRC_DIR}/Mechanisms/${PELELMEX_CHEMISTRY_MODEL}")
     set(PELE_PHYSICS_UTILITY_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Utility")
     set(PELE_PHYSICS_REACTIONS_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Reactions")
     set(PELE_PHYSICS_SOOT_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Soot")
-    set(AMREX_SUNDIALS_DIR ${AMREX_SUBMOD_LOCATION}/Src/Extern/SUNDIALS)
+    set(AMREX_SUNDIALS_DIR "${AMREX_SUBMOD_LOCATION}/Src/Extern/SUNDIALS")
 
     if(CLANG_TIDY_EXE)
       set_target_properties(${pelelmex_lib_name} PROPERTIES CXX_CLANG_TIDY
@@ -159,7 +159,7 @@ function(build_pelelmex_lib pelelmex_lib_name)
                       ${PELE_PHYSICS_SOOT_DIR}/Constants_Soot.H
                       ${PELE_PHYSICS_SOOT_DIR}/SootData.H
                       ${PELE_PHYSICS_SOOT_DIR}/SootReactions.H)
-       target_include_directories(${pele=lmex_lib_name} PUBLIC ${PELE_PHYSICS_SOOT_DIR})
+       target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_SOOT_DIR})
     endif()
 
     include(AMReXBuildInfo)

CMakeLists.txt

@@ -113,3 +113,4 @@ add_subdirectory(Exec)
 enable_testing()
 include(CTest)
 add_subdirectory(Tests)
+add_subdirectory(${CMAKE_SOURCE_DIR}/Submodules/PelePhysics/Testing/Exec/Radiation)

Exec/Make.PeleLMeX

@@ -53,11 +53,8 @@ ifeq ($(USE_SOOT), TRUE)
    DEFINES += -DNUM_SOOT_MOMENTS=$(NUM_SOOT_MOMENTS)
 endif
 
-ifeq ($(USE_PELERAD), TRUE)
-   DEFINES += -DPELELM_USE_RAD
-   ifeq ($(USE_HIP), TRUE)
-       DEFINES += -DPELERAD_USE_HIP
-   endif
+ifeq ($(USE_RADIATION), TRUE)
+   DEFINES += -DPELELM_USE_RADIATION
 endif
 
 Bpack += $(foreach dir, $(LMdirs), $(PELELMEX_HOME)/$(dir)/Make.package)
@@ -126,8 +123,10 @@ ifeq ($(USE_SOOT), TRUE)
    Bpack += $(SOOT_HOME)/Make.package
    Blocs += $(SOOT_HOME)/Source/Soot_Models
 endif
-ifeq ($(USE_PELERAD), TRUE)
-   Blocs += $(PELERAD_HOME)/src
+ifeq ($(USE_RADIATION), TRUE)
+   RADIATION_HOME = $(PELE_PHYSICS_HOME)/Source/Radiation
+   Bpack += $(RADIATION_HOME)/Make.package
+   Blocs += $(RADIATION_HOME)
 endif
 
 #---------------

Exec/RegTests/EB_EnclosedVortex/input.2d-regt

@@ -81,6 +81,6 @@ nodal_proj.verbose = 0
 #nodal_proj.rtol = 1.0e-10
 #nodal_proj.atol = 1.0e-10
 #nodal_proj.mg_max_coarsening_level = 2
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EB_EnclosedVortex/input.2d-regt_incomp

@@ -66,6 +66,6 @@ nodal_proj.verbose = 0
 #nodal_proj.rtol = 1.0e-10
 #nodal_proj.atol = 1.0e-10
 #nodal_proj.mg_max_coarsening_level = 2
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EB_EnclosedVortex/input.3d-regt

@@ -88,6 +88,6 @@ nodal_proj.mg_max_coarsening_level = 2
 
 #fabarray.mfiter_tile_size = 1024 1024 1024
 
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EB_FlowPastCylinder/input.2d-regt_isoT

@@ -65,6 +65,6 @@ eb2.small_volfrac = 1.0e-4
 #amr.temp.value_greater = 305
 #amr.temp.field_name    = temp
 
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EB_FlowPastCylinder/input.3d-regtY

@@ -65,6 +65,6 @@ eb2.small_volfrac = 1.0e-4
 #
 fabarray.mfiter_tile_size = 1024 1024 1024
 
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EB_PipeFlow/input.3d-regt

@@ -62,6 +62,6 @@ fabarray.mfiter_tile_size = 1024 1024 1024
 
 nodal_proj.verbose = 0
 nodal_proj.mg_max_coarsening_level = 2
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EnclosedFlame/enclosedflame-2d.inp

@@ -81,6 +81,6 @@ amr.yH.field_name    = Y(H)
 #--------------------LINEAR SOLVER CONTROL------------------------
 nodal_proj.verbose = 2
 nodal_proj.rtol = 2.0e-11
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EnclosedFlame/enclosedflame-3d.inp

@@ -80,6 +80,6 @@ amr.yH.field_name    = Y(H)
 #--------------------LINEAR SOLVER CONTROL------------------------
 nodal_proj.verbose = 0
 nodal_proj.rtol = 1.0e-11
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EnclosedFlame/input.2d-regt

@@ -81,6 +81,6 @@ amr.yH.field_name    = Y(H)
 #--------------------LINEAR SOLVER CONTROL------------------------
 nodal_proj.verbose = 2
 nodal_proj.rtol = 2.0e-11
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/EnclosedFlame/input.3d-regt

@@ -80,6 +80,6 @@ amr.yH.field_name    = Y(H)
 #--------------------LINEAR SOLVER CONTROL------------------------
 nodal_proj.verbose = 0
 nodal_proj.rtol = 1.0e-11
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+#amrex.fpe_trap_invalid = 1
+#amrex.fpe_trap_zero = 1
+#amrex.fpe_trap_overflow = 1

Exec/RegTests/SootRadTest/GNUmakefile

@@ -2,15 +2,15 @@
 DIM             = 2
 DEBUG           = FALSE
 PRECISION       = DOUBLE
-VERBOSE         = FALSE
-TINY_PROFILE    = TRUE
+TINY_PROFILE    = FALSE
 
 # Compilation
 COMP            = llvm
-USE_MPI         = TRUE
+USE_MPI         = FALSE
 USE_OMP         = FALSE
 USE_CUDA        = FALSE
 USE_HIP         = FALSE
+USE_SYCL        = FALSE
 
 # PeleLMeX
 USE_EFIELD      = FALSE
@@ -20,15 +20,9 @@ Chemistry_Model = SootReaction
 Eos_Model	= Fuego
 Transport_Model = Simple
 
+USE_RADIATION = TRUE
 USE_SOOT = TRUE
 NUM_SOOT_MOMENTS = 3
 
-USE_PELERAD = TRUE
-ifeq ($(USE_PELERAD), TRUE)
-ifeq ($(USE_HIP), TRUE)
-LIBRARIES += -lstdc++fs
-endif
-endif
-
 PELELMEX_HOME ?= ../../..
 include $(PELELMEX_HOME)/Exec/Make.PeleLMeX

Exec/RegTests/SootRadTest/README.md

@@ -1,12 +1,2 @@
 ## SootRadTest
-Testing of the coupling between PeleLMeX and PeleMP soot and radiation modules.
-
-For now, PeleRad must be clone separately for this test case. In a convenient
-location, run:
-
-git clone https://github.com/AMReX-Combustion/PeleRad.git
-export PELERAD_HOME=$(pwd)/PeleRad
-
-Please specify the radiation database path in the input file before running.
-
-echo "pelerad.kppath = "$PELERAD_HOME/data/kpDB/"" >> first-input
+Testing of the coupling between PeleLMeX and soot and radiation modules.

Exec/RegTests/SootRadTest/first-input.inp

@@ -19,7 +19,7 @@ amr.n_error_buf     = 1 1 2 2          # number of buffer cells in error est
 amr.grid_eff        = 0.7              # what constitutes an efficient grid
 amr.ref_ratio = 2 2 2
 amr.blocking_factor = 8
-amr.max_grid_size = 128
+amr.max_grid_size = 64
 
 #--------------------------- Problem -------------------------------
 prob.P_mean = 98700.
@@ -86,9 +86,10 @@ amr.gradT.adjacent_difference_greater = 30.
 amr.gradT.field_name    = temp
 
 amrex.regtest_reduction = 1
-amrex.fpe_trap_invalid = 1
-amrex.fpe_trap_zero = 1
-amrex.fpe_trap_overflow = 1
+amrex.signal_handling = 0
+#amrex.fpe_trap_invalid = 0
+#amrex.fpe_trap_zero = 0
+#amrex.fpe_trap_overflow = 0
 
 #--------------------RADIATION MODELING------------------------
 peleLM.do_rad_solve = 1
@@ -109,5 +110,5 @@ pelerad.linop_maxorder = 2
 pelerad.max_fmg_iter = 0
 pelerad.lo_bc = Robin Periodic Periodic
 pelerad.hi_bc = Robin Periodic Periodic
-#please set the pelerad.kppath
-pelerad.kppath = /lustre/orion/cmb138/proj-shared/w0g/PR/PeleRad/data/kpDB/
+#The trailing / might matter here:
+pelerad.kppath = kpDB/