Skip to content

Commit

Permalink
Adding the radiation module (#313)
Browse files Browse the repository at this point in the history
* prepare the PR

* add the case to PeleLMeX

* add the data file

* fix assert for derrhomrhoy

* changes cvode solve_type for sootrad

* sootrad into ci

* rad: eb robin amrex was merged dont need special clone anymore

* instructions to clone PeleRad in Rad case README

* Edits for Marc's comments

* add a runtime flag for plot

* dt is not needed in the radiation calculation

* fix spelling

* update pelerad test to mostly use submods

* update rad gnumakefile

---------

Co-authored-by: Bruce Perry <[email protected]>
Co-authored-by: Bruce Perry <[email protected]>
  • Loading branch information
3 people authored Dec 7, 2023
1 parent c8c1cab commit 01790c2
Show file tree
Hide file tree
Showing 17 changed files with 805 additions and 1 deletion.
42 changes: 42 additions & 0 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,48 @@ jobs:
run: |
mpirun -n 2 ./PeleLMeX3d.gnu.MPI.ex input.3d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1 amr.n_cell=128 32 32
# Build and Run the SootRadTest RegTest with GNU9.3 and MPI support
SOOTRAD:
name: [email protected] MPI Run [SootRadTest]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: |
Utils/CloneDeps.sh
git clone --branch main https://github.com/AMReX-Combustion/PeleRad.git Submodules/PeleRad
- name: Build Release
env:
PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
working-directory: ./Exec/RegTests/SootRadTest/
run: |
make TPL COMP=gnu USE_MPI=TRUE TINY_PROFILE=FALSE
make -j 2 COMP=gnu USE_MPI=TRUE TINY_PROFILE=FALSE
- name: Run Release
env:
PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
working-directory: ./Exec/RegTests/SootRadTest/
run: |
eval DATA_PATH=$PELERAD_HOME
mpirun -n 2 ./PeleLMeX2d.gnu.MPI.ex first-input.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2 pelerad.kppath="$DATA_PATH/data/kpDB/"
- name: Build Debug
env:
PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
working-directory: ./Exec/RegTests/SootRadTest/
run: |
make TPL COMP=gnu USE_MPI=TRUE DEBUG=TRUE TINY_PROFILE=FALSE
make -j 2 COMP=gnu USE_MPI=TRUE DEBUG=TRUE TINY_PROFILE=FALSE
- name: Run Debug
env:
PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
working-directory: ./Exec/RegTests/SootRadTest/
run: |
eval DATA_PATH=$PELERAD_HOME
mpirun -n 2 ./PeleLMeX2d.gnu.DEBUG.MPI.ex first-input.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2 pelerad.kppath="$DATA_PATH/data/kpDB/"
# Build and Run the EB_BackwardStepFlame RegTest with GNU9.3 and MPI support
EBBFS:
Expand Down
10 changes: 10 additions & 0 deletions Exec/Make.PeleLMeX
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,13 @@ ifeq ($(USE_SOOT), TRUE)
DEFINES += -DNUM_SOOT_MOMENTS=$(NUM_SOOT_MOMENTS)
endif

ifeq ($(USE_PELERAD), TRUE)
DEFINES += -DPELELM_USE_RAD
ifeq ($(USE_HIP), TRUE)
DEFINES += -DPELERAD_USE_HIP
endif
endif

Bpack += $(foreach dir, $(LMdirs), $(PELELMEX_HOME)/$(dir)/Make.package)
Blocs += $(foreach dir, $(LMdirs), $(PELELMEX_HOME)/$(dir))

Expand Down Expand Up @@ -142,6 +149,9 @@ ifeq ($(USE_SOOT), TRUE)
Bpack += $(PELEMP_HOME)/Source/Soot_Models/Make.package
Blocs += $(PELEMP_HOME)/Source/Soot_Models
endif
ifeq ($(USE_PELERAD), TRUE)
Blocs += $(PELERAD_HOME)/src
endif

#---------------
# Includes
Expand Down
34 changes: 34 additions & 0 deletions Exec/RegTests/SootRadTest/GNUmakefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# AMReX
DIM = 2
DEBUG = FALSE
PRECISION = DOUBLE
VERBOSE = FALSE
TINY_PROFILE = TRUE

# Compilation
COMP = llvm
USE_MPI = TRUE
USE_OMP = FALSE
USE_CUDA = FALSE
USE_HIP = FALSE

# PeleLMeX
USE_EFIELD = FALSE

# PelePhysics
Chemistry_Model = SootReaction
Eos_Model = Fuego
Transport_Model = Simple

USE_SOOT = TRUE
NUM_SOOT_MOMENTS = 3

USE_PELERAD = TRUE
ifeq ($(USE_PELERAD), TRUE)
ifeq ($(USE_HIP), TRUE)
LIBRARIES += -lstdc++fs
endif
endif

PELELMEX_HOME ?= ../../..
include $(PELELMEX_HOME)/Exec/Make.PeleLMeX
12 changes: 12 additions & 0 deletions Exec/RegTests/SootRadTest/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
## SootRadTest
Testing of the coupling between PeleLMeX and PeleMP soot and radiation modules.

For now, PeleRad must be clone separately for this test case. In a convenient
location, run:

git clone https://github.com/AMReX-Combustion/PeleRad.git
export PELERAD_HOME=$(pwd)/PeleRad

Please specify the radiation database path in the input file before running.

echo "pelerad.kppath = "$PELERAD_HOME/data/kpDB/"" >> first-input
256 changes: 256 additions & 0 deletions Exec/RegTests/SootRadTest/datafile_init/mueller_burner.dat

Large diffs are not rendered by default.

113 changes: 113 additions & 0 deletions Exec/RegTests/SootRadTest/first-input.inp
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#----------------------DOMAIN DEFINITION------------------------
geometry.is_periodic = 0 1 1 # For each dir, 0: non-perio, 1: periodic
geometry.coord_sys = 0 # 0 => cart, 1 => RZ
geometry.prob_lo = 0.0 0.0 0.0 # x_lo y_lo (z_lo)
geometry.prob_hi = 0.04 0.0025 0.0025 # x_hi y_hi (z_hi)

# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<<
# Interior, Inflow, Outflow, Symmetry,
# SlipWallAdiab, NoSlipWallAdiab, SlipWallIsotherm, NoSlipWallIsotherm
peleLM.lo_bc = Inflow Interior Interior
peleLM.hi_bc = Outflow Interior Interior

#-------------------------AMR CONTROL----------------------------
amr.n_cell = 128 8 8 # Level 0 number of cells in each direction
amr.v = 1 # AMR verbose
amr.max_level = 2 # maximum level number allowed
amr.regrid_int = 4 # how often to regrid
amr.n_error_buf = 1 1 2 2 # number of buffer cells in error est
amr.grid_eff = 0.7 # what constitutes an efficient grid
amr.ref_ratio = 2 2 2
amr.blocking_factor = 8
amr.max_grid_size = 128

#--------------------------- Problem -------------------------------
prob.P_mean = 98700.
prob.standoff = 0.0
pmf.datafile = "datafile_init/mueller_burner.dat"
pmf.do_cellAverage = 0

#--------------------SOOT MODELING------------------------
peleLM.do_soot_solve = 1
soot.incept_pah = A2 # Soot inception species
soot.v = 0
soot.temp_cutoff = 290.
soot.conserve_mass = false
soot.num_subcycles = 10
soot.max_subcycles = 1000

#-------------------------PeleLM CONTROL----------------------------
peleLM.v = 1
peleLM.incompressible = 0
peleLM.use_wbar = 0
peleLM.sdc_iterMax = 1
peleLM.floor_species = 0
peleLM.advection_scheme = Godunov_BDS

peleLM.do_temporals = 0
peleLM.temporal_int = 2
peleLM.mass_balance = 1
peleLM.num_init_iter = 1
peleLM.plot_react = 0

#amr.restart = chk00005
#amr.check_int = 2000
amr.plot_per = 1.E-3
amr.dt_shrink = 0.1
amr.max_step = 10000
amr.stop_time = 0.022
amr.cfl = 0.3
amr.derive_plot_vars = rhoRT mass_fractions rhominsumrhoY
#amr.fixed_dt = 0.008
#amr.fixed_dt = 1.E-6

# --------------- INPUTS TO CHEMISTRY REACTOR ---------------
peleLM.chem_integrator = "ReactorCvode"
peleLM.use_typ_vals_chem = 0 # Use species/temp typical values in CVODE
# ode.rtol = 1.0e-6 # Relative tolerance of the chemical solve
# ode.atol = 1.0e-5 # Absolute tolerance factor applied on typical values
#cvode.solve_type = GMRES
cvode.solve_type = dense_direct # CVODE Linear solve type (for Newton direction)
#cvode.solve_type = magma_direct # CVODE Linear solve type (for Newton direction)
#cvode.max_order = 4 # CVODE max BDF order.
#ode.atol = 1.E-12

mac_proj.verbose = 0
mac_proj.atol = 1.E-14
mac_proj.rtol = 1.E-11
nodal_proj.verbose = 0
nodal_proj.atol = 6.0e-14 # tolerance for projections
nodal_proj.rtol = 6.0e-11

#--------------------REFINEMENT CONTROL------------------------
amr.refinement_indicators = gradT
amr.gradT.max_level = 2
amr.gradT.adjacent_difference_greater = 30.
amr.gradT.field_name = temp

amrex.regtest_reduction = 1
amrex.fpe_trap_invalid = 1
amrex.fpe_trap_zero = 1
amrex.fpe_trap_overflow = 1

#--------------------RADIATION MODELING------------------------
peleLM.do_rad_solve = 1
pelerad.composite_solve = 1
pelerad.use_hypre = 0
pelerad.verbose = 0
pelerad.max_iter = 200
pelerad.max_coarsening_level = 10
pelerad.reltol = 1.0e-3
pelerad.abstol = 1.0e-3
pelerad.bottom_reltol = 1.0e-6
pelerad.bottom_abstol = 1.0e-6
pelerad.agglomeration = 1
pelerad.consolidation = 0
pelerad.bottom_verbose = 0
pelerad.maxorder = 2
pelerad.linop_maxorder = 2
pelerad.max_fmg_iter = 0
pelerad.lo_bc = Robin Periodic Periodic
pelerad.hi_bc = Robin Periodic Periodic
#please set the pelerad.kppath
pelerad.kppath = /lustre/orion/cmb138/proj-shared/w0g/PR/PeleRad/data/kpDB/
140 changes: 140 additions & 0 deletions Exec/RegTests/SootRadTest/pelelmex_prob.H
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
#ifndef PELELMEX_PROB_H
#define PELELMEX_PROB_H

#include <AMReX_Geometry.H>
#include <AMReX_FArrayBox.H>
#include <AMReX_ParmParse.H>

#include <pelelmex_prob_parm.H>
#include <PMF.H>
#include <PMFData.H>

#include <PeleLMeX_Index.H>
#include <PelePhysics.H>
#include "SootModel.H"

AMREX_GPU_DEVICE
AMREX_FORCE_INLINE
void
pelelmex_initdata(
int i,
int j,
int k,
int /*is_incompressible*/,
amrex::Array4<amrex::Real> const& state,
amrex::Array4<amrex::Real> const& aux,
amrex::GeometryData const& geomdata,
ProbParm const& prob_parm,
pele::physics::PMF::PmfData::DataContainer const* pmf_data)
{

const amrex::Real* prob_lo = geomdata.ProbLo();
const amrex::Real* prob_hi = geomdata.ProbHi();
const amrex::Real* dx = geomdata.CellSize();

AMREX_D_TERM(const amrex::Real x = prob_lo[0] + (i + 0.5) * dx[0];
, const amrex::Real y = prob_lo[1] + (j + 0.5) * dx[1];
, const amrex::Real z = prob_lo[2] + (k + 0.5) * dx[2];);
amrex::GpuArray<amrex::Real, NUM_SPECIES> massfrac = {{0.0}};
amrex::GpuArray<amrex::Real, NUM_SPECIES + 4> pmf_vals = {{0.0}};
amrex::Real x1 = (x - prob_parm.standoff - 0.5 * dx[0]) * 100.;
amrex::Real x2 = (x - prob_parm.standoff + 0.5 * dx[0]) * 100.;
pele::physics::PMF::pmf(pmf_data, x1, x2, pmf_vals);
state(i, j, k, TEMP) = pmf_vals[1];
amrex::Real norm = 0.;
for (int n = 0; n < NUM_SPECIES; n++) {
massfrac[n] = amrex::max(0., amrex::min(1., pmf_vals[3 + n]));
norm += massfrac[n];
}
for (int n = 0; n < NUM_SPECIES; ++n) {
massfrac[n] = massfrac[n] / norm;
}
AMREX_D_TERM(state(i, j, k, VELX) = pmf_vals[0] * 1.E-2;
, state(i, j, k, VELY) = 0.;, state(i, j, k, VELZ) = 0.;);
amrex::Real rho_cgs;
auto P_cgs = prob_parm.P_mean * 10.;

auto eos = pele::physics::PhysicsType::eos();
eos.PYT2R(P_cgs, massfrac.data(), state(i, j, k, TEMP), rho_cgs);
state(i, j, k, DENSITY) = rho_cgs * 1.0e3; // CGS -> MKS conversion

eos.TY2H(state(i, j, k, TEMP), massfrac.data(), state(i, j, k, RHOH));
state(i, j, k, RHOH) = state(i, j, k, RHOH) * 1.0e-4 *
state(i, j, k, DENSITY); // CGS -> MKS conversion

for (int n = 0; n < NUM_SPECIES; n++) {
state(i, j, k, FIRSTSPEC + n) = massfrac[n] * state(i, j, k, DENSITY);
}
for (int is = 0; is < NUM_SOOT_MOMENTS + 1; ++is) {
state(i, j, k, FIRSTSOOT + is) = prob_parm.soot_vals[is];
}
}

AMREX_GPU_DEVICE
AMREX_FORCE_INLINE
void
bcnormal(
const amrex::Real x[AMREX_SPACEDIM],
const int /*m_nAux*/,
amrex::Real s_ext[NVAR],
const int idir,
const int sgn,
const amrex::Real time,
amrex::GeometryData const& geomdata,
ProbParm const& prob_parm,
pele::physics::PMF::PmfData::DataContainer const* pmf_data)
{
const amrex::Real* prob_lo = geomdata.ProbLo();
amrex::GpuArray<amrex::Real, NUM_SPECIES + 4> pmf_vals = {{0.0}};
amrex::Real massfrac[NUM_SPECIES] = {0.0};
if (sgn == 1) {
pele::physics::PMF::pmf(pmf_data, prob_lo[idir], prob_lo[idir], pmf_vals);
AMREX_D_TERM(s_ext[VELX] = pmf_vals[0] * 1.E-2;, s_ext[VELY] = 0.0;
, s_ext[VELZ] = 0.0;);

s_ext[TEMP] = pmf_vals[1];

for (int n = 0; n < NUM_SPECIES; n++) {
massfrac[n] = pmf_vals[3 + n];
}

amrex::Real rho_cgs, P_cgs, RhoH_temp;
P_cgs = prob_parm.P_mean * 10.0;

auto eos = pele::physics::PhysicsType::eos();
eos.PYT2R(P_cgs, massfrac, s_ext[TEMP], rho_cgs);
s_ext[DENSITY] = rho_cgs * 1.0e3;

eos.TY2H(s_ext[TEMP], massfrac, RhoH_temp);
s_ext[RHOH] = RhoH_temp * 1.0e-4 * s_ext[DENSITY]; // CGS -> MKS conversion

for (int n = 0; n < NUM_SPECIES; n++) {
s_ext[FIRSTSPEC + n] = massfrac[n] * s_ext[DENSITY];
}
for (int is = 0; is < NUM_SOOT_MOMENTS + 1; ++is) {
s_ext[FIRSTSOOT + is] = prob_parm.soot_vals[is];
}
}
}

AMREX_GPU_DEVICE
AMREX_FORCE_INLINE
void
zero_visc(
int i,
int j,
int k,
amrex::Array4<amrex::Real> const& beta,
amrex::GeometryData const& geomdata,
amrex::Box const& domainBox,
const int dir,
const int beta_comp,
const int nComp)
{
amrex::ignore_unused(
i, j, k, beta, geomdata, domainBox, dir, beta_comp, nComp);
// We treat species when beta_comp == 0 and nComp == NUM_SPECIES
// otherwise this routine could be called for other face diffusivity (Temp,
// velocity, ...)
}
#endif
20 changes: 20 additions & 0 deletions Exec/RegTests/SootRadTest/pelelmex_prob.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#include <PeleLMeX.H>
#include <pelelmex_prob.H>

void
PeleLM::readProbParm()
{
// Parse params
amrex::ParmParse pp("prob");
pp.query("P_mean", PeleLM::prob_parm->P_mean);
pp.query("standoff", PeleLM::prob_parm->standoff);
PeleLM::pmf_data.initialize();
amrex::Real moments[NUM_SOOT_MOMENTS + 1] = {0.0};
if (PeleLM::do_soot_solve) {
SootData* const sd = PeleLM::soot_model->getSootData();
sd->initialSmallMomVals(moments);
}
for (int n = 0; n < NUM_SOOT_MOMENTS + 1; ++n) {
PeleLM::prob_parm->soot_vals[n] = moments[n];
}
}
Loading

0 comments on commit 01790c2

Please sign in to comment.