Skip to content

Commit

Permalink
Generalize fillpatching (#1916)
Browse files Browse the repository at this point in the history 
* generalize fp, etc

* fix re-used variable
  • Loading branch information
@asalmgren
asalmgren authored Oct 29, 2024
1 parent 1d7b18b commit 4a4c136
Show file tree
Hide file tree
Showing 18 changed files with 780 additions and 362 deletions.
1 change: 1 addition & 0 deletions CMake/BuildERFExe.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ function(build_erf_lib erf_lib_name)
${SRC_DIR}/TimeIntegration/ERF_fast_rhs_N.cpp
${SRC_DIR}/TimeIntegration/ERF_fast_rhs_T.cpp
${SRC_DIR}/TimeIntegration/ERF_fast_rhs_MT.cpp
${SRC_DIR}/Utils/ERF_AverageDown.cpp
${SRC_DIR}/Utils/ERF_ChopGrids.cpp
${SRC_DIR}/Utils/ERF_MomentumToVelocity.cpp
${SRC_DIR}/Utils/ERF_TerrainMetrics.cpp
Expand Down
126 changes: 122 additions & 4 deletions Source/BoundaryConditions/ERF_BoundaryConditions_basestate.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "AMReX_PhysBCFunct.H"
#include <ERF_PhysBCFunct.H>
#include <ERF_Constants.H>

using namespace amrex;

Expand All @@ -15,9 +16,14 @@ void ERFPhysBCFunct_base::impose_lateral_basestate_bcs (const Array4<Real>& dest
int ncomp, const IntVect& nghost)
{
BL_PROFILE_VAR("impose_lateral_base_bcs()",impose_lateral_base_bcs);
//
// Note that the "bx" that comes in here has already been grown in the lateral directions
// but not in the vertical
//

const int* bxlo = bx.loVect();
const int* bxhi = bx.hiVect();

const int* dlo = domain.loVect();
const int* dhi = domain.hiVect();

Expand Down Expand Up @@ -63,6 +69,84 @@ void ERFPhysBCFunct_base::impose_lateral_basestate_bcs (const Array4<Real>& dest
// Populate ghost cells on lo-x and hi-x domain boundaries
Box bx_xlo(bx); bx_xlo.setBig (0,dom_lo.x-nghost[0]);
Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+nghost[0]);

ParallelFor(
bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
{
int dest_comp = n;
int l_bc_type = bc_ptr[n].lo(0);
int iflip = dom_lo.x - 1 - i;
if (l_bc_type == ERFBCType::foextrap) {
dest_arr(i,j,k,dest_comp) = dest_arr(dom_lo.x,j,k,dest_comp);
} else if (l_bc_type == ERFBCType::open) {
dest_arr(i,j,k,dest_comp) = dest_arr(dom_lo.x,j,k,dest_comp);
} else if (l_bc_type == ERFBCType::reflect_even) {
dest_arr(i,j,k,dest_comp) = dest_arr(iflip,j,k,dest_comp);
}
},
bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
{
int dest_comp = n;
int h_bc_type = bc_ptr[n].hi(0);
int iflip = 2*dom_hi.x + 1 - i;
if (h_bc_type == ERFBCType::foextrap) {
dest_arr(i,j,k,dest_comp) = dest_arr(dom_hi.x,j,k,dest_comp);
} else if (h_bc_type == ERFBCType::open) {
dest_arr(i,j,k,dest_comp) = dest_arr(dom_hi.x,j,k,dest_comp);
} else if (h_bc_type == ERFBCType::reflect_even) {
dest_arr(i,j,k,dest_comp) = dest_arr(iflip,j,k,dest_comp);
}
}
);
}

if (!is_periodic_in_y)
{
// Populate ghost cells on lo-y and hi-y domain boundaries
Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-nghost[1]);
Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+nghost[1]);
if (bx_ylo.smallEnd(2) != domain.smallEnd(2)) bx_ylo.growLo(2,nghost[2]);
if (bx_ylo.bigEnd(2) != domain.bigEnd(2)) bx_ylo.growHi(2,nghost[2]);
if (bx_yhi.smallEnd(2) != domain.smallEnd(2)) bx_yhi.growLo(2,nghost[2]);
if (bx_yhi.bigEnd(2) != domain.bigEnd(2)) bx_yhi.growHi(2,nghost[2]);
ParallelFor(
bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
{
int dest_comp = n;
int l_bc_type = bc_ptr[n].lo(1);
int jflip = dom_lo.y - 1 - j;
if (l_bc_type == ERFBCType::foextrap) {
dest_arr(i,j,k,dest_comp) = dest_arr(i,dom_lo.y,k,dest_comp);
} else if (l_bc_type == ERFBCType::open) {
dest_arr(i,j,k,dest_comp) = dest_arr(i,dom_lo.y,k,dest_comp);
} else if (l_bc_type == ERFBCType::reflect_even) {
dest_arr(i,j,k,dest_comp) = dest_arr(i,jflip,k,dest_comp);
}

},
bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
{
int dest_comp = n;
int h_bc_type = bc_ptr[n].hi(1);
int jflip = 2*dom_hi.y + 1 - j;
if (h_bc_type == ERFBCType::foextrap) {
dest_arr(i,j,k,dest_comp) = dest_arr(i,dom_hi.y,k,dest_comp);
} else if (h_bc_type == ERFBCType::open) {
dest_arr(i,j,k,dest_comp) = dest_arr(i,dom_hi.y,k,dest_comp);
} else if (h_bc_type == ERFBCType::reflect_even) {
dest_arr(i,j,k,dest_comp) = dest_arr(i,jflip,k,dest_comp);
}
}
);
}

// Next do ghost cells in x-direction but not reaching out in y
// The corners we miss here will be covered in the y-loop below or by periodicity
if (!is_periodic_in_x)
{
// Populate ghost cells on lo-x and hi-x domain boundaries
Box bx_xlo(bx); bx_xlo.setBig (0,dom_lo.x-1);
Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
if (bx_xlo.smallEnd(2) != domain.smallEnd(2)) bx_xlo.growLo(2,nghost[2]);
if (bx_xlo.bigEnd(2) != domain.bigEnd(2)) bx_xlo.growHi(2,nghost[2]);
if (bx_xhi.smallEnd(2) != domain.smallEnd(2)) bx_xhi.growLo(2,nghost[2]);
Expand All @@ -79,6 +163,11 @@ void ERFPhysBCFunct_base::impose_lateral_basestate_bcs (const Array4<Real>& dest
dest_arr(i,j,k,dest_comp) = dest_arr(dom_lo.x,j,k,dest_comp);
} else if (l_bc_type == ERFBCType::reflect_even) {
dest_arr(i,j,k,dest_comp) = dest_arr(iflip,j,k,dest_comp);
} else if (l_bc_type == ERFBCType::reflect_odd) {
dest_arr(i,j,k,dest_comp) = -dest_arr(iflip,j,k,dest_comp);
} else if (l_bc_type == ERFBCType::hoextrapcc) {
Real delta_i = (dom_lo.x - i);
dest_arr(i,j,k,dest_comp) = (1.0 + delta_i)*dest_arr(dom_lo.x,j,k,dest_comp) - delta_i*dest_arr(dom_lo.x+1,j,k,dest_comp) ;
}
},
bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
Expand All @@ -92,6 +181,11 @@ void ERFPhysBCFunct_base::impose_lateral_basestate_bcs (const Array4<Real>& dest
dest_arr(i,j,k,dest_comp) = dest_arr(dom_hi.x,j,k,dest_comp);
} else if (h_bc_type == ERFBCType::reflect_even) {
dest_arr(i,j,k,dest_comp) = dest_arr(iflip,j,k,dest_comp);
} else if (h_bc_type == ERFBCType::reflect_odd) {
dest_arr(i,j,k,dest_comp) = -dest_arr(iflip,j,k,dest_comp);
} else if (h_bc_type == ERFBCType::hoextrapcc) {
Real delta_i = (i - dom_hi.x);
dest_arr(i,j,k,dest_comp) = (1.0 + delta_i)*dest_arr(dom_hi.x,j,k,dest_comp) - delta_i*dest_arr(dom_hi.x-1,j,k,dest_comp) ;
}
}
);
Expand All @@ -100,8 +194,8 @@ void ERFPhysBCFunct_base::impose_lateral_basestate_bcs (const Array4<Real>& dest
if (!is_periodic_in_y)
{
// Populate ghost cells on lo-y and hi-y domain boundaries
Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-nghost[1]);
Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+nghost[1]);
Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-1);
Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
if (bx_ylo.smallEnd(2) != domain.smallEnd(2)) bx_ylo.growLo(2,nghost[2]);
if (bx_ylo.bigEnd(2) != domain.bigEnd(2)) bx_ylo.growHi(2,nghost[2]);
if (bx_yhi.smallEnd(2) != domain.smallEnd(2)) bx_yhi.growLo(2,nghost[2]);
Expand All @@ -118,6 +212,11 @@ void ERFPhysBCFunct_base::impose_lateral_basestate_bcs (const Array4<Real>& dest
dest_arr(i,j,k,dest_comp) = dest_arr(i,dom_lo.y,k,dest_comp);
} else if (l_bc_type == ERFBCType::reflect_even) {
dest_arr(i,j,k,dest_comp) = dest_arr(i,jflip,k,dest_comp);
} else if (l_bc_type == ERFBCType::reflect_odd) {
dest_arr(i,j,k,dest_comp) = -dest_arr(i,jflip,k,dest_comp);
} else if (l_bc_type == ERFBCType::hoextrapcc) {
Real delta_j = (dom_lo.y - j);
dest_arr(i,j,k,dest_comp) = (1.0 + delta_j)*dest_arr(i,dom_lo.y,k,dest_comp) - delta_j*dest_arr(i,dom_lo.y+1,k,dest_comp) ;
}

},
Expand All @@ -132,6 +231,11 @@ void ERFPhysBCFunct_base::impose_lateral_basestate_bcs (const Array4<Real>& dest
dest_arr(i,j,k,dest_comp) = dest_arr(i,dom_hi.y,k,dest_comp);
} else if (h_bc_type == ERFBCType::reflect_even) {
dest_arr(i,j,k,dest_comp) = dest_arr(i,jflip,k,dest_comp);
} else if (h_bc_type == ERFBCType::reflect_odd) {
dest_arr(i,j,k,dest_comp) = -dest_arr(i,jflip,k,dest_comp);
} else if (h_bc_type == ERFBCType::hoextrapcc) {
Real delta_j = (j - dom_hi.y);
dest_arr(i,j,k,dest_comp) = (1.0 + delta_j)*dest_arr(i,dom_hi.y,k,dest_comp) - delta_j*dest_arr(i,dom_hi.y-1,k,dest_comp);
}
}
);
Expand All @@ -147,16 +251,30 @@ void ERFPhysBCFunct_base::impose_vertical_basestate_bcs (const Array4<Real>& des
const auto& dom_lo = lbound(domain);
const auto& dom_hi = ubound(domain);

const Real hz = Real(0.5) * m_geom.CellSize(2);

Box bx_zlo1(bx); bx_zlo1.setBig(2,dom_lo.z-1); if (bx_zlo1.ok()) bx_zlo1.setSmall(2,dom_lo.z-1);
ParallelFor(
bx_zlo1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
{
dest_arr(i,j,k,BaseState::r0_comp) = dest_arr(i,j,dom_lo.z,BaseState::r0_comp);
dest_arr(i,j,k,BaseState::p0_comp) = p_0 -
dest_arr(i,j,k,BaseState::r0_comp) * hz * 9.81;
dest_arr(i,j,k,BaseState::pi0_comp) = dest_arr(i,j,dom_lo.z,BaseState::pi0_comp);
dest_arr(i,j,k,BaseState::th0_comp) = dest_arr(i,j,dom_lo.z,BaseState::th0_comp);
}
);

Box bx_zlo(bx); bx_zlo.setBig(2,dom_lo.z-2);
Box bx_zhi(bx); bx_zhi.setSmall(2,dom_hi.z+2);
Box bx_zhi(bx); bx_zhi.setSmall(2,dom_hi.z+1);
ParallelFor(
bx_zlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
{
dest_arr(i,j,k,n) = dest_arr(i,j,dom_lo.z-1,n);
},
bx_zhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
{
dest_arr(i,j,k,n) = dest_arr(i,j,dom_hi.z+1,n);
dest_arr(i,j,k,n) = dest_arr(i,j,dom_hi.z,n);
}
);
}
17 changes: 12 additions & 5 deletions Source/BoundaryConditions/ERF_FillCoarsePatch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,18 @@ ERF::FillCoarsePatch (int lev, Real time)
//****************************************************************************************************************
//
bool cons_only = false;
FillPatch(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
&vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
{&vars_new[lev-1][Vars::cons],
&rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
false, cons_only);
if (lev == 1) {
FillPatch(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
&vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
cons_only);
} else {
FillPatch(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
&vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
{&vars_new[lev-1][Vars::cons],
&rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
base_state[lev-1], base_state[lev-1],
false, cons_only);
}

//
// ************************************************
Expand Down
Loading

0 comments on commit 4a4c136

Please sign in to comment.