Skip to content

Commit

Permalink
Remove all elixirs (#1418)
Browse files Browse the repository at this point in the history 
* fix how we locate particles with terrain

* Replace all elixirs; now use The_Async_Arena
  • Loading branch information
@asalmgren
asalmgren authored Feb 1, 2024
1 parent f2184e1 commit 6330c12
Show file tree
Hide file tree
Showing 13 changed files with 34 additions and 75 deletions.
5 changes: 2 additions & 3 deletions Source/Advection/AdvectionSrcForMom.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,8 @@ AdvectionSrcForMom (const Box& bxx, const Box& bxy, const Box& bxz,
// compute mapfactor inverses
Box box2d_u(bxx); box2d_u.setRange(2,0); box2d_u.grow({3,3,0});
Box box2d_v(bxy); box2d_v.setRange(2,0); box2d_v.grow({3,3,0});
FArrayBox mf_u_invFAB(box2d_u); FArrayBox mf_v_invFAB(box2d_v);
Elixir mf_u_inv_eli = mf_u_invFAB.elixir();
Elixir mf_v_inv_eli = mf_v_invFAB.elixir();
FArrayBox mf_u_invFAB(box2d_u,1,The_Async_Arena());
FArrayBox mf_v_invFAB(box2d_v,1,The_Async_Arena());
const Array4<Real>& mf_u_inv = mf_u_invFAB.array();
const Array4<Real>& mf_v_inv = mf_v_invFAB.array();

Expand Down
5 changes: 1 addition & 4 deletions Source/BoundaryConditions/ERF_FillPatcher.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,16 +249,13 @@ void ERFFillPatcher::InterpCell (MultiFab& fine,
}
}

FArrayBox ccfab(cslope_bx, ncomp*AMREX_SPACEDIM);
FArrayBox ccfab(cslope_bx, ncomp*AMREX_SPACEDIM, The_Async_Arena());
Array4<Real> const& tmp = ccfab.array();
Array4<Real const> const& ctmp = ccfab.const_array();

#ifdef AMREX_USE_GPU
AsyncArray<BCRec> async_bcr(bcr.data(), (run_on_gpu) ? ncomp : 0);
BCRec const* bcrp = (run_on_gpu) ? async_bcr.data() : bcr.data();

Elixir cceli;
if (run_on_gpu) { cceli = ccfab.elixir(); }
#else
BCRec const* bcrp = bcr.data();
#endif
Expand Down
4 changes: 2 additions & 2 deletions Source/Diffusion/ComputeStress_T.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ ComputeStressConsVisc_T (Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Real mu_eff,
FArrayBox temp;
Box gbx = bxcc; // Note: bxcc have been grown in x/y only.
gbx.grow(IntVect(0,0,1));
temp.resize(gbx,1);
temp.resize(gbx,1, The_Async_Arena());
Array4<Real> rhoAlpha = temp.array();
if (cell_data) {
ParallelFor(gbx,
Expand Down Expand Up @@ -319,7 +319,7 @@ ComputeStressVarVisc_T (Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Real mu_eff,
FArrayBox temp;
Box gbx = bxcc; // Note: bxcc have been grown in x/y only.
gbx.grow(IntVect(0,0,1));
temp.resize(gbx,1);
temp.resize(gbx,1, The_Async_Arena());
Array4<Real> rhoAlpha = temp.array();
if (cell_data) {
ParallelFor(gbx,
Expand Down
6 changes: 2 additions & 4 deletions Source/Diffusion/NumericalDiffusion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,8 @@ NumericalDiffusion (const Box& bx,

// Average map factors to correct locations
Box planebx(bx); planebx.setSmall(2,0); planebx.setBig(2,0);
FArrayBox mf_x_bar; mf_x_bar.resize(planebx,1);
FArrayBox mf_y_bar; mf_y_bar.resize(planebx,1);
Elixir mfx_eli = mf_x_bar.elixir();
Elixir mfy_eli = mf_y_bar.elixir();
FArrayBox mf_x_bar; mf_x_bar.resize(planebx,1,The_Async_Arena());
FArrayBox mf_y_bar; mf_y_bar.resize(planebx,1,The_Async_Arena());
const Array4<Real>& mfx_arr = mf_x_bar.array();
const Array4<Real>& mfy_arr = mf_y_bar.array();
ParallelFor(planebx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
Expand Down
3 changes: 1 addition & 2 deletions Source/ERF.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1327,8 +1327,7 @@ ERF::MakeDiagnosticAverage (Vector<Real>& h_havg, MultiFab& S, int n)

auto fab_arr = S[mfi].array();

FArrayBox fab_reduce(box, 1);
Elixir elx_reduce = fab_reduce.elixir();
FArrayBox fab_reduce(box, 1, The_Async_Arena());
auto arr_reduce = fab_reduce.array();

ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
Expand Down
10 changes: 3 additions & 7 deletions Source/TimeIntegration/ERF_fast_rhs_MT.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,13 +221,9 @@ void erf_fast_rhs_MT (int step, int nrk,
} // if step
} // end profile

RHS_fab.resize(tbz,1);
soln_fab.resize(tbz,1);
temp_rhs_fab.resize(tbz,2);

Elixir rCeli = RHS_fab.elixir();
Elixir sCeli = soln_fab.elixir();
Elixir temp_rhs_eli = temp_rhs_fab.elixir();
RHS_fab.resize (tbz,1, The_Async_Arena());
soln_fab.resize (tbz,1, The_Async_Arena());
temp_rhs_fab.resize(tbz,2, The_Async_Arena());

auto const& RHS_a = RHS_fab.array();
auto const& soln_a = soln_fab.array();
Expand Down
7 changes: 2 additions & 5 deletions Source/TimeIntegration/ERF_fast_rhs_N.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,19 +301,16 @@ void erf_fast_rhs_N (int step, int nrk,
const Array4<const Real>& mf_v = mapfac_v->const_array(mfi);

FArrayBox RHS_fab;
RHS_fab.resize(tbz,1);
RHS_fab.resize(tbz,1, The_Async_Arena());

FArrayBox soln_fab;
soln_fab.resize(tbz,1);
soln_fab.resize(tbz,1, The_Async_Arena());

auto const& RHS_a = RHS_fab.array();
auto const& soln_a = soln_fab.array();

auto const& temp_rhs_arr = temp_rhs.array(mfi);

Elixir rCeli = RHS_fab.elixir();
Elixir sCeli = soln_fab.elixir();

auto const& coeffA_a = coeff_A_mf.array(mfi);
auto const& inv_coeffB_a = inv_coeff_B_mf.array(mfi);
auto const& coeffC_a = coeff_C_mf.array(mfi);
Expand Down
9 changes: 3 additions & 6 deletions Source/TimeIntegration/ERF_fast_rhs_T.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,13 +354,10 @@ void erf_fast_rhs_T (int step, int nrk,
FArrayBox temp_rhs_fab;
FArrayBox RHS_fab;
FArrayBox soln_fab;
RHS_fab.resize(tbz,1);
soln_fab.resize(tbz,1);
temp_rhs_fab.resize(tbz,2);

Elixir rCeli = RHS_fab.elixir();
Elixir sCeli = soln_fab.elixir();
Elixir temp_rhs_eli = temp_rhs_fab.elixir();
RHS_fab.resize (tbz,1,The_Async_Arena());
soln_fab.resize (tbz,1,The_Async_Arena());
temp_rhs_fab.resize(tbz,2,The_Async_Arena());

auto const& RHS_a = RHS_fab.array();
auto const& soln_a = soln_fab.array();
Expand Down
3 changes: 1 addition & 2 deletions Source/TimeIntegration/ERF_make_fast_coeffs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,7 @@ void make_fast_coeffs (int /*level*/,
const Array4<const Real>& r0_ca = r0->const_array(mfi);
const Array4<const Real>& pi0_ca = pi0->const_array(mfi); const Array4<const Real>& pi_stage_ca = pi_stage.const_array(mfi);

FArrayBox gam_fab; gam_fab.resize(surroundingNodes(bx,2),1);
Elixir gEli = gam_fab.elixir();
FArrayBox gam_fab; gam_fab.resize(surroundingNodes(bx,2),1,The_Async_Arena());

auto const& coeffA_a = coeff_A_mf.array(mfi);
auto const& coeffB_a = coeff_B_mf.array(mfi);
Expand Down
9 changes: 3 additions & 6 deletions Source/TimeIntegration/ERF_slow_rhs_inc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,10 +233,8 @@ void erf_slow_rhs_inc (int /*level*/, int nrk,
// Temporary storage for tiling/OMP
FArrayBox S11,S22,S33;
FArrayBox S12,S13,S23;
S11.resize(bxcc,1); S22.resize(bxcc,1); S33.resize(bxcc,1);
S12.resize(tbxxy,1); S13.resize(tbxxz,1); S23.resize(tbxyz,1);
Elixir S11_eli = S11.elixir(); Elixir S22_eli = S22.elixir(); Elixir S33_eli = S33.elixir();
Elixir S12_eli = S12.elixir(); Elixir S13_eli = S13.elixir(); Elixir S23_eli = S23.elixir();
S11.resize( bxcc,1,The_Async_Arena()); S22.resize( bxcc,1,The_Async_Arena()); S33.resize( bxcc,1,The_Async_Arena());
S12.resize(tbxxy,1,The_Async_Arena()); S13.resize(tbxxz,1,The_Async_Arena()); S23.resize(tbxyz,1,The_Async_Arena());
Array4<Real> s11 = S11.array(); Array4<Real> s22 = S22.array(); Array4<Real> s33 = S33.array();
Array4<Real> s12 = S12.array(); Array4<Real> s13 = S13.array(); Array4<Real> s23 = S23.array();

Expand All @@ -250,8 +248,7 @@ void erf_slow_rhs_inc (int /*level*/, int nrk,
if (l_use_terrain) {
// Terrain non-symmetric terms
FArrayBox S21,S31,S32;
S21.resize(tbxxy,1); S31.resize(tbxxz,1); S32.resize(tbxyz,1);
Elixir S21_eli = S21.elixir(); Elixir S31_eli = S31.elixir(); Elixir S32_eli = S32.elixir();
S21.resize(tbxxy,1,The_Async_Arena()); S31.resize(tbxxz,1,The_Async_Arena()); S32.resize(tbxyz,1,The_Async_Arena());
Array4<Real> s21 = S21.array(); Array4<Real> s31 = S31.array(); Array4<Real> s32 = S32.array();
Array4<Real> tau21 = Tau21->array(mfi); Array4<Real> tau31 = Tau31->array(mfi); Array4<Real> tau32 = Tau32->array(mfi);

Expand Down
6 changes: 2 additions & 4 deletions Source/TimeIntegration/ERF_slow_rhs_post.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,10 +439,8 @@ void erf_slow_rhs_post (int level, int finest_level,

// Temporary FABs for storage (owned/filled on all ranks)
FArrayBox QV_xlo, QV_xhi, QV_ylo, QV_yhi;
QV_xlo.resize(bx_xlo,1); QV_xhi.resize(bx_xhi,1);
QV_ylo.resize(bx_ylo,1); QV_yhi.resize(bx_yhi,1);
Elixir QV_xlo_eli = QV_xlo.elixir(); Elixir QV_xhi_eli = QV_xhi.elixir();
Elixir QV_ylo_eli = QV_ylo.elixir(); Elixir QV_yhi_eli = QV_yhi.elixir();
QV_xlo.resize(bx_xlo,1,The_Async_Arena()); QV_xhi.resize(bx_xhi,1,The_Async_Arena());
QV_ylo.resize(bx_ylo,1,The_Async_Arena()); QV_yhi.resize(bx_yhi,1,The_Async_Arena());

// Get Array4 of interpolated values
Array4<Real> arr_xlo = QV_xlo.array(); Array4<Real> arr_xhi = QV_xhi.array();
Expand Down
12 changes: 4 additions & 8 deletions Source/TimeIntegration/ERF_slow_rhs_pre.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,10 +246,8 @@ void erf_slow_rhs_pre (int level, int finest_level,
// Temporary storage for tiling/OMP
FArrayBox S11,S22,S33;
FArrayBox S12,S13,S23;
S11.resize(bxcc,1); S22.resize(bxcc,1); S33.resize(bxcc,1);
S12.resize(tbxxy,1); S13.resize(tbxxz,1); S23.resize(tbxyz,1);
Elixir S11_eli = S11.elixir(); Elixir S22_eli = S22.elixir(); Elixir S33_eli = S33.elixir();
Elixir S12_eli = S12.elixir(); Elixir S13_eli = S13.elixir(); Elixir S23_eli = S23.elixir();
S11.resize( bxcc,1,The_Async_Arena()); S22.resize(bxcc,1,The_Async_Arena()); S33.resize(bxcc,1,The_Async_Arena());
S12.resize(tbxxy,1,The_Async_Arena()); S13.resize(tbxxz,1,The_Async_Arena()); S23.resize(tbxyz,1,The_Async_Arena());
Array4<Real> s11 = S11.array(); Array4<Real> s22 = S22.array(); Array4<Real> s33 = S33.array();
Array4<Real> s12 = S12.array(); Array4<Real> s13 = S13.array(); Array4<Real> s23 = S23.array();

Expand All @@ -263,8 +261,7 @@ void erf_slow_rhs_pre (int level, int finest_level,
if (l_use_terrain) {
// Terrain non-symmetric terms
FArrayBox S21,S31,S32;
S21.resize(tbxxy,1); S31.resize(tbxxz,1); S32.resize(tbxyz,1);
Elixir S21_eli = S21.elixir(); Elixir S31_eli = S31.elixir(); Elixir S32_eli = S32.elixir();
S21.resize(tbxxy,1,The_Async_Arena()); S31.resize(tbxxz,1,The_Async_Arena()); S32.resize(tbxyz,1,The_Async_Arena());
Array4<Real> s21 = S21.array(); Array4<Real> s31 = S31.array(); Array4<Real> s32 = S32.array();
Array4<Real> tau21 = Tau21->array(mfi); Array4<Real> tau31 = Tau31->array(mfi); Array4<Real> tau32 = Tau32->array(mfi);

Expand Down Expand Up @@ -556,8 +553,7 @@ void erf_slow_rhs_pre (int level, int finest_level,
// Perturbational pressure field
//-----------------------------------------
Box gbx = mfi.tilebox(); gbx.grow(IntVect(1,1,0));
FArrayBox pprime; pprime.resize(gbx,1);
Elixir pp_eli = pprime.elixir();
FArrayBox pprime; pprime.resize(gbx,1,The_Async_Arena());
const Array4<Real > & pp_arr = pprime.array();
{
BL_PROFILE("slow_rhs_pre_pprime");
Expand Down
30 changes: 8 additions & 22 deletions Source/Utils/InteriorGhostCells.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,36 +191,22 @@ wrfbdy_compute_interior_ghost_rhs (const std::string& init_type,

// Size the FABs
if (ivar == ivarU) {
U_xlo.resize(bx_xlo,1); U_xhi.resize(bx_xhi,1);
U_ylo.resize(bx_ylo,1); U_yhi.resize(bx_yhi,1);
U_xlo.resize(bx_xlo,1,The_Async_Arena()); U_xhi.resize(bx_xhi,1,The_Async_Arena());
U_ylo.resize(bx_ylo,1,The_Async_Arena()); U_yhi.resize(bx_yhi,1,The_Async_Arena());
} else if (ivar == ivarV) {
V_xlo.resize(bx_xlo,1); V_xhi.resize(bx_xhi,1);
V_ylo.resize(bx_ylo,1); V_yhi.resize(bx_yhi,1);
V_xlo.resize(bx_xlo,1,The_Async_Arena()); V_xhi.resize(bx_xhi,1,The_Async_Arena());
V_ylo.resize(bx_ylo,1,The_Async_Arena()); V_yhi.resize(bx_yhi,1,The_Async_Arena());
} else if (ivar == ivarR) {
R_xlo.resize(bx_xlo,1); R_xhi.resize(bx_xhi,1);
R_ylo.resize(bx_ylo,1); R_yhi.resize(bx_yhi,1);
R_xlo.resize(bx_xlo,1,The_Async_Arena()); R_xhi.resize(bx_xhi,1,The_Async_Arena());
R_ylo.resize(bx_ylo,1,The_Async_Arena()); R_yhi.resize(bx_yhi,1,The_Async_Arena());
} else if (ivar == ivarT){
T_xlo.resize(bx_xlo,1); T_xhi.resize(bx_xhi,1);
T_ylo.resize(bx_ylo,1); T_yhi.resize(bx_yhi,1);
T_xlo.resize(bx_xlo,1,The_Async_Arena()); T_xhi.resize(bx_xhi,1,The_Async_Arena());
T_ylo.resize(bx_ylo,1,The_Async_Arena()); T_yhi.resize(bx_yhi,1,The_Async_Arena());
} else {
continue;
}
} // ivar

// Elixir to avoid destruction before kernel end
Elixir U_xlo_eli = U_xlo.elixir(); Elixir U_xhi_eli = U_xhi.elixir();
Elixir U_ylo_eli = U_ylo.elixir(); Elixir U_yhi_eli = U_yhi.elixir();

Elixir V_xlo_eli = V_xlo.elixir(); Elixir V_xhi_eli = V_xhi.elixir();
Elixir V_ylo_eli = V_ylo.elixir(); Elixir V_yhi_eli = V_yhi.elixir();

Elixir R_xlo_eli = R_xlo.elixir(); Elixir R_xhi_eli = R_xhi.elixir();
Elixir R_ylo_eli = R_ylo.elixir(); Elixir R_yhi_eli = R_yhi.elixir();

Elixir T_xlo_eli = T_xlo.elixir(); Elixir T_xhi_eli = T_xhi.elixir();
Elixir T_ylo_eli = T_ylo.elixir(); Elixir T_yhi_eli = T_yhi.elixir();


// Populate FABs from bdy interpolation
//==========================================================
for (int ivar(ivarU); ivar < BdyEnd; ivar++) {
Expand Down

0 comments on commit 6330c12

Please sign in to comment.