Skip to content

Commit

Permalink
Merge branch 'development' of github.com:seahorce-scidac/ROMSX into d…
Browse files Browse the repository at this point in the history 
…evelopment
  • Loading branch information
@hklion
hklion committed Dec 14, 2023
2 parents 3cde801 + b432f69 commit a779c97
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 46 deletions.
19 changes: 13 additions & 6 deletions Exec/Upwelling/prob.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,12 +127,10 @@ init_custom_prob(

ParallelFor(bx, [=, parms=parms] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
{
// Geometry (note we must include these here to get the data on device)
// const auto prob_lo = geomdata.ProbLo();
// const auto dx = geomdata.CellSize();
const auto prob_lo = geomdata.ProbLo();
const auto prob_hi = geomdata.ProbHi();
const auto dx = geomdata.CellSize();

// const Real x = prob_lo[0] + (i + 0.5) * dx[0];
// const Real y = prob_lo[1] + (j + 0.5) * dx[1];
const Real z = z_r(i,j,k);

state(i, j, k, Temp_comp) = 1.;
Expand All @@ -143,7 +141,16 @@ init_custom_prob(
}

// Set scalar = 0 everywhere
state(i, j, k, Scalar_comp) = 0.0;
const Real xcent = 0.5*(prob_lo[0] + prob_hi[0]);
const Real ycent = 0.5*(prob_lo[1] + prob_hi[1]);

const Real x = prob_lo[0] + (i + 0.5) * dx[0] - xcent;
const Real y = prob_lo[1] + (j + 0.5) * dx[1] - ycent;
const Real r2 = x*x + y*y;
const Real rad = 0.1 * (prob_hi[0]-prob_lo[0]);
const Real radsq = rad*rad;

state(i, j, k, Scalar_comp) = (r2 < radsq) ? 1.0 : 0.0;
});

// Construct a box that is on x-faces
Expand Down
6 changes: 4 additions & 2 deletions Source/Initialization/ROMSX_init.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ ROMSX::set_2darrays (int lev)
});
}

vec_ubar[lev]->FillBoundary(geom[lev].periodicity());
vec_vbar[lev]->FillBoundary(geom[lev].periodicity());
// DEBUGGING NOTE -- Upwelling fails if these are commented out
const Real time = 0.0;
FillPatch(lev,time, *vec_ubar[lev], GetVecOfPtrs(vec_ubar));
FillPatch(lev,time, *vec_vbar[lev], GetVecOfPtrs(vec_vbar));
}
21 changes: 12 additions & 9 deletions Source/ROMSX.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,35 +361,38 @@ ROMSX::set_bathymetry(int lev)
{
init_custom_bathymetry(geom[lev], *vec_hOfTheConfusingName[lev], *vec_Zt_avg1[lev], solverChoice);

vec_hOfTheConfusingName[lev]->FillBoundary(geom[lev].periodicity());
vec_Zt_avg1[lev]->FillBoundary(geom[lev].periodicity());
Real time = 0.0;
FillPatch(lev, time, *vec_Zt_avg1[lev], GetVecOfPtrs(vec_Zt_avg1));
FillPatch(lev, time, *vec_hOfTheConfusingName[lev], GetVecOfPtrs(vec_hOfTheConfusingName));
}

void
ROMSX::set_vmix(int lev) {
init_custom_vmix(geom[lev], *vec_Akv[lev], *vec_Akt[lev], *vec_z_w[lev], solverChoice);

vec_Akv[lev]->FillBoundary(geom[lev].periodicity());
vec_Akt[lev]->FillBoundary(geom[lev].periodicity());
Real time = 0.0;
FillPatch(lev, time, *vec_Akv[lev], GetVecOfPtrs(vec_Akv));
FillPatch(lev, time, *vec_Akt[lev], GetVecOfPtrs(vec_Akt));
}

void
ROMSX::set_hmixcoef(int lev)
{
init_custom_hmix(geom[lev], *vec_visc2_p[lev], *vec_visc2_r[lev], *vec_diff2[lev], solverChoice);

vec_visc2_p[lev]->FillBoundary(geom[lev].periodicity());
vec_visc2_r[lev]->FillBoundary(geom[lev].periodicity());
vec_diff2[lev]->FillBoundary(geom[lev].periodicity());
Real time = 0.0;
FillPatch(lev, time, *vec_visc2_p[lev], GetVecOfPtrs(vec_visc2_p));
FillPatch(lev, time, *vec_visc2_r[lev], GetVecOfPtrs(vec_visc2_r));
FillPatch(lev, time, *vec_diff2[lev] , GetVecOfPtrs(vec_diff2));
}

void
ROMSX::set_smflux(int lev, Real time)
{
init_custom_smflux(geom[lev], time, *vec_sustr[lev], *vec_svstr[lev], solverChoice);

vec_sustr[lev]->FillBoundary(geom[lev].periodicity());
vec_svstr[lev]->FillBoundary(geom[lev].periodicity());
// FillPatch(lev, time, *vec_sustr[lev], GetVecOfPtrs(vec_sustr));
// FillPatch(lev, time, *vec_svstr[lev], GetVecOfPtrs(vec_svstr));
}

void
Expand Down
7 changes: 4 additions & 3 deletions Source/TimeIntegration/ROMSX_advance_3d.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,9 @@ ROMSX::advance_3d (int lev, MultiFab& mf_cons,
#endif
}

mf_Huon->FillBoundary(geom[lev].periodicity());
mf_Hvom->FillBoundary(geom[lev].periodicity());
// WE BELIEVE THESE VALUES SHOULD ALREADY BE FILLED
// mf_Huon->FillBoundary(geom[lev].periodicity());
// mf_Hvom->FillBoundary(geom[lev].periodicity());

// ************************************************************************
// This should fill both temp and salt with temp/salt currently in cons_old
Expand Down Expand Up @@ -323,7 +324,7 @@ ROMSX::advance_3d (int lev, MultiFab& mf_cons,

} // mfi

mf_cons.FillBoundary(geom[lev].periodicity());
FillPatch(lev, t_new[lev], mf_cons, cons_new);

for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
{
Expand Down
9 changes: 4 additions & 5 deletions Source/TimeIntegration/ROMSX_advance_3d_ml.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,10 @@ void ROMSX::advance_3d_ml (int lev, Real dt_lev)
vec_Akv[lev], vec_Akt[lev], vec_Hz[lev], vec_Huon[lev], vec_Hvom[lev],
vec_z_w[lev], vec_hOfTheConfusingName[lev], N, dt_lev);

vec_ubar[lev]->FillBoundary(geom[lev].periodicity());
vec_vbar[lev]->FillBoundary(geom[lev].periodicity());

vec_t3[lev]->FillBoundary(geom[lev].periodicity());
vec_s3[lev]->FillBoundary(geom[lev].periodicity());
FillPatch(lev, t_new[lev], *vec_ubar[lev], GetVecOfPtrs(vec_ubar));
FillPatch(lev, t_new[lev], *vec_vbar[lev], GetVecOfPtrs(vec_vbar));
FillPatch(lev, t_new[lev], *vec_t3[lev], GetVecOfPtrs(vec_t3));
FillPatch(lev, t_new[lev], *vec_s3[lev], GetVecOfPtrs(vec_s3));

// Fill in three ways: 1) interpolate from coarse grid if lev > 0; 2) fill from physical boundaries;
// 3) fine-fine fill of ghost cells with FillBoundary call
Expand Down
38 changes: 17 additions & 21 deletions Source/TimeIntegration/ROMSX_setup_step.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,6 @@ ROMSX::setup_step (int lev, Real time, Real dt_lev)
std::unique_ptr<MultiFab>& mf_bvstr = vec_bvstr[lev];
MultiFab mf_rw(ba,dm,1,IntVect(NGROW,NGROW,0));

MultiFab mf_W(ba,dm,1,IntVect(NGROW+1,NGROW+1,0));
mf_W.setVal(0.0);

std::unique_ptr<MultiFab>& mf_visc2_p = vec_visc2_p[lev];
std::unique_ptr<MultiFab>& mf_visc2_r = vec_visc2_r[lev];

Expand All @@ -82,19 +79,15 @@ ROMSX::setup_step (int lev, Real time, Real dt_lev)

mf_DC.setVal(0);

U_old.FillBoundary(geom[lev].periodicity());
U_new.FillBoundary(geom[lev].periodicity());
V_old.FillBoundary(geom[lev].periodicity());
V_new.FillBoundary(geom[lev].periodicity());
mf_W.FillBoundary(geom[lev].periodicity());
FillPatch(lev, time, *cons_old[lev], cons_old);
FillPatch(lev, time, *xvel_old[lev], xvel_old);
FillPatch(lev, time, *yvel_old[lev], yvel_old);

S_old.FillBoundary(geom[lev].periodicity());
S_new.FillBoundary(geom[lev].periodicity());
FillPatch(lev, time, *cons_new[lev], cons_new);
FillPatch(lev, time, *xvel_new[lev], xvel_new);
FillPatch(lev, time, *yvel_new[lev], yvel_new);

mf_rw.setVal(0.0);
mf_W.setVal(0.0);
U_old.FillBoundary(geom[lev].periodicity());
V_old.FillBoundary(geom[lev].periodicity());
mf_rufrc->setVal(0);
mf_rvfrc->setVal(0);

Expand Down Expand Up @@ -237,7 +230,10 @@ ROMSX::setup_step (int lev, Real time, Real dt_lev)
rho_eos(gbx2,state_old,rho,rhoA,rhoS,Hz,z_w,h,N);
}

if(solverChoice.use_prestep) {
MultiFab mf_W(ba,dm,1,IntVect(NGROW+1,NGROW+1,0));
mf_W.setVal(0.0);

if (solverChoice.use_prestep) {
const int nnew = 0;
prestep(lev, U_old, V_old, U_new, V_new,
mf_ru, mf_rv,
Expand All @@ -246,7 +242,7 @@ ROMSX::setup_step (int lev, Real time, Real dt_lev)
mf_bvstr, iic, ntfirst, nnew, nstp, nrhs, N, dt_lev);
}


// We use FillBoundary not FillPatch here since mf_W is single-level scratch space
mf_W.FillBoundary(geom[lev].periodicity());

for ( MFIter mfi(S_old, TilingIfNotGPU()); mfi.isValid(); ++mfi )
Expand Down Expand Up @@ -436,12 +432,12 @@ ROMSX::setup_step (int lev, Real time, Real dt_lev)
// Update Akv with new depth. NOTE: this happens before set_zeta in ROMS
set_vmix(lev);

S_old.FillBoundary(geom[lev].periodicity());
S_new.FillBoundary(geom[lev].periodicity());
FillPatch(lev, time, *cons_old[lev], cons_old);
FillPatch(lev, time, *cons_new[lev], cons_new);

vec_t3[lev]->FillBoundary(geom[lev].periodicity());
vec_s3[lev]->FillBoundary(geom[lev].periodicity());
FillPatch(lev, time, *vec_t3[lev], GetVecOfPtrs(vec_t3));
FillPatch(lev, time, *vec_s3[lev], GetVecOfPtrs(vec_s3));

vec_Huon[lev]->FillBoundary(geom[lev].periodicity());
vec_Hvom[lev]->FillBoundary(geom[lev].periodicity());
FillPatch(lev, time, *vec_Huon[lev], GetVecOfPtrs(vec_Huon));
FillPatch(lev, time, *vec_Hvom[lev], GetVecOfPtrs(vec_Hvom));
}

0 comments on commit a779c97

Please sign in to comment.