Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] MultiFabRegister: throw in get #5356

Open
wants to merge 8 commits into
base: development
Choose a base branch
from
Open

[WIP] MultiFabRegister: throw in get #5356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2c2f48c
MultiFabRegister: `throw` in get
ax3l Oct 2, 2024
06bb292
Merge `ECP-WarpX:development` into `ax3l:mf-register-throw`
EZoni Oct 18, 2024
6a5ea04
Add `skip_level_0` argument for coarse-patch fields
EZoni Oct 19, 2024
29e08fe
Fix bug in `WarpX::SyncCurrent`
EZoni Oct 21, 2024
a88403e
Remove `return` statement after throwing runtime error
EZoni Oct 21, 2024
eddd8f7
Fix bug in `PML::PushPSATD`
EZoni Oct 21, 2024
8749799
Merge `ECP-WarpX:development` into `ax3l:mf-register-throw`
EZoni Oct 21, 2024
620d6ee
Remove debugging info left over
EZoni Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Source/BoundaryConditions/PML.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1298,16 +1298,16 @@ PML::PushPSATD (ablastr::fields::MultiFabRegister& fields, const int lev)
{
ablastr::fields::VectorField pml_E_fp = fields.get_alldirs(FieldType::pml_E_fp, lev);
ablastr::fields::VectorField pml_B_fp = fields.get_alldirs(FieldType::pml_B_fp, lev);
ablastr::fields::ScalarField pml_F_fp = fields.get(FieldType::pml_F_fp, lev);
ablastr::fields::ScalarField pml_G_fp = fields.get(FieldType::pml_G_fp, lev);
ablastr::fields::ScalarField pml_F_fp = (fields.has(FieldType::pml_F_fp, lev)) ? fields.get(FieldType::pml_F_fp, lev) : nullptr;
ablastr::fields::ScalarField pml_G_fp = (fields.has(FieldType::pml_G_fp, lev)) ? fields.get(FieldType::pml_G_fp, lev) : nullptr;

// Update the fields on the fine and coarse patch
PushPMLPSATDSinglePatch(lev, *spectral_solver_fp, pml_E_fp, pml_B_fp, pml_F_fp, pml_G_fp, m_fill_guards_fields);
if (spectral_solver_cp) {
ablastr::fields::VectorField pml_E_cp = fields.get_alldirs(FieldType::pml_E_cp, lev);
ablastr::fields::VectorField pml_B_cp = fields.get_alldirs(FieldType::pml_B_cp, lev);
ablastr::fields::ScalarField pml_F_cp = fields.get(FieldType::pml_F_cp, lev);
ablastr::fields::ScalarField pml_G_cp = fields.get(FieldType::pml_G_cp, lev);
ablastr::fields::ScalarField pml_F_cp = (fields.has(FieldType::pml_F_cp, lev)) ? fields.get(FieldType::pml_F_cp, lev) : nullptr;
ablastr::fields::ScalarField pml_G_cp = (fields.has(FieldType::pml_G_cp, lev)) ? fields.get(FieldType::pml_G_cp, lev) : nullptr;
PushPMLPSATDSinglePatch(lev, *spectral_solver_cp, pml_E_cp, pml_B_cp, pml_F_cp, pml_G_cp, m_fill_guards_fields);
}
}
Expand Down
32 changes: 18 additions & 14 deletions Source/Evolve/WarpXEvolve.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time)

using warpx::fields::FieldType;

bool const skip_level_0 = true;

const int rho_mid = spectral_solver_fp[0]->m_spectral_index.rho_mid;
const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new;

Expand Down Expand Up @@ -796,8 +798,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time)
PSATDBackwardTransformEBavg(
m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level)
m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_level_0),
m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_level_0)
);
}

Expand Down Expand Up @@ -868,11 +870,13 @@ WarpX::OneStep_sub1 (Real cur_time)

using warpx::fields::FieldType;

bool const skip_level_0 = true;

// i) Push particles and fields on the fine patch (first fine step)
PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf);
RestrictCurrentFromFineToCoarsePatch(
m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev);
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0), fine_lev);
RestrictRhoFromFineToCoarsePatch(fine_lev);
if (use_filter) {
ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev);
Expand All @@ -883,7 +887,7 @@ WarpX::OneStep_sub1 (Real cur_time)

ApplyFilterandSumBoundaryRho(
m_fields.get_mr_levels(FieldType::rho_fp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0),
fine_lev, PatchType::fine, 0, 2*ncomps);

EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf);
Expand Down Expand Up @@ -914,12 +918,12 @@ WarpX::OneStep_sub1 (Real cur_time)
StoreCurrent(coarse_lev);
AddCurrentFromFineLevelandSumBoundary(
m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level), coarse_lev);
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0),
m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_level_0), coarse_lev);
AddRhoFromFineLevelandSumBoundary(
m_fields.get_mr_levels(FieldType::rho_fp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level),
m_fields.get_mr_levels(FieldType::rho_buf, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0),
m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_level_0),
coarse_lev, 0, ncomps);

EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf);
Expand Down Expand Up @@ -950,15 +954,15 @@ WarpX::OneStep_sub1 (Real cur_time)
PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf);
RestrictCurrentFromFineToCoarsePatch(
m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev);
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0), fine_lev);
RestrictRhoFromFineToCoarsePatch(fine_lev);
if (use_filter) {
ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev);
}
SumBoundaryJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev, Geom(fine_lev).periodicity());
ApplyFilterandSumBoundaryRho(
m_fields.get_mr_levels(FieldType::rho_fp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0),
fine_lev, PatchType::fine, 0, ncomps);

EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf);
Expand Down Expand Up @@ -988,13 +992,13 @@ WarpX::OneStep_sub1 (Real cur_time)
RestoreCurrent(coarse_lev);
AddCurrentFromFineLevelandSumBoundary(
m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0),
m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_level_0),
coarse_lev);
AddRhoFromFineLevelandSumBoundary(
m_fields.get_mr_levels(FieldType::rho_fp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level),
m_fields.get_mr_levels(FieldType::rho_buf, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0),
m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_level_0),
coarse_lev, ncomps, ncomps);

EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]);
Expand Down
4 changes: 3 additions & 1 deletion Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ void LabFrameExplicitES::ComputeSpaceChargeField (
using ablastr::fields::MultiLevelVectorField;
using warpx::fields::FieldType;

bool const skip_level_0 = true;

const MultiLevelScalarField rho_fp = fields.get_mr_levels(FieldType::rho_fp, max_level);
const MultiLevelScalarField rho_cp = fields.get_mr_levels(FieldType::rho_cp, max_level);
const MultiLevelScalarField rho_cp = fields.get_mr_levels(FieldType::rho_cp, max_level, skip_level_0);
const MultiLevelScalarField phi_fp = fields.get_mr_levels(FieldType::phi_fp, max_level);
const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level);

Expand Down
10 changes: 7 additions & 3 deletions Source/FieldSolver/WarpXPushFieldsEM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -721,6 +721,8 @@ WarpX::PushPSATD ()
"PushFieldsEM: PSATD solver selected but not built");
#else

bool const skip_level_0 = true;

const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old;
const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new;

Expand Down Expand Up @@ -852,8 +854,8 @@ WarpX::PushPSATD ()
if (WarpX::fft_do_time_averaging) {
auto Efield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level);
auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level);
auto Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level);
auto Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level);
auto Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_level_0);
auto Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_level_0);
PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp);
}
if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); }
Expand Down Expand Up @@ -1098,6 +1100,8 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t

WARPX_PROFILE("WarpX::EvolveG()");

bool const skip_level_0 = true;

// Evolve G field in regular cells
if (patch_type == PatchType::fine)
{
Expand All @@ -1108,7 +1112,7 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t
}
else // coarse patch
{
ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs(FieldType::Bfield_cp, finest_level);
ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs(FieldType::Bfield_cp, finest_level, skip_level_0);
m_fdtd_solver_cp[lev]->EvolveG(
m_fields.get(FieldType::G_cp, lev),
Bfield_cp_new[lev], a_dt);
Expand Down
4 changes: 3 additions & 1 deletion Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,11 +195,13 @@ void WarpX::HybridPICDepositInitialRhoAndJ ()
{
using warpx::fields::FieldType;

bool const skip_level_0 = true;

ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels(FieldType::hybrid_rho_fp_temp, finest_level);
ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs(FieldType::hybrid_current_fp_temp, finest_level);
mypc->DepositCharge(rho_fp_temp, 0._rt);
mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt);
SyncRho(rho_fp_temp, m_fields.get_mr_levels(FieldType::rho_cp, finest_level), m_fields.get_mr_levels(FieldType::rho_buf, finest_level));
SyncRho(rho_fp_temp, m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0), m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_level_0));
SyncCurrent("hybrid_current_fp_temp");
for (int lev=0; lev <= finest_level; ++lev) {
// SyncCurrent does not include a call to FillBoundary, but it is needed
Expand Down
23 changes: 15 additions & 8 deletions Source/Parallelization/WarpXComm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -836,6 +836,8 @@ WarpX::FillBoundaryE_avg(int lev, IntVect ng)
void
WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng)
{
bool const skip_level_0 = true;

if (patch_type == PatchType::fine)
{
if (do_pml && pml[lev]->ok())
Expand Down Expand Up @@ -865,7 +867,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng)
WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented");
}

ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level);
ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_level_0);

const amrex::Periodicity& cperiod = Geom(lev-1).periodicity();
if ( safe_guard_cells ) {
Expand Down Expand Up @@ -896,6 +898,8 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng)
{
using ablastr::fields::Direction;

bool const skip_level_0 = true;

if (patch_type == PatchType::fine)
{
if (do_pml && pml[lev]->ok())
Expand Down Expand Up @@ -925,7 +929,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng)
WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented");
}

ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level);
ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_level_0);

const amrex::Periodicity& cperiod = Geom(lev-1).periodicity();
if ( safe_guard_cells ){
Expand Down Expand Up @@ -1077,12 +1081,14 @@ WarpX::SyncCurrent (const std::string& current_fp_string)

WARPX_PROFILE("WarpX::SyncCurrent()");

bool const skip_level_0 = true;

ablastr::fields::MultiLevelVectorField const& J_fp = m_fields.get_mr_levels_alldirs(current_fp_string, finest_level);

// If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid
if (do_current_centering)
{
ablastr::fields::MultiLevelVectorField const& J_fp_nodal = m_fields.get_mr_levels_alldirs(FieldType::current_fp_nodal, finest_level+1);
ablastr::fields::MultiLevelVectorField const& J_fp_nodal = m_fields.get_mr_levels_alldirs(FieldType::current_fp_nodal, finest_level);

AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1,
"warpx.do_current_centering=1 not supported with more than one fine levels");
Expand Down Expand Up @@ -1192,7 +1198,7 @@ WarpX::SyncCurrent (const std::string& current_fp_string)
}
});
// Now it's safe to apply filter and sumboundary on J_cp
ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level);
ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0);
if (use_filter)
{
ApplyFilterJ(J_cp, lev+1, idim);
Expand All @@ -1207,14 +1213,14 @@ WarpX::SyncCurrent (const std::string& current_fp_string)
// filtering depends on the level. This is also done before any
// same-level communication because it's easier this way to
// avoid double counting.
ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level);
ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0);
J_cp[lev][Direction{idim}]->setVal(0.0);
ablastr::coarsen::average::Coarsen(*J_cp[lev][Direction{idim}],
*J_fp[lev][Direction{idim}],
refRatio(lev-1));
if (m_fields.has(FieldType::current_buf, Direction{idim}, lev))
{
ablastr::fields::MultiLevelVectorField const& J_buffer = m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level);
ablastr::fields::MultiLevelVectorField const& J_buffer = m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_level_0);

IntVect const& ng = J_cp[lev][Direction{idim}]->nGrowVect();
AMREX_ASSERT(ng.allLE(J_buffer[lev][Direction{idim}]->nGrowVect()));
Expand All @@ -1241,14 +1247,15 @@ WarpX::SyncCurrent (const std::string& current_fp_string)

void
WarpX::SyncRho () {
bool const skip_level_0 = true;
const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has(FieldType::rho_fp, 0) ?
m_fields.get_mr_levels(FieldType::rho_fp, finest_level) :
ablastr::fields::MultiLevelScalarField{static_cast<size_t>(finest_level+1)};
const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has(FieldType::rho_cp, 1) ?
m_fields.get_mr_levels(FieldType::rho_cp, finest_level) :
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0) :
ablastr::fields::MultiLevelScalarField{static_cast<size_t>(finest_level+1)};
const ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has(FieldType::rho_buf, 1) ?
m_fields.get_mr_levels(FieldType::rho_buf, finest_level) :
m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_level_0) :
ablastr::fields::MultiLevelScalarField{static_cast<size_t>(finest_level+1)};

SyncRho(rho_fp, rho_cp, rho_buf);
Expand Down
6 changes: 4 additions & 2 deletions Source/Utils/WarpXMovingWindow.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ WarpX::MoveWindow (const int step, bool move_j)
using ablastr::fields::Direction;
using warpx::fields::FieldType;

bool const skip_level_0 = true;

if (step == start_moving_window_step) {
amrex::Print() << Utils::TextMsg::Info("Starting moving window");
}
Expand Down Expand Up @@ -276,8 +278,8 @@ WarpX::MoveWindow (const int step, bool move_j)
shiftMF(*m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost);
shiftMF(*m_fields.get(FieldType::Efield_aux, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost);
if (fft_do_time_averaging) {
ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level);
ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level);
ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_level_0);
ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_level_0);
shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost,
m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser);
shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost,
Expand Down
Loading
Loading