Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] MultiFabRegister: throw in get #5356

Open
wants to merge 8 commits into
base: development
Choose a base branch
from
Open

[WIP] MultiFabRegister: throw in get #5356

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2c2f48c
MultiFabRegister: `throw` in get
ax3l Oct 2, 2024
06bb292
Merge `ECP-WarpX:development` into `ax3l:mf-register-throw`
EZoni Oct 18, 2024
6a5ea04
Add `skip_level_0` argument for coarse-patch fields
EZoni Oct 19, 2024
29e08fe
Fix bug in `WarpX::SyncCurrent`
EZoni Oct 21, 2024
a88403e
Remove `return` statement after throwing runtime error
EZoni Oct 21, 2024
eddd8f7
Fix bug in `PML::PushPSATD`
EZoni Oct 21, 2024
8749799
Merge `ECP-WarpX:development` into `ax3l:mf-register-throw`
EZoni Oct 21, 2024
620d6ee
Remove debugging info left over
EZoni Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions Source/Evolve/WarpXEvolve.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time)

using warpx::fields::FieldType;

bool const skip_level_0 = true;

const int rho_mid = spectral_solver_fp[0]->m_spectral_index.rho_mid;
const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new;

Expand Down Expand Up @@ -796,8 +798,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time)
PSATDBackwardTransformEBavg(
m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level)
m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_level_0),
m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_level_0)
);
}

Expand Down Expand Up @@ -868,11 +870,13 @@ WarpX::OneStep_sub1 (Real cur_time)

using warpx::fields::FieldType;

bool const skip_level_0 = true;

// i) Push particles and fields on the fine patch (first fine step)
PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf);
RestrictCurrentFromFineToCoarsePatch(
m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev);
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0), fine_lev);
RestrictRhoFromFineToCoarsePatch(fine_lev);
if (use_filter) {
ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev);
Expand All @@ -883,7 +887,7 @@ WarpX::OneStep_sub1 (Real cur_time)

ApplyFilterandSumBoundaryRho(
m_fields.get_mr_levels(FieldType::rho_fp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0),
fine_lev, PatchType::fine, 0, 2*ncomps);

EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf);
Expand Down Expand Up @@ -914,12 +918,12 @@ WarpX::OneStep_sub1 (Real cur_time)
StoreCurrent(coarse_lev);
AddCurrentFromFineLevelandSumBoundary(
m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level), coarse_lev);
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0),
m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_level_0), coarse_lev);
AddRhoFromFineLevelandSumBoundary(
m_fields.get_mr_levels(FieldType::rho_fp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level),
m_fields.get_mr_levels(FieldType::rho_buf, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0),
m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_level_0),
coarse_lev, 0, ncomps);

EvolveB(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf);
Expand Down Expand Up @@ -950,15 +954,15 @@ WarpX::OneStep_sub1 (Real cur_time)
PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf);
RestrictCurrentFromFineToCoarsePatch(
m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev);
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0), fine_lev);
RestrictRhoFromFineToCoarsePatch(fine_lev);
if (use_filter) {
ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev);
}
SumBoundaryJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev, Geom(fine_lev).periodicity());
ApplyFilterandSumBoundaryRho(
m_fields.get_mr_levels(FieldType::rho_fp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0),
fine_lev, PatchType::fine, 0, ncomps);

EvolveB(fine_lev, PatchType::fine, 0.5_rt*dt[fine_lev], DtType::FirstHalf);
Expand Down Expand Up @@ -988,13 +992,13 @@ WarpX::OneStep_sub1 (Real cur_time)
RestoreCurrent(coarse_lev);
AddCurrentFromFineLevelandSumBoundary(
m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level),
m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0),
m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_level_0),
coarse_lev);
AddRhoFromFineLevelandSumBoundary(
m_fields.get_mr_levels(FieldType::rho_fp, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level),
m_fields.get_mr_levels(FieldType::rho_buf, finest_level),
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0),
m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_level_0),
coarse_lev, ncomps, ncomps);

EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]);
Expand Down
4 changes: 3 additions & 1 deletion Source/FieldSolver/ElectrostaticSolvers/LabFrameExplicitES.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ void LabFrameExplicitES::ComputeSpaceChargeField (
using ablastr::fields::MultiLevelVectorField;
using warpx::fields::FieldType;

bool const skip_level_0 = true;

const MultiLevelScalarField rho_fp = fields.get_mr_levels(FieldType::rho_fp, max_level);
const MultiLevelScalarField rho_cp = fields.get_mr_levels(FieldType::rho_cp, max_level);
const MultiLevelScalarField rho_cp = fields.get_mr_levels(FieldType::rho_cp, max_level, skip_level_0);
const MultiLevelScalarField phi_fp = fields.get_mr_levels(FieldType::phi_fp, max_level);
const MultiLevelVectorField Efield_fp = fields.get_mr_levels_alldirs(FieldType::Efield_fp, max_level);

Expand Down
10 changes: 7 additions & 3 deletions Source/FieldSolver/WarpXPushFieldsEM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -721,6 +721,8 @@ WarpX::PushPSATD ()
"PushFieldsEM: PSATD solver selected but not built");
#else

bool const skip_level_0 = true;

const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old;
const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new;

Expand Down Expand Up @@ -852,8 +854,8 @@ WarpX::PushPSATD ()
if (WarpX::fft_do_time_averaging) {
auto Efield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_fp, finest_level);
auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_fp, finest_level);
auto Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level);
auto Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level);
auto Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_level_0);
auto Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_level_0);
PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp);
}
if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); }
Expand Down Expand Up @@ -1098,6 +1100,8 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t

WARPX_PROFILE("WarpX::EvolveG()");

bool const skip_level_0 = true;

// Evolve G field in regular cells
if (patch_type == PatchType::fine)
{
Expand All @@ -1108,7 +1112,7 @@ WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_t
}
else // coarse patch
{
ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs(FieldType::Bfield_cp, finest_level);
ablastr::fields::MultiLevelVectorField const& Bfield_cp_new = m_fields.get_mr_levels_alldirs(FieldType::Bfield_cp, finest_level, skip_level_0);
m_fdtd_solver_cp[lev]->EvolveG(
m_fields.get(FieldType::G_cp, lev),
Bfield_cp_new[lev], a_dt);
Expand Down
4 changes: 3 additions & 1 deletion Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,11 +195,13 @@ void WarpX::HybridPICDepositInitialRhoAndJ ()
{
using warpx::fields::FieldType;

bool const skip_level_0 = true;

ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels(FieldType::hybrid_rho_fp_temp, finest_level);
ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs(FieldType::hybrid_current_fp_temp, finest_level);
mypc->DepositCharge(rho_fp_temp, 0._rt);
mypc->DepositCurrent(current_fp_temp, dt[0], 0._rt);
SyncRho(rho_fp_temp, m_fields.get_mr_levels(FieldType::rho_cp, finest_level), m_fields.get_mr_levels(FieldType::rho_buf, finest_level));
SyncRho(rho_fp_temp, m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0), m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_level_0));
SyncCurrent("hybrid_current_fp_temp");
for (int lev=0; lev <= finest_level; ++lev) {
// SyncCurrent does not include a call to FillBoundary, but it is needed
Expand Down
21 changes: 14 additions & 7 deletions Source/Parallelization/WarpXComm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -836,6 +836,8 @@ WarpX::FillBoundaryE_avg(int lev, IntVect ng)
void
WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng)
{
bool const skip_level_0 = true;

if (patch_type == PatchType::fine)
{
if (do_pml && pml[lev]->ok())
Expand Down Expand Up @@ -865,7 +867,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng)
WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented");
}

ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level);
ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_level_0);

const amrex::Periodicity& cperiod = Geom(lev-1).periodicity();
if ( safe_guard_cells ) {
Expand Down Expand Up @@ -896,6 +898,8 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng)
{
using ablastr::fields::Direction;

bool const skip_level_0 = true;

if (patch_type == PatchType::fine)
{
if (do_pml && pml[lev]->ok())
Expand Down Expand Up @@ -925,7 +929,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng)
WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented");
}

ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level);
ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_level_0);

const amrex::Periodicity& cperiod = Geom(lev-1).periodicity();
if ( safe_guard_cells ){
Expand Down Expand Up @@ -1077,6 +1081,8 @@ WarpX::SyncCurrent (const std::string& current_fp_string)

WARPX_PROFILE("WarpX::SyncCurrent()");

bool const skip_level_0 = true;

ablastr::fields::MultiLevelVectorField const& J_fp = m_fields.get_mr_levels_alldirs(current_fp_string, finest_level);

// If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid
Expand Down Expand Up @@ -1192,7 +1198,7 @@ WarpX::SyncCurrent (const std::string& current_fp_string)
}
});
// Now it's safe to apply filter and sumboundary on J_cp
ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level);
ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0);
if (use_filter)
{
ApplyFilterJ(J_cp, lev+1, idim);
Expand All @@ -1207,14 +1213,14 @@ WarpX::SyncCurrent (const std::string& current_fp_string)
// filtering depends on the level. This is also done before any
// same-level communication because it's easier this way to
// avoid double counting.
ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level);
ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level, skip_level_0);
J_cp[lev][Direction{idim}]->setVal(0.0);
ablastr::coarsen::average::Coarsen(*J_cp[lev][Direction{idim}],
*J_fp[lev][Direction{idim}],
refRatio(lev-1));
if (m_fields.has(FieldType::current_buf, Direction{idim}, lev))
{
ablastr::fields::MultiLevelVectorField const& J_buffer = m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level);
ablastr::fields::MultiLevelVectorField const& J_buffer = m_fields.get_mr_levels_alldirs(FieldType::current_buf, finest_level, skip_level_0);

IntVect const& ng = J_cp[lev][Direction{idim}]->nGrowVect();
AMREX_ASSERT(ng.allLE(J_buffer[lev][Direction{idim}]->nGrowVect()));
Expand All @@ -1241,14 +1247,15 @@ WarpX::SyncCurrent (const std::string& current_fp_string)

void
WarpX::SyncRho () {
bool const skip_level_0 = true;
const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.has(FieldType::rho_fp, 0) ?
m_fields.get_mr_levels(FieldType::rho_fp, finest_level) :
ablastr::fields::MultiLevelScalarField{static_cast<size_t>(finest_level+1)};
const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.has(FieldType::rho_cp, 1) ?
m_fields.get_mr_levels(FieldType::rho_cp, finest_level) :
m_fields.get_mr_levels(FieldType::rho_cp, finest_level, skip_level_0) :
ablastr::fields::MultiLevelScalarField{static_cast<size_t>(finest_level+1)};
const ablastr::fields::MultiLevelScalarField rho_buf = m_fields.has(FieldType::rho_buf, 1) ?
m_fields.get_mr_levels(FieldType::rho_buf, finest_level) :
m_fields.get_mr_levels(FieldType::rho_buf, finest_level, skip_level_0) :
ablastr::fields::MultiLevelScalarField{static_cast<size_t>(finest_level+1)};

SyncRho(rho_fp, rho_cp, rho_buf);
Expand Down
6 changes: 4 additions & 2 deletions Source/Utils/WarpXMovingWindow.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ WarpX::MoveWindow (const int step, bool move_j)
using ablastr::fields::Direction;
using warpx::fields::FieldType;

bool const skip_level_0 = true;

if (step == start_moving_window_step) {
amrex::Print() << Utils::TextMsg::Info("Starting moving window");
}
Expand Down Expand Up @@ -276,8 +278,8 @@ WarpX::MoveWindow (const int step, bool move_j)
shiftMF(*m_fields.get(FieldType::Bfield_aux, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost);
shiftMF(*m_fields.get(FieldType::Efield_aux, Direction{dim}, lev), geom[lev], num_shift, dir, lev, do_update_cost);
if (fft_do_time_averaging) {
ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level);
ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level);
ablastr::fields::MultiLevelVectorField Efield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Efield_avg_cp, finest_level, skip_level_0);
ablastr::fields::MultiLevelVectorField Bfield_avg_cp = m_fields.get_mr_levels_alldirs(FieldType::Bfield_avg_cp, finest_level, skip_level_0);
shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost,
m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser);
shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost,
Expand Down
1 change: 1 addition & 0 deletions Source/WarpX.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2593,6 +2593,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm
}

// Create the MultiFabs for the current
amrex::Print() << "allocate current_cp" << std::endl;
EZoni marked this conversation as resolved.
Show resolved Hide resolved
m_fields.alloc_init(FieldType::current_cp, Direction{0}, lev, amrex::convert(cba, jx_nodal_flag), dm, ncomps, ngJ, 0.0_rt);
m_fields.alloc_init(FieldType::current_cp, Direction{1}, lev, amrex::convert(cba, jy_nodal_flag), dm, ncomps, ngJ, 0.0_rt);
m_fields.alloc_init(FieldType::current_cp, Direction{2}, lev, amrex::convert(cba, jz_nodal_flag), dm, ncomps, ngJ, 0.0_rt);
Expand Down
36 changes: 24 additions & 12 deletions Source/ablastr/fields/MultiFabRegister.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,24 +479,28 @@ namespace ablastr::fields
[[nodiscard]] MultiLevelScalarField
get_mr_levels (
T name,
int finest_level
int finest_level,
bool skip_level_0=false
)
{
return internal_get_mr_levels(
getExtractedName(name),
finest_level
finest_level,
skip_level_0
);
}
template<typename T>
[[nodiscard]] ConstMultiLevelScalarField
get_mr_levels (
T name,
int finest_level
int finest_level,
bool skip_level_0=false
) const
{
return internal_get_mr_levels(
getExtractedName(name),
finest_level
finest_level,
skip_level_0
);
}
//@}
Expand Down Expand Up @@ -550,24 +554,28 @@ namespace ablastr::fields
[[nodiscard]] MultiLevelVectorField
get_mr_levels_alldirs (
T name,
int finest_level
int finest_level,
bool skip_level_0=false
)
{
return internal_get_mr_levels_alldirs(
getExtractedName(name),
finest_level
finest_level,
skip_level_0
);
}
template<typename T>
[[nodiscard]] ConstMultiLevelVectorField
get_mr_levels_alldirs (
T name,
int finest_level
int finest_level,
bool skip_level_0=false
) const
{
return internal_get_mr_levels_alldirs(
getExtractedName(name),
finest_level
finest_level,
skip_level_0
);
}
//@}
Expand Down Expand Up @@ -762,12 +770,14 @@ namespace ablastr::fields
[[nodiscard]] MultiLevelScalarField
internal_get_mr_levels (
std::string const & name,
int finest_level
int finest_level,
bool skip_level_0
);
[[nodiscard]] ConstMultiLevelScalarField
internal_get_mr_levels (
std::string const & name,
int finest_level
int finest_level,
bool skip_level_0
) const;
[[nodiscard]] VectorField
internal_get_alldirs (
Expand All @@ -782,12 +792,14 @@ namespace ablastr::fields
[[nodiscard]] MultiLevelVectorField
internal_get_mr_levels_alldirs (
std::string const & name,
int finest_level
int finest_level,
bool skip_level_0
);
[[nodiscard]] ConstMultiLevelVectorField
internal_get_mr_levels_alldirs (
std::string const & name,
int finest_level
int finest_level,
bool skip_level_0
) const;

void
Expand Down
Loading
Loading