From 7f4d79453f6b3f1650214aab639775b9fd42e08f Mon Sep 17 00:00:00 2001 From: Ann Almgren Date: Mon, 13 Nov 2023 09:31:30 -0800 Subject: [PATCH] fix CUDA issue --- Source/ERF.H | 6 +++--- Source/ERF.cpp | 37 ++++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Source/ERF.H b/Source/ERF.H index 63b39887d..455eb8a0a 100644 --- a/Source/ERF.H +++ b/Source/ERF.H @@ -330,6 +330,9 @@ public: void init_from_metgrid (int lev); #endif // ERF_USE_NETCDF + // more flexible version of AverageDown() that lets you average down across multiple levels + void AverageDownTo (int crse_lev, int scomp, int ncomp); // NOLINT + private: /////////////////////////// @@ -362,9 +365,6 @@ private: void initialize_integrator (int lev, amrex::MultiFab& cons_mf, amrex::MultiFab& vel_mf); - // more flexible version of AverageDown() that lets you average down across multiple levels - void AverageDownTo (int crse_lev, int scomp, int ncomp); // NOLINT - // Compute a vector of new MultiFabs by copying from valid region and filling ghost cells - // works for single level and 2-level cases (fill fine grid ghost by interpolating from coarse) void FillPatch (int lev, amrex::Real time, const amrex::Vector& mf, bool fillset=true); diff --git a/Source/ERF.cpp b/Source/ERF.cpp index 7dda2b551..82b8a4da2 100644 --- a/Source/ERF.cpp +++ b/Source/ERF.cpp @@ -1280,9 +1280,6 @@ ERF::AverageDownTo (int crse_lev, int scomp, int ncomp) // NOLINT AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay || solverChoice.coupling_type == CouplingType::Mixed); - int l_scomp = scomp; - int l_ncomp = ncomp; - for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) { const BoxArray& ba(vars_new[crse_lev][var_idx].boxArray()); if (ba[0].type() == IntVect::TheZeroVector()) @@ -1297,14 +1294,14 @@ ERF::AverageDownTo (int crse_lev, int scomp, int ncomp) // NOLINT const Array4 mapfac_arr = mapfac_m[lev]->const_array(mfi); if (solverChoice.use_terrain) { const Array4 detJ_arr = detJ_cc[lev]->const_array(mfi); - ParallelFor(bx, l_ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept { - cons_arr(i,j,k,l_scomp+n) *= detJ_arr(i,j,k) / (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)); + cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)); }); } else { - ParallelFor(bx, l_ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept { - cons_arr(i,j,k,l_scomp+n) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)); + cons_arr(i,j,k,scomp+n) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)); }); } } // mfi @@ -1322,14 +1319,14 @@ ERF::AverageDownTo (int crse_lev, int scomp, int ncomp) // NOLINT const Array4 mapfac_arr = mapfac_m[lev]->const_array(mfi); if (solverChoice.use_terrain) { const Array4 detJ_arr = detJ_cc[lev]->const_array(mfi); - ParallelFor(bx, l_ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept { - cons_arr(i,j,k,l_scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)) / detJ_arr(i,j,k); + cons_arr(i,j,k,scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)) / detJ_arr(i,j,k); }); } else { - ParallelFor(bx, l_ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept + ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept { - cons_arr(i,j,k,l_scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)); + cons_arr(i,j,k,scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)); }); } } // mfi @@ -1348,6 +1345,9 @@ ERF::AverageDownTo (int crse_lev, int scomp, int ncomp) // NOLINT void ERF::Construct_ERFFillPatchers (int lev) { + AMREX_ALWAYS_ASSERT(solver.coupling_type == CouplingType::OneWay || + solver.coupling_type == CouplingType::Mixed); + auto& fine_new = vars_new[lev]; auto& crse_new = vars_new[lev-1]; auto& ba_fine = fine_new[Vars::cons].boxArray(); @@ -1355,10 +1355,12 @@ ERF::Construct_ERFFillPatchers (int lev) auto& dm_fine = fine_new[Vars::cons].DistributionMap(); auto& dm_crse = crse_new[Vars::cons].DistributionMap(); - // NOTE: crse-fine set/relaxation only done on Rho/RhoTheta + // NOTE: if "Mixed", then crse-fine set/relaxation only done on Rho/RhoTheta + int ncomp = (solver.coupling_type == CouplingType::OneWay) ? NVAR : 2; + FPr_c.emplace_back(ba_fine, dm_fine, geom[lev] , ba_crse, dm_crse, geom[lev-1], - -cf_width, -cf_set_width, 2, &cell_cons_interp); + -cf_width, -cf_set_width, ncomp, &cell_cons_interp); FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] , convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1], -cf_width, -cf_set_width, 1, &face_linear_interp); @@ -1373,6 +1375,9 @@ ERF::Construct_ERFFillPatchers (int lev) void ERF::Define_ERFFillPatchers (int lev) { + AMREX_ALWAYS_ASSERT(solver.coupling_type == CouplingType::OneWay || + solver.coupling_type == CouplingType::Mixed); + auto& fine_new = vars_new[lev]; auto& crse_new = vars_new[lev-1]; auto& ba_fine = fine_new[Vars::cons].boxArray(); @@ -1380,10 +1385,12 @@ ERF::Define_ERFFillPatchers (int lev) auto& dm_fine = fine_new[Vars::cons].DistributionMap(); auto& dm_crse = crse_new[Vars::cons].DistributionMap(); - // NOTE: crse-fine set/relaxation only done on Rho/RhoTheta + // NOTE: if "Mixed", then crse-fine set/relaxation only done on Rho/RhoTheta + int ncomp = (solver.coupling_type == CouplingType::OneWay) ? NVAR : 2; + FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev] , ba_crse, dm_crse, geom[lev-1], - -cf_width, -cf_set_width, 2, &cell_cons_interp); + -cf_width, -cf_set_width, ncomp, &cell_cons_interp); FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] , convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1], -cf_width, -cf_set_width, 1, &face_linear_interp);