From d6027cf82c988de854f1b78390057f861127fe85 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 13 Feb 2024 14:50:51 -0800 Subject: [PATCH 01/83] Initial setup for the inflow-outflow BC that manages both kind of cells in the same boundary --- amr-wind/boundary_conditions/BCInterface.cpp | 7 ++++ amr-wind/boundary_conditions/CMakeLists.txt | 1 + .../boundary_conditions/MassInflowOutflowBC.H | 33 +++++++++++++++++++ .../MassInflowOutflowBC.cpp | 27 +++++++++++++++ amr-wind/diffusion/incflo_diffusion.cpp | 1 + amr-wind/incflo_enums.H | 1 + 6 files changed, 70 insertions(+) create mode 100644 amr-wind/boundary_conditions/MassInflowOutflowBC.H create mode 100644 amr-wind/boundary_conditions/MassInflowOutflowBC.cpp diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index b6ca50ab21..007a59ce4d 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -64,6 +64,8 @@ void BCIface::read_bctype() ibctype[ori] = BC::pressure_outflow; } else if ((bcstr == "mass_inflow") || (bcstr == "mi")) { ibctype[ori] = BC::mass_inflow; + } else if ((bcstr == "mass_inflow_outflow") || (bcstr == "mio")) { + ibctype[ori] = BC::mass_inflow_outflow; } else if ((bcstr == "no_slip_wall") || (bcstr == "nsw")) { ibctype[ori] = BC::no_slip_wall; } else if ((bcstr == "slip_wall") || (bcstr == "sw")) { @@ -194,6 +196,11 @@ void BCVelocity::set_bcrec() } break; + case BC::mass_inflow_outflow: + set_bcrec_lo(dir, amrex::BCType::user_1); + set_bcrec_hi(dir, amrex::BCType::user_1); + break; + case BC::slip_wall: case BC::wall_model: if (side == amrex::Orientation::low) { diff --git a/amr-wind/boundary_conditions/CMakeLists.txt b/amr-wind/boundary_conditions/CMakeLists.txt index bdc98bd3f6..26af39ca57 100644 --- a/amr-wind/boundary_conditions/CMakeLists.txt +++ b/amr-wind/boundary_conditions/CMakeLists.txt @@ -4,6 +4,7 @@ target_sources(${amr_wind_lib_name} BCInterface.cpp FixedGradientBC.cpp scalar_bcs.cpp + MassInflowOutflowBC.cpp ) add_subdirectory(wall_models) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.H b/amr-wind/boundary_conditions/MassInflowOutflowBC.H new file mode 100644 index 0000000000..77d724915c --- /dev/null +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.H @@ -0,0 +1,33 @@ +#ifndef MASSINFLOWOUTFLOWBC_H +#define MASSINFLOWOUTFLOWBC_H + +#include "amr-wind/core/FieldBCOps.H" +#include "amr-wind/core/FieldRepo.H" + +#include "AMReX_Orientation.H" + +namespace amr_wind { + +/** Can handle both inflow and outflow cells in the same boundary + * \ingroup field_bc + * + * + */ +class MassInflowOutflowBC : public FieldBCIface +{ +public: + MassInflowOutflowBC(Field& field, amrex::Orientation ori); + + void operator()(Field& field, const FieldState /*rho_state*/) override; + +private: + Field& m_field; + + amrex::Orientation m_ori; +}; + +} // namespace amr_wind + + +#endif + diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp new file mode 100644 index 0000000000..dd484ca590 --- /dev/null +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -0,0 +1,27 @@ + +#include "amr-wind/boundary_conditions/MassInflowOutflowBC.H" + +namespace amr_wind { +namespace { +AMREX_FORCE_INLINE +amrex::Box lower_boundary_faces(const amrex::Box& b, int dir) +{ + amrex::IntVect lo(b.smallEnd()); + amrex::IntVect hi(b.bigEnd()); + int sm = lo[dir]; + lo.setVal(dir, sm - 1); + hi.setVal(dir, sm - 1); + amrex::IndexType bxtype(b.ixType()); + bxtype.set(dir); + return {lo, hi, bxtype}; +} +} // namespace + +MassInflowOutflowBC::MassInflowOutflowBC(Field& field, amrex::Orientation ori) + : m_field(field), m_ori(ori) +{} + +void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_state*/) +{} + +} // namespace amr_wind \ No newline at end of file diff --git a/amr-wind/diffusion/incflo_diffusion.cpp b/amr-wind/diffusion/incflo_diffusion.cpp index a26afaac53..910562d60a 100644 --- a/amr-wind/diffusion/incflo_diffusion.cpp +++ b/amr-wind/diffusion/incflo_diffusion.cpp @@ -20,6 +20,7 @@ Vector> get_diffuse_tensor_bc( switch (bc) { case BC::pressure_inflow: case BC::pressure_outflow: + case BC::mass_inflow_outflow: case BC::zero_gradient: { // All three components are Neumann r[0][dir] = LinOpBCType::Neumann; diff --git a/amr-wind/incflo_enums.H b/amr-wind/incflo_enums.H index b73ff06b11..888ff0ae70 100644 --- a/amr-wind/incflo_enums.H +++ b/amr-wind/incflo_enums.H @@ -7,6 +7,7 @@ enum struct BC { pressure_inflow, pressure_outflow, mass_inflow, + mass_inflow_outflow, zero_gradient, no_slip_wall, slip_wall, From e9cd5a9cf0aaa19d1ddd6c51dd0b7dc15f72a69d Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 13 Feb 2024 17:05:56 -0800 Subject: [PATCH 02/83] add comment for header ifdef --- amr-wind/boundary_conditions/MassInflowOutflowBC.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.H b/amr-wind/boundary_conditions/MassInflowOutflowBC.H index 77d724915c..c57ed9be01 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.H +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.H @@ -29,5 +29,5 @@ private: } // namespace amr_wind -#endif +#endif /* MASSINFLOWOUTFLOWBC_H */ From 659ba97a093f3296692e0d13a15d1b3d0af0ad50 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 22 Feb 2024 15:14:11 -0800 Subject: [PATCH 03/83] set up advection and projection BCs --- amr-wind/boundary_conditions/BCInterface.cpp | 7 +- amr-wind/convection/incflo_godunov_weno.H | 570 ++++++++++++++++++ .../incflo_apply_nodal_projection.cpp | 1 + 3 files changed, 576 insertions(+), 2 deletions(-) create mode 100644 amr-wind/convection/incflo_godunov_weno.H diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index 007a59ce4d..12cefe623b 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -197,8 +197,11 @@ void BCVelocity::set_bcrec() break; case BC::mass_inflow_outflow: - set_bcrec_lo(dir, amrex::BCType::user_1); - set_bcrec_hi(dir, amrex::BCType::user_1); + if (side == amrex::Orientation::low) { + set_bcrec_lo(dir, amrex::BCType::user_1); + } else { + set_bcrec_hi(dir, amrex::BCType::user_1); + } break; case BC::slip_wall: diff --git a/amr-wind/convection/incflo_godunov_weno.H b/amr-wind/convection/incflo_godunov_weno.H new file mode 100644 index 0000000000..576bac7a45 --- /dev/null +++ b/amr-wind/convection/incflo_godunov_weno.H @@ -0,0 +1,570 @@ +#ifndef GODUNOV_WENO_H +#define GODUNOV_WENO_H + +#include +#include + +/* This header file contains the inlined __host__ __device__ functions required + for the scalar advection routines for 3D Godunov. It also contains function + declarations for controlling host functions. */ + +namespace { + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real weno5( + const amrex::Real sm2, + const amrex::Real sm1, + const amrex::Real s, + const amrex::Real sp1, + const amrex::Real sp2, + bool weno_js) +{ + constexpr amrex::Real eps = 1.e-6; + + const amrex::Real beta1 = + 13.0 / 12.0 * (sm2 - 2.0 * sm1 + s) * (sm2 - 2.0 * sm1 + s) + + 0.25 * (sm2 - 4.0 * sm1 + 3.0 * s) * (sm2 - 4.0 * sm1 + 3.0 * s); + const amrex::Real beta2 = + 13.0 / 12.0 * (sm1 - 2.0 * s + sp1) * (sm1 - 2.0 * s + sp1) + + 0.25 * (sm1 - sp1) * (sm1 - sp1); + const amrex::Real beta3 = + 13.0 / 12.0 * (s - 2.0 * sp1 + sp2) * (s - 2.0 * sp1 + sp2) + + 0.25 * (3.0 * s - 4.0 * sp1 + sp2) * (3.0 * s - 4.0 * sp1 + sp2); + + amrex::Real omega1, omega2, omega3; + + if (weno_js) { + omega1 = 0.1 / (eps + beta1); + omega2 = 0.6 / (eps + beta2); + omega3 = 0.3 / (eps + beta3); + } else { + const amrex::Real t5 = std::abs(beta3 - beta1); + omega1 = 0.1 * (1.0 + t5 / (eps + beta1)); + omega2 = 0.6 * (1.0 + t5 / (eps + beta2)); + omega3 = 0.3 * (1.0 + t5 / (eps + beta3)); + } + + const amrex::Real omega = omega1 + omega2 + omega3; + + const amrex::Real v_1 = 2.0 * sm2 - 7.0 * sm1 + 11.0 * s; + const amrex::Real v_2 = -sm1 + 5.0 * s + 2.0 * sp1; + const amrex::Real v_3 = 2.0 * s + 5.0 * sp1 - sp2; + + return (omega1 * v_1 + omega2 * v_2 + omega3 * v_3) / (6.0 * omega); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( + const int i, + const int j, + const int k, + const int n, + amrex::Real& sm, + amrex::Real& sp, + const amrex::Real& sedge1, + const amrex::Real& sedge2, + const amrex::Array4& s, + const amrex::Real velm, + const amrex::Real velp, + const int bclo, + const int bchi, + const int domlo, + const int domhi) +{ + using namespace amrex; + + if (bclo == BCType::ext_dir || bclo == BCType::hoextrap + || (bclo == BCType::user_1 && velm >= 0.0)) { + if (i == domlo) { + sp = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); + + sm = s(domlo - 1, j, k, n); + + } else if (i == domlo + 1) { + + sm = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); + + sp = sedge2; + } + } + + if (bchi == BCType::ext_dir || bchi == BCType::hoextrap + || (bchi == BCType::user_1 && velp <= 0.0)) { + if (i == domhi) { + sm = -0.2 * s(domhi + 1, j, k, n) + 0.75 * s(domhi, j, k, n) + + 0.5 * s(domhi - 1, j, k, n) - 0.05 * s(domhi - 2, j, k, n); + + sp = s(domhi + 1, j, k, n); + + } else if (i == domhi - 1) { + + sp = -0.2 * s(domhi + 1, j, k, n) + 0.75 * s(domhi, j, k, n) + + 0.5 * s(domhi - 1, j, k, n) - 0.05 * s(domhi - 2, j, k, n); + + sm = sedge1; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_ybc( + const int i, + const int j, + const int k, + const int n, + amrex::Real& sm, + amrex::Real& sp, + const amrex::Real& sedge1, + const amrex::Real& sedge2, + const amrex::Array4& s, + const amrex::Real velm, + const amrex::Real velp, + const int bclo, + const int bchi, + const int domlo, + const int domhi) +{ + using namespace amrex; + + if (bclo == BCType::ext_dir || bclo == BCType::hoextrap + || (bclo == BCType::user_1 && velm >= 0.0)) { + if (j == domlo) { + sp = -0.2 * s(i, domlo - 1, k, n) + 0.75 * s(i, domlo, k, n) + + 0.5 * s(i, domlo + 1, k, n) - 0.05 * s(i, domlo + 2, k, n); + + sm = s(i, domlo - 1, k, n); + + } else if (j == domlo + 1) { + + sm = -0.2 * s(i, domlo - 1, k, n) + 0.75 * s(i, domlo, k, n) + + 0.5 * s(i, domlo + 1, k, n) - 0.05 * s(i, domlo + 2, k, n); + + sp = sedge2; + } + } + + if (bchi == BCType::ext_dir || bchi == BCType::hoextrap + || (bchi == BCType::user_1 && velp <= 0.0)) { + if (j == domhi) { + sm = -0.2 * s(i, domhi + 1, k, n) + 0.75 * s(i, domhi, k, n) + + 0.5 * s(i, domhi - 1, k, n) - 0.05 * s(i, domhi - 2, k, n); + + sp = s(i, domhi + 1, k, n); + + } else if (j == domhi - 1) { + + sp = -0.2 * s(i, domhi + 1, k, n) + 0.75 * s(i, domhi, k, n) + + 0.5 * s(i, domhi - 1, k, n) - 0.05 * s(i, domhi - 2, k, n); + + sm = sedge1; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_zbc( + const int i, + const int j, + const int k, + const int n, + amrex::Real& sm, + amrex::Real& sp, + const amrex::Real& sedge1, + const amrex::Real& sedge2, + const amrex::Array4& s, + const amrex::Real velm, + const amrex::Real velp, + const int bclo, + const int bchi, + const int domlo, + const int domhi) +{ + using namespace amrex; + + if (bclo == BCType::ext_dir || bclo == BCType::hoextrap + || (bclo == BCType::user_1 && velm >= 0.0)) { + if (k == domlo) { + sp = -0.2 * s(i, j, domlo - 1, n) + 0.75 * s(i, j, domlo, n) + + 0.5 * s(i, j, domlo + 1, n) - 0.05 * s(i, j, domlo + 2, n); + + sm = s(i, j, domlo - 1, n); + + } else if (k == domlo + 1) { + + sm = -0.2 * s(i, j, domlo - 1, n) + 0.75 * s(i, j, domlo, n) + + 0.5 * s(i, j, domlo + 1, n) - 0.05 * s(i, j, domlo + 2, n); + + sp = sedge2; + } + } + + if (bchi == BCType::ext_dir || bchi == BCType::hoextrap + || (bchi == BCType::user_1 && velp <= 0.0)) { + if (k == domhi) { + sm = -0.2 * s(i, j, domhi + 1, n) + 0.75 * s(i, j, domhi, n) + + 0.5 * s(i, j, domhi - 1, n) - 0.05 * s(i, j, domhi - 2, n); + + sp = s(i, j, domhi + 1, n); + + } else if (k == domhi - 1) { + + sp = -0.2 * s(i, j, domhi + 1, n) + 0.75 * s(i, j, domhi, n) + + 0.5 * s(i, j, domhi - 1, n) - 0.05 * s(i, j, domhi - 2, n); + + sm = sedge1; + } + } +} + +// This version is called before the MAC projection, when we use the +// cell-centered velocity +// for upwinding +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_x( + const int i, + const int j, + const int k, + const int n, + const amrex::Real dtdx, + const amrex::Real v_ad, + const amrex::Array4& S, + const amrex::Array4& Im, + const amrex::Array4& Ip, + const amrex::BCRec& bc, + const int domlo, + const int domhi, + const bool weno_js) +{ + + using namespace amrex; + + constexpr amrex::Real small_vel = 1e-8; + + amrex::Real sm2 = S(i - 2, j, k, n); + amrex::Real sm1 = S(i - 1, j, k, n); + amrex::Real s0 = S(i, j, k, n); + amrex::Real sp1 = S(i + 1, j, k, n); + amrex::Real sp2 = S(i + 2, j, k, n); + + // right of i-1/2 + amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT + // left of i+1/2 + amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT + + amrex::Real sm = sedge1; + amrex::Real sp = sedge2; + + Godunov_weno_xbc( + i, j, k, n, sm, sp, sedge1, sedge2, S, v_ad, v_ad, bc.lo(0), bc.hi(0), + domlo, domhi); + + amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); + + amrex::Real sigma = std::abs(v_ad) * dtdx; + + // upwind + if (v_ad > small_vel) { + Ip(i, j, k, n) = + sp - (0.5 * sigma) * ((sp - sm) - (1.0 - 2.0 / 3.0 * sigma) * s6); + Im(i, j, k, n) = S(i, j, k, n); + } else if (v_ad < -small_vel) { + Ip(i, j, k, n) = S(i, j, k, n); + Im(i, j, k, n) = + sm + (0.5 * sigma) * ((sp - sm) + (1.0 - 2.0 / 3.0 * sigma) * s6); + } else { + Ip(i, j, k, n) = S(i, j, k, n); + Im(i, j, k, n) = S(i, j, k, n); + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_y( + const int i, + const int j, + const int k, + const int n, + const amrex::Real dtdy, + const amrex::Real v_ad, + const amrex::Array4& S, + const amrex::Array4& Im, + const amrex::Array4& Ip, + const amrex::BCRec& bc, + const int domlo, + const int domhi, + const bool weno_js) +{ + using namespace amrex; + + constexpr amrex::Real small_vel = 1e-8; + + amrex::Real sm2 = S(i, j - 2, k, n); + amrex::Real sm1 = S(i, j - 1, k, n); + amrex::Real s0 = S(i, j, k, n); + amrex::Real sp1 = S(i, j + 1, k, n); + amrex::Real sp2 = S(i, j + 2, k, n); + + // right of j-1/2 + amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT + // left of j+1/2 + amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT + + amrex::Real sm = sedge1; + amrex::Real sp = sedge2; + + Godunov_weno_ybc( + i, j, k, n, sm, sp, sedge1, sedge2, S, v_ad, v_ad, bc.lo(1), bc.hi(1), + domlo, domhi); + + amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); + + amrex::Real sigma = std::abs(v_ad) * dtdy; + + if (v_ad > small_vel) { + Ip(i, j, k, n) = + sp - (0.5 * sigma) * ((sp - sm) - (1.0 - 2.0 / 3.0 * sigma) * s6); + Im(i, j, k, n) = S(i, j, k, n); + } else if (v_ad < -small_vel) { + Ip(i, j, k, n) = S(i, j, k, n); + Im(i, j, k, n) = + sm + (0.5 * sigma) * ((sp - sm) + (1.0 - 2.0 / 3.0 * sigma) * s6); + } else { + Ip(i, j, k, n) = S(i, j, k, n); + Im(i, j, k, n) = S(i, j, k, n); + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_z( + const int i, + const int j, + const int k, + const int n, + const amrex::Real dtdz, + const amrex::Real v_ad, + const amrex::Array4& S, + const amrex::Array4& Im, + const amrex::Array4& Ip, + const amrex::BCRec& bc, + const int domlo, + const int domhi, + const bool weno_js) +{ + using namespace amrex; + + constexpr amrex::Real small_vel = 1e-8; + + amrex::Real sm2 = S(i, j, k - 2, n); + amrex::Real sm1 = S(i, j, k - 1, n); + amrex::Real s0 = S(i, j, k, n); + amrex::Real sp1 = S(i, j, k + 1, n); + amrex::Real sp2 = S(i, j, k + 2, n); + + // right of k-1/2 + amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT + // left of k+1/2 + amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT + + amrex::Real sm = sedge1; + amrex::Real sp = sedge2; + + Godunov_weno_zbc( + i, j, k, n, sm, sp, sedge1, sedge2, S, v_ad, v_ad, bc.lo(2), bc.hi(2), + domlo, domhi); + + amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); + + amrex::Real sigma = std::abs(v_ad) * dtdz; + + if (v_ad > small_vel) { + Ip(i, j, k, n) = + sp - (0.5 * sigma) * ((sp - sm) - (1.0 - 2.0 / 3.0 * sigma) * s6); + Im(i, j, k, n) = S(i, j, k, n); + } else if (v_ad < -small_vel) { + Ip(i, j, k, n) = S(i, j, k, n); + Im(i, j, k, n) = + sm + (0.5 * sigma) * ((sp - sm) + (1.0 - 2.0 / 3.0 * sigma) * s6); + } else { + Ip(i, j, k, n) = S(i, j, k, n); + Im(i, j, k, n) = S(i, j, k, n); + } +} + +// This version is called after the MAC projection, when we use the +// MAC-projected velocity +// for upwinding +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_x( + const int i, + const int j, + const int k, + const int n, + const amrex::Real dt, + const amrex::Real dx, + amrex::Real& Im, + amrex::Real& Ip, + const amrex::Array4& S, + const amrex::Array4& vel_edge, + const amrex::BCRec& bc, + const int domlo, + const int domhi, + const bool weno_js) +{ + + using namespace amrex; + + constexpr amrex::Real small_vel = 1e-8; + + amrex::Real sm2 = S(i - 2, j, k, n); + amrex::Real sm1 = S(i - 1, j, k, n); + amrex::Real s0 = S(i, j, k, n); + amrex::Real sp1 = S(i + 1, j, k, n); + amrex::Real sp2 = S(i + 2, j, k, n); + + // right of i-1/2 + amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT + // left of i+1/2 + amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT + + amrex::Real sm = sedge1; + amrex::Real sp = sedge2; + + Godunov_weno_xbc( + i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), vel_edge(i+1, j, k), + bc.lo(0), bc.hi(0), domlo, domhi); + + amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); + + amrex::Real sigmap = std::abs(vel_edge(i + 1, j, k)) * dt / dx; + amrex::Real sigmam = std::abs(vel_edge(i, j, k)) * dt / dx; + + if (vel_edge(i + 1, j, k) > small_vel) { + Ip = sp - + (0.5 * sigmap) * ((sp - sm) - (1.e0 - 2.e0 / 3.e0 * sigmap) * s6); + } else { + Ip = S(i, j, k, n); + } + + if (vel_edge(i, j, k) < -small_vel) { + Im = sm + + (0.5 * sigmam) * ((sp - sm) + (1.e0 - 2.e0 / 3.e0 * sigmam) * s6); + } else { + Im = S(i, j, k, n); + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_y( + const int i, + const int j, + const int k, + const int n, + const amrex::Real dt, + const amrex::Real dx, + amrex::Real& Im, + amrex::Real& Ip, + const amrex::Array4& S, + const amrex::Array4& vel_edge, + const amrex::BCRec& bc, + const int domlo, + const int domhi, + const bool weno_js) +{ + + using namespace amrex; + + constexpr amrex::Real small_vel = 1e-8; + + amrex::Real sm2 = S(i, j - 2, k, n); + amrex::Real sm1 = S(i, j - 1, k, n); + amrex::Real s0 = S(i, j, k, n); + amrex::Real sp1 = S(i, j + 1, k, n); + amrex::Real sp2 = S(i, j + 2, k, n); + + // right of j-1/2 + amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT + // left of j+1/2 + amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT + + amrex::Real sm = sedge1; + amrex::Real sp = sedge2; + + Godunov_weno_ybc( + i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), vel_edge(i, j+1, k), + bc.lo(1), bc.hi(1), domlo, domhi); + + amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); + + amrex::Real sigmap = std::abs(vel_edge(i, j + 1, k)) * dt / dx; + amrex::Real sigmam = std::abs(vel_edge(i, j, k)) * dt / dx; + + if (vel_edge(i, j + 1, k) > small_vel) { + Ip = sp - + (0.5 * sigmap) * ((sp - sm) - (1.e0 - 2.e0 / 3.e0 * sigmap) * s6); + } else { + Ip = S(i, j, k, n); + } + + if (vel_edge(i, j, k) < -small_vel) { + Im = sm + + (0.5 * sigmam) * ((sp - sm) + (1.e0 - 2.e0 / 3.e0 * sigmam) * s6); + } else { + Im = S(i, j, k, n); + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_z( + const int i, + const int j, + const int k, + const int n, + const amrex::Real dt, + const amrex::Real dx, + amrex::Real& Im, + amrex::Real& Ip, + const amrex::Array4& S, + const amrex::Array4& vel_edge, + const amrex::BCRec& bc, + const int domlo, + const int domhi, + const bool weno_js) +{ + + using namespace amrex; + + constexpr amrex::Real small_vel = 1e-8; + + amrex::Real sm2 = S(i, j, k - 2, n); + amrex::Real sm1 = S(i, j, k - 1, n); + amrex::Real s0 = S(i, j, k, n); + amrex::Real sp1 = S(i, j, k + 1, n); + amrex::Real sp2 = S(i, j, k + 2, n); + + // right of k-1/2 + amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT + // left of k+1/2 + amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT + + amrex::Real sm = sedge1; + amrex::Real sp = sedge2; + + Godunov_weno_zbc( + i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), vel_edge(i, j, k+1), + bc.lo(2), bc.hi(2), domlo, domhi); + + amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); + + amrex::Real sigmap = std::abs(vel_edge(i, j, k + 1)) * dt / dx; + amrex::Real sigmam = std::abs(vel_edge(i, j, k)) * dt / dx; + + if (vel_edge(i, j, k + 1) > small_vel) { + Ip = sp - + (0.5 * sigmap) * ((sp - sm) - (1.e0 - 2.e0 / 3.e0 * sigmap) * s6); + } else { + Ip = S(i, j, k, n); + } + + if (vel_edge(i, j, k) < -small_vel) { + Im = sm + + (0.5 * sigmam) * ((sp - sm) + (1.e0 - 2.e0 / 3.e0 * sigmam) * s6); + } else { + Im = S(i, j, k, n); + } +} + +} // namespace + +#endif + diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index e8bb944559..4aa1803652 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -46,6 +46,7 @@ amr_wind::nodal_projection::get_projection_bc( r[dir] = LinOpBCType::Dirichlet; break; } + case BC::mass_inflow_outflow: case BC::mass_inflow: { r[dir] = LinOpBCType::inflow; break; From 407a1209b5b042d74fb0034567512964e063ecf6 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 28 Feb 2024 18:07:31 -0800 Subject: [PATCH 04/83] temporary fix to use an external hydro repo --- cmake/amr-wind-utils.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/amr-wind-utils.cmake b/cmake/amr-wind-utils.cmake index 3168836304..bc95f9ea09 100644 --- a/cmake/amr-wind-utils.cmake +++ b/cmake/amr-wind-utils.cmake @@ -79,6 +79,7 @@ macro(init_amrex_hydro) else() set(CMAKE_PREFIX_PATH ${AMReX-Hydro_DIR} ${CMAKE_PREFIX_PATH}) find_package(AMReX-Hydro CONFIG REQUIRED) + add_subdirectory(${AMReX-Hydro_DIR} build) message(STATUS "Found AMReX-Hydro = ${AMReX-Hydro_DIR}") endif() endmacro(init_amrex_hydro) From e8cb980b9c5382bab44bf6256f69c5882d312827 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 28 Feb 2024 18:09:41 -0800 Subject: [PATCH 05/83] initial changes to add the outflow correction --- amr-wind/equation_systems/icns/icns_advection.H | 2 ++ amr-wind/equation_systems/icns/icns_advection.cpp | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/amr-wind/equation_systems/icns/icns_advection.H b/amr-wind/equation_systems/icns/icns_advection.H index a72d6d24d0..024f32e512 100644 --- a/amr-wind/equation_systems/icns/icns_advection.H +++ b/amr-wind/equation_systems/icns/icns_advection.H @@ -49,6 +49,8 @@ private: void init_projector(const FaceFabPtrVec& /*beta*/) noexcept; void init_projector(const amrex::Real /*beta*/) noexcept; + void enforce_solvability(const amrex::Vector>& a_umac) noexcept; + FieldRepo& m_repo; PhysicsMgr& m_phy_mgr; std::unique_ptr m_mac_proj; diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index e381e54c36..d86e7af3c6 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -64,6 +64,17 @@ MacProjOp::MacProjOp( m_has_overset = m_has_overset && !disable_ovst_mac; } +void MacProjOp::enforce_solvability ( + const amrex::Vector>& a_umac +) noexcept +{ + auto& velocity = m_repo.get_field("velocity"); + amrex::BCRec const* bc_type = velocity.bcrec_device().data(); + const amrex::Vector& geom = m_repo.mesh().Geom(); + + m_mac_proj->enforceSolvability(a_umac, bc_type, geom); +} + void MacProjOp::init_projector(const MacProjOp::FaceFabPtrVec& beta) noexcept { m_mac_proj = std::make_unique( @@ -266,6 +277,7 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } + enforce_solvability(mac_vec); m_mac_proj->setUMAC(mac_vec); if (m_has_overset) { From b549cb6f1b3b4d8c3e4a78c423e52553adbea94f Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 7 Mar 2024 17:18:36 -0800 Subject: [PATCH 06/83] additional changes to enable mass in-out BC, along with udf capabilities --- amr-wind/boundary_conditions/BCInterface.H | 2 +- amr-wind/boundary_conditions/BCInterface.cpp | 36 ++++++++++++++++--- amr-wind/boundary_conditions/scalar_bcs.H | 2 +- amr-wind/boundary_conditions/scalar_bcs.cpp | 22 ++++++++---- amr-wind/boundary_conditions/velocity_bcs.H | 22 ++++++++---- amr-wind/core/FieldBCOps.H | 3 +- amr-wind/core/FieldFillPatchOps.H | 3 +- amr-wind/diffusion/incflo_diffusion.cpp | 1 + .../equation_systems/icns/icns_advection.cpp | 2 +- amr-wind/physics/udfs/CustomVelocity.H | 26 +++++++------- amr-wind/physics/udfs/CustomVelocity.cpp | 29 +++++++-------- 11 files changed, 101 insertions(+), 47 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.H b/amr-wind/boundary_conditions/BCInterface.H index b6d815e82d..630ed07fa0 100644 --- a/amr-wind/boundary_conditions/BCInterface.H +++ b/amr-wind/boundary_conditions/BCInterface.H @@ -49,7 +49,7 @@ public: virtual void operator()(const amrex::Real value = 0.0); //! User-defined functions for Dirichlet-type boundaries - std::pair get_dirichlet_udfs(); + std::array get_dirichlet_udfs(); protected: //! Setup AMReX mathematical BC types diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index 12cefe623b..c0367d20b0 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -105,15 +105,18 @@ void BCIface::set_bcfuncs() } } -std::pair BCIface::get_dirichlet_udfs() +std::array BCIface::get_dirichlet_udfs() { const auto& fname = m_field.name(); const auto& bctype = m_field.bc_type(); const std::string inflow_key = fname + ".inflow_type"; + const std::string inflow_outflow_key = fname + ".inflow_outflow_type"; const std::string wall_key = fname + ".wall_type"; std::string inflow_udf{"ConstDirichlet"}; + std::string inflow_outflow_udf{"ConstDirichlet"}; std::string wall_udf{"ConstDirichlet"}; bool has_inflow_udf = false; + bool has_inflow_outflow_udf = false; bool has_wall_udf = false; for (amrex::OrientationIter oit; oit != nullptr; ++oit) { @@ -137,6 +140,21 @@ std::pair BCIface::get_dirichlet_udfs() } } + if (bct == BC::mass_inflow_outflow) { + if (pp.contains(inflow_outflow_key.c_str())) { + std::string val; + pp.get(inflow_outflow_key.c_str(), val); + + if (has_inflow_outflow_udf && (inflow_outflow_udf != val)) { + amrex::Abort( + "BC: Inflow-outflow UDF must be same for all inflow-outflow faces"); + } else { + inflow_outflow_udf = val; + has_inflow_outflow_udf = true; + } + } + } + if (bct == BC::slip_wall) { if (pp.contains(wall_key.c_str())) { std::string val; @@ -153,7 +171,7 @@ std::pair BCIface::get_dirichlet_udfs() } } - return {inflow_udf, wall_udf}; + return {inflow_udf, inflow_outflow_udf, wall_udf}; } void BCVelocity::set_bcrec() @@ -297,6 +315,14 @@ void BCScalar::set_bcrec() } break; + case BC::mass_inflow_outflow: + if (side == amrex::Orientation::low) { + set_bcrec_lo(dir, amrex::BCType::user_1); + } else { + set_bcrec_hi(dir, amrex::BCType::user_1); + } + break; + case BC::slip_wall: case BC::wall_model: case BC::fixed_gradient: @@ -320,7 +346,8 @@ void BCScalar::read_values() auto& bcval = m_field.bc_values(); const int ndim = m_field.num_comp(); const auto udfs = get_dirichlet_udfs(); - const auto const_dirichlet_inflow = udfs.first == "ConstDirichlet"; + const auto const_dirichlet_inflow = udfs[0] == "ConstDirichlet"; + const auto const_dirichlet_inflow_outflow = udfs[1] == "ConstDirichlet"; for (amrex::OrientationIter oit; oit != nullptr; ++oit) { auto ori = oit(); @@ -328,7 +355,8 @@ void BCScalar::read_values() const auto bct = bctype[ori]; amrex::ParmParse pp(bcid); - if ((bct == BC::mass_inflow) && (const_dirichlet_inflow)) { + if (((bct == BC::mass_inflow) && (const_dirichlet_inflow)) || + ((bct == BC::mass_inflow_outflow) && (const_dirichlet_inflow_outflow))) { pp.getarr(fname.c_str(), bcval[ori], 0, ndim); } else { pp.queryarr(fname.c_str(), bcval[ori], 0, ndim); diff --git a/amr-wind/boundary_conditions/scalar_bcs.H b/amr-wind/boundary_conditions/scalar_bcs.H index d27adaac6c..3566c134aa 100644 --- a/amr-wind/boundary_conditions/scalar_bcs.H +++ b/amr-wind/boundary_conditions/scalar_bcs.H @@ -29,7 +29,7 @@ void register_scalar_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const std::pair& udfs); + const std::array udfs); } // namespace amr_wind::scalar_bc diff --git a/amr-wind/boundary_conditions/scalar_bcs.cpp b/amr-wind/boundary_conditions/scalar_bcs.cpp index e2e9adbf81..71d8e544f3 100644 --- a/amr-wind/boundary_conditions/scalar_bcs.cpp +++ b/amr-wind/boundary_conditions/scalar_bcs.cpp @@ -5,12 +5,15 @@ void register_scalar_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const std::pair& udfs) + const std::array udfs) { - const std::string inflow_udf = udfs.first; - const std::string wall_udf = udfs.second; + const std::string inflow_udf = udfs[0]; + const std::string inflow_outflow_udf = udfs[1]; + const std::string wall_udf = udfs[2]; - if ((inflow_udf == "ConstDirichlet") && (wall_udf == "ConstDirichlet")) { + if ((inflow_udf == "ConstDirichlet") && + (inflow_outflow_udf == "ConstDirichlet") && + (wall_udf == "ConstDirichlet")) { return; } @@ -19,7 +22,14 @@ void register_scalar_dirichlet( "Scalar BC: Only constant dirichlet supported for Wall BC"); } - register_inflow_scalar_dirichlet( - field, inflow_udf, mesh, time); + if (inflow_udf != "ConstDirichlet") { + register_inflow_scalar_dirichlet( + field, inflow_udf, mesh, time); + } + + if (inflow_outflow_udf != "ConstDirichlet") { + register_inflow_scalar_dirichlet( + field, inflow_outflow_udf, mesh, time); + } } } // namespace amr_wind::scalar_bc diff --git a/amr-wind/boundary_conditions/velocity_bcs.H b/amr-wind/boundary_conditions/velocity_bcs.H index 6d36fa1e83..5b676e1f62 100644 --- a/amr-wind/boundary_conditions/velocity_bcs.H +++ b/amr-wind/boundary_conditions/velocity_bcs.H @@ -49,12 +49,15 @@ void register_velocity_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const std::pair& udfs) + const std::array udfs) { - const std::string inflow_udf = udfs.first; - const std::string wall_udf = udfs.second; + const std::string inflow_udf = udfs[0]; + const std::string inflow_outflow_udf = udfs[1]; + const std::string wall_udf = udfs[2]; - if ((inflow_udf == "ConstDirichlet") && (wall_udf == "ConstDirichlet")) { + if ((inflow_udf == "ConstDirichlet") && + (inflow_outflow_udf == "ConstDirichlet") && + (wall_udf == "ConstDirichlet")) { return; } @@ -63,8 +66,15 @@ void register_velocity_dirichlet( "Velocity BC: Only constant dirichlet supported for Wall BC"); } - register_inflow_vel_dirichlet( - field, inflow_udf, mesh, time); + if (inflow_udf != "ConstDirichlet") { + register_inflow_vel_dirichlet( + field, inflow_udf, mesh, time); + } + + if (inflow_outflow_udf != "ConstDirichlet") { + register_inflow_vel_dirichlet( + field, inflow_outflow_udf, mesh, time); + } } } // namespace amr_wind::vel_bc diff --git a/amr-wind/core/FieldBCOps.H b/amr-wind/core/FieldBCOps.H index bdd2f43121..dcd6d30adb 100644 --- a/amr-wind/core/FieldBCOps.H +++ b/amr-wind/core/FieldBCOps.H @@ -182,7 +182,8 @@ struct DirichletOp continue; } - if (m_bc_type[ori] == BC::mass_inflow) { + if ((m_bc_type[ori] == BC::mass_inflow) || + (m_bc_type[ori] == BC::mass_inflow_outflow)) { m_inflow_op( iv, field, geom, time, ori, n, dcomp, orig_comp); } else { diff --git a/amr-wind/core/FieldFillPatchOps.H b/amr-wind/core/FieldFillPatchOps.H index 56bde83b2d..a733e2196e 100644 --- a/amr-wind/core/FieldFillPatchOps.H +++ b/amr-wind/core/FieldFillPatchOps.H @@ -370,7 +370,8 @@ public: for (amrex::OrientationIter oit; oit != nullptr; ++oit) { const auto ori = oit(); - if (bctype[ori] != BC::mass_inflow) { + if ((bctype[ori] != BC::mass_inflow) && + (bctype[ori] != BC::mass_inflow_outflow)) { continue; } diff --git a/amr-wind/diffusion/incflo_diffusion.cpp b/amr-wind/diffusion/incflo_diffusion.cpp index 910562d60a..20886e85db 100644 --- a/amr-wind/diffusion/incflo_diffusion.cpp +++ b/amr-wind/diffusion/incflo_diffusion.cpp @@ -78,6 +78,7 @@ get_diffuse_scalar_bc(amr_wind::Field& scalar, Orientation::Side side) noexcept switch (bc) { case BC::pressure_inflow: case BC::pressure_outflow: + case BC::mass_inflow_outflow: case BC::zero_gradient: case BC::symmetric_wall: case BC::slip_wall: { diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index d86e7af3c6..06961bd3c2 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -277,7 +277,7 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } - enforce_solvability(mac_vec); + //enforce_solvability(mac_vec); m_mac_proj->setUMAC(mac_vec); if (m_has_overset) { diff --git a/amr-wind/physics/udfs/CustomVelocity.H b/amr-wind/physics/udfs/CustomVelocity.H index b1f7fe014f..cddf7771dd 100644 --- a/amr-wind/physics/udfs/CustomVelocity.H +++ b/amr-wind/physics/udfs/CustomVelocity.H @@ -17,31 +17,33 @@ struct CustomVelocity // clang-format off // Declare parameters here if needed. For example: // amrex::Real foo{1.0}; + amrex::Real pvel{1.0}; amrex::Real mvel{0.5}; // amrex::GpuArray bar = {0.0}; // clang-format on AMREX_GPU_DEVICE inline void operator()( - const amrex::IntVect& /*iv*/, - amrex::Array4 const& /*field*/, - amrex::GeometryData const& /*geom*/, - const amrex::Real /*time*/, - amrex::Orientation /*ori*/, - const int /*comp*/, - const int /*dcomp*/, - const int /*orig_comp*/) const + const amrex::IntVect& iv, + amrex::Array4 const& field, + amrex::GeometryData const& geom, + const amrex::Real time, + amrex::Orientation ori, + const int comp, + const int dcomp, + const int orig_comp) const { // Compute quantities to set the field values. For example: // clang-format off - // const auto* problo = geom.ProbLo(); - // const auto* dx = geom.CellSize(); + const auto* problo = geom.ProbLo(); + const auto* dx = geom.CellSize(); // const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; // const auto y = problo[1] + (iv[1] + 0.5) * dx[1]; - // const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; + const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; // const amrex::GpuArray vel = {1.0, 0.0, 0.0}; // Once the above is done, fill the field as: - // field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; + const amrex::GpuArray vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; + field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; // clang-format on } }; diff --git a/amr-wind/physics/udfs/CustomVelocity.cpp b/amr-wind/physics/udfs/CustomVelocity.cpp index 0aad96ba76..b7a2dbeed0 100644 --- a/amr-wind/physics/udfs/CustomVelocity.cpp +++ b/amr-wind/physics/udfs/CustomVelocity.cpp @@ -17,21 +17,22 @@ CustomVelocity::CustomVelocity(const Field& /*fld*/) // CustomVelocity.foo = 1.0 // clang-format off - //{ - // const int ncomp = fld.num_comp(); - // amrex::ParmParse pp("CustomVelocity"); - // pp.query("foo", m_op.foo); - // amrex::Vector vel(0.0, ncomp); - // pp.getarr("velocity", vel); - // AMREX_ALWAYS_ASSERT(vel.size() == ncomp); - // for (int i = 0; i < ncomp; ++i) { - // m_op.bar[i] = vel[i]; - // } - //} + { + //const int ncomp = fld.num_comp(); + amrex::ParmParse pp("CustomVelocity"); + pp.query("pvel", m_op.pvel); + pp.query("mvel", m_op.mvel); + //amrex::Vector vel(0.0, ncomp); + //pp.getarr("velocity", vel); + //AMREX_ALWAYS_ASSERT(vel.size() == ncomp); + //for (int i = 0; i < ncomp; ++i) { + // m_op.bar[i] = vel[i]; + //} + } // clang-format on - amrex::Abort( - "Please define the body of this function and the corresponding struct " - "in the header file before using it. Then remove this message"); + //amrex::Abort( + // "Please define the body of this function and the corresponding struct " + // "in the header file before using it. Then remove this message"); } } // namespace amr_wind::udf From fc6f538805e46aa20d572a2f0d4cb1f251f92364 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 4 Apr 2024 13:45:04 -0700 Subject: [PATCH 07/83] Mukul's test files for the in-out BC --- bctest/cmake_mpi.sh | 16 +++++++++++ bctest/input_inflow | 66 +++++++++++++++++++++++++++++++++++++++++++++ bctest/input_inout | 66 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100755 bctest/cmake_mpi.sh create mode 100644 bctest/input_inflow create mode 100644 bctest/input_inout diff --git a/bctest/cmake_mpi.sh b/bctest/cmake_mpi.sh new file mode 100755 index 0000000000..1cb287eb36 --- /dev/null +++ b/bctest/cmake_mpi.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Example CMake config script for an OSX laptop with MPICH + +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ + -DCMAKE_CXX_COMPILER:STRING=mpicxx \ + -DCMAKE_C_COMPILER:STRING=mpicc \ + -DCMAKE_Fortran_COMPILER:STRING=mpifort \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DAMR_WIND_USE_INTERNAL_AMREX_HYDRO:BOOL=OFF \ + -DAMReX-Hydro_DIR:STRING=$HOME/AMReX-Hydro \ + -DAMR_WIND_ENABLE_MPI:BOOL=ON \ + -DAMR_WIND_ENABLE_TESTS:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + .. && make -j8 diff --git a/bctest/input_inflow b/bctest/input_inflow new file mode 100644 index 0000000000..bdc43e3605 --- /dev/null +++ b/bctest/input_inflow @@ -0,0 +1,66 @@ +# This is a 2D poiseuille flow when run to steady state + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# SIMULATION STOP # +#.......................................# +time.stop_time = 2.00 # Max (simulated) time to evolve +time.max_step = -1000 # Max number of time steps + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# TIME STEP COMPUTATION # +#.......................................# +time.fixed_dt = 0.02 # Use this constant dt if > 0 +time.cfl = 0.95 # CFL factor + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# INPUT AND OUTPUT # +#.......................................# +time.plot_interval = 100 # Steps between plot files +time.checkpoint_interval = -100 # Steps between checkpoint files + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# PHYSICS # +#.......................................# +incflo.density = 1.0 # Reference density +incflo.use_godunov = 1 +incflo.diffusion_type = 1 +incflo.godunov_type = "weno_z" + +transport.viscosity = 0.0 +turbulence.model = Laminar + +ICNS.source_terms = BodyForce +BodyForce.magnitude = 0 0 0 +incflo.physics = ChannelFlow +ChannelFlow.density = 1.0 +ChannelFlow.Mean_Velocity = 0.0 + +io.output_default_variables = 1 + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# ADAPTIVE MESH REFINEMENT # +#.......................................# +amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel +amr.blocking_factor = 4 +amr.max_level = 0 # Max AMR level in hierarchy + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# GEOMETRY # +#.......................................# +geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates +geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates +geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) + +# Boundary conditions +xlo.type = "mass_inflow" +xlo.density = 1.0 +xlo.velocity.inflow_type = CustomVelocity + +xhi.type = "mass_inflow" +xhi.density = 1.0 +xhi.velocity.inflow_type = CustomVelocity +CustomVelocity.mvel = -1.0 +CustomVelocity.pvel = 1.0 + +zlo.type = "slip_wall" +zhi.type = "slip_wall" diff --git a/bctest/input_inout b/bctest/input_inout new file mode 100644 index 0000000000..0b2a59b24a --- /dev/null +++ b/bctest/input_inout @@ -0,0 +1,66 @@ +# This is a 2D poiseuille flow when run to steady state + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# SIMULATION STOP # +#.......................................# +time.stop_time = 2.00 # Max (simulated) time to evolve +time.max_step = -1000 # Max number of time steps + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# TIME STEP COMPUTATION # +#.......................................# +time.fixed_dt = 0.02 # Use this constant dt if > 0 +time.cfl = 0.95 # CFL factor + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# INPUT AND OUTPUT # +#.......................................# +time.plot_interval = 100 # Steps between plot files +time.checkpoint_interval = -100 # Steps between checkpoint files + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# PHYSICS # +#.......................................# +incflo.density = 1.0 # Reference density +incflo.use_godunov = 1 +incflo.diffusion_type = 1 +incflo.godunov_type = "weno_z" + +transport.viscosity = 0.0 +turbulence.model = Laminar + +ICNS.source_terms = BodyForce +BodyForce.magnitude = 0 0 0 +incflo.physics = ChannelFlow +ChannelFlow.density = 1.0 +ChannelFlow.Mean_Velocity = 0.0 + +io.output_default_variables = 1 + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# ADAPTIVE MESH REFINEMENT # +#.......................................# +amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel +amr.blocking_factor = 4 +amr.max_level = 0 # Max AMR level in hierarchy + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# GEOMETRY # +#.......................................# +geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates +geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates +geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) + +# Boundary conditions +xlo.type = "mass_inflow_outflow" +xlo.density = 1.0 +xlo.velocity.inflow_outflow_type = CustomVelocity + +xhi.type = "mass_inflow_outflow" +xhi.density = 1.0 +xhi.velocity.inflow_outflow_type = CustomVelocity +CustomVelocity.mvel = -1.0 +CustomVelocity.pvel = 1.0 + +zlo.type = "slip_wall" +zhi.type = "slip_wall" From 757dd953d1a75f2fcac9356a58d4f125b85c47a5 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 4 Apr 2024 13:45:48 -0700 Subject: [PATCH 08/83] ignoring all plt and chk files --- .gitignore | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 41e9483250..1660ba99ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,15 @@ exec/*.ex exec/tmp_build_dir/ exec/Backtrace.* -exec/plt*/ -exec/chk*/ exec/movie.visit exec/*.vtp exec/*.pvtp test/*.ex test/tmp_build_dir/ test/Backtrace.* -test/*plt*/ -test/chk*/ test/AMR-WindGoldFiles +*plt*/ +chk*/ .DS_store spack* .clangd From 6d388b71347200a43bcb6c746d3a5cf8fe4eb300 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 4 Apr 2024 17:55:30 -0700 Subject: [PATCH 09/83] set up garbage values at outflow for testing --- amr-wind/physics/udfs/CustomVelocity.H | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/amr-wind/physics/udfs/CustomVelocity.H b/amr-wind/physics/udfs/CustomVelocity.H index cddf7771dd..c920315c0c 100644 --- a/amr-wind/physics/udfs/CustomVelocity.H +++ b/amr-wind/physics/udfs/CustomVelocity.H @@ -36,13 +36,21 @@ struct CustomVelocity // clang-format off const auto* problo = geom.ProbLo(); const auto* dx = geom.CellSize(); - // const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; + const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; // const auto y = problo[1] + (iv[1] + 0.5) * dx[1]; const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; // const amrex::GpuArray vel = {1.0, 0.0, 0.0}; // Once the above is done, fill the field as: - const amrex::GpuArray vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; + amrex::GpuArray vel; + if (x <= 0.5) { + // at the low-x boundary, +ve vel in top portion, garbage in the rest + vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; + } else { + // at the high-x boundary, -ve vel in bottom portion, garbage in the rest + vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; + } + field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; // clang-format on } From bd6c9feaff42f7194600f12ede9e6a1b0507952e Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 4 Apr 2024 17:57:06 -0700 Subject: [PATCH 10/83] intermediate changes for custom fill at MIO boundaries --- amr-wind/boundary_conditions/BCInterface.cpp | 6 ++ .../MassInflowOutflowBC.cpp | 56 ++++++++++++++++++- amr-wind/core/FieldBCOps.H | 3 +- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index c0367d20b0..03b4591f60 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -1,6 +1,7 @@ #include "amr-wind/boundary_conditions/BCInterface.H" #include "amr-wind/core/FieldRepo.H" #include "amr-wind/boundary_conditions/FixedGradientBC.H" +#include "amr-wind/boundary_conditions/MassInflowOutflowBC.H" #include "AMReX_ParmParse.H" namespace amr_wind { @@ -102,6 +103,11 @@ void BCIface::set_bcfuncs() if (bct == BC::fixed_gradient) { m_field.register_custom_bc(ori); } + + if (bct == BC::mass_inflow_outflow) { + amrex::Print() << "howdyyyyyyyyyyyy" << std::endl; + m_field.register_custom_bc(ori); + } } } diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index dd484ca590..e22df4a680 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -22,6 +22,60 @@ MassInflowOutflowBC::MassInflowOutflowBC(Field& field, amrex::Orientation ori) {} void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_state*/) -{} +{ + const auto& repo = m_field.repo(); + const auto& velocity = repo.get_field("velocity"); + //const auto bcvals = m_field.bc_values_device(); + const int ncomp = m_field.num_comp(); + const int idx = static_cast(m_ori); + const int idim = m_ori.coordDir(); + const auto islow = m_ori.isLow(); + const auto ishigh = m_ori.isHigh(); + const int nlevels = m_field.repo().num_active_levels(); + //const amrex::Array index + + amrex::Print() << "******* applying MIO at orientation: " << idx << std::endl; + + for (int lev = 0; lev < nlevels; ++lev) { + const auto& domain = repo.mesh().Geom(lev).Domain(); + + amrex::MFItInfo mfi_info{}; + if (amrex::Gpu::notInLaunchRegion()) { + mfi_info.SetDynamic(true); + } +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { + const auto& bx = mfi.validbox(); + const auto& bc_a = m_field(lev).array(mfi); + const auto& vel = velocity(lev).array(mfi); + + if (islow && (bx.smallEnd(idim) == domain.smallEnd(idim))) { + amrex::ParallelFor( + lower_boundary_faces(bx, idim), + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + if (vel(i, j, k, idim) < 0) { + for (int n = 0; n < ncomp; ++n) { + bc_a(i, j, k, n) = bc_a(i+1, j, k, n); + } + } + }); + } + + if (ishigh && (bx.bigEnd(idim) == domain.bigEnd(idim))) { + amrex::ParallelFor( + amrex::bdryHi(bx, idim), + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + if (vel(i, j, k, idim) > 0) { + for (int n = 0; n < ncomp; ++n) { + bc_a(i, j, k, n) = bc_a(i-1, j, k, n); + } + } + }); + } + } + } +} } // namespace amr_wind \ No newline at end of file diff --git a/amr-wind/core/FieldBCOps.H b/amr-wind/core/FieldBCOps.H index dcd6d30adb..bdd2f43121 100644 --- a/amr-wind/core/FieldBCOps.H +++ b/amr-wind/core/FieldBCOps.H @@ -182,8 +182,7 @@ struct DirichletOp continue; } - if ((m_bc_type[ori] == BC::mass_inflow) || - (m_bc_type[ori] == BC::mass_inflow_outflow)) { + if (m_bc_type[ori] == BC::mass_inflow) { m_inflow_op( iv, field, geom, time, ori, n, dcomp, orig_comp); } else { From e8eae208aa143690e3893958426a39b90b9271aa Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 8 Apr 2024 16:24:16 -0700 Subject: [PATCH 11/83] added options to toggle initial iterations for testing --- bctest/input_inout | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bctest/input_inout b/bctest/input_inout index 0b2a59b24a..d7949afd88 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -4,7 +4,7 @@ # SIMULATION STOP # #.......................................# time.stop_time = 2.00 # Max (simulated) time to evolve -time.max_step = -1000 # Max number of time steps +time.max_step = 1 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # @@ -26,6 +26,9 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" +#incflo.initial_iterations = 0 +#incflo.do_initial_proj = false + transport.viscosity = 0.0 turbulence.model = Laminar From ec5f4bbe4483457c0eb3fcdd1c49b67e0c2b8652 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 8 Apr 2024 17:01:03 -0700 Subject: [PATCH 12/83] rebasing upstream changes --- amr-wind/boundary_conditions/BCInterface.cpp | 4 +- .../MassInflowOutflowBC.cpp | 40 +++++++------------ amr-wind/physics/ChannelFlow.cpp | 12 ++++++ amr-wind/physics/udfs/CustomVelocity.H | 1 + 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index 03b4591f60..db8ed1dd3d 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -104,10 +104,12 @@ void BCIface::set_bcfuncs() m_field.register_custom_bc(ori); } - if (bct == BC::mass_inflow_outflow) { + if ((m_field.name() == "velocity") + && (bct == BC::mass_inflow_outflow)) { amrex::Print() << "howdyyyyyyyyyyyy" << std::endl; m_field.register_custom_bc(ori); } + } } diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index e22df4a680..d09df05013 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -3,18 +3,7 @@ namespace amr_wind { namespace { -AMREX_FORCE_INLINE -amrex::Box lower_boundary_faces(const amrex::Box& b, int dir) -{ - amrex::IntVect lo(b.smallEnd()); - amrex::IntVect hi(b.bigEnd()); - int sm = lo[dir]; - lo.setVal(dir, sm - 1); - hi.setVal(dir, sm - 1); - amrex::IndexType bxtype(b.ixType()); - bxtype.set(dir); - return {lo, hi, bxtype}; -} + } // namespace MassInflowOutflowBC::MassInflowOutflowBC(Field& field, amrex::Orientation ori) @@ -32,10 +21,10 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st const auto islow = m_ori.isLow(); const auto ishigh = m_ori.isHigh(); const int nlevels = m_field.repo().num_active_levels(); - //const amrex::Array index - - amrex::Print() << "******* applying MIO at orientation: " << idx << std::endl; + const bool ib = (idim == 0), jb = (idim == 1), kb = (idim == 2); + amrex::Print() << "******* applying MIO fill-patch at orientation: " << idx << std::endl; + amrex::Print() << nlevels << " level(s)" << std::endl << std::endl; for (int lev = 0; lev < nlevels; ++lev) { const auto& domain = repo.mesh().Geom(lev).Domain(); @@ -53,24 +42,23 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st if (islow && (bx.smallEnd(idim) == domain.smallEnd(idim))) { amrex::ParallelFor( - lower_boundary_faces(bx, idim), - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if (vel(i, j, k, idim) < 0) { - for (int n = 0; n < ncomp; ++n) { - bc_a(i, j, k, n) = bc_a(i+1, j, k, n); - } + amrex::bdryLo(bx, idim), ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + //amrex::Print() << i << " " << j << " " << k << std::endl; + //amrex::Print() << i-ib << " " << j-jb << " " << k-kb << std::endl; + //amrex::Print() << vel(i-ib, j-jb, k-kb, idim) << " " << bc_a(i-ib, j-jb, k-kb, n) << " " << bc_a(i, j, k, n) << std::endl << std::endl; + if (vel(i-ib, j-jb, k-kb, idim) < 0) { + bc_a(i-ib, j-jb, k-kb, n) = bc_a(i, j, k, n); } }); } if (ishigh && (bx.bigEnd(idim) == domain.bigEnd(idim))) { amrex::ParallelFor( - amrex::bdryHi(bx, idim), - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + amrex::bdryHi(bx, idim), ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { if (vel(i, j, k, idim) > 0) { - for (int n = 0; n < ncomp; ++n) { - bc_a(i, j, k, n) = bc_a(i-1, j, k, n); - } + bc_a(i, j, k, n) = bc_a(i-ib, j-jb, k-kb, n); } }); } diff --git a/amr-wind/physics/ChannelFlow.cpp b/amr-wind/physics/ChannelFlow.cpp index f49d99e57e..d75483d2b8 100644 --- a/amr-wind/physics/ChannelFlow.cpp +++ b/amr-wind/physics/ChannelFlow.cpp @@ -5,6 +5,7 @@ #include "AMReX_ParmParse.H" #include "amr-wind/utilities/trig_ops.H" #include "amr-wind/utilities/DirectionSelector.H" +#include "amr-wind/boundary_conditions/MassInflowOutflowBC.H" namespace amr_wind::channel_flow { @@ -451,6 +452,17 @@ void ChannelFlow::post_init_actions() std::abs(body_force[2]) < 1e-16, "body force in z should be zero if using a wall model"); } +/* + const auto& ibctype = velocity.bc_type(); + for (amrex::OrientationIter oit; oit != nullptr; ++oit) { + auto ori = oit(); + const auto bct = ibctype[ori]; + + if (bct == BC::mass_inflow_outflow) { + amrex::Print() << "howdyyyyyyyyyyyy" << std::endl; + velocity.register_custom_bc(ori); + } + }*/ } void ChannelFlow::post_advance_work() diff --git a/amr-wind/physics/udfs/CustomVelocity.H b/amr-wind/physics/udfs/CustomVelocity.H index c920315c0c..bcd0a746bb 100644 --- a/amr-wind/physics/udfs/CustomVelocity.H +++ b/amr-wind/physics/udfs/CustomVelocity.H @@ -32,6 +32,7 @@ struct CustomVelocity const int dcomp, const int orig_comp) const { + //amrex::Print() << "****** filling custom velocity at " << iv << std::endl; // Compute quantities to set the field values. For example: // clang-format off const auto* problo = geom.ProbLo(); From 71e095d4d9b4852aeac1d84711f926788949bd2b Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 12 Apr 2024 18:16:27 -0700 Subject: [PATCH 13/83] adding print statements for testing --- amr-wind/boundary_conditions/BCInterface.cpp | 2 +- .../MassInflowOutflowBC.cpp | 4 +- amr-wind/boundary_conditions/velocity_bcs.H | 1 + .../convection/incflo_godunov_advection.cpp | 551 ++++++++++++++++++ .../convection/incflo_godunov_predict.cpp | 517 ++++++++++++++++ amr-wind/physics/udfs/CustomVelocity.H | 6 +- amr-wind/setup/init.cpp | 2 + 7 files changed, 1079 insertions(+), 4 deletions(-) create mode 100644 amr-wind/convection/incflo_godunov_advection.cpp create mode 100644 amr-wind/convection/incflo_godunov_predict.cpp diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index db8ed1dd3d..d545b08499 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -106,7 +106,7 @@ void BCIface::set_bcfuncs() if ((m_field.name() == "velocity") && (bct == BC::mass_inflow_outflow)) { - amrex::Print() << "howdyyyyyyyyyyyy" << std::endl; + amrex::Print() << "******** registering MIO custom Neumann BC" << std::endl; m_field.register_custom_bc(ori); } diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index d09df05013..51d7d2897e 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -23,8 +23,8 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st const int nlevels = m_field.repo().num_active_levels(); const bool ib = (idim == 0), jb = (idim == 1), kb = (idim == 2); - amrex::Print() << "******* applying MIO fill-patch at orientation: " << idx << std::endl; - amrex::Print() << nlevels << " level(s)" << std::endl << std::endl; + amrex::Print() << "******* applying MIO custom Neumann fills at orientation: " << idx << std::endl; + //amrex::Print() << nlevels << " level(s)" << std::endl << std::endl; for (int lev = 0; lev < nlevels; ++lev) { const auto& domain = repo.mesh().Geom(lev).Domain(); diff --git a/amr-wind/boundary_conditions/velocity_bcs.H b/amr-wind/boundary_conditions/velocity_bcs.H index 5b676e1f62..4d7595890f 100644 --- a/amr-wind/boundary_conditions/velocity_bcs.H +++ b/amr-wind/boundary_conditions/velocity_bcs.H @@ -38,6 +38,7 @@ void register_inflow_vel_dirichlet( mesh, time, InflowOp(field)); } else if (inflow_udf == "CustomVelocity") { using InflowOp = BCOpCreator; + amrex::Print() << "******* registering CustomVelocity" << std::endl; field.register_fill_patch_op>( mesh, time, InflowOp(field)); } else { diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp new file mode 100644 index 0000000000..52471e3e7f --- /dev/null +++ b/amr-wind/convection/incflo_godunov_advection.cpp @@ -0,0 +1,551 @@ +#include "amr-wind/convection/incflo_godunov_ppm.H" +#include "amr-wind/convection/incflo_godunov_minmod.H" +#include "amr-wind/convection/incflo_godunov_upwind.H" +#include "amr-wind/convection/Godunov.H" +#include + +using namespace amrex; + +void godunov::compute_fluxes( + int lev, + Box const& bx, + int ncomp, + Array4 const& fx, + Array4 const& fy, + Array4 const& fz, + Array4 const& q, + Array4 const& umac, + Array4 const& vmac, + Array4 const& wmac, + Array4 const& fq, + BCRec const* pbc, + int const* iconserv, + Real* p, + Vector geom, + Real dt, + godunov::scheme godunov_scheme, + bool use_forces_in_trans) +{ + + /* iconserv functionality: (would be better served by two flags) + * ------------------------------------------------------------------------ + * == 0 : non-conservative formulation of interpolation, fluxes not + * multiplied by MAC velocity + * == 1 : conservative formulation of interpolation, fluxes include factor + * of MAC velocity + * (!= 1 && != 0) : conservative formulation of interpolation, fluxes not + * multiplied by MAC velocity */ + + BL_PROFILE("amr-wind::godunov::compute_fluxes"); + Box const& xbx = amrex::surroundingNodes(bx, 0); + Box const& ybx = amrex::surroundingNodes(bx, 1); + Box const& zbx = amrex::surroundingNodes(bx, 2); + Box const& bxg1 = amrex::grow(bx, 1); + Box xebox = Box(xbx).grow(1, 1).grow(2, 1); + Box yebox = Box(ybx).grow(0, 1).grow(2, 1); + Box zebox = Box(zbx).grow(0, 1).grow(1, 1); + + const Real dx = geom[lev].CellSize(0); + const Real dy = geom[lev].CellSize(1); + const Real dz = geom[lev].CellSize(2); + Real l_dt = dt; + Real dtdx = l_dt / dx; + Real dtdy = l_dt / dy; + Real dtdz = l_dt / dz; + + Box const& domain = geom[lev].Domain(); + const auto dlo = amrex::lbound(domain); + const auto dhi = amrex::ubound(domain); + + Array4 Imx = makeArray4(p, bxg1, ncomp); + p += Imx.size(); + Array4 Ipx = makeArray4(p, bxg1, ncomp); + p += Ipx.size(); + Array4 Imy = makeArray4(p, bxg1, ncomp); + p += Imy.size(); + Array4 Ipy = makeArray4(p, bxg1, ncomp); + p += Ipy.size(); + Array4 Imz = makeArray4(p, bxg1, ncomp); + p += Imz.size(); + Array4 Ipz = makeArray4(p, bxg1, ncomp); + p += Ipz.size(); + Array4 xlo = makeArray4(p, xebox, ncomp); + p += xlo.size(); + Array4 xhi = makeArray4(p, xebox, ncomp); + p += xhi.size(); + Array4 ylo = makeArray4(p, yebox, ncomp); + p += ylo.size(); + Array4 yhi = makeArray4(p, yebox, ncomp); + p += yhi.size(); + Array4 zlo = makeArray4(p, zebox, ncomp); + p += zlo.size(); + Array4 zhi = makeArray4(p, zebox, ncomp); + p += zhi.size(); + Array4 xyzlo = makeArray4(p, bxg1, ncomp); + p += xyzlo.size(); + Array4 xyzhi = makeArray4(p, bxg1, ncomp); + + // Use PPM to generate Im and Ip */ + switch (godunov_scheme) { + case godunov::scheme::MINMOD: { + amrex::ParallelFor( + bxg1, ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + Godunov_minmod_fpu_x( + i, j, k, n, l_dt, dx, Imx(i, j, k, n), Ipx(i, j, k, n), q, + umac, pbc[n], dlo.x, dhi.x); + Godunov_minmod_fpu_y( + i, j, k, n, l_dt, dy, Imy(i, j, k, n), Ipy(i, j, k, n), q, + vmac, pbc[n], dlo.y, dhi.y); + Godunov_minmod_fpu_z( + i, j, k, n, l_dt, dz, Imz(i, j, k, n), Ipz(i, j, k, n), q, + wmac, pbc[n], dlo.z, dhi.z); + }); + break; + } + case godunov::scheme::UPWIND: { + amrex::ParallelFor( + bxg1, ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + Godunov_upwind_fpu( + i, j, k, n, Imx(i, j, k, n), Ipx(i, j, k, n), q); + Godunov_upwind_fpu( + i, j, k, n, Imy(i, j, k, n), Ipy(i, j, k, n), q); + Godunov_upwind_fpu( + i, j, k, n, Imz(i, j, k, n), Ipz(i, j, k, n), q); + }); + break; + } + default: { + amrex::Abort( + "This code path only used in multiphase simulations with MINMOD " + "and UPWIND"); + } + } + + amrex::ParallelFor( + xebox, ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + constexpr Real small_vel = 1.e-8; + + Real uad = umac(i, j, k); + Real fux = (std::abs(uad) < small_vel) ? 0. : 1.; + bool uval = uad >= 0.; + // divu = 0 + // Real cons1 = (iconserv[n]) ? -0.5*l_dt*q(i-1,j,k,n)*divu(i-1,j,k) + // : 0.; Real cons2 = (iconserv[n]) ? -0.5*l_dt*q(i ,j,k,n)*divu(i + // ,j,k) : 0.; + Real lo = Ipx(i - 1, j, k, n); // + cons1; + Real hi = Imx(i, j, k, n); // + cons2; + if (use_forces_in_trans && fq) { + lo += 0.5 * l_dt * fq(i - 1, j, k, n); + hi += 0.5 * l_dt * fq(i, j, k, n); + } + + auto bc = pbc[n]; + + Godunov_trans_xbc( + i, j, k, n, q, lo, hi, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + xlo(i, j, k, n) = lo; + xhi(i, j, k, n) = hi; + Real st = (uval) ? lo : hi; + Imx(i, j, k, n) = fux * st + (1. - fux) * 0.5 * (hi + lo); + }, + yebox, ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + constexpr Real small_vel = 1.e-8; + + Real vad = vmac(i, j, k); + Real fuy = (std::abs(vad) < small_vel) ? 0. : 1.; + bool vval = vad >= 0.; + // divu = 0 + // Real cons1 = (iconserv[n]) ? -0.5*l_dt*q(i,j-1,k,n)*divu(i,j-1,k) + // : 0.; Real cons2 = (iconserv[n]) ? -0.5*l_dt*q(i,j ,k,n)*divu(i,j + // ,k) : 0.; + Real lo = Ipy(i, j - 1, k, n); // + cons1; + Real hi = Imy(i, j, k, n); // + cons2; + if (use_forces_in_trans && fq) { + lo += 0.5 * l_dt * fq(i, j - 1, k, n); + hi += 0.5 * l_dt * fq(i, j, k, n); + } + + auto bc = pbc[n]; + + Godunov_trans_ybc( + i, j, k, n, q, lo, hi, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + + ylo(i, j, k, n) = lo; + yhi(i, j, k, n) = hi; + Real st = (vval) ? lo : hi; + Imy(i, j, k, n) = fuy * st + (1. - fuy) * 0.5 * (hi + lo); + }, + zebox, ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + constexpr Real small_vel = 1.e-8; + + Real wad = wmac(i, j, k); + Real fuz = (std::abs(wad) < small_vel) ? 0. : 1.; + bool wval = wad >= 0.; + auto bc = pbc[n]; + // divu = 0 + // Real cons1 = (iconserv[n]) ? -0.5*l_dt*q(i,j,k-1,n)*divu(i,j,k-1) + // : 0.; Real cons2 = (iconserv[n]) ? -0.5*l_dt*q(i,j,k + // ,n)*divu(i,j,k ) : 0.; + Real lo = Ipz(i, j, k - 1, n); // + cons1; + Real hi = Imz(i, j, k, n); // + cons2; + if (use_forces_in_trans && fq) { + lo += 0.5 * l_dt * fq(i, j, k - 1, n); + hi += 0.5 * l_dt * fq(i, j, k, n); + } + + Godunov_trans_zbc( + i, j, k, n, q, lo, hi, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + + zlo(i, j, k, n) = lo; + zhi(i, j, k, n) = hi; + Real st = (wval) ? lo : hi; + Imz(i, j, k, n) = fuz * st + (1. - fuz) * 0.5 * (hi + lo); + }); + + Array4 xedge = Imx; + Array4 yedge = Imy; + Array4 zedge = Imz; + + // We can reuse the space in Ipx, Ipy and Ipz. + + // + // x-direction + // + Box const& xbxtmp = amrex::grow(bx, 0, 1); + Array4 yzlo = + makeArray4(xyzlo.dataPtr(), amrex::surroundingNodes(xbxtmp, 1), ncomp); + Array4 zylo = + makeArray4(xyzhi.dataPtr(), amrex::surroundingNodes(xbxtmp, 2), ncomp); + amrex::ParallelFor( + Box(zylo), ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + const auto bc = pbc[n]; + Real l_zylo, l_zyhi; + Godunov_corner_couple_zy( + l_zylo, l_zyhi, i, j, k, n, l_dt, dy, iconserv[n] != 0, + zlo(i, j, k, n), zhi(i, j, k, n), q, vmac, yedge); + + Real wad = wmac(i, j, k); + Godunov_trans_zbc( + i, j, k, n, q, l_zylo, l_zyhi, wad, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); + + constexpr Real small_vel = 1.e-8; + + Real st = (wad >= 0.) ? l_zylo : l_zyhi; + Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; + zylo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_zyhi + l_zylo); + }, + Box(yzlo), ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + const auto bc = pbc[n]; + Real l_yzlo, l_yzhi; + Godunov_corner_couple_yz( + l_yzlo, l_yzhi, i, j, k, n, l_dt, dz, iconserv[n] != 0, + ylo(i, j, k, n), yhi(i, j, k, n), q, wmac, zedge); + + Real vad = vmac(i, j, k); + Godunov_trans_ybc( + i, j, k, n, q, l_yzlo, l_yzhi, vad, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); + + constexpr Real small_vel = 1.e-8; + + Real st = (vad >= 0.) ? l_yzlo : l_yzhi; + Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; + yzlo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_yzhi + l_yzlo); + }); + // + + amrex::ParallelFor( + xbx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + Real stl, sth; + constexpr Real small_vel = 1.e-8; + if (iconserv[n] != 0) { + stl = xlo(i, j, k, n) - + (0.5 * dtdy) * + (yzlo(i - 1, j + 1, k, n) * vmac(i - 1, j + 1, k) - + yzlo(i - 1, j, k, n) * vmac(i - 1, j, k)) - + (0.5 * dtdz) * + (zylo(i - 1, j, k + 1, n) * wmac(i - 1, j, k + 1) - + zylo(i - 1, j, k, n) * wmac(i - 1, j, k)) + + (0.5 * dtdy) * q(i - 1, j, k, n) * + (vmac(i - 1, j + 1, k) - vmac(i - 1, j, k)) + + (0.5 * dtdz) * q(i - 1, j, k, n) * + (wmac(i - 1, j, k + 1) - wmac(i - 1, j, k)); + + sth = xhi(i, j, k, n) - + (0.5 * dtdy) * (yzlo(i, j + 1, k, n) * vmac(i, j + 1, k) - + yzlo(i, j, k, n) * vmac(i, j, k)) - + (0.5 * dtdz) * (zylo(i, j, k + 1, n) * wmac(i, j, k + 1) - + zylo(i, j, k, n) * wmac(i, j, k)) + + (0.5 * dtdy) * q(i, j, k, n) * + (vmac(i, j + 1, k) - vmac(i, j, k)) + + (0.5 * dtdz) * q(i, j, k, n) * + (wmac(i, j, k + 1) - wmac(i, j, k)); + + } else { + stl = xlo(i, j, k, n) - + (0.25 * dtdy) * + (vmac(i - 1, j + 1, k) + vmac(i - 1, j, k)) * + (yzlo(i - 1, j + 1, k, n) - yzlo(i - 1, j, k, n)) - + (0.25 * dtdz) * + (wmac(i - 1, j, k + 1) + wmac(i - 1, j, k)) * + (zylo(i - 1, j, k + 1, n) - zylo(i - 1, j, k, n)); + + sth = xhi(i, j, k, n) - + (0.25 * dtdy) * (vmac(i, j + 1, k) + vmac(i, j, k)) * + (yzlo(i, j + 1, k, n) - yzlo(i, j, k, n)) - + (0.25 * dtdz) * (wmac(i, j, k + 1) + wmac(i, j, k)) * + (zylo(i, j, k + 1, n) - zylo(i, j, k, n)); + } + + stl += (!use_forces_in_trans && fq) + ? 0.5 * l_dt * fq(i - 1, j, k, n) + : 0.; + sth += + (!use_forces_in_trans && fq) ? 0.5 * l_dt * fq(i, j, k, n) : 0.; + + auto bc = pbc[n]; +if ((i==0) && (j==0) && (k==0)) + amrex::Print() << "******* now calling Godunov_cc_xbc_lo/hi from compute_fluxes" << std::endl; + Godunov_cc_xbc_lo(i, j, k, n, q, stl, sth, umac, bc.lo(0), dlo.x); + Godunov_cc_xbc_hi(i, j, k, n, q, stl, sth, umac, bc.hi(0), dhi.x); + + Real qx = (umac(i, j, k) >= 0.) ? stl : sth; + qx = (std::abs(umac(i, j, k)) < small_vel) ? 0.5 * (stl + sth) : qx; + + if (iconserv[n] == 1) { + fx(i, j, k, n) = umac(i, j, k) * qx; + } else { + fx(i, j, k, n) = qx; + } + }); + + // + // y-direction + // + Box const& ybxtmp = amrex::grow(bx, 1, 1); + Array4 xzlo = + makeArray4(xyzlo.dataPtr(), amrex::surroundingNodes(ybxtmp, 0), ncomp); + Array4 zxlo = + makeArray4(xyzhi.dataPtr(), amrex::surroundingNodes(ybxtmp, 2), ncomp); + amrex::ParallelFor( + Box(xzlo), ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + const auto bc = pbc[n]; + Real l_xzlo, l_xzhi; + Godunov_corner_couple_xz( + l_xzlo, l_xzhi, i, j, k, n, l_dt, dz, iconserv[n] != 0, + xlo(i, j, k, n), xhi(i, j, k, n), q, wmac, zedge); + + Real uad = umac(i, j, k); + Godunov_trans_xbc( + i, j, k, n, q, l_xzlo, l_xzhi, uad, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); + + constexpr Real small_vel = 1.e-8; + + Real st = (uad >= 0.) ? l_xzlo : l_xzhi; + Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; + xzlo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_xzhi + l_xzlo); + }, + Box(zxlo), ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + const auto bc = pbc[n]; + Real l_zxlo, l_zxhi; + Godunov_corner_couple_zx( + l_zxlo, l_zxhi, i, j, k, n, l_dt, dx, iconserv[n] != 0, + zlo(i, j, k, n), zhi(i, j, k, n), q, umac, xedge); + + Real wad = wmac(i, j, k); + Godunov_trans_zbc( + i, j, k, n, q, l_zxlo, l_zxhi, wad, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); + + constexpr Real small_vel = 1.e-8; + + Real st = (wad >= 0.) ? l_zxlo : l_zxhi; + Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; + zxlo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_zxhi + l_zxlo); + }); + + amrex::ParallelFor( + ybx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + Real stl, sth; + constexpr Real small_vel = 1.e-8; + + if (iconserv[n] != 0) { + stl = ylo(i, j, k, n) - + (0.5 * dtdx) * + (xzlo(i + 1, j - 1, k, n) * umac(i + 1, j - 1, k) - + xzlo(i, j - 1, k, n) * umac(i, j - 1, k)) - + (0.5 * dtdz) * + (zxlo(i, j - 1, k + 1, n) * wmac(i, j - 1, k + 1) - + zxlo(i, j - 1, k, n) * wmac(i, j - 1, k)) + + (0.5 * dtdx) * q(i, j - 1, k, n) * + (umac(i + 1, j - 1, k) - umac(i, j - 1, k)) + + (0.5 * dtdz) * q(i, j - 1, k, n) * + (wmac(i, j - 1, k + 1) - wmac(i, j - 1, k)); + + sth = yhi(i, j, k, n) - + (0.5 * dtdx) * (xzlo(i + 1, j, k, n) * umac(i + 1, j, k) - + xzlo(i, j, k, n) * umac(i, j, k)) - + (0.5 * dtdz) * (zxlo(i, j, k + 1, n) * wmac(i, j, k + 1) - + zxlo(i, j, k, n) * wmac(i, j, k)) + + (0.5 * dtdx) * q(i, j, k, n) * + (umac(i + 1, j, k) - umac(i, j, k)) + + (0.5 * dtdz) * q(i, j, k, n) * + (wmac(i, j, k + 1) - wmac(i, j, k)); + } else { + stl = ylo(i, j, k, n) - + (0.25 * dtdx) * + (umac(i + 1, j - 1, k) + umac(i, j - 1, k)) * + (xzlo(i + 1, j - 1, k, n) - xzlo(i, j - 1, k, n)) - + (0.25 * dtdz) * + (wmac(i, j - 1, k + 1) + wmac(i, j - 1, k)) * + (zxlo(i, j - 1, k + 1, n) - zxlo(i, j - 1, k, n)); + + sth = yhi(i, j, k, n) - + (0.25 * dtdx) * (umac(i + 1, j, k) + umac(i, j, k)) * + (xzlo(i + 1, j, k, n) - xzlo(i, j, k, n)) - + (0.25 * dtdz) * (wmac(i, j, k + 1) + wmac(i, j, k)) * + (zxlo(i, j, k + 1, n) - zxlo(i, j, k, n)); + } + + stl += (!use_forces_in_trans && fq) + ? 0.5 * l_dt * fq(i, j - 1, k, n) + : 0.; + sth += + (!use_forces_in_trans && fq) ? 0.5 * l_dt * fq(i, j, k, n) : 0.; + + auto bc = pbc[n]; + Godunov_cc_ybc_lo(i, j, k, n, q, stl, sth, vmac, bc.lo(1), dlo.y); + Godunov_cc_ybc_hi(i, j, k, n, q, stl, sth, vmac, bc.hi(1), dhi.y); + + Real qy = (vmac(i, j, k) >= 0.) ? stl : sth; + qy = (std::abs(vmac(i, j, k)) < small_vel) ? 0.5 * (stl + sth) : qy; + + if (iconserv[n] == 1) { + fy(i, j, k, n) = vmac(i, j, k) * qy; + } else { + fy(i, j, k, n) = qy; + } + }); + + // + // z-direction + // + Box const& zbxtmp = amrex::grow(bx, 2, 1); + Array4 xylo = + makeArray4(xyzlo.dataPtr(), amrex::surroundingNodes(zbxtmp, 0), ncomp); + Array4 yxlo = + makeArray4(xyzhi.dataPtr(), amrex::surroundingNodes(zbxtmp, 1), ncomp); + amrex::ParallelFor( + Box(xylo), ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + const auto bc = pbc[n]; + Real l_xylo, l_xyhi; + Godunov_corner_couple_xy( + l_xylo, l_xyhi, i, j, k, n, l_dt, dy, iconserv[n] != 0, + xlo(i, j, k, n), xhi(i, j, k, n), q, vmac, yedge); + + Real uad = umac(i, j, k); + Godunov_trans_xbc( + i, j, k, n, q, l_xylo, l_xyhi, uad, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); + + constexpr Real small_vel = 1.e-8; + + Real st = (uad >= 0.) ? l_xylo : l_xyhi; + Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; + xylo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_xyhi + l_xylo); + }, + Box(yxlo), ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + const auto bc = pbc[n]; + Real l_yxlo, l_yxhi; + Godunov_corner_couple_yx( + l_yxlo, l_yxhi, i, j, k, n, l_dt, dx, iconserv[n] != 0, + ylo(i, j, k, n), yhi(i, j, k, n), q, umac, xedge); + + Real vad = vmac(i, j, k); + Godunov_trans_ybc( + i, j, k, n, q, l_yxlo, l_yxhi, vad, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); + + constexpr Real small_vel = 1.e-8; + + Real st = (vad >= 0.) ? l_yxlo : l_yxhi; + Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; + yxlo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_yxhi + l_yxlo); + }); + + amrex::ParallelFor( + zbx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + Real stl, sth; + constexpr Real small_vel = 1.e-8; + if (iconserv[n] != 0) { + stl = zlo(i, j, k, n) - + (0.5 * dtdx) * + (xylo(i + 1, j, k - 1, n) * umac(i + 1, j, k - 1) - + xylo(i, j, k - 1, n) * umac(i, j, k - 1)) - + (0.5 * dtdy) * + (yxlo(i, j + 1, k - 1, n) * vmac(i, j + 1, k - 1) - + yxlo(i, j, k - 1, n) * vmac(i, j, k - 1)) + + (0.5 * dtdx) * q(i, j, k - 1, n) * + (umac(i + 1, j, k - 1) - umac(i, j, k - 1)) + + (0.5 * dtdy) * q(i, j, k - 1, n) * + (vmac(i, j + 1, k - 1) - vmac(i, j, k - 1)); + + sth = zhi(i, j, k, n) - + (0.5 * dtdx) * (xylo(i + 1, j, k, n) * umac(i + 1, j, k) - + xylo(i, j, k, n) * umac(i, j, k)) - + (0.5 * dtdy) * (yxlo(i, j + 1, k, n) * vmac(i, j + 1, k) - + yxlo(i, j, k, n) * vmac(i, j, k)) + + (0.5 * dtdx) * q(i, j, k, n) * + (umac(i + 1, j, k) - umac(i, j, k)) + + (0.5 * dtdy) * q(i, j, k, n) * + (vmac(i, j + 1, k) - vmac(i, j, k)); + } else { + stl = zlo(i, j, k, n) - + (0.25 * dtdx) * + (umac(i + 1, j, k - 1) + umac(i, j, k - 1)) * + (xylo(i + 1, j, k - 1, n) - xylo(i, j, k - 1, n)) - + (0.25 * dtdy) * + (vmac(i, j + 1, k - 1) + vmac(i, j, k - 1)) * + (yxlo(i, j + 1, k - 1, n) - yxlo(i, j, k - 1, n)); + + sth = zhi(i, j, k, n) - + (0.25 * dtdx) * (umac(i + 1, j, k) + umac(i, j, k)) * + (xylo(i + 1, j, k, n) - xylo(i, j, k, n)) - + (0.25 * dtdy) * (vmac(i, j + 1, k) + vmac(i, j, k)) * + (yxlo(i, j + 1, k, n) - yxlo(i, j, k, n)); + } + + stl += (!use_forces_in_trans && fq) + ? 0.5 * l_dt * fq(i, j, k - 1, n) + : 0.; + sth += + (!use_forces_in_trans && fq) ? 0.5 * l_dt * fq(i, j, k, n) : 0.; + + auto bc = pbc[n]; + Godunov_cc_zbc_lo(i, j, k, n, q, stl, sth, wmac, bc.lo(2), dlo.z); + Godunov_cc_zbc_hi(i, j, k, n, q, stl, sth, wmac, bc.hi(2), dhi.z); + + Real qz = (wmac(i, j, k) >= 0.) ? stl : sth; + qz = (std::abs(wmac(i, j, k)) < small_vel) ? 0.5 * (stl + sth) : qz; + + if (iconserv[n] == 1) { + fz(i, j, k, n) = wmac(i, j, k) * qz; + } else { + fz(i, j, k, n) = qz; + } + }); +} + diff --git a/amr-wind/convection/incflo_godunov_predict.cpp b/amr-wind/convection/incflo_godunov_predict.cpp new file mode 100644 index 0000000000..c5bd10dc30 --- /dev/null +++ b/amr-wind/convection/incflo_godunov_predict.cpp @@ -0,0 +1,517 @@ +#include +#include "amr-wind/convection/Godunov.H" +#include "amr-wind/convection/incflo_godunov_ppm.H" + +using namespace amrex; + +void godunov::make_trans_velocities( + int lev, + Box const& xbx, + Box const& ybx, + Box const& zbx, + Array4 const& u_ad, + Array4 const& v_ad, + Array4 const& w_ad, + Array4 const& Imx, + Array4 const& Ipx, + Array4 const& Imy, + Array4 const& Ipy, + Array4 const& Imz, + Array4 const& Ipz, + Array4 const& vel, + Array4 const& f, + Vector geom, + Real dt, + amrex::Gpu::DeviceVector& bcrec_device, + bool godunov_use_forces_in_trans) + +{ + BL_PROFILE("amr-wind::godunov::make_trans_velocities"); + Real l_dt = dt; + bool l_use_forces_in_trans = godunov_use_forces_in_trans; + + const Box& domain = geom[lev].Domain(); + const Dim3 dlo = amrex::lbound(domain); + const Dim3 dhi = amrex::ubound(domain); + + BCRec const* pbc = bcrec_device.data(); + + amrex::ParallelFor( + xbx, ybx, zbx, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // We only care about x-velocity on x-faces here + constexpr int n = 0; + + Real lo, hi; + if (l_use_forces_in_trans) { + lo = Ipx(i - 1, j, k, n) + 0.5 * l_dt * f(i - 1, j, k, n); + hi = Imx(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); + } else { + lo = Ipx(i - 1, j, k, n); + hi = Imx(i, j, k, n); + } + + auto bc = pbc[n]; + Godunov_trans_xbc( + i, j, k, n, vel, lo, hi, lo, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + + constexpr Real small_vel = 1e-8; + + Real st = ((lo + hi) >= 0.) ? lo : hi; + bool ltm = + ((lo <= 0. && hi >= 0.) || (std::abs(lo + hi) < small_vel)); + u_ad(i, j, k) = ltm ? 0. : st; + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // We only care about y-velocity on y-faces here + constexpr int n = 1; + + Real lo, hi; + if (l_use_forces_in_trans) { + lo = Ipy(i, j - 1, k, n) + 0.5 * l_dt * f(i, j - 1, k, n); + hi = Imy(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); + } else { + lo = Ipy(i, j - 1, k, n); + hi = Imy(i, j, k, n); + } + + auto bc = pbc[n]; + Godunov_trans_ybc( + i, j, k, n, vel, lo, hi, lo, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + + constexpr Real small_vel = 1e-8; + + Real st = ((lo + hi) >= 0.) ? lo : hi; + bool ltm = + ((lo <= 0. && hi >= 0.) || (std::abs(lo + hi) < small_vel)); + v_ad(i, j, k) = ltm ? 0. : st; + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + // We only care about z-velocity on z-faces here + constexpr int n = 2; + + Real lo, hi; + if (l_use_forces_in_trans) { + lo = Ipz(i, j, k - 1, n) + 0.5 * l_dt * f(i, j, k - 1, n); + hi = Imz(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); + } else { + lo = Ipz(i, j, k - 1, n); + hi = Imz(i, j, k, n); + } + + auto bc = pbc[n]; + Godunov_trans_zbc( + i, j, k, n, vel, lo, hi, lo, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + + constexpr Real small_vel = 1e-8; + + Real st = ((lo + hi) >= 0.) ? lo : hi; + bool ltm = + ((lo <= 0. && hi >= 0.) || (std::abs(lo + hi) < small_vel)); + w_ad(i, j, k) = ltm ? 0. : st; + }); +} + +void godunov::predict_godunov( + int lev, + Box const& bx, + int ncomp, + Box const& xbx, + Box const& ybx, + Box const& zbx, + Array4 const& qx, + Array4 const& qy, + Array4 const& qz, + Array4 const& q, + Array4 const& u_ad, + Array4 const& v_ad, + Array4 const& w_ad, + Array4 const& Imx, + Array4 const& Ipx, + Array4 const& Imy, + Array4 const& Ipy, + Array4 const& Imz, + Array4 const& Ipz, + Array4 const& f, + Real* p, + Vector geom, + Real dt, + amrex::Gpu::DeviceVector& bcrec_device, + bool godunov_use_forces_in_trans) +{ + BL_PROFILE("amr-wind::godunov::predict_godunov"); + Real l_dt = dt; + bool l_use_forces_in_trans = godunov_use_forces_in_trans; + + const Box& domain = geom[lev].Domain(); + const Dim3 dlo = amrex::lbound(domain); + const Dim3 dhi = amrex::ubound(domain); + Real dx = geom[lev].CellSize(0); + Real dy = geom[lev].CellSize(1); + Real dz = geom[lev].CellSize(2); + + BCRec const* pbc = bcrec_device.data(); + + Box xebox = Box(bx).grow(1, 1).grow(2, 1).surroundingNodes(0); + Box yebox = Box(bx).grow(0, 1).grow(2, 1).surroundingNodes(1); + Box zebox = Box(bx).grow(0, 1).grow(1, 1).surroundingNodes(2); + Array4 xlo = makeArray4(p, xebox, ncomp); + p += xlo.size(); + Array4 xhi = makeArray4(p, xebox, ncomp); + p += xhi.size(); + Array4 ylo = makeArray4(p, yebox, ncomp); + p += ylo.size(); + Array4 yhi = makeArray4(p, yebox, ncomp); + p += yhi.size(); + Array4 zlo = makeArray4(p, zebox, ncomp); + p += zlo.size(); + Array4 zhi = makeArray4(p, zebox, ncomp); + p += zhi.size(); // NOLINT: Value not read warning + + amrex::ParallelFor( + xebox, ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + Real lo, hi; + if (l_use_forces_in_trans) { + lo = Ipx(i - 1, j, k, n) + 0.5 * l_dt * f(i - 1, j, k, n); + hi = Imx(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); + } else { + lo = Ipx(i - 1, j, k, n); + hi = Imx(i, j, k, n); + } + + Real uad = u_ad(i, j, k); + auto bc = pbc[n]; + + Godunov_trans_xbc( + i, j, k, n, q, lo, hi, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + + xlo(i, j, k, n) = lo; + xhi(i, j, k, n) = hi; + + constexpr Real small_vel = 1e-8; + + Real st = (uad >= 0.) ? lo : hi; + Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; + Imx(i, j, k, n) = + fu * st + (1.0 - fu) * 0.5 * (hi + lo); // store xedge + }, + yebox, ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + Real lo, hi; + if (l_use_forces_in_trans) { + lo = Ipy(i, j - 1, k, n) + 0.5 * l_dt * f(i, j - 1, k, n); + hi = Imy(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); + } else { + lo = Ipy(i, j - 1, k, n); + hi = Imy(i, j, k, n); + } + + Real vad = v_ad(i, j, k); + auto bc = pbc[n]; + + Godunov_trans_ybc( + i, j, k, n, q, lo, hi, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + + ylo(i, j, k, n) = lo; + yhi(i, j, k, n) = hi; + + constexpr Real small_vel = 1e-8; + + Real st = (vad >= 0.) ? lo : hi; + Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; + Imy(i, j, k, n) = + fu * st + (1.0 - fu) * 0.5 * (hi + lo); // store yedge + }, + zebox, ncomp, + [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + Real lo, hi; + if (l_use_forces_in_trans) { + lo = Ipz(i, j, k - 1, n) + 0.5 * l_dt * f(i, j, k - 1, n); + hi = Imz(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); + } else { + lo = Ipz(i, j, k - 1, n); + hi = Imz(i, j, k, n); + } + + Real wad = w_ad(i, j, k); + auto bc = pbc[n]; + + Godunov_trans_zbc( + i, j, k, n, q, lo, hi, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + + zlo(i, j, k, n) = lo; + zhi(i, j, k, n) = hi; + + constexpr Real small_vel = 1e-8; + + Real st = (wad >= 0.) ? lo : hi; + Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; + Imz(i, j, k, n) = + fu * st + (1.0 - fu) * 0.5 * (hi + lo); // store zedge + }); + + Array4 xedge = Imx; + Array4 yedge = Imy; + Array4 zedge = Imz; + + // We can reuse the space in Ipy and Ipz. + + // + // X-Flux + // + Box const xbxtmp = Box(xbx).enclosedCells().grow(0, 1); + Array4 yzlo = + makeArray4(Ipy.dataPtr(), amrex::surroundingNodes(xbxtmp, 1), 1); + Array4 zylo = + makeArray4(Ipz.dataPtr(), amrex::surroundingNodes(xbxtmp, 2), 1); + // Add d/dy term to z-faces + // Start with {zlo,zhi} --> {zylo, zyhi} and upwind using w_ad to {zylo} + // Add d/dz to y-faces + // Start with {ylo,yhi} --> {yzlo, yzhi} and upwind using v_ad to {yzlo} + amrex::ParallelFor( + Box(zylo), Box(yzlo), + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 0; + const auto bc = pbc[n]; + Real l_zylo, l_zyhi; + Godunov_corner_couple_zy( + l_zylo, l_zyhi, i, j, k, n, l_dt, dy, false, zlo(i, j, k, n), + zhi(i, j, k, n), q, v_ad, yedge); + + Real wad = w_ad(i, j, k); + Godunov_trans_zbc( + i, j, k, n, q, l_zylo, l_zyhi, wad, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); + + constexpr Real small_vel = 1.e-8; + + Real st = (wad >= 0.) ? l_zylo : l_zyhi; + Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; + zylo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_zyhi + l_zylo); + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 0; + const auto bc = pbc[n]; + Real l_yzlo, l_yzhi; + Godunov_corner_couple_yz( + l_yzlo, l_yzhi, i, j, k, n, l_dt, dz, false, ylo(i, j, k, n), + yhi(i, j, k, n), q, w_ad, zedge); + + Real vad = v_ad(i, j, k); + Godunov_trans_ybc( + i, j, k, n, q, l_yzlo, l_yzhi, vad, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); + + constexpr Real small_vel = 1.e-8; + + Real st = (vad >= 0.) ? l_yzlo : l_yzhi; + Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; + yzlo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_yzhi + l_yzlo); + }); + // + amrex::ParallelFor(xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 0; + auto bc = pbc[n]; + Real stl = + xlo(i, j, k, n) - + (0.25 * l_dt / dy) * (v_ad(i - 1, j + 1, k) + v_ad(i - 1, j, k)) * + (yzlo(i - 1, j + 1, k) - yzlo(i - 1, j, k)) - + (0.25 * l_dt / dz) * (w_ad(i - 1, j, k + 1) + w_ad(i - 1, j, k)) * + (zylo(i - 1, j, k + 1) - zylo(i - 1, j, k)); + Real sth = xhi(i, j, k, n) - + (0.25 * l_dt / dy) * (v_ad(i, j + 1, k) + v_ad(i, j, k)) * + (yzlo(i, j + 1, k) - yzlo(i, j, k)) - + (0.25 * l_dt / dz) * (w_ad(i, j, k + 1) + w_ad(i, j, k)) * + (zylo(i, j, k + 1) - zylo(i, j, k)); + if (!l_use_forces_in_trans) { + stl += 0.5 * l_dt * f(i - 1, j, k, n); + sth += 0.5 * l_dt * f(i, j, k, n); + } +if ((i==0) && (j==0) && (k==0)) + amrex::Print() << "******* now calling Godunov_cc_xbc_lo/hi from predict_godunov" << std::endl; + Godunov_cc_xbc_lo(i, j, k, n, q, stl, sth, u_ad, bc.lo(0), dlo.x); + Godunov_cc_xbc_hi(i, j, k, n, q, stl, sth, u_ad, bc.hi(0), dhi.x); + + constexpr Real small_vel = 1.e-8; + + Real st = ((stl + sth) >= 0.) ? stl : sth; + bool ltm = + ((stl <= 0. && sth >= 0.) || (std::abs(stl + sth) < small_vel)); + qx(i, j, k) = ltm ? 0. : st; + }); + + // + // Y-Flux + // + Box const ybxtmp = Box(ybx).enclosedCells().grow(1, 1); + Array4 xzlo = + makeArray4(Ipy.dataPtr(), amrex::surroundingNodes(ybxtmp, 0), 1); + Array4 zxlo = + makeArray4(Ipz.dataPtr(), amrex::surroundingNodes(ybxtmp, 2), 1); + + // Add d/dz to x-faces + // Start with {xlo,xhi} --> {xzlo, xzhi} and upwind using u_ad to {xzlo} + // Add d/dx term to z-faces + // Start with {zlo,zhi} --> {zxlo, zxhi} and upwind using w_ad to {zxlo} + amrex::ParallelFor( + Box(xzlo), Box(zxlo), + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 1; + const auto bc = pbc[n]; + Real l_xzlo, l_xzhi; + Godunov_corner_couple_xz( + l_xzlo, l_xzhi, i, j, k, n, l_dt, dz, false, xlo(i, j, k, n), + xhi(i, j, k, n), q, w_ad, zedge); + + Real uad = u_ad(i, j, k); + Godunov_trans_xbc( + i, j, k, n, q, l_xzlo, l_xzhi, uad, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); + + constexpr Real small_vel = 1.e-8; + + Real st = (uad >= 0.) ? l_xzlo : l_xzhi; + Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; + xzlo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_xzhi + l_xzlo); + }, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 1; + const auto bc = pbc[n]; + Real l_zxlo, l_zxhi; + Godunov_corner_couple_zx( + l_zxlo, l_zxhi, i, j, k, n, l_dt, dx, false, zlo(i, j, k, n), + zhi(i, j, k, n), q, u_ad, xedge); + + Real wad = w_ad(i, j, k); + Godunov_trans_zbc( + i, j, k, n, q, l_zxlo, l_zxhi, wad, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); + + constexpr Real small_vel = 1.e-8; + + Real st = (wad >= 0.) ? l_zxlo : l_zxhi; + Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; + zxlo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_zxhi + l_zxlo); + }); + // + amrex::ParallelFor(ybx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 1; + auto bc = pbc[n]; + Real stl = + ylo(i, j, k, n) - + (0.25 * l_dt / dx) * (u_ad(i + 1, j - 1, k) + u_ad(i, j - 1, k)) * + (xzlo(i + 1, j - 1, k) - xzlo(i, j - 1, k)) - + (0.25 * l_dt / dz) * (w_ad(i, j - 1, k + 1) + w_ad(i, j - 1, k)) * + (zxlo(i, j - 1, k + 1) - zxlo(i, j - 1, k)); + Real sth = yhi(i, j, k, n) - + (0.25 * l_dt / dx) * (u_ad(i + 1, j, k) + u_ad(i, j, k)) * + (xzlo(i + 1, j, k) - xzlo(i, j, k)) - + (0.25 * l_dt / dz) * (w_ad(i, j, k + 1) + w_ad(i, j, k)) * + (zxlo(i, j, k + 1) - zxlo(i, j, k)); + if (!l_use_forces_in_trans) { + stl += 0.5 * l_dt * f(i, j - 1, k, n); + sth += 0.5 * l_dt * f(i, j, k, n); + } + + Godunov_cc_ybc_lo(i, j, k, n, q, stl, sth, v_ad, bc.lo(1), dlo.y); + Godunov_cc_ybc_hi(i, j, k, n, q, stl, sth, v_ad, bc.hi(1), dhi.y); + + constexpr Real small_vel = 1.e-8; + + Real st = ((stl + sth) >= 0.) ? stl : sth; + bool ltm = + ((stl <= 0. && sth >= 0.) || (std::abs(stl + sth) < small_vel)); + qy(i, j, k) = ltm ? 0. : st; + }); + + // + // Z-Flux + // + Box const zbxtmp = Box(zbx).enclosedCells().grow(2, 1); + Array4 xylo = + makeArray4(Ipy.dataPtr(), amrex::surroundingNodes(zbxtmp, 0), 1); + Array4 yxlo = + makeArray4(Ipz.dataPtr(), amrex::surroundingNodes(zbxtmp, 1), 1); + + amrex::ParallelFor( + Box(xylo), Box(yxlo), + // + // Add d/dy term to x-faces + // Start with {xlo,xhi} --> {xylo, xyhi} and upwind using u_ad to {xylo} + // + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 2; + const auto bc = pbc[n]; + Real l_xylo, l_xyhi; + Godunov_corner_couple_xy( + l_xylo, l_xyhi, i, j, k, n, l_dt, dy, false, xlo(i, j, k, n), + xhi(i, j, k, n), q, v_ad, yedge); + + Real uad = u_ad(i, j, k); + Godunov_trans_xbc( + i, j, k, n, q, l_xylo, l_xyhi, uad, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); + + constexpr Real small_vel = 1.e-8; + + Real st = (uad >= 0.) ? l_xylo : l_xyhi; + Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; + xylo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_xyhi + l_xylo); + }, + // + // Add d/dx term to y-faces + // Start with {ylo,yhi} --> {yxlo, yxhi} and upwind using v_ad to {yxlo} + // + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 2; + const auto bc = pbc[n]; + Real l_yxlo, l_yxhi; + Godunov_corner_couple_yx( + l_yxlo, l_yxhi, i, j, k, n, l_dt, dx, false, ylo(i, j, k, n), + yhi(i, j, k, n), q, u_ad, xedge); + + Real vad = v_ad(i, j, k); + Godunov_trans_ybc( + i, j, k, n, q, l_yxlo, l_yxhi, vad, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); + + constexpr Real small_vel = 1.e-8; + + Real st = (vad >= 0.) ? l_yxlo : l_yxhi; + Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; + yxlo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_yxhi + l_yxlo); + }); + // + amrex::ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + constexpr int n = 2; + auto bc = pbc[n]; + Real stl = + zlo(i, j, k, n) - + (0.25 * l_dt / dx) * (u_ad(i + 1, j, k - 1) + u_ad(i, j, k - 1)) * + (xylo(i + 1, j, k - 1) - xylo(i, j, k - 1)) - + (0.25 * l_dt / dy) * (v_ad(i, j + 1, k - 1) + v_ad(i, j, k - 1)) * + (yxlo(i, j + 1, k - 1) - yxlo(i, j, k - 1)); + Real sth = zhi(i, j, k, n) - + (0.25 * l_dt / dx) * (u_ad(i + 1, j, k) + u_ad(i, j, k)) * + (xylo(i + 1, j, k) - xylo(i, j, k)) - + (0.25 * l_dt / dy) * (v_ad(i, j + 1, k) + v_ad(i, j, k)) * + (yxlo(i, j + 1, k) - yxlo(i, j, k)); + + if (!l_use_forces_in_trans) { + stl += 0.5 * l_dt * f(i, j, k - 1, n); + sth += 0.5 * l_dt * f(i, j, k, n); + } + + Godunov_cc_zbc_lo(i, j, k, n, q, stl, sth, w_ad, bc.lo(2), dlo.z); + Godunov_cc_zbc_hi(i, j, k, n, q, stl, sth, w_ad, bc.hi(2), dhi.z); + + constexpr Real small_vel = 1.e-8; + + Real st = ((stl + sth) >= 0.) ? stl : sth; + bool ltm = + ((stl <= 0. && sth >= 0.) || (std::abs(stl + sth) < small_vel)); + qz(i, j, k) = ltm ? 0. : st; + }); +} + diff --git a/amr-wind/physics/udfs/CustomVelocity.H b/amr-wind/physics/udfs/CustomVelocity.H index bcd0a746bb..3203cf8b98 100644 --- a/amr-wind/physics/udfs/CustomVelocity.H +++ b/amr-wind/physics/udfs/CustomVelocity.H @@ -32,7 +32,9 @@ struct CustomVelocity const int dcomp, const int orig_comp) const { - //amrex::Print() << "****** filling custom velocity at " << iv << std::endl; + if ((iv[0] == -1) && (iv[1] == -1) && (iv[2] == 0)) + amrex::Print() << "****** filling custom velocity at " << iv + << " and at other ghost cells... " << std::endl; // Compute quantities to set the field values. For example: // clang-format off const auto* problo = geom.ProbLo(); @@ -47,9 +49,11 @@ struct CustomVelocity if (x <= 0.5) { // at the low-x boundary, +ve vel in top portion, garbage in the rest vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; + //vel = {((z >= 0.5) ? pvel : -1.0), 0.0, 0.0}; } else { // at the high-x boundary, -ve vel in bottom portion, garbage in the rest vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; + //vel = {((z < 0.5) ? mvel : 1.0), 0.0, 0.0}; } field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; diff --git a/amr-wind/setup/init.cpp b/amr-wind/setup/init.cpp index b669a8b2af..1896e3ec13 100644 --- a/amr-wind/setup/init.cpp +++ b/amr-wind/setup/init.cpp @@ -79,7 +79,9 @@ void incflo::InitialIterations() { auto& vel = icns().fields().field; vel.copy_state(amr_wind::FieldState::Old, amr_wind::FieldState::New); + amrex::Print() << "****** starting fillpatch from InitialIterations()" << std::endl; vel.state(amr_wind::FieldState::Old).fillpatch(m_time.current_time()); + amrex::Print() << "****** completed fillpatch from InitialIterations()" << std::endl; if (m_sim.pde_manager().constant_density()) { auto& rho = density(); From 07c43232b946c34ef5805f27f3a54a7284d46287 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 15 Apr 2024 15:15:08 -0700 Subject: [PATCH 14/83] inplementing custom Neumann fill-patch correctly before the nodal projection --- .../MassInflowOutflowBC.cpp | 25 ++++++++++++------- amr-wind/physics/udfs/CustomVelocity.H | 2 ++ .../incflo_apply_nodal_projection.cpp | 10 ++++++++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index 51d7d2897e..a537b2bfed 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -36,29 +36,36 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { - const auto& bx = mfi.validbox(); + auto bx = mfi.validbox(); + bx.grow({!ib, !jb, !kb}); const auto& bc_a = m_field(lev).array(mfi); const auto& vel = velocity(lev).array(mfi); if (islow && (bx.smallEnd(idim) == domain.smallEnd(idim))) { amrex::ParallelFor( - amrex::bdryLo(bx, idim), ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - //amrex::Print() << i << " " << j << " " << k << std::endl; - //amrex::Print() << i-ib << " " << j-jb << " " << k-kb << std::endl; - //amrex::Print() << vel(i-ib, j-jb, k-kb, idim) << " " << bc_a(i-ib, j-jb, k-kb, n) << " " << bc_a(i, j, k, n) << std::endl << std::endl; + amrex::bdryLo(bx, idim), + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (vel(i-ib, j-jb, k-kb, idim) < 0) { + for (int n = 0; n < ncomp; n++) { bc_a(i-ib, j-jb, k-kb, n) = bc_a(i, j, k, n); + } } + + /*for (int n = 0; n < ncomp; n++) { + amrex::Print() << i << " " << j << " " << k << " " << n << std::endl; + amrex::Print() << "result: " << vel(i-ib, j-jb, k-kb, idim) << " " << bc_a(i-ib, j-jb, k-kb, n) << " " << bc_a(i, j, k, n) << std::endl << std::endl; + }*/ }); } if (ishigh && (bx.bigEnd(idim) == domain.bigEnd(idim))) { amrex::ParallelFor( - amrex::bdryHi(bx, idim), ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { + amrex::bdryHi(bx, idim), + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { if (vel(i, j, k, idim) > 0) { - bc_a(i, j, k, n) = bc_a(i-ib, j-jb, k-kb, n); + for (int n = 0; n < ncomp; n++) { + bc_a(i, j, k, n) = bc_a(i-ib, j-jb, k-kb, n); + } } }); } diff --git a/amr-wind/physics/udfs/CustomVelocity.H b/amr-wind/physics/udfs/CustomVelocity.H index 3203cf8b98..748377d98d 100644 --- a/amr-wind/physics/udfs/CustomVelocity.H +++ b/amr-wind/physics/udfs/CustomVelocity.H @@ -49,10 +49,12 @@ struct CustomVelocity if (x <= 0.5) { // at the low-x boundary, +ve vel in top portion, garbage in the rest vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; + //vel = {((z >= 0.5) ? pvel : 0.0), 0.0, 0.0}; //vel = {((z >= 0.5) ? pvel : -1.0), 0.0, 0.0}; } else { // at the high-x boundary, -ve vel in bottom portion, garbage in the rest vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; + //vel = {((z < 0.5) ? mvel : 0.0), 0.0, 0.0}; //vel = {((z < 0.5) ? mvel : 1.0), 0.0, 0.0}; } diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 4aa1803652..30885e8354 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -6,6 +6,9 @@ #include "amr-wind/core/field_ops.H" #include "amr-wind/projection/nodal_projection_ops.H" +#include "amr-wind/utilities/IOManager.H" + + using namespace amrex; void amr_wind::nodal_projection::set_inflow_velocity( @@ -333,6 +336,13 @@ void incflo::ApplyProjection( } } + // need to apply custom Neumann funcs for inflow-outflow BC + // after setting the inflow vels above + if (!proj_for_small_dt and !incremental) { + velocity.apply_bc_funcs(amr_wind::FieldState::New); + } + + if (is_anelastic) { for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < velocity.num_comp(); ++idim) { From f7602a573f148cbf19b7ee2d0c15c50cdc025223 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 18 Apr 2024 18:03:30 -0700 Subject: [PATCH 15/83] intermediate changes in the BC routines for the in-out BC --- amr-wind/convection/incflo_godunov_ppm.H | 591 +++++++++++++++++++++++ 1 file changed, 591 insertions(+) create mode 100644 amr-wind/convection/incflo_godunov_ppm.H diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H new file mode 100644 index 0000000000..7171a9624a --- /dev/null +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -0,0 +1,591 @@ +#ifndef GODUNOV_PPM_H +#define GODUNOV_PPM_H + +#include +#include + +/* This header file contains the inlined __host__ __device__ functions required + for the scalar advection routines for 3D Godunov. It also contains function + declarations for controlling host functions. */ + +namespace { + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + amrex::Real& uedge, + const int bclo, + const int bchi, + const int domlo, + const int domhi) +{ + using namespace amrex; + + // Low X + if (i <= domlo) { + if (bclo == BCType::ext_dir || + (bclo == BCType::user_1 && uedge > 0.0)) { + Print() << "trans_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n + << " as uedge is " << uedge << std::endl; + // IAMR does this but it breaks lo/hi symmetry + // Real st = (uedge <= small_vel) ? hi : s(domlo-1,j,k,n); + // So here we do something simpler... + Real st = s(domlo - 1, j, k, n); + lo = st; + hi = st; + } else if ( + bclo == BCType::foextrap || bclo == BCType::hoextrap || + bclo == BCType::reflect_even || + (bclo == BCType::user_1 && uedge <= 0.0)) { + Print() << "trans_xbc_domlo_foextrap triggered at " << i << " " << j << " " << k << " " << n + << " as uedge is " << uedge << std::endl; + lo = hi; + + } else if (bclo == BCType::reflect_odd) { + hi = 0.; + lo = 0.; + } else { + Print() << "trans_xbc_domlo_nothing triggered at " << i << " " << j << " " << k << " " << n + << " as uedge is " << uedge << std::endl; + } + } + + // High X + if (i > domhi) { + if (bchi == BCType::ext_dir || + (bchi == BCType::user_1 && uedge < 0.0)) { // is this correct use of uedge here?? + Print() << "trans_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n + << " as uedge is " << uedge << std::endl; + // IAMR does this but it breaks lo/hi symmetry + // Real st = (uedge >= -small_vel)? lo : s(domhi+1,j,k,n); + // So here we do something simpler... + Real st = s(domhi + 1, j, k, n); + lo = st; + hi = st; + } else if ( + bchi == BCType::foextrap || bchi == BCType::hoextrap || + bchi == BCType::reflect_even || + (bchi == BCType::user_1 && uedge >= 0.0)) { + Print() << "trans_xbc_domhi_foextrap triggered at " << i << " " << j << " " << k << " " << n + << " as uedge is " << uedge << std::endl; + hi = lo; + + } else if (bchi == BCType::reflect_odd) { + hi = 0.; + lo = 0.; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_ybc( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + amrex::Real /* vedge */, + const int bclo, + const int bchi, + const int domlo, + const int domhi) +{ + using namespace amrex; + + // Low Y + if (j <= domlo) { + if (bclo == BCType::ext_dir) { + // IAMR does this but it breaks lo/hi symmetry + // Real st = (vedge <= small_vel) ? hi : s(i,domlo-1,k,n); + // So here we do something simpler... + Real st = s(i, domlo - 1, k, n); + lo = st; + hi = st; + } + + else if ( + bclo == BCType::foextrap || bclo == BCType::hoextrap || + bclo == BCType::reflect_even) { + lo = hi; + + } else if (bclo == BCType::reflect_odd) { + hi = 0.; + lo = 0.; + } + } + + // High Y + if (j > domhi) { + if (bchi == BCType::ext_dir) { + // IAMR does this but it breaks lo/hi symmetry + // Real st = (vedge >= -small_vel)? lo : s(i,domhi+1,k,n); + // So here we do something simpler... + Real st = s(i, domhi + 1, k, n); + lo = st; + hi = st; + } + + else if ( + bchi == BCType::foextrap || bchi == BCType::hoextrap || + bchi == BCType::reflect_even) { + hi = lo; + + } else if (bchi == BCType::reflect_odd) { + hi = 0.; + lo = 0.; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + amrex::Real /* wedge */, + const int bclo, + const int bchi, + const int domlo, + const int domhi) +{ + using namespace amrex; + + // Low Z + if (k <= domlo) { + if (bclo == BCType::ext_dir) { + // IAMR does this but it breaks lo/hi symmetry + // Real st = (wedge <= small_vel) ? hi : s(i,j,domlo-1,n); + // So here we do something simpler... + Real st = s(i, j, domlo - 1, n); + lo = st; + hi = st; + } + + else if ( + bclo == BCType::foextrap || bclo == BCType::hoextrap || + bclo == BCType::reflect_even) { + lo = hi; + + } else if (bclo == BCType::reflect_odd) { + hi = 0.; + lo = 0.; + } + } + + // High Z + if (k > domhi) { + if (bchi == BCType::ext_dir) { + // IAMR does this but it breaks lo/hi symmetry + // Real st = (wedge >= -small_vel)? lo : s(i,j,domhi+1,n); + // So here we do something simpler... + Real st = s(i, j, domhi + 1, n); + lo = st; + hi = st; + } else if ( + bchi == BCType::foextrap || bchi == BCType::hoextrap || + bchi == BCType::reflect_even) { + hi = lo; + + } else if (bchi == BCType::reflect_odd) { + hi = 0.; + lo = 0.; + } + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + const amrex::Array4& uedge, + const int bclo, + const int domlo) +{ + using namespace amrex; + + if (i == domlo) { + if ((bclo == BCType::ext_dir && uedge(i, j, k) >= 0.) || + (bclo == BCType::user_1 && uedge(i, j, k) >= 0.)) { + hi = s(domlo - 1, j, k, n); + lo = hi; + } else if ( + bclo == BCType::foextrap || bclo == BCType::hoextrap || + bclo == BCType::reflect_even || + (bclo == BCType::ext_dir && uedge(i, j, k) < 0) || + (bclo == BCType::user_1 && uedge(i, j, k) < 0)) { + lo = hi; + + } else if (bclo == BCType::reflect_odd) { + hi = 0.; + lo = hi; + } + } else { + return; + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + const amrex::Array4& uedge, + const int bchi, + const int domhi) +{ + using namespace amrex; + + if (i == domhi + 1) { + if ((bchi == BCType::ext_dir && uedge(i, j, k) <= 0.) || + (bchi == BCType::user_1 && uedge(i, j, k) <= 0.)) { + lo = s(domhi + 1, j, k, n); + hi = lo; + } else if ( + bchi == BCType::foextrap || bchi == BCType::hoextrap || + bchi == BCType::reflect_even || + (bchi == BCType::ext_dir && uedge(i, j, k) > 0) || + (bchi == BCType::user_1 && uedge(i, j, k) > 0)) { + hi = lo; + + } else if (bchi == BCType::reflect_odd) { + lo = 0.; + hi = lo; + } + } else { + return; + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_ybc_lo( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + const amrex::Array4& vedge, + const int bclo, + const int domlo) +{ + using namespace amrex; + + if (j == domlo) { + if (bclo == BCType::ext_dir && vedge(i, j, k) >= 0.) { + hi = s(i, domlo - 1, k, n); + lo = hi; + } else if ( + bclo == BCType::foextrap || bclo == BCType::hoextrap || + bclo == BCType::reflect_even || + (bclo == BCType::ext_dir && vedge(i, j, k) < 0)) { + lo = hi; + + } else if (bclo == BCType::reflect_odd) { + hi = 0.; + lo = hi; + } + } else { + return; + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_ybc_hi( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + const amrex::Array4& vedge, + const int bchi, + const int domhi) +{ + using namespace amrex; + + if (j == domhi + 1) { + if (bchi == BCType::ext_dir && vedge(i, j, k) <= 0.) { + lo = s(i, domhi + 1, k, n); + hi = lo; + } else if ( + bchi == BCType::foextrap || bchi == BCType::hoextrap || + bchi == BCType::reflect_even || + (bchi == BCType::ext_dir && vedge(i, j, k) > 0)) { + hi = lo; + + } else if (bchi == BCType::reflect_odd) { + lo = 0.; + hi = lo; + } + } else { + return; + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_zbc_lo( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + const amrex::Array4& wedge, + const int bclo, + const int domlo) +{ + using namespace amrex; + + if (k == domlo) { + if (bclo == BCType::ext_dir && wedge(i, j, k) >= 0.) { + hi = s(i, j, domlo - 1, n); + lo = hi; + } + + else if ( + bclo == BCType::foextrap || bclo == BCType::hoextrap || + bclo == BCType::reflect_even || + (bclo == BCType::ext_dir && wedge(i, j, k) < 0)) { + lo = hi; + + } else if (bclo == BCType::reflect_odd) { + hi = 0.; + lo = hi; + } + } else { + return; + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_zbc_hi( + const int i, + const int j, + const int k, + const int n, + const amrex::Array4& s, + amrex::Real& lo, + amrex::Real& hi, + const amrex::Array4& wedge, + const int bchi, + const int domhi) +{ + using namespace amrex; + + if (k == domhi + 1) { + if (bchi == BCType::ext_dir && wedge(i, j, k) <= 0.) { + lo = s(i, j, domhi + 1, n); + hi = lo; + } else if ( + bchi == BCType::foextrap || bchi == BCType::hoextrap || + bchi == BCType::reflect_even || + (bchi == BCType::ext_dir && wedge(i, j, k) > 0)) { + hi = lo; + + } else if (bchi == BCType::reflect_odd) { + lo = 0.; + hi = lo; + } + } else { + return; + } +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_yx( + amrex::Real& lo1, + amrex::Real& hi1, + int i, + int j, + int k, + int n, + amrex::Real dt, + amrex::Real dx, + bool /*iconserv*/, + amrex::Real lo, + amrex::Real hi, + amrex::Array4 const& s, + amrex::Array4 const& mac, + amrex::Array4 const& state) +{ + lo1 = lo - + dt / (amrex::Real(3.0) * dx) * + (state(i + 1, j - 1, k, n) * mac(i + 1, j - 1, k) - + state(i, j - 1, k, n) * mac(i, j - 1, k)) + + dt / (amrex::Real(3.0) * dx) * s(i, j - 1, k, n) * + (mac(i + 1, j - 1, k) - mac(i, j - 1, k)); + hi1 = hi - + dt / (amrex::Real(3.0) * dx) * + (state(i + 1, j, k, n) * mac(i + 1, j, k) - + state(i, j, k, n) * mac(i, j, k)) + + dt / (amrex::Real(3.0) * dx) * s(i, j, k, n) * + (mac(i + 1, j, k) - mac(i, j, k)); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_zx( + amrex::Real& lo1, + amrex::Real& hi1, + int i, + int j, + int k, + int n, + amrex::Real dt, + amrex::Real dx, + bool /*iconserv*/, + amrex::Real lo, + amrex::Real hi, + amrex::Array4 const& s, + amrex::Array4 const& mac, + amrex::Array4 const& state) +{ + lo1 = lo - + dt / (amrex::Real(3.0) * dx) * + (state(i + 1, j, k - 1, n) * mac(i + 1, j, k - 1) - + state(i, j, k - 1, n) * mac(i, j, k - 1)) + + dt / (amrex::Real(3.0) * dx) * s(i, j, k - 1, n) * + (mac(i + 1, j, k - 1) - mac(i, j, k - 1)); + hi1 = hi - + dt / (amrex::Real(3.0) * dx) * + (state(i + 1, j, k, n) * mac(i + 1, j, k) - + state(i, j, k, n) * mac(i, j, k)) + + dt / (amrex::Real(3.0) * dx) * s(i, j, k, n) * + (mac(i + 1, j, k) - mac(i, j, k)); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_xy( + amrex::Real& lo1, + amrex::Real& hi1, + int i, + int j, + int k, + int n, + amrex::Real dt, + amrex::Real dy, + bool /*iconserv*/, + amrex::Real lo, + amrex::Real hi, + amrex::Array4 const& s, + amrex::Array4 const& mac, + amrex::Array4 const& state) +{ + lo1 = lo - + dt / (amrex::Real(3.0) * dy) * + (state(i - 1, j + 1, k, n) * mac(i - 1, j + 1, k) - + state(i - 1, j, k, n) * mac(i - 1, j, k)) + + dt / (amrex::Real(3.0) * dy) * s(i - 1, j, k, n) * + (mac(i - 1, j + 1, k) - mac(i - 1, j, k)); + hi1 = hi - + dt / (amrex::Real(3.0) * dy) * + (state(i, j + 1, k, n) * mac(i, j + 1, k) - + state(i, j, k, n) * mac(i, j, k)) + + dt / (amrex::Real(3.0) * dy) * s(i, j, k, n) * + (mac(i, j + 1, k) - mac(i, j, k)); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_zy( + amrex::Real& lo1, + amrex::Real& hi1, + int i, + int j, + int k, + int n, + amrex::Real dt, + amrex::Real dy, + bool /*iconserv*/, + amrex::Real lo, + amrex::Real hi, + amrex::Array4 const& s, + amrex::Array4 const& mac, + amrex::Array4 const& state) +{ + lo1 = lo - + dt / (amrex::Real(3.0) * dy) * + (state(i, j + 1, k - 1, n) * mac(i, j + 1, k - 1) - + state(i, j, k - 1, n) * mac(i, j, k - 1)) + + dt / (amrex::Real(3.0) * dy) * s(i, j, k - 1, n) * + (mac(i, j + 1, k - 1) - mac(i, j, k - 1)); + hi1 = hi - + dt / (amrex::Real(3.0) * dy) * + (state(i, j + 1, k, n) * mac(i, j + 1, k) - + state(i, j, k, n) * mac(i, j, k)) + + dt / (amrex::Real(3.0) * dy) * s(i, j, k, n) * + (mac(i, j + 1, k) - mac(i, j, k)); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_xz( + amrex::Real& lo1, + amrex::Real& hi1, + int i, + int j, + int k, + int n, + amrex::Real dt, + amrex::Real dz, + bool /*iconserv*/, + amrex::Real lo, + amrex::Real hi, + amrex::Array4 const& s, + amrex::Array4 const& mac, + amrex::Array4 const& state) +{ + lo1 = lo - + dt / (amrex::Real(3.0) * dz) * + (state(i - 1, j, k + 1, n) * mac(i - 1, j, k + 1) - + state(i - 1, j, k, n) * mac(i - 1, j, k)) + + dt / (amrex::Real(3.0) * dz) * s(i - 1, j, k, n) * + (mac(i - 1, j, k + 1) - mac(i - 1, j, k)); + hi1 = hi - + dt / (amrex::Real(3.0) * dz) * + (state(i, j, k + 1, n) * mac(i, j, k + 1) - + state(i, j, k, n) * mac(i, j, k)) + + dt / (amrex::Real(3.0) * dz) * s(i, j, k, n) * + (mac(i, j, k + 1) - mac(i, j, k)); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_yz( + amrex::Real& lo1, + amrex::Real& hi1, + int i, + int j, + int k, + int n, + amrex::Real dt, + amrex::Real dz, + bool /*iconserv*/, + amrex::Real lo, + amrex::Real hi, + amrex::Array4 const& s, + amrex::Array4 const& mac, + amrex::Array4 const& state) +{ + lo1 = lo - + dt / (amrex::Real(3.0) * dz) * + (state(i, j - 1, k + 1, n) * mac(i, j - 1, k + 1) - + state(i, j - 1, k, n) * mac(i, j - 1, k)) + + dt / (amrex::Real(3.0) * dz) * s(i, j - 1, k, n) * + (mac(i, j - 1, k + 1) - mac(i, j - 1, k)); + hi1 = hi - + dt / (amrex::Real(3.0) * dz) * + (state(i, j, k + 1, n) * mac(i, j, k + 1) - + state(i, j, k, n) * mac(i, j, k)) + + dt / (amrex::Real(3.0) * dz) * s(i, j, k, n) * + (mac(i, j, k + 1) - mac(i, j, k)); +} + +} // namespace +#endif + From c8b445e68d90c998d3cd6d991bfc9d5dad708db2 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 19 Apr 2024 17:03:40 -0700 Subject: [PATCH 16/83] debug build cmake config --- bctest/cmake_mpi_debug.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 bctest/cmake_mpi_debug.sh diff --git a/bctest/cmake_mpi_debug.sh b/bctest/cmake_mpi_debug.sh new file mode 100755 index 0000000000..57700d293b --- /dev/null +++ b/bctest/cmake_mpi_debug.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Example CMake config script for an OSX laptop with MPICH + +cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ + -DCMAKE_CXX_COMPILER:STRING=mpicxx \ + -DCMAKE_C_COMPILER:STRING=mpicc \ + -DCMAKE_Fortran_COMPILER:STRING=mpifort \ + -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DAMR_WIND_USE_INTERNAL_AMREX_HYDRO:BOOL=OFF \ + -DAMReX-Hydro_DIR:STRING=$HOME/AMReX-Hydro \ + -DAMR_WIND_ENABLE_MPI:BOOL=ON \ + -DAMR_WIND_ENABLE_TESTS:BOOL=OFF \ + -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ + .. && make -j8 From 1ca5a9078a9f74207d0d622b81bc8e33fb853a1a Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 19 Apr 2024 17:04:37 -0700 Subject: [PATCH 17/83] input file updates --- bctest/input_inout | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bctest/input_inout b/bctest/input_inout index d7949afd88..021245be95 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -3,13 +3,13 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 2.00 # Max (simulated) time to evolve +time.stop_time = 5.00 # Max (simulated) time to evolve time.max_step = 1 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # #.......................................# -time.fixed_dt = 0.02 # Use this constant dt if > 0 +time.fixed_dt = 0.01 #0.02 # Use this constant dt if > 0 time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -46,6 +46,7 @@ io.output_default_variables = 1 amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel amr.blocking_factor = 4 amr.max_level = 0 # Max AMR level in hierarchy +fabarray.mfiter_tile_size = 1024 1024 1024 #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # GEOMETRY # From da539802b8df327abc6723535b00f0ee9208f323 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 19 Apr 2024 17:08:02 -0700 Subject: [PATCH 18/83] playing around with if-else on zero velocity values at the bndry --- amr-wind/convection/incflo_godunov_ppm.H | 27 +++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index 7171a9624a..3c3cbc22f7 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -30,8 +30,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( if (i <= domlo) { if (bclo == BCType::ext_dir || (bclo == BCType::user_1 && uedge > 0.0)) { - Print() << "trans_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n - << " as uedge is " << uedge << std::endl; + //Print() << "trans_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n + // << " as uedge is " << uedge << std::endl; // IAMR does this but it breaks lo/hi symmetry // Real st = (uedge <= small_vel) ? hi : s(domlo-1,j,k,n); // So here we do something simpler... @@ -42,16 +42,13 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || (bclo == BCType::user_1 && uedge <= 0.0)) { - Print() << "trans_xbc_domlo_foextrap triggered at " << i << " " << j << " " << k << " " << n - << " as uedge is " << uedge << std::endl; + //Print() << "trans_xbc_domlo_foextrap triggered at " << i << " " << j << " " << k << " " << n + // << " as uedge is " << uedge << std::endl; lo = hi; } else if (bclo == BCType::reflect_odd) { hi = 0.; lo = 0.; - } else { - Print() << "trans_xbc_domlo_nothing triggered at " << i << " " << j << " " << k << " " << n - << " as uedge is " << uedge << std::endl; } } @@ -59,8 +56,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( if (i > domhi) { if (bchi == BCType::ext_dir || (bchi == BCType::user_1 && uedge < 0.0)) { // is this correct use of uedge here?? - Print() << "trans_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n - << " as uedge is " << uedge << std::endl; + //Print() << "trans_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n + // << " as uedge is " << uedge << std::endl; // IAMR does this but it breaks lo/hi symmetry // Real st = (uedge >= -small_vel)? lo : s(domhi+1,j,k,n); // So here we do something simpler... @@ -71,8 +68,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || (bchi == BCType::user_1 && uedge >= 0.0)) { - Print() << "trans_xbc_domhi_foextrap triggered at " << i << " " << j << " " << k << " " << n - << " as uedge is " << uedge << std::endl; + //Print() << "trans_xbc_domhi_foextrap triggered at " << i << " " << j << " " << k << " " << n + // << " as uedge is " << uedge << std::endl; hi = lo; } else if (bchi == BCType::reflect_odd) { @@ -218,14 +215,14 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( if (i == domlo) { if ((bclo == BCType::ext_dir && uedge(i, j, k) >= 0.) || - (bclo == BCType::user_1 && uedge(i, j, k) >= 0.)) { + (bclo == BCType::user_1 && uedge(i, j, k) > 0.)) { hi = s(domlo - 1, j, k, n); lo = hi; } else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || (bclo == BCType::ext_dir && uedge(i, j, k) < 0) || - (bclo == BCType::user_1 && uedge(i, j, k) < 0)) { + (bclo == BCType::user_1 && uedge(i, j, k) <= 0)) { lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -253,14 +250,14 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( if (i == domhi + 1) { if ((bchi == BCType::ext_dir && uedge(i, j, k) <= 0.) || - (bchi == BCType::user_1 && uedge(i, j, k) <= 0.)) { + (bchi == BCType::user_1 && uedge(i, j, k) < 0.)) { lo = s(domhi + 1, j, k, n); hi = lo; } else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || (bchi == BCType::ext_dir && uedge(i, j, k) > 0) || - (bchi == BCType::user_1 && uedge(i, j, k) > 0)) { + (bchi == BCType::user_1 && uedge(i, j, k) >= 0)) { hi = lo; } else if (bchi == BCType::reflect_odd) { From 99f4cf8a1b8cc222f1444ce52fce9cb996a45706 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 19 Apr 2024 17:09:01 -0700 Subject: [PATCH 19/83] activating enforce solvability routine --- amr-wind/equation_systems/icns/icns_advection.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 06961bd3c2..1fcc66921a 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -71,7 +71,7 @@ void MacProjOp::enforce_solvability ( auto& velocity = m_repo.get_field("velocity"); amrex::BCRec const* bc_type = velocity.bcrec_device().data(); const amrex::Vector& geom = m_repo.mesh().Geom(); - + amrex::Print() << "Calling enforceSolvability from AMR-Wind" << std::endl; m_mac_proj->enforceSolvability(a_umac, bc_type, geom); } @@ -277,7 +277,7 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } - //enforce_solvability(mac_vec); + enforce_solvability(mac_vec); m_mac_proj->setUMAC(mac_vec); if (m_has_overset) { From 3c88c9d98fd4e647e5cd62e6718dc8c86b569ee8 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 23 Apr 2024 15:24:10 -0700 Subject: [PATCH 20/83] redefining ext_dir and foextrap conditions for the in-out BC --- amr-wind/convection/incflo_godunov_ppm.H | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index 3c3cbc22f7..ddb72efeee 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -29,7 +29,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( // Low X if (i <= domlo) { if (bclo == BCType::ext_dir || - (bclo == BCType::user_1 && uedge > 0.0)) { + (bclo == BCType::user_1 && uedge >= 0.0)) { //Print() << "trans_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << uedge << std::endl; // IAMR does this but it breaks lo/hi symmetry @@ -41,7 +41,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( } else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || - (bclo == BCType::user_1 && uedge <= 0.0)) { + (bclo == BCType::user_1 && uedge < 0.0)) { //Print() << "trans_xbc_domlo_foextrap triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << uedge << std::endl; lo = hi; @@ -55,7 +55,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( // High X if (i > domhi) { if (bchi == BCType::ext_dir || - (bchi == BCType::user_1 && uedge < 0.0)) { // is this correct use of uedge here?? + (bchi == BCType::user_1 && uedge <= 0.0)) { // is this correct use of uedge here?? //Print() << "trans_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << uedge << std::endl; // IAMR does this but it breaks lo/hi symmetry @@ -67,7 +67,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( } else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || - (bchi == BCType::user_1 && uedge >= 0.0)) { + (bchi == BCType::user_1 && uedge > 0.0)) { //Print() << "trans_xbc_domhi_foextrap triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << uedge << std::endl; hi = lo; @@ -215,14 +215,14 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( if (i == domlo) { if ((bclo == BCType::ext_dir && uedge(i, j, k) >= 0.) || - (bclo == BCType::user_1 && uedge(i, j, k) > 0.)) { + (bclo == BCType::user_1 && uedge(i, j, k) >= 0.)) { hi = s(domlo - 1, j, k, n); lo = hi; } else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || (bclo == BCType::ext_dir && uedge(i, j, k) < 0) || - (bclo == BCType::user_1 && uedge(i, j, k) <= 0)) { + (bclo == BCType::user_1 && uedge(i, j, k) < 0)) { lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -250,14 +250,14 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( if (i == domhi + 1) { if ((bchi == BCType::ext_dir && uedge(i, j, k) <= 0.) || - (bchi == BCType::user_1 && uedge(i, j, k) < 0.)) { + (bchi == BCType::user_1 && uedge(i, j, k) <= 0.)) { lo = s(domhi + 1, j, k, n); hi = lo; } else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || (bchi == BCType::ext_dir && uedge(i, j, k) > 0) || - (bchi == BCType::user_1 && uedge(i, j, k) >= 0)) { + (bchi == BCType::user_1 && uedge(i, j, k) > 0)) { hi = lo; } else if (bchi == BCType::reflect_odd) { From 7b0d5d9e3a7d9b0f5039b6703f9124f859f1fadf Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 23 Apr 2024 15:26:17 -0700 Subject: [PATCH 21/83] using physical custom velocity BCs instead of garbage --- amr-wind/physics/udfs/CustomVelocity.H | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/amr-wind/physics/udfs/CustomVelocity.H b/amr-wind/physics/udfs/CustomVelocity.H index 748377d98d..a4dfbddf8b 100644 --- a/amr-wind/physics/udfs/CustomVelocity.H +++ b/amr-wind/physics/udfs/CustomVelocity.H @@ -48,14 +48,14 @@ struct CustomVelocity amrex::GpuArray vel; if (x <= 0.5) { // at the low-x boundary, +ve vel in top portion, garbage in the rest - vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; + //vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; //vel = {((z >= 0.5) ? pvel : 0.0), 0.0, 0.0}; - //vel = {((z >= 0.5) ? pvel : -1.0), 0.0, 0.0}; + vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; } else { // at the high-x boundary, -ve vel in bottom portion, garbage in the rest - vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; + //vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; //vel = {((z < 0.5) ? mvel : 0.0), 0.0, 0.0}; - //vel = {((z < 0.5) ? mvel : 1.0), 0.0, 0.0}; + vel = {((z < 0.5) ? mvel : pvel), 0.0, 0.0}; } field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; From 9e798e6a855705dc3c7e37a93bff730226a98c0c Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 2 May 2024 16:59:15 -0700 Subject: [PATCH 22/83] printing macvel and velocity plotfiles for debugging --- amr-wind/equation_systems/icns/icns_advection.cpp | 13 +++++++++++++ .../projection/incflo_apply_nodal_projection.cpp | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 1fcc66921a..30ab47ef7c 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -8,6 +8,10 @@ #include "AMReX_MultiFabUtil.H" #include "hydro_MacProjector.H" +#include "AMReX_PlotFileUtil.H" +#include "AMReX_MultiFabUtil.H" +#include "AMReX_MultiFab.H" + namespace amr_wind::pde { namespace { @@ -277,7 +281,14 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } +amrex::Array mac_arr = {&u_mac(0), &v_mac(0), &w_mac(0)}; +amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); + +amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); enforce_solvability(mac_vec); +amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); m_mac_proj->setUMAC(mac_vec); if (m_has_overset) { @@ -293,6 +304,8 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } else { m_mac_proj->project(m_options.rel_tol, m_options.abs_tol); } +amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); if (m_is_anelastic) { for (int lev = 0; lev < m_repo.num_active_levels(); ++lev) { diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 30885e8354..c156ed03b6 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -8,6 +8,7 @@ #include "amr-wind/utilities/IOManager.H" +#include "AMReX_PlotFileUtil.H" using namespace amrex; @@ -163,6 +164,7 @@ void incflo::ApplyProjection( auto& grad_p = m_repo.get_field("gp"); auto& pressure = m_repo.get_field("p"); auto& velocity = icns().fields().field; +amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); auto& velocity_old = icns().fields().field.state(amr_wind::FieldState::Old); amr_wind::Field const* mesh_fac = mesh_mapping @@ -469,6 +471,8 @@ void incflo::ApplyProjection( } } +amrex::WriteSingleLevelPlotfile("plt_vel_post_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); + // Get phi and fluxes auto phi = nodal_projector->getPhi(); auto gradphi = nodal_projector->getGradPhi(); From 6a219221cca7bea26e277cb69bc6f7a31802c3c0 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 6 May 2024 18:17:40 -0700 Subject: [PATCH 23/83] more print messages for debugging and changes in Godunov_trans_xbc and rebasing bug fix --- .../convection/incflo_godunov_advection.cpp | 6 ++-- amr-wind/convection/incflo_godunov_ppm.H | 29 ++++++++++++------- .../convection/incflo_godunov_predict.cpp | 13 +++++---- amr-wind/convection/incflo_godunov_weno.H | 11 ++++++- 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp index 52471e3e7f..8ff7822a87 100644 --- a/amr-wind/convection/incflo_godunov_advection.cpp +++ b/amr-wind/convection/incflo_godunov_advection.cpp @@ -145,7 +145,7 @@ void godunov::compute_fluxes( auto bc = pbc[n]; Godunov_trans_xbc( - i, j, k, n, q, lo, hi, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); xlo(i, j, k, n) = lo; xhi(i, j, k, n) = hi; Real st = (uval) ? lo : hi; @@ -346,7 +346,7 @@ if ((i==0) && (j==0) && (k==0)) Real uad = umac(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xzlo, l_xzhi, uad, bc.lo(0), bc.hi(0), dlo.x, + i, j, k, n, q, l_xzlo, l_xzhi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); constexpr Real small_vel = 1.e-8; @@ -457,7 +457,7 @@ if ((i==0) && (j==0) && (k==0)) Real uad = umac(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xylo, l_xyhi, uad, bc.lo(0), bc.hi(0), dlo.x, + i, j, k, n, q, l_xylo, l_xyhi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); constexpr Real small_vel = 1.e-8; diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index ddb72efeee..8db512fd72 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -18,20 +18,21 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( const amrex::Array4& s, amrex::Real& lo, amrex::Real& hi, - amrex::Real& uedge, + amrex::Real& velm, + amrex::Real& velp, const int bclo, const int bchi, const int domlo, const int domhi) { using namespace amrex; - +// is the "inverse" use of velm and velp justified? // Low X if (i <= domlo) { if (bclo == BCType::ext_dir || - (bclo == BCType::user_1 && uedge >= 0.0)) { + (bclo == BCType::user_1 && velp >= 0.0)) { //Print() << "trans_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << uedge << std::endl; + // << " as uedge is " << velp << std::endl; // IAMR does this but it breaks lo/hi symmetry // Real st = (uedge <= small_vel) ? hi : s(domlo-1,j,k,n); // So here we do something simpler... @@ -41,9 +42,9 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( } else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || - (bclo == BCType::user_1 && uedge < 0.0)) { + (bclo == BCType::user_1 && velp < 0.0)) { //Print() << "trans_xbc_domlo_foextrap triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << uedge << std::endl; + // << " as uedge is " << velp << std::endl; lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -55,9 +56,9 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( // High X if (i > domhi) { if (bchi == BCType::ext_dir || - (bchi == BCType::user_1 && uedge <= 0.0)) { // is this correct use of uedge here?? + (bchi == BCType::user_1 && velm <= 0.0)) { // is this correct use of uedge here?? //Print() << "trans_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << uedge << std::endl; + // << " as uedge is " << velm << std::endl; // IAMR does this but it breaks lo/hi symmetry // Real st = (uedge >= -small_vel)? lo : s(domhi+1,j,k,n); // So here we do something simpler... @@ -67,9 +68,9 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( } else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || - (bchi == BCType::user_1 && uedge > 0.0)) { + (bchi == BCType::user_1 && velm > 0.0)) { //Print() << "trans_xbc_domhi_foextrap triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << uedge << std::endl; + // << " as uedge is " << velm << std::endl; hi = lo; } else if (bchi == BCType::reflect_odd) { @@ -216,6 +217,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( if (i == domlo) { if ((bclo == BCType::ext_dir && uedge(i, j, k) >= 0.) || (bclo == BCType::user_1 && uedge(i, j, k) >= 0.)) { +//Print() << "cc_xbc_lo_extdir triggered at " << i << " " << j << " " << k << " " << n +// << " as uedge is " << uedge(i,j,k) << std::endl; hi = s(domlo - 1, j, k, n); lo = hi; } else if ( @@ -223,6 +226,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( bclo == BCType::reflect_even || (bclo == BCType::ext_dir && uedge(i, j, k) < 0) || (bclo == BCType::user_1 && uedge(i, j, k) < 0)) { +//Print() << "cc_xbc_lo_foextrap triggered at " << i << " " << j << " " << k << " " << n +// << " as uedge is " << uedge(i,j,k) << std::endl; lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -251,6 +256,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( if (i == domhi + 1) { if ((bchi == BCType::ext_dir && uedge(i, j, k) <= 0.) || (bchi == BCType::user_1 && uedge(i, j, k) <= 0.)) { +//Print() << "cc_xbc_hi_extdir triggered at " << i << " " << j << " " << k << " " << n +// << " as uedge is " << uedge(i,j,k) << std::endl; lo = s(domhi + 1, j, k, n); hi = lo; } else if ( @@ -258,6 +265,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( bchi == BCType::reflect_even || (bchi == BCType::ext_dir && uedge(i, j, k) > 0) || (bchi == BCType::user_1 && uedge(i, j, k) > 0)) { +//Print() << "cc_xbc_hi_foextrap triggered at " << i << " " << j << " " << k << " " << n +// << " as uedge is " << uedge(i,j,k) << std::endl; hi = lo; } else if (bchi == BCType::reflect_odd) { diff --git a/amr-wind/convection/incflo_godunov_predict.cpp b/amr-wind/convection/incflo_godunov_predict.cpp index c5bd10dc30..c8eda87d26 100644 --- a/amr-wind/convection/incflo_godunov_predict.cpp +++ b/amr-wind/convection/incflo_godunov_predict.cpp @@ -50,10 +50,13 @@ void godunov::make_trans_velocities( lo = Ipx(i - 1, j, k, n); hi = Imx(i, j, k, n); } - +/*if (((i==0) || (i==16)) && (j==3) && ((k==1) || (k==14))) { +Print() << "lo hi at " << i << " " << j << " " << k << " " << n << " " + << lo << " " << hi << std::endl; +}*/ auto bc = pbc[n]; Godunov_trans_xbc( - i, j, k, n, vel, lo, hi, lo, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + i, j, k, n, vel, lo, hi, lo, hi, bc.lo(0), bc.hi(0), dlo.x, dhi.x); constexpr Real small_vel = 1e-8; @@ -184,7 +187,7 @@ void godunov::predict_godunov( auto bc = pbc[n]; Godunov_trans_xbc( - i, j, k, n, q, lo, hi, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); xlo(i, j, k, n) = lo; xhi(i, j, k, n) = hi; @@ -366,7 +369,7 @@ if ((i==0) && (j==0) && (k==0)) Real uad = u_ad(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xzlo, l_xzhi, uad, bc.lo(0), bc.hi(0), dlo.x, + i, j, k, n, q, l_xzlo, l_xzhi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); constexpr Real small_vel = 1.e-8; @@ -450,7 +453,7 @@ if ((i==0) && (j==0) && (k==0)) Real uad = u_ad(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xylo, l_xyhi, uad, bc.lo(0), bc.hi(0), dlo.x, + i, j, k, n, q, l_xylo, l_xyhi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); constexpr Real small_vel = 1.e-8; diff --git a/amr-wind/convection/incflo_godunov_weno.H b/amr-wind/convection/incflo_godunov_weno.H index 576bac7a45..fcfe7ccc72 100644 --- a/amr-wind/convection/incflo_godunov_weno.H +++ b/amr-wind/convection/incflo_godunov_weno.H @@ -74,13 +74,16 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || (bclo == BCType::user_1 && velm >= 0.0)) { if (i == domlo) { +//Print() << "weno_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n +// << " as velm is " << velm << std::endl; sp = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); sm = s(domlo - 1, j, k, n); } else if (i == domlo + 1) { - +//Print() << "weno_xbc_domlop1_extdir triggered at " << i << " " << j << " " << k << " " << n +// << " as velm is " << velm << std::endl; sm = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); @@ -91,6 +94,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || (bchi == BCType::user_1 && velp <= 0.0)) { if (i == domhi) { +//Print() << "weno_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n +// << " as velp is " << velp << std::endl; sm = -0.2 * s(domhi + 1, j, k, n) + 0.75 * s(domhi, j, k, n) + 0.5 * s(domhi - 1, j, k, n) - 0.05 * s(domhi - 2, j, k, n); @@ -272,6 +277,10 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_x( Ip(i, j, k, n) = S(i, j, k, n); Im(i, j, k, n) = S(i, j, k, n); } +/*if (((i==0) || (i==15)) && (j==3) && ((k==1) || (k==14))) { +Print() << "Im Ip at " << i << " " << j << " " << k << " " << n << " " + << Im(i, j, k, n) << " " << Ip(i, j, k, n) << std::endl; +}*/ } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_y( From 82ccb5bf3b35a4fc3b5cf46632a6da54f84d7be2 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 13 May 2024 11:32:39 -0700 Subject: [PATCH 24/83] enabling Dirichlet operations for the in-out BC --- amr-wind/core/FieldBCOps.H | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/amr-wind/core/FieldBCOps.H b/amr-wind/core/FieldBCOps.H index bdd2f43121..b3d315fc21 100644 --- a/amr-wind/core/FieldBCOps.H +++ b/amr-wind/core/FieldBCOps.H @@ -166,9 +166,10 @@ struct DirichletOp auto ori = oit(); const int idir = ori.coordDir(); - // Check if this boundary is a dirichlet type + // Check if this boundary is a dirichlet or mixed type const auto bctyp = (ori.isLow() ? bc.lo(idir) : bc.hi(idir)); - if (bctyp != amrex::BCType::ext_dir) { + if ((bctyp != amrex::BCType::ext_dir) + && (bctyp != amrex::BCType::user_1)) { continue; } @@ -182,7 +183,8 @@ struct DirichletOp continue; } - if (m_bc_type[ori] == BC::mass_inflow) { + if ((m_bc_type[ori] == BC::mass_inflow) + || (m_bc_type[ori] == BC::mass_inflow_outflow)) { m_inflow_op( iv, field, geom, time, ori, n, dcomp, orig_comp); } else { From 3f894d56b56c3078318633454e8c77ad591bf970 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 13 May 2024 11:33:25 -0700 Subject: [PATCH 25/83] not doing the custom Neumann fills to corner cells --- amr-wind/boundary_conditions/MassInflowOutflowBC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index a537b2bfed..c23dd97290 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -37,7 +37,7 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st #endif for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { auto bx = mfi.validbox(); - bx.grow({!ib, !jb, !kb}); + //bx.grow({!ib, !jb, !kb}); // how to manage corner cells?? const auto& bc_a = m_field(lev).array(mfi); const auto& vel = velocity(lev).array(mfi); From cd765bcee94b42af3341c39e454e1ea58ed0e2fe Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 13 May 2024 11:38:00 -0700 Subject: [PATCH 26/83] input fileupdates --- bctest/input_inflow | 21 +++++++++++++-------- bctest/input_inout | 19 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/bctest/input_inflow b/bctest/input_inflow index bdc43e3605..c6032c074d 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -3,19 +3,19 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 2.00 # Max (simulated) time to evolve -time.max_step = -1000 # Max number of time steps +time.stop_time = 5.00 # Max (simulated) time to evolve +time.max_step = 0 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # #.......................................# -time.fixed_dt = 0.02 # Use this constant dt if > 0 -time.cfl = 0.95 # CFL factor +time.fixed_dt = 0.03 # Use this constant dt if > 0 +time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 100 # Steps between plot files +time.plot_interval = 1 #100 # Steps between plot files time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -26,14 +26,18 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" +incflo.initial_iterations = 1 +#incflo.do_initial_proj = false +incflo.verbose = 3 + transport.viscosity = 0.0 turbulence.model = Laminar ICNS.source_terms = BodyForce BodyForce.magnitude = 0 0 0 -incflo.physics = ChannelFlow -ChannelFlow.density = 1.0 -ChannelFlow.Mean_Velocity = 0.0 + +incflo.physics = FreeStream +FreeStream.velocity_type = CustomVelocity io.output_default_variables = 1 @@ -43,6 +47,7 @@ io.output_default_variables = 1 amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel amr.blocking_factor = 4 amr.max_level = 0 # Max AMR level in hierarchy +fabarray.mfiter_tile_size = 1024 1024 1024 #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # GEOMETRY # diff --git a/bctest/input_inout b/bctest/input_inout index 021245be95..6916969a78 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -4,19 +4,19 @@ # SIMULATION STOP # #.......................................# time.stop_time = 5.00 # Max (simulated) time to evolve -time.max_step = 1 #-1000 # Max number of time steps +time.max_step = 0 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # #.......................................# -time.fixed_dt = 0.01 #0.02 # Use this constant dt if > 0 -time.cfl = 0.95 # CFL factor +time.fixed_dt = 0.03 # Use this constant dt if > 0 +time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 100 # Steps between plot files -time.checkpoint_interval = -100 # Steps between checkpoint files +time.plot_interval = 1 # 100 # Steps between plot files +time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # PHYSICS # @@ -26,17 +26,18 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" -#incflo.initial_iterations = 0 +incflo.initial_iterations = 1 #incflo.do_initial_proj = false +incflo.verbose = 3 transport.viscosity = 0.0 turbulence.model = Laminar ICNS.source_terms = BodyForce BodyForce.magnitude = 0 0 0 -incflo.physics = ChannelFlow -ChannelFlow.density = 1.0 -ChannelFlow.Mean_Velocity = 0.0 + +incflo.physics = FreeStream +FreeStream.velocity_type = CustomVelocity io.output_default_variables = 1 From 6fedafa2b5b9e36862d499e8217dad8f683ebe2e Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 15 May 2024 09:33:04 -0700 Subject: [PATCH 27/83] adding back the Neumann fills for corner cells --- amr-wind/boundary_conditions/MassInflowOutflowBC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index c23dd97290..edf46796ec 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -37,7 +37,7 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st #endif for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { auto bx = mfi.validbox(); - //bx.grow({!ib, !jb, !kb}); // how to manage corner cells?? + bx.grow({!ib, !jb, !kb}); // how to manage corner cells?? const auto& bc_a = m_field(lev).array(mfi); const auto& vel = velocity(lev).array(mfi); From cda21b3e375825ab28ac90c763d9c2d63ee4e5a5 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 15 May 2024 09:40:24 -0700 Subject: [PATCH 28/83] clean up, adding/removing print statements --- amr-wind/equation_systems/icns/icns_advection.cpp | 6 +++--- amr-wind/physics/udfs/CustomVelocity.H | 15 ++++++++++----- .../projection/incflo_apply_nodal_projection.cpp | 7 +++++-- bctest/input_inflow | 10 +++++----- bctest/input_inout | 8 ++++---- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 30ab47ef7c..2530a07469 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -285,10 +285,10 @@ amrex::Array mac_arr = {&u_mac(0), &v_ma amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +//amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); enforce_solvability(mac_vec); amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +//amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); m_mac_proj->setUMAC(mac_vec); if (m_has_overset) { @@ -305,7 +305,7 @@ amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","v m_mac_proj->project(m_options.rel_tol, m_options.abs_tol); } amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +//amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); if (m_is_anelastic) { for (int lev = 0; lev < m_repo.num_active_levels(); ++lev) { diff --git a/amr-wind/physics/udfs/CustomVelocity.H b/amr-wind/physics/udfs/CustomVelocity.H index a4dfbddf8b..cc9a4e3814 100644 --- a/amr-wind/physics/udfs/CustomVelocity.H +++ b/amr-wind/physics/udfs/CustomVelocity.H @@ -17,7 +17,7 @@ struct CustomVelocity // clang-format off // Declare parameters here if needed. For example: // amrex::Real foo{1.0}; - amrex::Real pvel{1.0}; amrex::Real mvel{0.5}; + amrex::Real pvel{0.0}; amrex::Real mvel{0.0}; // amrex::GpuArray bar = {0.0}; // clang-format on @@ -33,8 +33,9 @@ struct CustomVelocity const int orig_comp) const { if ((iv[0] == -1) && (iv[1] == -1) && (iv[2] == 0)) - amrex::Print() << "****** filling custom velocity at " << iv - << " and at other ghost cells... " << std::endl; + amrex::Print() << "****** filling custom velocity at " << iv << std::endl; + if ((iv[0] == 1) && (iv[1] == 1) && (iv[2] == 1)) + amrex::Print() << "****** filling custom velocity at " << iv << std::endl; // Compute quantities to set the field values. For example: // clang-format off const auto* problo = geom.ProbLo(); @@ -46,6 +47,11 @@ struct CustomVelocity // Once the above is done, fill the field as: amrex::GpuArray vel; + vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; + //if (time == 0.0) vel[0] *= 0.1; + //amrex::Print() << "!!!! time is " << time << " and vel is " << vel[0] << std::endl; + field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; + /* if (x <= 0.5) { // at the low-x boundary, +ve vel in top portion, garbage in the rest //vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; @@ -56,9 +62,8 @@ struct CustomVelocity //vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; //vel = {((z < 0.5) ? mvel : 0.0), 0.0, 0.0}; vel = {((z < 0.5) ? mvel : pvel), 0.0, 0.0}; - } + }*/ - field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; // clang-format on } }; diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index c156ed03b6..761bdfd8eb 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -164,7 +164,7 @@ void incflo::ApplyProjection( auto& grad_p = m_repo.get_field("gp"); auto& pressure = m_repo.get_field("p"); auto& velocity = icns().fields().field; -amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); +//amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); auto& velocity_old = icns().fields().field.state(amr_wind::FieldState::Old); amr_wind::Field const* mesh_fac = mesh_mapping @@ -337,6 +337,8 @@ amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v"," m_sim.physics_manager(), velocity, lev, time, *vel[lev], 1); } } +//amrex::Print() << "*********** partition **********" << std::endl; +//amrex::Print() << velocity(0)[0]; // need to apply custom Neumann funcs for inflow-outflow BC // after setting the inflow vels above @@ -344,6 +346,7 @@ amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v"," velocity.apply_bc_funcs(amr_wind::FieldState::New); } +//amrex::Print() << velocity(0)[0]; if (is_anelastic) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -471,7 +474,7 @@ amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v"," } } -amrex::WriteSingleLevelPlotfile("plt_vel_post_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); +//amrex::WriteSingleLevelPlotfile("plt_vel_post_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); // Get phi and fluxes auto phi = nodal_projector->getPhi(); diff --git a/bctest/input_inflow b/bctest/input_inflow index c6032c074d..c55cb2ad5d 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -3,20 +3,20 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 5.00 # Max (simulated) time to evolve -time.max_step = 0 #-1000 # Max number of time steps +time.stop_time = 2.00 # Max (simulated) time to evolve +time.max_step = 1 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # #.......................................# -time.fixed_dt = 0.03 # Use this constant dt if > 0 +time.fixed_dt = 0.02 # Use this constant dt if > 0 time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 1 #100 # Steps between plot files -time.checkpoint_interval = -100 # Steps between checkpoint files +time.plot_interval = 100 # Steps between plot files +time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # PHYSICS # diff --git a/bctest/input_inout b/bctest/input_inout index 6916969a78..f3f5f205ab 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -3,19 +3,19 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 5.00 # Max (simulated) time to evolve -time.max_step = 0 #-1000 # Max number of time steps +time.stop_time = 2.00 # Max (simulated) time to evolve +time.max_step = -1 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # #.......................................# -time.fixed_dt = 0.03 # Use this constant dt if > 0 +time.fixed_dt = 0.02 # Use this constant dt if > 0 time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 1 # 100 # Steps between plot files +time.plot_interval = 100 # Steps between plot files time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# From 05a0207efc747afa3e5de1a0cefb7221d67d0e87 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 17 May 2024 14:12:44 -0700 Subject: [PATCH 29/83] clean up gitignore --- .gitignore | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 1660ba99ed..41e9483250 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,17 @@ exec/*.ex exec/tmp_build_dir/ exec/Backtrace.* +exec/plt*/ +exec/chk*/ exec/movie.visit exec/*.vtp exec/*.pvtp test/*.ex test/tmp_build_dir/ test/Backtrace.* +test/*plt*/ +test/chk*/ test/AMR-WindGoldFiles -*plt*/ -chk*/ .DS_store spack* .clangd From f28f80f943802eda42d2e97d0ba7a7dce20e9a36 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 17 May 2024 16:10:57 -0700 Subject: [PATCH 30/83] define new UDF TwoLayer and restore CustomVelocity for PR --- amr-wind/boundary_conditions/velocity_bcs.H | 7 +- amr-wind/physics/udfs/CMakeLists.txt | 1 + amr-wind/physics/udfs/CustomVelocity.H | 48 ++++-------- amr-wind/physics/udfs/CustomVelocity.cpp | 29 ++++--- amr-wind/physics/udfs/TwoLayer.H | 83 +++++++++++++++++++++ amr-wind/physics/udfs/TwoLayer.cpp | 38 ++++++++++ amr-wind/physics/udfs/UDF.cpp | 2 + bctest/input_inflow | 10 +-- bctest/input_inout | 10 +-- 9 files changed, 167 insertions(+), 61 deletions(-) create mode 100644 amr-wind/physics/udfs/TwoLayer.H create mode 100644 amr-wind/physics/udfs/TwoLayer.cpp diff --git a/amr-wind/boundary_conditions/velocity_bcs.H b/amr-wind/boundary_conditions/velocity_bcs.H index 4d7595890f..1c88ca176e 100644 --- a/amr-wind/boundary_conditions/velocity_bcs.H +++ b/amr-wind/boundary_conditions/velocity_bcs.H @@ -10,6 +10,7 @@ #include "amr-wind/physics/udfs/BurggrafLid.H" #include "amr-wind/physics/udfs/Rankine.H" #include "amr-wind/physics/udfs/CustomVelocity.H" +#include "amr-wind/physics/udfs/TwoLayer.H" namespace amr_wind::vel_bc { @@ -38,7 +39,11 @@ void register_inflow_vel_dirichlet( mesh, time, InflowOp(field)); } else if (inflow_udf == "CustomVelocity") { using InflowOp = BCOpCreator; - amrex::Print() << "******* registering CustomVelocity" << std::endl; + field.register_fill_patch_op>( + mesh, time, InflowOp(field)); + } else if (inflow_udf == "TwoLayer") { + using InflowOp = BCOpCreator; + amrex::Print() << "***** Registering TwoLayer UDF" << std::endl; field.register_fill_patch_op>( mesh, time, InflowOp(field)); } else { diff --git a/amr-wind/physics/udfs/CMakeLists.txt b/amr-wind/physics/udfs/CMakeLists.txt index f29413ba3b..1ab72fd445 100644 --- a/amr-wind/physics/udfs/CMakeLists.txt +++ b/amr-wind/physics/udfs/CMakeLists.txt @@ -8,4 +8,5 @@ target_sources(${amr_wind_lib_name} Rankine.cpp CustomVelocity.cpp CustomScalar.cpp + TwoLayer.cpp ) diff --git a/amr-wind/physics/udfs/CustomVelocity.H b/amr-wind/physics/udfs/CustomVelocity.H index cc9a4e3814..b1f7fe014f 100644 --- a/amr-wind/physics/udfs/CustomVelocity.H +++ b/amr-wind/physics/udfs/CustomVelocity.H @@ -17,53 +17,31 @@ struct CustomVelocity // clang-format off // Declare parameters here if needed. For example: // amrex::Real foo{1.0}; - amrex::Real pvel{0.0}; amrex::Real mvel{0.0}; // amrex::GpuArray bar = {0.0}; // clang-format on AMREX_GPU_DEVICE inline void operator()( - const amrex::IntVect& iv, - amrex::Array4 const& field, - amrex::GeometryData const& geom, - const amrex::Real time, - amrex::Orientation ori, - const int comp, - const int dcomp, - const int orig_comp) const + const amrex::IntVect& /*iv*/, + amrex::Array4 const& /*field*/, + amrex::GeometryData const& /*geom*/, + const amrex::Real /*time*/, + amrex::Orientation /*ori*/, + const int /*comp*/, + const int /*dcomp*/, + const int /*orig_comp*/) const { - if ((iv[0] == -1) && (iv[1] == -1) && (iv[2] == 0)) - amrex::Print() << "****** filling custom velocity at " << iv << std::endl; - if ((iv[0] == 1) && (iv[1] == 1) && (iv[2] == 1)) - amrex::Print() << "****** filling custom velocity at " << iv << std::endl; // Compute quantities to set the field values. For example: // clang-format off - const auto* problo = geom.ProbLo(); - const auto* dx = geom.CellSize(); - const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; + // const auto* problo = geom.ProbLo(); + // const auto* dx = geom.CellSize(); + // const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; // const auto y = problo[1] + (iv[1] + 0.5) * dx[1]; - const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; + // const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; // const amrex::GpuArray vel = {1.0, 0.0, 0.0}; // Once the above is done, fill the field as: - amrex::GpuArray vel; - vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; - //if (time == 0.0) vel[0] *= 0.1; - //amrex::Print() << "!!!! time is " << time << " and vel is " << vel[0] << std::endl; - field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; - /* - if (x <= 0.5) { - // at the low-x boundary, +ve vel in top portion, garbage in the rest - //vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; - //vel = {((z >= 0.5) ? pvel : 0.0), 0.0, 0.0}; - vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; - } else { - // at the high-x boundary, -ve vel in bottom portion, garbage in the rest - //vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; - //vel = {((z < 0.5) ? mvel : 0.0), 0.0, 0.0}; - vel = {((z < 0.5) ? mvel : pvel), 0.0, 0.0}; - }*/ - + // field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; // clang-format on } }; diff --git a/amr-wind/physics/udfs/CustomVelocity.cpp b/amr-wind/physics/udfs/CustomVelocity.cpp index b7a2dbeed0..0aad96ba76 100644 --- a/amr-wind/physics/udfs/CustomVelocity.cpp +++ b/amr-wind/physics/udfs/CustomVelocity.cpp @@ -17,22 +17,21 @@ CustomVelocity::CustomVelocity(const Field& /*fld*/) // CustomVelocity.foo = 1.0 // clang-format off - { - //const int ncomp = fld.num_comp(); - amrex::ParmParse pp("CustomVelocity"); - pp.query("pvel", m_op.pvel); - pp.query("mvel", m_op.mvel); - //amrex::Vector vel(0.0, ncomp); - //pp.getarr("velocity", vel); - //AMREX_ALWAYS_ASSERT(vel.size() == ncomp); - //for (int i = 0; i < ncomp; ++i) { - // m_op.bar[i] = vel[i]; - //} - } + //{ + // const int ncomp = fld.num_comp(); + // amrex::ParmParse pp("CustomVelocity"); + // pp.query("foo", m_op.foo); + // amrex::Vector vel(0.0, ncomp); + // pp.getarr("velocity", vel); + // AMREX_ALWAYS_ASSERT(vel.size() == ncomp); + // for (int i = 0; i < ncomp; ++i) { + // m_op.bar[i] = vel[i]; + // } + //} // clang-format on - //amrex::Abort( - // "Please define the body of this function and the corresponding struct " - // "in the header file before using it. Then remove this message"); + amrex::Abort( + "Please define the body of this function and the corresponding struct " + "in the header file before using it. Then remove this message"); } } // namespace amr_wind::udf diff --git a/amr-wind/physics/udfs/TwoLayer.H b/amr-wind/physics/udfs/TwoLayer.H new file mode 100644 index 0000000000..32ff8e6d6f --- /dev/null +++ b/amr-wind/physics/udfs/TwoLayer.H @@ -0,0 +1,83 @@ +#ifndef TWO_LAYER_H +#define TWO_LAYER_H + +#include "AMReX_Geometry.H" +#include "AMReX_Gpu.H" + +namespace amr_wind { + +class Field; + +namespace udf { + +struct TwoLayer +{ + struct DeviceOp + { + // clang-format off + // Declare parameters here if needed. For example: + // amrex::Real foo{1.0}; + // amrex::GpuArray bar = {0.0}; + // clang-format on + + // x-velocities of the top and bottom layer respectively + amrex::Real pvel{0.0}; amrex::Real mvel{0.0}; + + AMREX_GPU_DEVICE + inline void operator()( + const amrex::IntVect& iv, + amrex::Array4 const& field, + amrex::GeometryData const& geom, + const amrex::Real time, + amrex::Orientation ori, + const int comp, + const int dcomp, + const int orig_comp) const + { + if ((iv[0] == -1) && (iv[1] == -1) && (iv[2] == 0)) + amrex::Print() << "***** filling custom velocity at " << iv << "..." << std::endl; + if ((iv[0] == 1) && (iv[1] == 1) && (iv[2] == 1)) + amrex::Print() << "***** filling custom velocity at " << iv << "..." << std::endl; + // Compute quantities to set the field values. For example: + // clang-format off + const auto* problo = geom.ProbLo(); + const auto* dx = geom.CellSize(); + const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; + // const auto y = problo[1] + (iv[1] + 0.5) * dx[1]; + const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; + // const amrex::GpuArray vel = {1.0, 0.0, 0.0}; + + // Once the above is done, fill the field as: + amrex::GpuArray vel; + vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; + //if (time == 0.0) vel[0] *= 0.1; + field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; + /* + if (x <= 0.5) { + // at the low-x boundary, +ve vel in top portion, garbage in the rest + //vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; + //vel = {((z >= 0.5) ? pvel : 0.0), 0.0, 0.0}; + } else { + // at the high-x boundary, -ve vel in bottom portion, garbage in the rest + //vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; + //vel = {((z < 0.5) ? mvel : 0.0), 0.0, 0.0}; + }*/ + + // clang-format on + } + }; + using DeviceType = DeviceOp; + + static std::string identifier() { return "TwoLayer"; } + + explicit TwoLayer(const Field& fld); + + DeviceType device_instance() const { return m_op; } + + DeviceOp m_op; +}; + +} // namespace udf +} // namespace amr_wind + +#endif /* TWO_LAYER_H */ diff --git a/amr-wind/physics/udfs/TwoLayer.cpp b/amr-wind/physics/udfs/TwoLayer.cpp new file mode 100644 index 0000000000..f9a7be3f60 --- /dev/null +++ b/amr-wind/physics/udfs/TwoLayer.cpp @@ -0,0 +1,38 @@ +#include "amr-wind/physics/udfs/TwoLayer.H" +#include "amr-wind/core/Field.H" +#include "amr-wind/core/FieldRepo.H" +#include "amr-wind/core/vs/vector.H" +#include "amr-wind/equation_systems/icns/icns.H" + +#include "AMReX_ParmParse.H" + +namespace amr_wind::udf { + +TwoLayer::TwoLayer(const Field& /*fld*/) +{ + // This is a where the user can set some user defined variables + // This capability can be activated with the following in the input file: + // xlo.type = "mass_inflow" + // xlo.velocity.inflow_type = CustomVelocity + // CustomVelocity.foo = 1.0 + + // clang-format off + { + //const int ncomp = fld.num_comp(); + amrex::ParmParse pp("TwoLayer"); + pp.query("pvel", m_op.pvel); + pp.query("mvel", m_op.mvel); + //amrex::Vector vel(0.0, ncomp); + //pp.getarr("velocity", vel); + //AMREX_ALWAYS_ASSERT(vel.size() == ncomp); + //for (int i = 0; i < ncomp; ++i) { + // m_op.bar[i] = vel[i]; + //} + } + // clang-format on + //amrex::Abort( + // "Please define the body of this function and the corresponding struct " + // "in the header file before using it. Then remove this message"); +} + +} // namespace amr_wind::udf diff --git a/amr-wind/physics/udfs/UDF.cpp b/amr-wind/physics/udfs/UDF.cpp index 5372b27976..7a3427de26 100644 --- a/amr-wind/physics/udfs/UDF.cpp +++ b/amr-wind/physics/udfs/UDF.cpp @@ -7,6 +7,7 @@ #include "amr-wind/physics/udfs/Rankine.H" #include "amr-wind/physics/udfs/CustomVelocity.H" #include "amr-wind/physics/udfs/CustomScalar.H" +#include "amr-wind/physics/udfs/TwoLayer.H" #include "AMReX_ParmParse.H" @@ -63,5 +64,6 @@ template class UDFImpl; template class UDFImpl; template class UDFImpl; template class UDFImpl; +template class UDFImpl; } // namespace amr_wind::udf diff --git a/bctest/input_inflow b/bctest/input_inflow index c55cb2ad5d..2a02a0c234 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -37,7 +37,7 @@ ICNS.source_terms = BodyForce BodyForce.magnitude = 0 0 0 incflo.physics = FreeStream -FreeStream.velocity_type = CustomVelocity +FreeStream.velocity_type = TwoLayer io.output_default_variables = 1 @@ -59,13 +59,13 @@ geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) # Boundary conditions xlo.type = "mass_inflow" xlo.density = 1.0 -xlo.velocity.inflow_type = CustomVelocity +xlo.velocity.inflow_type = TwoLayer xhi.type = "mass_inflow" xhi.density = 1.0 -xhi.velocity.inflow_type = CustomVelocity -CustomVelocity.mvel = -1.0 -CustomVelocity.pvel = 1.0 +xhi.velocity.inflow_type = TwoLayer +TwoLayer.mvel = -1.0 +TwoLayer.pvel = 1.0 zlo.type = "slip_wall" zhi.type = "slip_wall" diff --git a/bctest/input_inout b/bctest/input_inout index f3f5f205ab..660057d681 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -37,7 +37,7 @@ ICNS.source_terms = BodyForce BodyForce.magnitude = 0 0 0 incflo.physics = FreeStream -FreeStream.velocity_type = CustomVelocity +FreeStream.velocity_type = TwoLayer io.output_default_variables = 1 @@ -59,13 +59,13 @@ geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) # Boundary conditions xlo.type = "mass_inflow_outflow" xlo.density = 1.0 -xlo.velocity.inflow_outflow_type = CustomVelocity +xlo.velocity.inflow_outflow_type = TwoLayer xhi.type = "mass_inflow_outflow" xhi.density = 1.0 -xhi.velocity.inflow_outflow_type = CustomVelocity -CustomVelocity.mvel = -1.0 -CustomVelocity.pvel = 1.0 +xhi.velocity.inflow_outflow_type = TwoLayer +TwoLayer.mvel = -1.0 +TwoLayer.pvel = 1.0 zlo.type = "slip_wall" zhi.type = "slip_wall" From edfe4b7e07421a43e9c5db6e85633ab96f835794 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 17 May 2024 18:01:58 -0700 Subject: [PATCH 31/83] clean up print messages etc for draft PR, and rebase --- amr-wind/boundary_conditions/BCInterface.cpp | 4 ++-- .../boundary_conditions/MassInflowOutflowBC.H | 4 ++-- .../MassInflowOutflowBC.cpp | 5 ++--- .../convection/incflo_godunov_advection.cpp | 4 ++-- amr-wind/convection/incflo_godunov_ppm.H | 19 ++++++++++--------- .../convection/incflo_godunov_predict.cpp | 9 +++------ amr-wind/convection/incflo_godunov_weno.H | 16 ++++++---------- .../equation_systems/icns/icns_advection.cpp | 18 +++++++++++------- amr-wind/physics/ChannelFlow.cpp | 12 ------------ .../incflo_apply_nodal_projection.cpp | 9 +++++---- amr-wind/setup/init.cpp | 2 -- bctest/input_inflow | 4 ++-- bctest/input_inout | 2 +- 13 files changed, 46 insertions(+), 62 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index d545b08499..2e0d304638 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -104,9 +104,9 @@ void BCIface::set_bcfuncs() m_field.register_custom_bc(ori); } - if ((m_field.name() == "velocity") + if ((m_field.name() == "velocity") // only velocity for now && (bct == BC::mass_inflow_outflow)) { - amrex::Print() << "******** registering MIO custom Neumann BC" << std::endl; + amrex::Print() << "***** Registering MIO custom Neumann BC" << std::endl; m_field.register_custom_bc(ori); } diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.H b/amr-wind/boundary_conditions/MassInflowOutflowBC.H index c57ed9be01..46f33ba063 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.H +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.H @@ -8,10 +8,10 @@ namespace amr_wind { -/** Can handle both inflow and outflow cells in the same boundary +/** Custom Neumann fills for the inflow-outflow BC * \ingroup field_bc * - * + * Used to fill the outflow boundary cells for mass-inflow-outflow BC */ class MassInflowOutflowBC : public FieldBCIface { diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index edf46796ec..e5e79ce845 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -23,8 +23,7 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st const int nlevels = m_field.repo().num_active_levels(); const bool ib = (idim == 0), jb = (idim == 1), kb = (idim == 2); - amrex::Print() << "******* applying MIO custom Neumann fills at orientation: " << idx << std::endl; - //amrex::Print() << nlevels << " level(s)" << std::endl << std::endl; + amrex::Print() << "***** Applying MIO custom Neumann fills at orientation: " << idx << std::endl; for (int lev = 0; lev < nlevels; ++lev) { const auto& domain = repo.mesh().Geom(lev).Domain(); @@ -37,7 +36,7 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st #endif for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { auto bx = mfi.validbox(); - bx.grow({!ib, !jb, !kb}); // how to manage corner cells?? + //bx.grow({!ib, !jb, !kb}); // how to manage corner cells?? const auto& bc_a = m_field(lev).array(mfi); const auto& vel = velocity(lev).array(mfi); diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp index 8ff7822a87..bbccf8c966 100644 --- a/amr-wind/convection/incflo_godunov_advection.cpp +++ b/amr-wind/convection/incflo_godunov_advection.cpp @@ -312,8 +312,8 @@ void godunov::compute_fluxes( (!use_forces_in_trans && fq) ? 0.5 * l_dt * fq(i, j, k, n) : 0.; auto bc = pbc[n]; -if ((i==0) && (j==0) && (k==0)) - amrex::Print() << "******* now calling Godunov_cc_xbc_lo/hi from compute_fluxes" << std::endl; +//if ((i==0) && (j==0) && (k==0)) +// amrex::Print() << "***** now calling Godunov_cc_xbc_lo/hi from compute_fluxes" << std::endl; Godunov_cc_xbc_lo(i, j, k, n, q, stl, sth, umac, bc.lo(0), dlo.x); Godunov_cc_xbc_hi(i, j, k, n, q, stl, sth, umac, bc.hi(0), dhi.x); diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index 8db512fd72..01e7f1dedb 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -26,7 +26,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( const int domhi) { using namespace amrex; -// is the "inverse" use of velm and velp justified? + + // for MIO BC, use vel+ at bclo and vel- at bchi for determining ext_dir/foextrap treatment // Low X if (i <= domlo) { if (bclo == BCType::ext_dir || @@ -217,8 +218,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( if (i == domlo) { if ((bclo == BCType::ext_dir && uedge(i, j, k) >= 0.) || (bclo == BCType::user_1 && uedge(i, j, k) >= 0.)) { -//Print() << "cc_xbc_lo_extdir triggered at " << i << " " << j << " " << k << " " << n -// << " as uedge is " << uedge(i,j,k) << std::endl; + //Print() << "cc_xbc_lo_extdir triggered at " << i << " " << j << " " << k << " " << n + // << " as uedge is " << uedge(i,j,k) << std::endl; hi = s(domlo - 1, j, k, n); lo = hi; } else if ( @@ -226,8 +227,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( bclo == BCType::reflect_even || (bclo == BCType::ext_dir && uedge(i, j, k) < 0) || (bclo == BCType::user_1 && uedge(i, j, k) < 0)) { -//Print() << "cc_xbc_lo_foextrap triggered at " << i << " " << j << " " << k << " " << n -// << " as uedge is " << uedge(i,j,k) << std::endl; + //Print() << "cc_xbc_lo_foextrap triggered at " << i << " " << j << " " << k << " " << n + // << " as uedge is " << uedge(i,j,k) << std::endl; lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -256,8 +257,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( if (i == domhi + 1) { if ((bchi == BCType::ext_dir && uedge(i, j, k) <= 0.) || (bchi == BCType::user_1 && uedge(i, j, k) <= 0.)) { -//Print() << "cc_xbc_hi_extdir triggered at " << i << " " << j << " " << k << " " << n -// << " as uedge is " << uedge(i,j,k) << std::endl; + //Print() << "cc_xbc_hi_extdir triggered at " << i << " " << j << " " << k << " " << n + // << " as uedge is " << uedge(i,j,k) << std::endl; lo = s(domhi + 1, j, k, n); hi = lo; } else if ( @@ -265,8 +266,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( bchi == BCType::reflect_even || (bchi == BCType::ext_dir && uedge(i, j, k) > 0) || (bchi == BCType::user_1 && uedge(i, j, k) > 0)) { -//Print() << "cc_xbc_hi_foextrap triggered at " << i << " " << j << " " << k << " " << n -// << " as uedge is " << uedge(i,j,k) << std::endl; + //Print() << "cc_xbc_hi_foextrap triggered at " << i << " " << j << " " << k << " " << n + // << " as uedge is " << uedge(i,j,k) << std::endl; hi = lo; } else if (bchi == BCType::reflect_odd) { diff --git a/amr-wind/convection/incflo_godunov_predict.cpp b/amr-wind/convection/incflo_godunov_predict.cpp index c8eda87d26..952340e583 100644 --- a/amr-wind/convection/incflo_godunov_predict.cpp +++ b/amr-wind/convection/incflo_godunov_predict.cpp @@ -50,10 +50,7 @@ void godunov::make_trans_velocities( lo = Ipx(i - 1, j, k, n); hi = Imx(i, j, k, n); } -/*if (((i==0) || (i==16)) && (j==3) && ((k==1) || (k==14))) { -Print() << "lo hi at " << i << " " << j << " " << k << " " << n << " " - << lo << " " << hi << std::endl; -}*/ + auto bc = pbc[n]; Godunov_trans_xbc( i, j, k, n, vel, lo, hi, lo, hi, bc.lo(0), bc.hi(0), dlo.x, dhi.x); @@ -331,8 +328,8 @@ void godunov::predict_godunov( stl += 0.5 * l_dt * f(i - 1, j, k, n); sth += 0.5 * l_dt * f(i, j, k, n); } -if ((i==0) && (j==0) && (k==0)) - amrex::Print() << "******* now calling Godunov_cc_xbc_lo/hi from predict_godunov" << std::endl; +//if ((i==0) && (j==0) && (k==0)) +// amrex::Print() << "***** now calling Godunov_cc_xbc_lo/hi from predict_godunov" << std::endl; Godunov_cc_xbc_lo(i, j, k, n, q, stl, sth, u_ad, bc.lo(0), dlo.x); Godunov_cc_xbc_hi(i, j, k, n, q, stl, sth, u_ad, bc.hi(0), dhi.x); diff --git a/amr-wind/convection/incflo_godunov_weno.H b/amr-wind/convection/incflo_godunov_weno.H index fcfe7ccc72..847cd5d835 100644 --- a/amr-wind/convection/incflo_godunov_weno.H +++ b/amr-wind/convection/incflo_godunov_weno.H @@ -74,16 +74,16 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || (bclo == BCType::user_1 && velm >= 0.0)) { if (i == domlo) { -//Print() << "weno_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n -// << " as velm is " << velm << std::endl; + //Print() << "weno_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n + // << " as velm is " << velm << std::endl; sp = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); sm = s(domlo - 1, j, k, n); } else if (i == domlo + 1) { -//Print() << "weno_xbc_domlop1_extdir triggered at " << i << " " << j << " " << k << " " << n -// << " as velm is " << velm << std::endl; + //Print() << "weno_xbc_domlop1_extdir triggered at " << i << " " << j << " " << k << " " << n + // << " as velm is " << velm << std::endl; sm = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); @@ -94,8 +94,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || (bchi == BCType::user_1 && velp <= 0.0)) { if (i == domhi) { -//Print() << "weno_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n -// << " as velp is " << velp << std::endl; + //Print() << "weno_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n + // << " as velp is " << velp << std::endl; sm = -0.2 * s(domhi + 1, j, k, n) + 0.75 * s(domhi, j, k, n) + 0.5 * s(domhi - 1, j, k, n) - 0.05 * s(domhi - 2, j, k, n); @@ -277,10 +277,6 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_x( Ip(i, j, k, n) = S(i, j, k, n); Im(i, j, k, n) = S(i, j, k, n); } -/*if (((i==0) || (i==15)) && (j==3) && ((k==1) || (k==14))) { -Print() << "Im Ip at " << i << " " << j << " " << k << " " << n << " " - << Im(i, j, k, n) << " " << Ip(i, j, k, n) << std::endl; -}*/ } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_y( diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 2530a07469..8537f7f99b 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -8,10 +8,11 @@ #include "AMReX_MultiFabUtil.H" #include "hydro_MacProjector.H" +// only for debugging #include "AMReX_PlotFileUtil.H" #include "AMReX_MultiFabUtil.H" #include "AMReX_MultiFab.H" - +// namespace amr_wind::pde { namespace { @@ -75,7 +76,7 @@ void MacProjOp::enforce_solvability ( auto& velocity = m_repo.get_field("velocity"); amrex::BCRec const* bc_type = velocity.bcrec_device().data(); const amrex::Vector& geom = m_repo.mesh().Geom(); - amrex::Print() << "Calling enforceSolvability from AMR-Wind" << std::endl; + amrex::Print() << "***** Calling enforceSolvability from AMR-Wind" << std::endl; m_mac_proj->enforceSolvability(a_umac, bc_type, geom); } @@ -281,14 +282,17 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } -amrex::Array mac_arr = {&u_mac(0), &v_mac(0), &w_mac(0)}; -amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); +//amrex::Array mac_arr = {&u_mac(0), &v_mac(0), &w_mac(0)}; +//amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); -amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); + enforce_solvability(mac_vec); -amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); + +//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); + m_mac_proj->setUMAC(mac_vec); if (m_has_overset) { @@ -304,7 +308,7 @@ amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); } else { m_mac_proj->project(m_options.rel_tol, m_options.abs_tol); } -amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); if (m_is_anelastic) { diff --git a/amr-wind/physics/ChannelFlow.cpp b/amr-wind/physics/ChannelFlow.cpp index d75483d2b8..f49d99e57e 100644 --- a/amr-wind/physics/ChannelFlow.cpp +++ b/amr-wind/physics/ChannelFlow.cpp @@ -5,7 +5,6 @@ #include "AMReX_ParmParse.H" #include "amr-wind/utilities/trig_ops.H" #include "amr-wind/utilities/DirectionSelector.H" -#include "amr-wind/boundary_conditions/MassInflowOutflowBC.H" namespace amr_wind::channel_flow { @@ -452,17 +451,6 @@ void ChannelFlow::post_init_actions() std::abs(body_force[2]) < 1e-16, "body force in z should be zero if using a wall model"); } -/* - const auto& ibctype = velocity.bc_type(); - for (amrex::OrientationIter oit; oit != nullptr; ++oit) { - auto ori = oit(); - const auto bct = ibctype[ori]; - - if (bct == BC::mass_inflow_outflow) { - amrex::Print() << "howdyyyyyyyyyyyy" << std::endl; - velocity.register_custom_bc(ori); - } - }*/ } void ChannelFlow::post_advance_work() diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 761bdfd8eb..d7059eb272 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -6,8 +6,7 @@ #include "amr-wind/core/field_ops.H" #include "amr-wind/projection/nodal_projection_ops.H" -#include "amr-wind/utilities/IOManager.H" - +// for debugging #include "AMReX_PlotFileUtil.H" using namespace amrex; @@ -340,11 +339,13 @@ void incflo::ApplyProjection( //amrex::Print() << "*********** partition **********" << std::endl; //amrex::Print() << velocity(0)[0]; - // need to apply custom Neumann funcs for inflow-outflow BC - // after setting the inflow vels above + // Need to apply custom Neumann funcs for inflow-outflow BC + // after setting the inflow vels above. if (!proj_for_small_dt and !incremental) { velocity.apply_bc_funcs(amr_wind::FieldState::New); } + // Currently doing this after setting the non-inflow boundaries to zero + // is causing issues at the corner cells. //amrex::Print() << velocity(0)[0]; diff --git a/amr-wind/setup/init.cpp b/amr-wind/setup/init.cpp index 1896e3ec13..b669a8b2af 100644 --- a/amr-wind/setup/init.cpp +++ b/amr-wind/setup/init.cpp @@ -79,9 +79,7 @@ void incflo::InitialIterations() { auto& vel = icns().fields().field; vel.copy_state(amr_wind::FieldState::Old, amr_wind::FieldState::New); - amrex::Print() << "****** starting fillpatch from InitialIterations()" << std::endl; vel.state(amr_wind::FieldState::Old).fillpatch(m_time.current_time()); - amrex::Print() << "****** completed fillpatch from InitialIterations()" << std::endl; if (m_sim.pde_manager().constant_density()) { auto& rho = density(); diff --git a/bctest/input_inflow b/bctest/input_inflow index 2a02a0c234..52dc2cac77 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -4,7 +4,7 @@ # SIMULATION STOP # #.......................................# time.stop_time = 2.00 # Max (simulated) time to evolve -time.max_step = 1 #-1000 # Max number of time steps +time.max_step = -1 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # @@ -26,7 +26,7 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" -incflo.initial_iterations = 1 +#incflo.initial_iterations = 1 #incflo.do_initial_proj = false incflo.verbose = 3 diff --git a/bctest/input_inout b/bctest/input_inout index 660057d681..81e076ea5b 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -26,7 +26,7 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" -incflo.initial_iterations = 1 +#incflo.initial_iterations = 1 #incflo.do_initial_proj = false incflo.verbose = 3 From 89c0add3b8687a5bd41ddee9569a21914f7b44f5 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 23 May 2024 15:44:12 -0700 Subject: [PATCH 32/83] printing information for debugging --- amr-wind/boundary_conditions/BCInterface.cpp | 3 ++- amr-wind/equation_systems/icns/icns_advection.cpp | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index 2e0d304638..cd005847c2 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -106,7 +106,8 @@ void BCIface::set_bcfuncs() if ((m_field.name() == "velocity") // only velocity for now && (bct == BC::mass_inflow_outflow)) { - amrex::Print() << "***** Registering MIO custom Neumann BC" << std::endl; + amrex::Print() << "***** Registering MIO custom Neumann BC for " + << m_field.name() << std::endl; m_field.register_custom_bc(ori); } diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 8537f7f99b..77611c01d3 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -287,9 +287,13 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) //amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +amrex::Print() << "!!! umac precorrect" << std::endl; +amrex::Print() << u_mac(0)[0]; enforce_solvability(mac_vec); +amrex::Print() << "!!! umac postcorrect" << std::endl; +amrex::Print() << u_mac(0)[0]; //amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); @@ -310,6 +314,8 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } //amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +amrex::Print() << "!!! umac postproject" << std::endl; +amrex::Print() << u_mac(0)[0]; if (m_is_anelastic) { for (int lev = 0; lev < m_repo.num_active_levels(); ++lev) { From 8dfc642a26d18431ae10fa238da8dad0dab6364a Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 23 May 2024 15:45:58 -0700 Subject: [PATCH 33/83] perturbing initial condition for testing and debugging --- amr-wind/physics/udfs/TwoLayer.H | 6 +++--- bctest/input_inout | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/amr-wind/physics/udfs/TwoLayer.H b/amr-wind/physics/udfs/TwoLayer.H index 32ff8e6d6f..86a966d652 100644 --- a/amr-wind/physics/udfs/TwoLayer.H +++ b/amr-wind/physics/udfs/TwoLayer.H @@ -42,15 +42,15 @@ struct TwoLayer // clang-format off const auto* problo = geom.ProbLo(); const auto* dx = geom.CellSize(); - const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; + // const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; // const auto y = problo[1] + (iv[1] + 0.5) * dx[1]; - const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; + const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; // not true for w-mac vels // const amrex::GpuArray vel = {1.0, 0.0, 0.0}; // Once the above is done, fill the field as: amrex::GpuArray vel; vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; - //if (time == 0.0) vel[0] *= 0.1; + if (time == 0.0) vel[0] *= 0.9; field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; /* if (x <= 0.5) { diff --git a/bctest/input_inout b/bctest/input_inout index 81e076ea5b..cec76e5771 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -4,7 +4,7 @@ # SIMULATION STOP # #.......................................# time.stop_time = 2.00 # Max (simulated) time to evolve -time.max_step = -1 #-1000 # Max number of time steps +time.max_step = 1 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # @@ -15,7 +15,7 @@ time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 100 # Steps between plot files +time.plot_interval = 1 # 100 # Steps between plot files time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -26,8 +26,8 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" -#incflo.initial_iterations = 1 -#incflo.do_initial_proj = false +incflo.initial_iterations = 0 +incflo.do_initial_proj = false incflo.verbose = 3 transport.viscosity = 0.0 From 124fc2309fbaa15030dffd70313c5af0939bf609 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 23 May 2024 16:02:03 -0700 Subject: [PATCH 34/83] resolved corner cell issue by adding a periodic boundary fill before nodal projection --- amr-wind/boundary_conditions/MassInflowOutflowBC.cpp | 7 ++++++- amr-wind/projection/incflo_apply_nodal_projection.cpp | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index e5e79ce845..7d1aa5011c 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -36,7 +36,7 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st #endif for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { auto bx = mfi.validbox(); - //bx.grow({!ib, !jb, !kb}); // how to manage corner cells?? + bx.grow({!ib, !jb, !kb}); // filling 1 ghost cell is enough? – check const auto& bc_a = m_field(lev).array(mfi); const auto& vel = velocity(lev).array(mfi); @@ -66,6 +66,11 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st bc_a(i, j, k, n) = bc_a(i-ib, j-jb, k-kb, n); } } + + /*for (int n = 0; n < ncomp; n++) { + amrex::Print() << i << " " << j << " " << k << " " << n << std::endl; + amrex::Print() << "result: " << vel(i-ib, j-jb, k-kb, idim) << " " << bc_a(i-ib, j-jb, k-kb, n) << " " << bc_a(i, j, k, n) << std::endl << std::endl; + }*/ }); } } diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index d7059eb272..1301c327b4 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -334,6 +334,9 @@ void incflo::ApplyProjection( if (!proj_for_small_dt and !incremental) { amr_wind::nodal_projection::set_inflow_velocity( m_sim.physics_manager(), velocity, lev, time, *vel[lev], 1); + + // fill periodic boundaries to avoid corner cell issues + vel[lev]->FillBoundary(geom[lev].periodicity()); } } //amrex::Print() << "*********** partition **********" << std::endl; @@ -344,8 +347,6 @@ void incflo::ApplyProjection( if (!proj_for_small_dt and !incremental) { velocity.apply_bc_funcs(amr_wind::FieldState::New); } - // Currently doing this after setting the non-inflow boundaries to zero - // is causing issues at the corner cells. //amrex::Print() << velocity(0)[0]; From 618ff2f45e0c8101a31b3787975dab9b33ac1afc Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 23 May 2024 16:45:20 -0700 Subject: [PATCH 35/83] change std::array to amrex::Array --- amr-wind/boundary_conditions/BCInterface.H | 2 +- amr-wind/boundary_conditions/BCInterface.cpp | 2 +- amr-wind/boundary_conditions/scalar_bcs.H | 2 +- amr-wind/boundary_conditions/scalar_bcs.cpp | 2 +- amr-wind/boundary_conditions/velocity_bcs.H | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.H b/amr-wind/boundary_conditions/BCInterface.H index 630ed07fa0..45c6e4e0ef 100644 --- a/amr-wind/boundary_conditions/BCInterface.H +++ b/amr-wind/boundary_conditions/BCInterface.H @@ -49,7 +49,7 @@ public: virtual void operator()(const amrex::Real value = 0.0); //! User-defined functions for Dirichlet-type boundaries - std::array get_dirichlet_udfs(); + amrex::Array get_dirichlet_udfs(); protected: //! Setup AMReX mathematical BC types diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index cd005847c2..3bc832dfbc 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -114,7 +114,7 @@ void BCIface::set_bcfuncs() } } -std::array BCIface::get_dirichlet_udfs() +amrex::Array BCIface::get_dirichlet_udfs() { const auto& fname = m_field.name(); const auto& bctype = m_field.bc_type(); diff --git a/amr-wind/boundary_conditions/scalar_bcs.H b/amr-wind/boundary_conditions/scalar_bcs.H index 3566c134aa..8be799856c 100644 --- a/amr-wind/boundary_conditions/scalar_bcs.H +++ b/amr-wind/boundary_conditions/scalar_bcs.H @@ -29,7 +29,7 @@ void register_scalar_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const std::array udfs); + const amrex::Array udfs); } // namespace amr_wind::scalar_bc diff --git a/amr-wind/boundary_conditions/scalar_bcs.cpp b/amr-wind/boundary_conditions/scalar_bcs.cpp index 71d8e544f3..b1f5988dca 100644 --- a/amr-wind/boundary_conditions/scalar_bcs.cpp +++ b/amr-wind/boundary_conditions/scalar_bcs.cpp @@ -5,7 +5,7 @@ void register_scalar_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const std::array udfs) + const amrex::Array udfs) { const std::string inflow_udf = udfs[0]; const std::string inflow_outflow_udf = udfs[1]; diff --git a/amr-wind/boundary_conditions/velocity_bcs.H b/amr-wind/boundary_conditions/velocity_bcs.H index 1c88ca176e..b6c9ffc0cb 100644 --- a/amr-wind/boundary_conditions/velocity_bcs.H +++ b/amr-wind/boundary_conditions/velocity_bcs.H @@ -55,7 +55,7 @@ void register_velocity_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const std::array udfs) + const amrex::Array udfs) { const std::string inflow_udf = udfs[0]; const std::string inflow_outflow_udf = udfs[1]; From f9eb0b7a993bd40ec90d8af6386e0216593f0546 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 24 May 2024 15:17:21 -0700 Subject: [PATCH 36/83] debugging for mac-velocity fill issue at i=0 --- amr-wind/equation_systems/icns/icns_advection.H | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.H b/amr-wind/equation_systems/icns/icns_advection.H index 024f32e512..b3fcea8597 100644 --- a/amr-wind/equation_systems/icns/icns_advection.H +++ b/amr-wind/equation_systems/icns/icns_advection.H @@ -215,7 +215,8 @@ struct AdvectionOp dof_field.fillpatch_sibling_fields( time + 0.5 * dt, u_mac.num_grow(), mac_vel); } - +//amrex::Print() << "!!! umac after fillpatch" << std::endl; +//amrex::Print() << u_mac(0)[0]; for (int lev = 0; lev < repo.num_active_levels(); ++lev) { u_mac(lev).FillBoundary(geom[lev].periodicity()); v_mac(lev).FillBoundary(geom[lev].periodicity()); From e77f20e54897aad15ff13a765982a5f0b64a3bfa Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 6 Jun 2024 11:48:44 -0700 Subject: [PATCH 37/83] replaced user_1 with new direction_dependent BCType in amrex --- amr-wind/boundary_conditions/BCInterface.cpp | 8 ++++---- amr-wind/convection/incflo_godunov_ppm.H | 16 ++++++++-------- amr-wind/convection/incflo_godunov_weno.H | 12 ++++++------ amr-wind/core/FieldBCOps.H | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index 3bc832dfbc..0bd620c568 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -225,9 +225,9 @@ void BCVelocity::set_bcrec() case BC::mass_inflow_outflow: if (side == amrex::Orientation::low) { - set_bcrec_lo(dir, amrex::BCType::user_1); + set_bcrec_lo(dir, amrex::BCType::direction_dependent); } else { - set_bcrec_hi(dir, amrex::BCType::user_1); + set_bcrec_hi(dir, amrex::BCType::direction_dependent); } break; @@ -326,9 +326,9 @@ void BCScalar::set_bcrec() case BC::mass_inflow_outflow: if (side == amrex::Orientation::low) { - set_bcrec_lo(dir, amrex::BCType::user_1); + set_bcrec_lo(dir, amrex::BCType::direction_dependent); } else { - set_bcrec_hi(dir, amrex::BCType::user_1); + set_bcrec_hi(dir, amrex::BCType::direction_dependent); } break; diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index 01e7f1dedb..42e71d534d 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -31,7 +31,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( // Low X if (i <= domlo) { if (bclo == BCType::ext_dir || - (bclo == BCType::user_1 && velp >= 0.0)) { + (bclo == BCType::direction_dependent && velp >= 0.0)) { //Print() << "trans_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << velp << std::endl; // IAMR does this but it breaks lo/hi symmetry @@ -43,7 +43,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( } else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || - (bclo == BCType::user_1 && velp < 0.0)) { + (bclo == BCType::direction_dependent && velp < 0.0)) { //Print() << "trans_xbc_domlo_foextrap triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << velp << std::endl; lo = hi; @@ -57,7 +57,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( // High X if (i > domhi) { if (bchi == BCType::ext_dir || - (bchi == BCType::user_1 && velm <= 0.0)) { // is this correct use of uedge here?? + (bchi == BCType::direction_dependent && velm <= 0.0)) { // is this correct use of uedge here?? //Print() << "trans_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << velm << std::endl; // IAMR does this but it breaks lo/hi symmetry @@ -69,7 +69,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( } else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || - (bchi == BCType::user_1 && velm > 0.0)) { + (bchi == BCType::direction_dependent && velm > 0.0)) { //Print() << "trans_xbc_domhi_foextrap triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << velm << std::endl; hi = lo; @@ -217,7 +217,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( if (i == domlo) { if ((bclo == BCType::ext_dir && uedge(i, j, k) >= 0.) || - (bclo == BCType::user_1 && uedge(i, j, k) >= 0.)) { + (bclo == BCType::direction_dependent && uedge(i, j, k) >= 0.)) { //Print() << "cc_xbc_lo_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << uedge(i,j,k) << std::endl; hi = s(domlo - 1, j, k, n); @@ -226,7 +226,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || (bclo == BCType::ext_dir && uedge(i, j, k) < 0) || - (bclo == BCType::user_1 && uedge(i, j, k) < 0)) { + (bclo == BCType::direction_dependent && uedge(i, j, k) < 0)) { //Print() << "cc_xbc_lo_foextrap triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << uedge(i,j,k) << std::endl; lo = hi; @@ -256,7 +256,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( if (i == domhi + 1) { if ((bchi == BCType::ext_dir && uedge(i, j, k) <= 0.) || - (bchi == BCType::user_1 && uedge(i, j, k) <= 0.)) { + (bchi == BCType::direction_dependent && uedge(i, j, k) <= 0.)) { //Print() << "cc_xbc_hi_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << uedge(i,j,k) << std::endl; lo = s(domhi + 1, j, k, n); @@ -265,7 +265,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || (bchi == BCType::ext_dir && uedge(i, j, k) > 0) || - (bchi == BCType::user_1 && uedge(i, j, k) > 0)) { + (bchi == BCType::direction_dependent && uedge(i, j, k) > 0)) { //Print() << "cc_xbc_hi_foextrap triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << uedge(i,j,k) << std::endl; hi = lo; diff --git a/amr-wind/convection/incflo_godunov_weno.H b/amr-wind/convection/incflo_godunov_weno.H index 847cd5d835..259934aba0 100644 --- a/amr-wind/convection/incflo_godunov_weno.H +++ b/amr-wind/convection/incflo_godunov_weno.H @@ -72,7 +72,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( using namespace amrex; if (bclo == BCType::ext_dir || bclo == BCType::hoextrap - || (bclo == BCType::user_1 && velm >= 0.0)) { + || (bclo == BCType::direction_dependent && velm >= 0.0)) { if (i == domlo) { //Print() << "weno_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as velm is " << velm << std::endl; @@ -92,7 +92,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( } if (bchi == BCType::ext_dir || bchi == BCType::hoextrap - || (bchi == BCType::user_1 && velp <= 0.0)) { + || (bchi == BCType::direction_dependent && velp <= 0.0)) { if (i == domhi) { //Print() << "weno_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as velp is " << velp << std::endl; @@ -131,7 +131,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_ybc( using namespace amrex; if (bclo == BCType::ext_dir || bclo == BCType::hoextrap - || (bclo == BCType::user_1 && velm >= 0.0)) { + || (bclo == BCType::direction_dependent && velm >= 0.0)) { if (j == domlo) { sp = -0.2 * s(i, domlo - 1, k, n) + 0.75 * s(i, domlo, k, n) + 0.5 * s(i, domlo + 1, k, n) - 0.05 * s(i, domlo + 2, k, n); @@ -148,7 +148,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_ybc( } if (bchi == BCType::ext_dir || bchi == BCType::hoextrap - || (bchi == BCType::user_1 && velp <= 0.0)) { + || (bchi == BCType::direction_dependent && velp <= 0.0)) { if (j == domhi) { sm = -0.2 * s(i, domhi + 1, k, n) + 0.75 * s(i, domhi, k, n) + 0.5 * s(i, domhi - 1, k, n) - 0.05 * s(i, domhi - 2, k, n); @@ -185,7 +185,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_zbc( using namespace amrex; if (bclo == BCType::ext_dir || bclo == BCType::hoextrap - || (bclo == BCType::user_1 && velm >= 0.0)) { + || (bclo == BCType::direction_dependent && velm >= 0.0)) { if (k == domlo) { sp = -0.2 * s(i, j, domlo - 1, n) + 0.75 * s(i, j, domlo, n) + 0.5 * s(i, j, domlo + 1, n) - 0.05 * s(i, j, domlo + 2, n); @@ -202,7 +202,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_zbc( } if (bchi == BCType::ext_dir || bchi == BCType::hoextrap - || (bchi == BCType::user_1 && velp <= 0.0)) { + || (bchi == BCType::direction_dependent && velp <= 0.0)) { if (k == domhi) { sm = -0.2 * s(i, j, domhi + 1, n) + 0.75 * s(i, j, domhi, n) + 0.5 * s(i, j, domhi - 1, n) - 0.05 * s(i, j, domhi - 2, n); diff --git a/amr-wind/core/FieldBCOps.H b/amr-wind/core/FieldBCOps.H index b3d315fc21..adcb23ecf0 100644 --- a/amr-wind/core/FieldBCOps.H +++ b/amr-wind/core/FieldBCOps.H @@ -169,7 +169,7 @@ struct DirichletOp // Check if this boundary is a dirichlet or mixed type const auto bctyp = (ori.isLow() ? bc.lo(idir) : bc.hi(idir)); if ((bctyp != amrex::BCType::ext_dir) - && (bctyp != amrex::BCType::user_1)) { + && (bctyp != amrex::BCType::direction_dependent)) { continue; } From 4c9aac47b558759e518e8bd04c9baa053402d74c Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 7 Jun 2024 09:59:32 -0700 Subject: [PATCH 38/83] commenting out debug statements for umac boundary fill checking --- amr-wind/equation_systems/icns/icns_advection.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 77611c01d3..49b58f6889 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -287,13 +287,13 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) //amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); -amrex::Print() << "!!! umac precorrect" << std::endl; -amrex::Print() << u_mac(0)[0]; +//amrex::Print() << "!!! umac precorrect" << std::endl; +//amrex::Print() << u_mac(0)[0]; enforce_solvability(mac_vec); -amrex::Print() << "!!! umac postcorrect" << std::endl; -amrex::Print() << u_mac(0)[0]; +//amrex::Print() << "!!! umac postcorrect" << std::endl; +//amrex::Print() << u_mac(0)[0]; //amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); @@ -314,8 +314,8 @@ amrex::Print() << u_mac(0)[0]; } //amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); //amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); -amrex::Print() << "!!! umac postproject" << std::endl; -amrex::Print() << u_mac(0)[0]; +//amrex::Print() << "!!! umac postproject" << std::endl; +//amrex::Print() << u_mac(0)[0]; if (m_is_anelastic) { for (int lev = 0; lev < m_repo.num_active_levels(); ++lev) { From 60bee5a0a3c393797f4728f22e9ed45fc876bf04 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 13 Jun 2024 15:30:44 -0700 Subject: [PATCH 39/83] reactivating some plotfile printing for debugging/digging --- .../equation_systems/icns/icns_advection.cpp | 16 ++++++++-------- .../projection/incflo_apply_nodal_projection.cpp | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 49b58f6889..ecdd8c59e8 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -282,11 +282,11 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } -//amrex::Array mac_arr = {&u_mac(0), &v_mac(0), &w_mac(0)}; -//amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); +amrex::Array mac_arr = {&u_mac(0), &v_mac(0), &w_mac(0)}; +amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); -//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -//amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); //amrex::Print() << "!!! umac precorrect" << std::endl; //amrex::Print() << u_mac(0)[0]; @@ -294,8 +294,8 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) //amrex::Print() << "!!! umac postcorrect" << std::endl; //amrex::Print() << u_mac(0)[0]; -//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -//amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); m_mac_proj->setUMAC(mac_vec); @@ -312,8 +312,8 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } else { m_mac_proj->project(m_options.rel_tol, m_options.abs_tol); } -//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -//amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); //amrex::Print() << "!!! umac postproject" << std::endl; //amrex::Print() << u_mac(0)[0]; diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 1301c327b4..9bb93f5997 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -163,7 +163,7 @@ void incflo::ApplyProjection( auto& grad_p = m_repo.get_field("gp"); auto& pressure = m_repo.get_field("p"); auto& velocity = icns().fields().field; -//amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); +amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); auto& velocity_old = icns().fields().field.state(amr_wind::FieldState::Old); amr_wind::Field const* mesh_fac = mesh_mapping @@ -476,7 +476,7 @@ void incflo::ApplyProjection( } } -//amrex::WriteSingleLevelPlotfile("plt_vel_post_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); +amrex::WriteSingleLevelPlotfile("plt_vel_post_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); // Get phi and fluxes auto phi = nodal_projector->getPhi(); From 074e71f3e8138cb03eae34101f9d96ca7c97221d Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Sun, 23 Jun 2024 08:39:28 -0700 Subject: [PATCH 40/83] modified function calls to reflect name changes in hydro --- .../equation_systems/icns/icns_advection.H | 2 +- .../equation_systems/icns/icns_advection.cpp | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/amr-wind/equation_systems/icns/icns_advection.H b/amr-wind/equation_systems/icns/icns_advection.H index b3fcea8597..d1ff5bdd4f 100644 --- a/amr-wind/equation_systems/icns/icns_advection.H +++ b/amr-wind/equation_systems/icns/icns_advection.H @@ -49,7 +49,7 @@ private: void init_projector(const FaceFabPtrVec& /*beta*/) noexcept; void init_projector(const amrex::Real /*beta*/) noexcept; - void enforce_solvability(const amrex::Vector>& a_umac) noexcept; + void enforce_inout_solvability(const amrex::Vector>& a_umac) noexcept; FieldRepo& m_repo; PhysicsMgr& m_phy_mgr; diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index ecdd8c59e8..b87ddcec81 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -69,7 +69,7 @@ MacProjOp::MacProjOp( m_has_overset = m_has_overset && !disable_ovst_mac; } -void MacProjOp::enforce_solvability ( +void MacProjOp::enforce_inout_solvability ( const amrex::Vector>& a_umac ) noexcept { @@ -77,7 +77,7 @@ void MacProjOp::enforce_solvability ( amrex::BCRec const* bc_type = velocity.bcrec_device().data(); const amrex::Vector& geom = m_repo.mesh().Geom(); amrex::Print() << "***** Calling enforceSolvability from AMR-Wind" << std::endl; - m_mac_proj->enforceSolvability(a_umac, bc_type, geom); + HydroUtils::enforceInOutSolvability(a_umac, bc_type, geom); } void MacProjOp::init_projector(const MacProjOp::FaceFabPtrVec& beta) noexcept @@ -282,20 +282,20 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } -amrex::Array mac_arr = {&u_mac(0), &v_mac(0), &w_mac(0)}; -amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); +//amrex::Array mac_arr = {&u_mac(0), &v_mac(0), &w_mac(0)}; +//amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); -amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +//amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); //amrex::Print() << "!!! umac precorrect" << std::endl; //amrex::Print() << u_mac(0)[0]; - enforce_solvability(mac_vec); + enforce_inout_solvability(mac_vec); //amrex::Print() << "!!! umac postcorrect" << std::endl; //amrex::Print() << u_mac(0)[0]; -amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +//amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); m_mac_proj->setUMAC(mac_vec); @@ -312,8 +312,8 @@ amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","v } else { m_mac_proj->project(m_options.rel_tol, m_options.abs_tol); } -amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); +//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); +//amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); //amrex::Print() << "!!! umac postproject" << std::endl; //amrex::Print() << u_mac(0)[0]; From 3cda624e98ee7f6069c0509a8841649e13a64c9c Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Sun, 23 Jun 2024 08:46:59 -0700 Subject: [PATCH 41/83] Added function to call the hydro routine for enforcing inflow-outflow solvability --- .../incflo_apply_nodal_projection.cpp | 25 +++++++++++++++++-- amr-wind/projection/nodal_projection_ops.H | 5 ++++ bctest/input_inflow | 8 +++--- bctest/input_inout | 16 ++++++------ 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 9bb93f5997..13aa4ae06c 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -5,6 +5,7 @@ #include "amr-wind/utilities/console_io.H" #include "amr-wind/core/field_ops.H" #include "amr-wind/projection/nodal_projection_ops.H" +#include "hydro_utils.H" // for debugging #include "AMReX_PlotFileUtil.H" @@ -80,6 +81,22 @@ void amr_wind::nodal_projection::apply_dirichlet_vel( }); } +void +amr_wind::nodal_projection::enforce_inout_solvability ( + amr_wind::Field& velocity, const Vector& geom, const int num_levels) +{ + BCRec const* bc_type = velocity.bcrec_device().data(); + Vector> vel_vec(num_levels); + + for (int lev = 0; lev < num_levels; ++lev) { + vel_vec[lev][0] = new MultiFab(velocity(lev), amrex::make_alias, 0, 1); + vel_vec[lev][1] = new MultiFab(velocity(lev), amrex::make_alias, 1, 1); + vel_vec[lev][2] = new MultiFab(velocity(lev), amrex::make_alias, 2, 1); + } + Print() << "***** Calling enforceSolvability from AMR-Wind" << std::endl; + HydroUtils::enforceInOutSolvability(vel_vec, bc_type, geom, true); +} + /** Perform nodal projection * * Computes the following decomposition: @@ -163,7 +180,7 @@ void incflo::ApplyProjection( auto& grad_p = m_repo.get_field("gp"); auto& pressure = m_repo.get_field("p"); auto& velocity = icns().fields().field; -amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); +//amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); auto& velocity_old = icns().fields().field.state(amr_wind::FieldState::Old); amr_wind::Field const* mesh_fac = mesh_mapping @@ -345,7 +362,11 @@ amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v"," // Need to apply custom Neumann funcs for inflow-outflow BC // after setting the inflow vels above. if (!proj_for_small_dt and !incremental) { + velocity.apply_bc_funcs(amr_wind::FieldState::New); + + amr_wind::nodal_projection::enforce_inout_solvability( + velocity, m_repo.mesh().Geom(), m_repo.num_active_levels()); } //amrex::Print() << velocity(0)[0]; @@ -476,7 +497,7 @@ amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v"," } } -amrex::WriteSingleLevelPlotfile("plt_vel_post_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); +//amrex::WriteSingleLevelPlotfile("plt_vel_post_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); // Get phi and fluxes auto phi = nodal_projector->getPhi(); diff --git a/amr-wind/projection/nodal_projection_ops.H b/amr-wind/projection/nodal_projection_ops.H index 5158fd17e7..e7f5df659d 100644 --- a/amr-wind/projection/nodal_projection_ops.H +++ b/amr-wind/projection/nodal_projection_ops.H @@ -26,6 +26,11 @@ Array get_projection_bc( void apply_dirichlet_vel( amrex::MultiFab& mf_velocity, amrex::iMultiFab& mf_iblank); +void enforce_inout_solvability ( + amr_wind::Field& velocity, + const Vector& geom, + int num_levels); + } // namespace amr_wind::nodal_projection #endif \ No newline at end of file diff --git a/bctest/input_inflow b/bctest/input_inflow index 52dc2cac77..cfa014c7ba 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -3,8 +3,8 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 2.00 # Max (simulated) time to evolve -time.max_step = -1 #-1000 # Max number of time steps +time.stop_time = 10.00 # Max (simulated) time to evolve +time.max_step = -10 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # @@ -15,7 +15,7 @@ time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 100 # Steps between plot files +time.plot_interval = 10 # 100 # Steps between plot files time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -26,7 +26,7 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" -#incflo.initial_iterations = 1 +#incflo.initial_iterations = 0 #incflo.do_initial_proj = false incflo.verbose = 3 diff --git a/bctest/input_inout b/bctest/input_inout index cec76e5771..5dc189cab2 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -3,19 +3,19 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 2.00 # Max (simulated) time to evolve -time.max_step = 1 #-1000 # Max number of time steps +time.stop_time = 10.00 # Max (simulated) time to evolve +time.max_step = -10 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # #.......................................# -time.fixed_dt = 0.02 # Use this constant dt if > 0 -time.cfl = 0.95 # CFL factor +time.fixed_dt = 0.02 # Use this constant dt if > 0 +time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 1 # 100 # Steps between plot files +time.plot_interval = 10 # 100 # Steps between plot files time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -26,9 +26,9 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" -incflo.initial_iterations = 0 -incflo.do_initial_proj = false -incflo.verbose = 3 +#incflo.initial_iterations = 0 +#incflo.do_initial_proj = false +incflo.verbose = 3 transport.viscosity = 0.0 turbulence.model = Laminar From 01e1dc099b925a5480be3302d7d944337af69b5a Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 24 Jun 2024 11:32:21 -0400 Subject: [PATCH 42/83] simpler one-way flow test --- bctest/oneway_inout | 74 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 bctest/oneway_inout diff --git a/bctest/oneway_inout b/bctest/oneway_inout new file mode 100644 index 0000000000..7182ac46fe --- /dev/null +++ b/bctest/oneway_inout @@ -0,0 +1,74 @@ +# This is a 2D poiseuille flow when run to steady state + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# SIMULATION STOP # +#.......................................# +time.stop_time = 5.00 # Max (simulated) time to evolve +time.max_step = 10 #-1000 # Max number of time steps + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# TIME STEP COMPUTATION # +#.......................................# +time.fixed_dt = 0.02 # Use this constant dt if > 0 +time.cfl = 0.95 # CFL factor + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# INPUT AND OUTPUT # +#.......................................# +time.plot_interval = 1 # 100 # Steps between plot files +time.checkpoint_interval = -100 # Steps between checkpoint files + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# PHYSICS # +#.......................................# +incflo.density = 1.0 # Reference density +incflo.use_godunov = 1 +incflo.diffusion_type = 1 +incflo.godunov_type = "weno_z" + +incflo.initial_iterations = 0 +incflo.do_initial_proj = false +incflo.verbose = 3 + +transport.viscosity = 0.0 +turbulence.model = Laminar + +ICNS.source_terms = BodyForce +BodyForce.magnitude = 0 0 0 + +incflo.physics = FreeStream +FreeStream.velocity_type = TwoLayer + +io.output_default_variables = 1 + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# ADAPTIVE MESH REFINEMENT # +#.......................................# +amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel +amr.blocking_factor = 4 +amr.max_level = 0 # Max AMR level in hierarchy +fabarray.mfiter_tile_size = 1024 1024 1024 + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# GEOMETRY # +#.......................................# +geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates +geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates +geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) + +# Boundary conditions +xlo.type = "mass_inflow_outflow" +#xlo.type = "mass_inflow" +xlo.density = 1.0 +xlo.velocity.inflow_outflow_type = TwoLayer +xlo.velocity.inflow_type = TwoLayer + +xhi.type = "mass_inflow_outflow" +xhi.density = 1.0 +xhi.velocity.inflow_outflow_type = TwoLayer +TwoLayer.mvel = 1.0 +TwoLayer.pvel = 1.0 +#xhi.type = "pressure_outflow" + +zlo.type = "slip_wall" +zhi.type = "slip_wall" From b0f9c18bbac8befcad00cb2372b029d57b735a7a Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 26 Jun 2024 12:04:10 -0400 Subject: [PATCH 43/83] accepting vector velocities for TwoLayer udf --- amr-wind/physics/udfs/TwoLayer.H | 19 +++++++++++++++---- amr-wind/physics/udfs/TwoLayer.cpp | 24 ++++++++++++++---------- bctest/input_inflow | 4 ++-- bctest/input_inout | 4 ++-- bctest/oneway_inout | 4 ++-- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/amr-wind/physics/udfs/TwoLayer.H b/amr-wind/physics/udfs/TwoLayer.H index 86a966d652..a990d71a81 100644 --- a/amr-wind/physics/udfs/TwoLayer.H +++ b/amr-wind/physics/udfs/TwoLayer.H @@ -20,8 +20,10 @@ struct TwoLayer // amrex::GpuArray bar = {0.0}; // clang-format on - // x-velocities of the top and bottom layer respectively - amrex::Real pvel{0.0}; amrex::Real mvel{0.0}; + // velocities of the top and bottom layer respectively + //amrex::Real pvel{0.0}; amrex::Real mvel{0.0}; + amrex::GpuArray pvel = {0.0}; + amrex::GpuArray mvel = {0.0}; AMREX_GPU_DEVICE inline void operator()( @@ -49,8 +51,17 @@ struct TwoLayer // Once the above is done, fill the field as: amrex::GpuArray vel; - vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; - if (time == 0.0) vel[0] *= 0.9; + //vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; + if (z>=0.5) { + vel = {pvel[0], pvel[1], 0.0}; + } else { + vel = {mvel[0], mvel[1], 0.0}; + } + + if (time == 0.0) { + vel[0] *= 0.9; + vel[1] *= 0.9; + } field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; /* if (x <= 0.5) { diff --git a/amr-wind/physics/udfs/TwoLayer.cpp b/amr-wind/physics/udfs/TwoLayer.cpp index f9a7be3f60..579f3d991e 100644 --- a/amr-wind/physics/udfs/TwoLayer.cpp +++ b/amr-wind/physics/udfs/TwoLayer.cpp @@ -8,7 +8,7 @@ namespace amr_wind::udf { -TwoLayer::TwoLayer(const Field& /*fld*/) +TwoLayer::TwoLayer(const Field& fld) { // This is a where the user can set some user defined variables // This capability can be activated with the following in the input file: @@ -18,16 +18,20 @@ TwoLayer::TwoLayer(const Field& /*fld*/) // clang-format off { - //const int ncomp = fld.num_comp(); + const int ncomp = fld.num_comp(); amrex::ParmParse pp("TwoLayer"); - pp.query("pvel", m_op.pvel); - pp.query("mvel", m_op.mvel); - //amrex::Vector vel(0.0, ncomp); - //pp.getarr("velocity", vel); - //AMREX_ALWAYS_ASSERT(vel.size() == ncomp); - //for (int i = 0; i < ncomp; ++i) { - // m_op.bar[i] = vel[i]; - //} + //pp.query("pvel", m_op.pvel); + //pp.query("mvel", m_op.mvel); + amrex::Vector pvel(0.0, ncomp); + amrex::Vector mvel(0.0, ncomp); + pp.getarr("pvel", pvel); + pp.getarr("mvel", mvel); + AMREX_ALWAYS_ASSERT(pvel.size() == ncomp); + AMREX_ALWAYS_ASSERT(mvel.size() == ncomp); + for (int i = 0; i < ncomp; ++i) { + m_op.pvel[i] = pvel[i]; + m_op.mvel[i] = mvel[i]; + } } // clang-format on //amrex::Abort( diff --git a/bctest/input_inflow b/bctest/input_inflow index cfa014c7ba..0249678ac8 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -64,8 +64,8 @@ xlo.velocity.inflow_type = TwoLayer xhi.type = "mass_inflow" xhi.density = 1.0 xhi.velocity.inflow_type = TwoLayer -TwoLayer.mvel = -1.0 -TwoLayer.pvel = 1.0 +TwoLayer.mvel = -1.0 0.0 0.0 +TwoLayer.pvel = 1.0 0.0 0.0 zlo.type = "slip_wall" zhi.type = "slip_wall" diff --git a/bctest/input_inout b/bctest/input_inout index 5dc189cab2..dbdcfbc2c4 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -64,8 +64,8 @@ xlo.velocity.inflow_outflow_type = TwoLayer xhi.type = "mass_inflow_outflow" xhi.density = 1.0 xhi.velocity.inflow_outflow_type = TwoLayer -TwoLayer.mvel = -1.0 -TwoLayer.pvel = 1.0 +TwoLayer.mvel = -1.0 0.0 0.0 +TwoLayer.pvel = 1.0 0.0 0.0 zlo.type = "slip_wall" zhi.type = "slip_wall" diff --git a/bctest/oneway_inout b/bctest/oneway_inout index 7182ac46fe..8e0f2a489f 100644 --- a/bctest/oneway_inout +++ b/bctest/oneway_inout @@ -66,8 +66,8 @@ xlo.velocity.inflow_type = TwoLayer xhi.type = "mass_inflow_outflow" xhi.density = 1.0 xhi.velocity.inflow_outflow_type = TwoLayer -TwoLayer.mvel = 1.0 -TwoLayer.pvel = 1.0 +TwoLayer.mvel = 1.0 0.0 0.0 +TwoLayer.pvel = 1.0 0.0 0.0 #xhi.type = "pressure_outflow" zlo.type = "slip_wall" From b00b97d6319ca7ff15496c54cde577334e2202fa Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 27 Jun 2024 11:22:13 -0400 Subject: [PATCH 44/83] add directional conditions in y and z-directions --- .../convection/incflo_godunov_advection.cpp | 12 ++-- amr-wind/convection/incflo_godunov_ppm.H | 56 ++++++++++++------- .../convection/incflo_godunov_predict.cpp | 16 +++--- 3 files changed, 51 insertions(+), 33 deletions(-) diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp index bbccf8c966..21e3df7458 100644 --- a/amr-wind/convection/incflo_godunov_advection.cpp +++ b/amr-wind/convection/incflo_godunov_advection.cpp @@ -172,7 +172,7 @@ void godunov::compute_fluxes( auto bc = pbc[n]; Godunov_trans_ybc( - i, j, k, n, q, lo, hi, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); ylo(i, j, k, n) = lo; yhi(i, j, k, n) = hi; @@ -199,7 +199,7 @@ void godunov::compute_fluxes( } Godunov_trans_zbc( - i, j, k, n, q, lo, hi, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); zlo(i, j, k, n) = lo; zhi(i, j, k, n) = hi; @@ -232,7 +232,7 @@ void godunov::compute_fluxes( Real wad = wmac(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zylo, l_zyhi, wad, bc.lo(2), bc.hi(2), dlo.z, + i, j, k, n, q, l_zylo, l_zyhi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); constexpr Real small_vel = 1.e-8; @@ -251,7 +251,7 @@ void godunov::compute_fluxes( Real vad = vmac(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yzlo, l_yzhi, vad, bc.lo(1), bc.hi(1), dlo.y, + i, j, k, n, q, l_yzlo, l_yzhi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); constexpr Real small_vel = 1.e-8; @@ -365,7 +365,7 @@ void godunov::compute_fluxes( Real wad = wmac(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zxlo, l_zxhi, wad, bc.lo(2), bc.hi(2), dlo.z, + i, j, k, n, q, l_zxlo, l_zxhi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); constexpr Real small_vel = 1.e-8; @@ -476,7 +476,7 @@ void godunov::compute_fluxes( Real vad = vmac(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yxlo, l_yxhi, vad, bc.lo(1), bc.hi(1), dlo.y, + i, j, k, n, q, l_yxlo, l_yxhi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); constexpr Real small_vel = 1.e-8; diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index 42e71d534d..8568effac3 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -57,7 +57,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( // High X if (i > domhi) { if (bchi == BCType::ext_dir || - (bchi == BCType::direction_dependent && velm <= 0.0)) { // is this correct use of uedge here?? + (bchi == BCType::direction_dependent && velm <= 0.0)) { //Print() << "trans_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n // << " as uedge is " << velm << std::endl; // IAMR does this but it breaks lo/hi symmetry @@ -89,7 +89,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_ybc( const amrex::Array4& s, amrex::Real& lo, amrex::Real& hi, - amrex::Real /* vedge */, + amrex::Real& velm, + amrex::Real& velp, const int bclo, const int bchi, const int domlo, @@ -99,7 +100,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_ybc( // Low Y if (j <= domlo) { - if (bclo == BCType::ext_dir) { + if (bclo == BCType::ext_dir || + (bclo == BCType::direction_dependent && velp >= 0.0)) { // IAMR does this but it breaks lo/hi symmetry // Real st = (vedge <= small_vel) ? hi : s(i,domlo-1,k,n); // So here we do something simpler... @@ -110,7 +112,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_ybc( else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || - bclo == BCType::reflect_even) { + bclo == BCType::reflect_even || + (bclo == BCType::direction_dependent && velp < 0.0)) { lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -121,7 +124,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_ybc( // High Y if (j > domhi) { - if (bchi == BCType::ext_dir) { + if (bchi == BCType::ext_dir || + (bchi == BCType::direction_dependent && velm <= 0.0)) { // IAMR does this but it breaks lo/hi symmetry // Real st = (vedge >= -small_vel)? lo : s(i,domhi+1,k,n); // So here we do something simpler... @@ -132,7 +136,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_ybc( else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || - bchi == BCType::reflect_even) { + bchi == BCType::reflect_even || + (bchi == BCType::direction_dependent && velm > 0.0)) { hi = lo; } else if (bchi == BCType::reflect_odd) { @@ -150,7 +155,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( const amrex::Array4& s, amrex::Real& lo, amrex::Real& hi, - amrex::Real /* wedge */, + amrex::Real& velm, + amrex::Real& velp, const int bclo, const int bchi, const int domlo, @@ -160,7 +166,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( // Low Z if (k <= domlo) { - if (bclo == BCType::ext_dir) { + if (bclo == BCType::ext_dir || + (bclo == BCType::direction_dependent && velp >= 0.0)) { // IAMR does this but it breaks lo/hi symmetry // Real st = (wedge <= small_vel) ? hi : s(i,j,domlo-1,n); // So here we do something simpler... @@ -171,7 +178,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || - bclo == BCType::reflect_even) { + bclo == BCType::reflect_even || + (bclo == BCType::direction_dependent && velp < 0.0)) { lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -182,7 +190,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( // High Z if (k > domhi) { - if (bchi == BCType::ext_dir) { + if (bchi == BCType::ext_dir || + (bchi == BCType::direction_dependent && velm <= 0.0)) { // IAMR does this but it breaks lo/hi symmetry // Real st = (wedge >= -small_vel)? lo : s(i,j,domhi+1,n); // So here we do something simpler... @@ -191,7 +200,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( hi = st; } else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || - bchi == BCType::reflect_even) { + bchi == BCType::reflect_even || + (bchi == BCType::direction_dependent && velm > 0.0)) { hi = lo; } else if (bchi == BCType::reflect_odd) { @@ -294,13 +304,15 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_ybc_lo( using namespace amrex; if (j == domlo) { - if (bclo == BCType::ext_dir && vedge(i, j, k) >= 0.) { + if ((bclo == BCType::ext_dir && vedge(i, j, k) >= 0.) || + (bclo == BCType::direction_dependent && vedge(i, j, k) >= 0.)) { hi = s(i, domlo - 1, k, n); lo = hi; } else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || - (bclo == BCType::ext_dir && vedge(i, j, k) < 0)) { + (bclo == BCType::ext_dir && vedge(i, j, k) < 0) || + (bclo == BCType::direction_dependent && vedge(i, j, k) < 0)) { lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -327,13 +339,15 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_ybc_hi( using namespace amrex; if (j == domhi + 1) { - if (bchi == BCType::ext_dir && vedge(i, j, k) <= 0.) { + if ((bchi == BCType::ext_dir && vedge(i, j, k) <= 0.) || + (bchi == BCType::direction_dependent && vedge(i, j, k) <= 0.)) { lo = s(i, domhi + 1, k, n); hi = lo; } else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || - (bchi == BCType::ext_dir && vedge(i, j, k) > 0)) { + (bchi == BCType::ext_dir && vedge(i, j, k) > 0) || + (bchi == BCType::direction_dependent && vedge(i, j, k) > 0)) { hi = lo; } else if (bchi == BCType::reflect_odd) { @@ -360,7 +374,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_zbc_lo( using namespace amrex; if (k == domlo) { - if (bclo == BCType::ext_dir && wedge(i, j, k) >= 0.) { + if ((bclo == BCType::ext_dir && wedge(i, j, k) >= 0.) || + (bclo == BCType::direction_dependent && wedge(i, j, k) >= 0.)) { hi = s(i, j, domlo - 1, n); lo = hi; } @@ -368,7 +383,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_zbc_lo( else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || - (bclo == BCType::ext_dir && wedge(i, j, k) < 0)) { + (bclo == BCType::ext_dir && wedge(i, j, k) < 0) || + (bclo == BCType::direction_dependent && wedge(i, j, k) < 0)) { lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -395,13 +411,15 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_zbc_hi( using namespace amrex; if (k == domhi + 1) { - if (bchi == BCType::ext_dir && wedge(i, j, k) <= 0.) { + if ((bchi == BCType::ext_dir && wedge(i, j, k) <= 0.) || + (bchi == BCType::direction_dependent && wedge(i, j, k) <= 0.)) { lo = s(i, j, domhi + 1, n); hi = lo; } else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || - (bchi == BCType::ext_dir && wedge(i, j, k) > 0)) { + (bchi == BCType::ext_dir && wedge(i, j, k) > 0) || + (bchi == BCType::direction_dependent && wedge(i, j, k) > 0)) { hi = lo; } else if (bchi == BCType::reflect_odd) { diff --git a/amr-wind/convection/incflo_godunov_predict.cpp b/amr-wind/convection/incflo_godunov_predict.cpp index 952340e583..adafb59157 100644 --- a/amr-wind/convection/incflo_godunov_predict.cpp +++ b/amr-wind/convection/incflo_godunov_predict.cpp @@ -77,7 +77,7 @@ void godunov::make_trans_velocities( auto bc = pbc[n]; Godunov_trans_ybc( - i, j, k, n, vel, lo, hi, lo, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + i, j, k, n, vel, lo, hi, lo, hi, bc.lo(1), bc.hi(1), dlo.y, dhi.y); constexpr Real small_vel = 1e-8; @@ -101,7 +101,7 @@ void godunov::make_trans_velocities( auto bc = pbc[n]; Godunov_trans_zbc( - i, j, k, n, vel, lo, hi, lo, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + i, j, k, n, vel, lo, hi, lo, hi, bc.lo(2), bc.hi(2), dlo.z, dhi.z); constexpr Real small_vel = 1e-8; @@ -211,7 +211,7 @@ void godunov::predict_godunov( auto bc = pbc[n]; Godunov_trans_ybc( - i, j, k, n, q, lo, hi, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); ylo(i, j, k, n) = lo; yhi(i, j, k, n) = hi; @@ -238,7 +238,7 @@ void godunov::predict_godunov( auto bc = pbc[n]; Godunov_trans_zbc( - i, j, k, n, q, lo, hi, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); zlo(i, j, k, n) = lo; zhi(i, j, k, n) = hi; @@ -281,7 +281,7 @@ void godunov::predict_godunov( Real wad = w_ad(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zylo, l_zyhi, wad, bc.lo(2), bc.hi(2), dlo.z, + i, j, k, n, q, l_zylo, l_zyhi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); constexpr Real small_vel = 1.e-8; @@ -300,7 +300,7 @@ void godunov::predict_godunov( Real vad = v_ad(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yzlo, l_yzhi, vad, bc.lo(1), bc.hi(1), dlo.y, + i, j, k, n, q, l_yzlo, l_yzhi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); constexpr Real small_vel = 1.e-8; @@ -385,7 +385,7 @@ void godunov::predict_godunov( Real wad = w_ad(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zxlo, l_zxhi, wad, bc.lo(2), bc.hi(2), dlo.z, + i, j, k, n, q, l_zxlo, l_zxhi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); constexpr Real small_vel = 1.e-8; @@ -473,7 +473,7 @@ void godunov::predict_godunov( Real vad = v_ad(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yxlo, l_yxhi, vad, bc.lo(1), bc.hi(1), dlo.y, + i, j, k, n, q, l_yxlo, l_yxhi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); constexpr Real small_vel = 1.e-8; From d4ac92b9e32d5bdedf7dbd4c39f0fc034db363e3 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 28 Jun 2024 17:30:51 -0700 Subject: [PATCH 45/83] change input files to have flow along both x and y --- bctest/input_inflow | 19 ++++++++++++++----- bctest/input_inout | 19 ++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/bctest/input_inflow b/bctest/input_inflow index 0249678ac8..ac040a022e 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -3,13 +3,13 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 10.00 # Max (simulated) time to evolve +time.stop_time = 20.00 # Max (simulated) time to evolve time.max_step = -10 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # #.......................................# -time.fixed_dt = 0.02 # Use this constant dt if > 0 +time.fixed_dt = 0.01 # Use this constant dt if > 0 time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -54,9 +54,12 @@ fabarray.mfiter_tile_size = 1024 1024 1024 #.......................................# geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates -geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) +geometry.is_periodic = 0 0 0 # Periodicity x y z (0/1) # Boundary conditions +TwoLayer.mvel = -1.0 -2.0 0.0 +TwoLayer.pvel = 1.0 2.0 0.0 + xlo.type = "mass_inflow" xlo.density = 1.0 xlo.velocity.inflow_type = TwoLayer @@ -64,8 +67,14 @@ xlo.velocity.inflow_type = TwoLayer xhi.type = "mass_inflow" xhi.density = 1.0 xhi.velocity.inflow_type = TwoLayer -TwoLayer.mvel = -1.0 0.0 0.0 -TwoLayer.pvel = 1.0 0.0 0.0 + +ylo.type = "mass_inflow" +ylo.density = 1.0 +ylo.velocity.inflow_type = TwoLayer + +yhi.type = "mass_inflow" +yhi.density = 1.0 +yhi.velocity.inflow_type = TwoLayer zlo.type = "slip_wall" zhi.type = "slip_wall" diff --git a/bctest/input_inout b/bctest/input_inout index dbdcfbc2c4..886c47112e 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -3,13 +3,13 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 10.00 # Max (simulated) time to evolve +time.stop_time = 20.00 # Max (simulated) time to evolve time.max_step = -10 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # #.......................................# -time.fixed_dt = 0.02 # Use this constant dt if > 0 +time.fixed_dt = 0.01 # Use this constant dt if > 0 time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -54,9 +54,12 @@ fabarray.mfiter_tile_size = 1024 1024 1024 #.......................................# geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates -geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) +geometry.is_periodic = 0 0 0 # Periodicity x y z (0/1) # Boundary conditions +TwoLayer.mvel = -1.0 -2.0 0.0 +TwoLayer.pvel = 1.0 2.0 0.0 + xlo.type = "mass_inflow_outflow" xlo.density = 1.0 xlo.velocity.inflow_outflow_type = TwoLayer @@ -64,8 +67,14 @@ xlo.velocity.inflow_outflow_type = TwoLayer xhi.type = "mass_inflow_outflow" xhi.density = 1.0 xhi.velocity.inflow_outflow_type = TwoLayer -TwoLayer.mvel = -1.0 0.0 0.0 -TwoLayer.pvel = 1.0 0.0 0.0 + +ylo.type = "mass_inflow_outflow" +ylo.density = 1.0 +ylo.velocity.inflow_outflow_type = TwoLayer + +yhi.type = "mass_inflow_outflow" +yhi.density = 1.0 +yhi.velocity.inflow_outflow_type = TwoLayer zlo.type = "slip_wall" zhi.type = "slip_wall" From 6e66714b0c9d9f9e992c07efd76ddc635400aa2e Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 8 Jul 2024 11:14:53 -0700 Subject: [PATCH 46/83] renamed input file to match with naming of other test files --- ...{oneway_inout => channel_godunov_wenoz_inout} | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) rename bctest/{oneway_inout => channel_godunov_wenoz_inout} (86%) diff --git a/bctest/oneway_inout b/bctest/channel_godunov_wenoz_inout similarity index 86% rename from bctest/oneway_inout rename to bctest/channel_godunov_wenoz_inout index 8e0f2a489f..9e90909f6e 100644 --- a/bctest/oneway_inout +++ b/bctest/channel_godunov_wenoz_inout @@ -4,7 +4,7 @@ # SIMULATION STOP # #.......................................# time.stop_time = 5.00 # Max (simulated) time to evolve -time.max_step = 10 #-1000 # Max number of time steps +time.max_step = -10 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # @@ -15,7 +15,7 @@ time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 1 # 100 # Steps between plot files +time.plot_interval = 10 # 100 # Steps between plot files time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -26,8 +26,8 @@ incflo.use_godunov = 1 incflo.diffusion_type = 1 incflo.godunov_type = "weno_z" -incflo.initial_iterations = 0 -incflo.do_initial_proj = false +#incflo.initial_iterations = 0 +#incflo.do_initial_proj = false incflo.verbose = 3 transport.viscosity = 0.0 @@ -57,18 +57,16 @@ geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) # Boundary conditions +TwoLayer.mvel = -1.0 0.0 0.0 +TwoLayer.pvel = 1.0 0.0 0.0 + xlo.type = "mass_inflow_outflow" -#xlo.type = "mass_inflow" xlo.density = 1.0 xlo.velocity.inflow_outflow_type = TwoLayer -xlo.velocity.inflow_type = TwoLayer xhi.type = "mass_inflow_outflow" xhi.density = 1.0 xhi.velocity.inflow_outflow_type = TwoLayer -TwoLayer.mvel = 1.0 0.0 0.0 -TwoLayer.pvel = 1.0 0.0 0.0 -#xhi.type = "pressure_outflow" zlo.type = "slip_wall" zhi.type = "slip_wall" From 6ff0782442f2ac93ea26689545a9906beb6c3ef4 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 9 Jul 2024 12:06:23 -0700 Subject: [PATCH 47/83] simplify x-y planar flow cases for debugging --- bctest/input_inflow | 2 +- bctest/input_inout | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bctest/input_inflow b/bctest/input_inflow index ac040a022e..dff7c8fe8f 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -4,7 +4,7 @@ # SIMULATION STOP # #.......................................# time.stop_time = 20.00 # Max (simulated) time to evolve -time.max_step = -10 #-1000 # Max number of time steps +time.max_step = 1 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # diff --git a/bctest/input_inout b/bctest/input_inout index 886c47112e..f42b55ef1e 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -4,7 +4,7 @@ # SIMULATION STOP # #.......................................# time.stop_time = 20.00 # Max (simulated) time to evolve -time.max_step = -10 #-1000 # Max number of time steps +time.max_step = 10 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # @@ -57,7 +57,7 @@ geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates geometry.is_periodic = 0 0 0 # Periodicity x y z (0/1) # Boundary conditions -TwoLayer.mvel = -1.0 -2.0 0.0 +TwoLayer.mvel = 1.0 2.0 0.0 TwoLayer.pvel = 1.0 2.0 0.0 xlo.type = "mass_inflow_outflow" From 927cd94c253b35cdf9aa5efaa720bc634ec42e4e Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 9 Jul 2024 12:07:31 -0700 Subject: [PATCH 48/83] printing time from UDF fordebugging --- amr-wind/physics/udfs/TwoLayer.H | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/amr-wind/physics/udfs/TwoLayer.H b/amr-wind/physics/udfs/TwoLayer.H index a990d71a81..b136a48b22 100644 --- a/amr-wind/physics/udfs/TwoLayer.H +++ b/amr-wind/physics/udfs/TwoLayer.H @@ -40,6 +40,10 @@ struct TwoLayer amrex::Print() << "***** filling custom velocity at " << iv << "..." << std::endl; if ((iv[0] == 1) && (iv[1] == 1) && (iv[2] == 1)) amrex::Print() << "***** filling custom velocity at " << iv << "..." << std::endl; + + if ((iv[0] == -1) && (iv[1] == -1) && (iv[2] == 0)) + amrex::Print() << "***** Time is " << time << "..." << std::endl; + // Compute quantities to set the field values. For example: // clang-format off const auto* problo = geom.ProbLo(); From 201d0a58a483be35fa3bee2a4f3aad867c37aeec Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 9 Jul 2024 16:38:05 -0700 Subject: [PATCH 49/83] simplifying mass_inflow case for asymmetry test --- bctest/input_inflow | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bctest/input_inflow b/bctest/input_inflow index dff7c8fe8f..2b8d73f19e 100644 --- a/bctest/input_inflow +++ b/bctest/input_inflow @@ -4,7 +4,7 @@ # SIMULATION STOP # #.......................................# time.stop_time = 20.00 # Max (simulated) time to evolve -time.max_step = 1 #-1000 # Max number of time steps +time.max_step = 2 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # @@ -57,8 +57,8 @@ geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates geometry.is_periodic = 0 0 0 # Periodicity x y z (0/1) # Boundary conditions -TwoLayer.mvel = -1.0 -2.0 0.0 -TwoLayer.pvel = 1.0 2.0 0.0 +TwoLayer.mvel = -1.0 0.0 0.0 +TwoLayer.pvel = 1.0 0.0 0.0 xlo.type = "mass_inflow" xlo.density = 1.0 From d18f42e664fd213cb59f70fb3b277ad5bfb9b692 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 12 Jul 2024 17:19:07 -0700 Subject: [PATCH 50/83] increasing run time for the test to allow for convergence --- bctest/channel_godunov_wenoz_inout | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bctest/channel_godunov_wenoz_inout b/bctest/channel_godunov_wenoz_inout index 9e90909f6e..29886c3c3d 100644 --- a/bctest/channel_godunov_wenoz_inout +++ b/bctest/channel_godunov_wenoz_inout @@ -3,7 +3,7 @@ #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # SIMULATION STOP # #.......................................# -time.stop_time = 5.00 # Max (simulated) time to evolve +time.stop_time = 10.0 # Max (simulated) time to evolve time.max_step = -10 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# From a92c52f299b514cb4526582573c9cad1325d4e0b Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 30 Jul 2024 09:44:06 -0700 Subject: [PATCH 51/83] input file update --- bctest/input_inout | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bctest/input_inout b/bctest/input_inout index f42b55ef1e..20385c7b14 100644 --- a/bctest/input_inout +++ b/bctest/input_inout @@ -4,7 +4,7 @@ # SIMULATION STOP # #.......................................# time.stop_time = 20.00 # Max (simulated) time to evolve -time.max_step = 10 #-1000 # Max number of time steps +time.max_step = -10 #-1000 # Max number of time steps #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # TIME STEP COMPUTATION # @@ -15,7 +15,7 @@ time.cfl = 0.95 # CFL factor #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# # INPUT AND OUTPUT # #.......................................# -time.plot_interval = 10 # 100 # Steps between plot files +time.plot_interval = 50 # 100 # Steps between plot files time.checkpoint_interval = -100 # Steps between checkpoint files #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# @@ -57,7 +57,7 @@ geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates geometry.is_periodic = 0 0 0 # Periodicity x y z (0/1) # Boundary conditions -TwoLayer.mvel = 1.0 2.0 0.0 +TwoLayer.mvel = -1.0 -2.0 0.0 TwoLayer.pvel = 1.0 2.0 0.0 xlo.type = "mass_inflow_outflow" From 5830c00e0dfe135ba80fbc6fbc7decc1e744b4ea Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 30 Jul 2024 10:37:13 -0700 Subject: [PATCH 52/83] clean up, remove print statements meant for debugging --- amr-wind/boundary_conditions/BCInterface.cpp | 3 +- .../MassInflowOutflowBC.cpp | 11 ------ amr-wind/boundary_conditions/velocity_bcs.H | 1 - .../convection/incflo_godunov_advection.cpp | 2 -- amr-wind/convection/incflo_godunov_ppm.H | 24 ++++--------- .../convection/incflo_godunov_predict.cpp | 3 +- amr-wind/convection/incflo_godunov_weno.H | 7 +--- .../equation_systems/icns/icns_advection.H | 3 +- .../equation_systems/icns/icns_advection.cpp | 24 +------------ amr-wind/physics/udfs/TwoLayer.H | 35 ++----------------- amr-wind/physics/udfs/TwoLayer.cpp | 11 ------ .../incflo_apply_nodal_projection.cpp | 9 ----- 12 files changed, 13 insertions(+), 120 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index 0bd620c568..67281c3f02 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -106,8 +106,7 @@ void BCIface::set_bcfuncs() if ((m_field.name() == "velocity") // only velocity for now && (bct == BC::mass_inflow_outflow)) { - amrex::Print() << "***** Registering MIO custom Neumann BC for " - << m_field.name() << std::endl; + m_field.register_custom_bc(ori); } diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index 7d1aa5011c..f345d2a81f 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -23,7 +23,6 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st const int nlevels = m_field.repo().num_active_levels(); const bool ib = (idim == 0), jb = (idim == 1), kb = (idim == 2); - amrex::Print() << "***** Applying MIO custom Neumann fills at orientation: " << idx << std::endl; for (int lev = 0; lev < nlevels; ++lev) { const auto& domain = repo.mesh().Geom(lev).Domain(); @@ -49,11 +48,6 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st bc_a(i-ib, j-jb, k-kb, n) = bc_a(i, j, k, n); } } - - /*for (int n = 0; n < ncomp; n++) { - amrex::Print() << i << " " << j << " " << k << " " << n << std::endl; - amrex::Print() << "result: " << vel(i-ib, j-jb, k-kb, idim) << " " << bc_a(i-ib, j-jb, k-kb, n) << " " << bc_a(i, j, k, n) << std::endl << std::endl; - }*/ }); } @@ -66,11 +60,6 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st bc_a(i, j, k, n) = bc_a(i-ib, j-jb, k-kb, n); } } - - /*for (int n = 0; n < ncomp; n++) { - amrex::Print() << i << " " << j << " " << k << " " << n << std::endl; - amrex::Print() << "result: " << vel(i-ib, j-jb, k-kb, idim) << " " << bc_a(i-ib, j-jb, k-kb, n) << " " << bc_a(i, j, k, n) << std::endl << std::endl; - }*/ }); } } diff --git a/amr-wind/boundary_conditions/velocity_bcs.H b/amr-wind/boundary_conditions/velocity_bcs.H index b6c9ffc0cb..acc26a3919 100644 --- a/amr-wind/boundary_conditions/velocity_bcs.H +++ b/amr-wind/boundary_conditions/velocity_bcs.H @@ -43,7 +43,6 @@ void register_inflow_vel_dirichlet( mesh, time, InflowOp(field)); } else if (inflow_udf == "TwoLayer") { using InflowOp = BCOpCreator; - amrex::Print() << "***** Registering TwoLayer UDF" << std::endl; field.register_fill_patch_op>( mesh, time, InflowOp(field)); } else { diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp index 21e3df7458..24a2d47a97 100644 --- a/amr-wind/convection/incflo_godunov_advection.cpp +++ b/amr-wind/convection/incflo_godunov_advection.cpp @@ -312,8 +312,6 @@ void godunov::compute_fluxes( (!use_forces_in_trans && fq) ? 0.5 * l_dt * fq(i, j, k, n) : 0.; auto bc = pbc[n]; -//if ((i==0) && (j==0) && (k==0)) -// amrex::Print() << "***** now calling Godunov_cc_xbc_lo/hi from compute_fluxes" << std::endl; Godunov_cc_xbc_lo(i, j, k, n, q, stl, sth, umac, bc.lo(0), dlo.x); Godunov_cc_xbc_hi(i, j, k, n, q, stl, sth, umac, bc.hi(0), dhi.x); diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index 8568effac3..3dce0fee8a 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -32,20 +32,18 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( if (i <= domlo) { if (bclo == BCType::ext_dir || (bclo == BCType::direction_dependent && velp >= 0.0)) { - //Print() << "trans_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << velp << std::endl; // IAMR does this but it breaks lo/hi symmetry // Real st = (uedge <= small_vel) ? hi : s(domlo-1,j,k,n); // So here we do something simpler... Real st = s(domlo - 1, j, k, n); lo = st; hi = st; - } else if ( + } + + else if ( bclo == BCType::foextrap || bclo == BCType::hoextrap || bclo == BCType::reflect_even || (bclo == BCType::direction_dependent && velp < 0.0)) { - //Print() << "trans_xbc_domlo_foextrap triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << velp << std::endl; lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -58,20 +56,18 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( if (i > domhi) { if (bchi == BCType::ext_dir || (bchi == BCType::direction_dependent && velm <= 0.0)) { - //Print() << "trans_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << velm << std::endl; // IAMR does this but it breaks lo/hi symmetry // Real st = (uedge >= -small_vel)? lo : s(domhi+1,j,k,n); // So here we do something simpler... Real st = s(domhi + 1, j, k, n); lo = st; hi = st; - } else if ( + } + + else if ( bchi == BCType::foextrap || bchi == BCType::hoextrap || bchi == BCType::reflect_even || (bchi == BCType::direction_dependent && velm > 0.0)) { - //Print() << "trans_xbc_domhi_foextrap triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << velm << std::endl; hi = lo; } else if (bchi == BCType::reflect_odd) { @@ -228,8 +224,6 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( if (i == domlo) { if ((bclo == BCType::ext_dir && uedge(i, j, k) >= 0.) || (bclo == BCType::direction_dependent && uedge(i, j, k) >= 0.)) { - //Print() << "cc_xbc_lo_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << uedge(i,j,k) << std::endl; hi = s(domlo - 1, j, k, n); lo = hi; } else if ( @@ -237,8 +231,6 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( bclo == BCType::reflect_even || (bclo == BCType::ext_dir && uedge(i, j, k) < 0) || (bclo == BCType::direction_dependent && uedge(i, j, k) < 0)) { - //Print() << "cc_xbc_lo_foextrap triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << uedge(i,j,k) << std::endl; lo = hi; } else if (bclo == BCType::reflect_odd) { @@ -267,8 +259,6 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( if (i == domhi + 1) { if ((bchi == BCType::ext_dir && uedge(i, j, k) <= 0.) || (bchi == BCType::direction_dependent && uedge(i, j, k) <= 0.)) { - //Print() << "cc_xbc_hi_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << uedge(i,j,k) << std::endl; lo = s(domhi + 1, j, k, n); hi = lo; } else if ( @@ -276,8 +266,6 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( bchi == BCType::reflect_even || (bchi == BCType::ext_dir && uedge(i, j, k) > 0) || (bchi == BCType::direction_dependent && uedge(i, j, k) > 0)) { - //Print() << "cc_xbc_hi_foextrap triggered at " << i << " " << j << " " << k << " " << n - // << " as uedge is " << uedge(i,j,k) << std::endl; hi = lo; } else if (bchi == BCType::reflect_odd) { diff --git a/amr-wind/convection/incflo_godunov_predict.cpp b/amr-wind/convection/incflo_godunov_predict.cpp index adafb59157..fe4670f57f 100644 --- a/amr-wind/convection/incflo_godunov_predict.cpp +++ b/amr-wind/convection/incflo_godunov_predict.cpp @@ -328,8 +328,7 @@ void godunov::predict_godunov( stl += 0.5 * l_dt * f(i - 1, j, k, n); sth += 0.5 * l_dt * f(i, j, k, n); } -//if ((i==0) && (j==0) && (k==0)) -// amrex::Print() << "***** now calling Godunov_cc_xbc_lo/hi from predict_godunov" << std::endl; + Godunov_cc_xbc_lo(i, j, k, n, q, stl, sth, u_ad, bc.lo(0), dlo.x); Godunov_cc_xbc_hi(i, j, k, n, q, stl, sth, u_ad, bc.hi(0), dhi.x); diff --git a/amr-wind/convection/incflo_godunov_weno.H b/amr-wind/convection/incflo_godunov_weno.H index 259934aba0..8887829459 100644 --- a/amr-wind/convection/incflo_godunov_weno.H +++ b/amr-wind/convection/incflo_godunov_weno.H @@ -74,16 +74,13 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || (bclo == BCType::direction_dependent && velm >= 0.0)) { if (i == domlo) { - //Print() << "weno_xbc_domlo_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as velm is " << velm << std::endl; sp = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); sm = s(domlo - 1, j, k, n); } else if (i == domlo + 1) { - //Print() << "weno_xbc_domlop1_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as velm is " << velm << std::endl; + sm = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); @@ -94,8 +91,6 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || (bchi == BCType::direction_dependent && velp <= 0.0)) { if (i == domhi) { - //Print() << "weno_xbc_domhi_extdir triggered at " << i << " " << j << " " << k << " " << n - // << " as velp is " << velp << std::endl; sm = -0.2 * s(domhi + 1, j, k, n) + 0.75 * s(domhi, j, k, n) + 0.5 * s(domhi - 1, j, k, n) - 0.05 * s(domhi - 2, j, k, n); diff --git a/amr-wind/equation_systems/icns/icns_advection.H b/amr-wind/equation_systems/icns/icns_advection.H index d1ff5bdd4f..68cf1b895e 100644 --- a/amr-wind/equation_systems/icns/icns_advection.H +++ b/amr-wind/equation_systems/icns/icns_advection.H @@ -215,8 +215,7 @@ struct AdvectionOp dof_field.fillpatch_sibling_fields( time + 0.5 * dt, u_mac.num_grow(), mac_vel); } -//amrex::Print() << "!!! umac after fillpatch" << std::endl; -//amrex::Print() << u_mac(0)[0]; + for (int lev = 0; lev < repo.num_active_levels(); ++lev) { u_mac(lev).FillBoundary(geom[lev].periodicity()); v_mac(lev).FillBoundary(geom[lev].periodicity()); diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index b87ddcec81..2b04c5a84f 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -8,11 +8,6 @@ #include "AMReX_MultiFabUtil.H" #include "hydro_MacProjector.H" -// only for debugging -#include "AMReX_PlotFileUtil.H" -#include "AMReX_MultiFabUtil.H" -#include "AMReX_MultiFab.H" -// namespace amr_wind::pde { namespace { @@ -76,7 +71,7 @@ void MacProjOp::enforce_inout_solvability ( auto& velocity = m_repo.get_field("velocity"); amrex::BCRec const* bc_type = velocity.bcrec_device().data(); const amrex::Vector& geom = m_repo.mesh().Geom(); - amrex::Print() << "***** Calling enforceSolvability from AMR-Wind" << std::endl; + HydroUtils::enforceInOutSolvability(a_umac, bc_type, geom); } @@ -282,21 +277,8 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } -//amrex::Array mac_arr = {&u_mac(0), &v_mac(0), &w_mac(0)}; -//amrex::MultiFab mac_vec_cc(amrex::convert((u_mac(0)).boxArray(), amrex::IntVect{0,0,0}), (u_mac(0)).DistributionMap(), 3, 0); - -//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -//amrex::WriteSingleLevelPlotfile("plt_macvel_precorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); -//amrex::Print() << "!!! umac precorrect" << std::endl; -//amrex::Print() << u_mac(0)[0]; - enforce_inout_solvability(mac_vec); -//amrex::Print() << "!!! umac postcorrect" << std::endl; -//amrex::Print() << u_mac(0)[0]; -//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -//amrex::WriteSingleLevelPlotfile("plt_macvel_postcorrect", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); - m_mac_proj->setUMAC(mac_vec); if (m_has_overset) { @@ -312,10 +294,6 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } else { m_mac_proj->project(m_options.rel_tol, m_options.abs_tol); } -//amrex::average_face_to_cellcenter(mac_vec_cc, 0, mac_arr); -//amrex::WriteSingleLevelPlotfile("plt_macvel_postproject", mac_vec_cc, {"umac","vmac","wmac"}, geom[0], 0.0, 0); -//amrex::Print() << "!!! umac postproject" << std::endl; -//amrex::Print() << u_mac(0)[0]; if (m_is_anelastic) { for (int lev = 0; lev < m_repo.num_active_levels(); ++lev) { diff --git a/amr-wind/physics/udfs/TwoLayer.H b/amr-wind/physics/udfs/TwoLayer.H index b136a48b22..b4db35bb98 100644 --- a/amr-wind/physics/udfs/TwoLayer.H +++ b/amr-wind/physics/udfs/TwoLayer.H @@ -14,12 +14,6 @@ struct TwoLayer { struct DeviceOp { - // clang-format off - // Declare parameters here if needed. For example: - // amrex::Real foo{1.0}; - // amrex::GpuArray bar = {0.0}; - // clang-format on - // velocities of the top and bottom layer respectively //amrex::Real pvel{0.0}; amrex::Real mvel{0.0}; amrex::GpuArray pvel = {0.0}; @@ -36,49 +30,24 @@ struct TwoLayer const int dcomp, const int orig_comp) const { - if ((iv[0] == -1) && (iv[1] == -1) && (iv[2] == 0)) - amrex::Print() << "***** filling custom velocity at " << iv << "..." << std::endl; - if ((iv[0] == 1) && (iv[1] == 1) && (iv[2] == 1)) - amrex::Print() << "***** filling custom velocity at " << iv << "..." << std::endl; - - if ((iv[0] == -1) && (iv[1] == -1) && (iv[2] == 0)) - amrex::Print() << "***** Time is " << time << "..." << std::endl; - - // Compute quantities to set the field values. For example: - // clang-format off const auto* problo = geom.ProbLo(); const auto* dx = geom.CellSize(); - // const auto x = problo[0] + (iv[0] + 0.5) * dx[0]; - // const auto y = problo[1] + (iv[1] + 0.5) * dx[1]; const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; // not true for w-mac vels - // const amrex::GpuArray vel = {1.0, 0.0, 0.0}; - // Once the above is done, fill the field as: amrex::GpuArray vel; - //vel = {((z >= 0.5) ? pvel : mvel), 0.0, 0.0}; if (z>=0.5) { vel = {pvel[0], pvel[1], 0.0}; } else { vel = {mvel[0], mvel[1], 0.0}; } + // perturb by some factor at time 0 if (time == 0.0) { vel[0] *= 0.9; vel[1] *= 0.9; } - field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; - /* - if (x <= 0.5) { - // at the low-x boundary, +ve vel in top portion, garbage in the rest - //vel = {((z >= 0.5) ? pvel : -1e10), 0.0, 0.0}; - //vel = {((z >= 0.5) ? pvel : 0.0), 0.0, 0.0}; - } else { - // at the high-x boundary, -ve vel in bottom portion, garbage in the rest - //vel = {((z < 0.5) ? mvel : 1e10), 0.0, 0.0}; - //vel = {((z < 0.5) ? mvel : 0.0), 0.0, 0.0}; - }*/ - // clang-format on + field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; } }; using DeviceType = DeviceOp; diff --git a/amr-wind/physics/udfs/TwoLayer.cpp b/amr-wind/physics/udfs/TwoLayer.cpp index 579f3d991e..a23964aaf9 100644 --- a/amr-wind/physics/udfs/TwoLayer.cpp +++ b/amr-wind/physics/udfs/TwoLayer.cpp @@ -10,13 +10,6 @@ namespace amr_wind::udf { TwoLayer::TwoLayer(const Field& fld) { - // This is a where the user can set some user defined variables - // This capability can be activated with the following in the input file: - // xlo.type = "mass_inflow" - // xlo.velocity.inflow_type = CustomVelocity - // CustomVelocity.foo = 1.0 - - // clang-format off { const int ncomp = fld.num_comp(); amrex::ParmParse pp("TwoLayer"); @@ -33,10 +26,6 @@ TwoLayer::TwoLayer(const Field& fld) m_op.mvel[i] = mvel[i]; } } - // clang-format on - //amrex::Abort( - // "Please define the body of this function and the corresponding struct " - // "in the header file before using it. Then remove this message"); } } // namespace amr_wind::udf diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 13aa4ae06c..6a430f6803 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -7,9 +7,6 @@ #include "amr-wind/projection/nodal_projection_ops.H" #include "hydro_utils.H" -// for debugging -#include "AMReX_PlotFileUtil.H" - using namespace amrex; void amr_wind::nodal_projection::set_inflow_velocity( @@ -356,8 +353,6 @@ void incflo::ApplyProjection( vel[lev]->FillBoundary(geom[lev].periodicity()); } } -//amrex::Print() << "*********** partition **********" << std::endl; -//amrex::Print() << velocity(0)[0]; // Need to apply custom Neumann funcs for inflow-outflow BC // after setting the inflow vels above. @@ -369,8 +364,6 @@ void incflo::ApplyProjection( velocity, m_repo.mesh().Geom(), m_repo.num_active_levels()); } -//amrex::Print() << velocity(0)[0]; - if (is_anelastic) { for (int lev = 0; lev <= finest_level; ++lev) { for (int idim = 0; idim < velocity.num_comp(); ++idim) { @@ -497,8 +490,6 @@ void incflo::ApplyProjection( } } -//amrex::WriteSingleLevelPlotfile("plt_vel_post_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); - // Get phi and fluxes auto phi = nodal_projector->getPhi(); auto gradphi = nodal_projector->getGradPhi(); From 833928e1f5df65342fb51ccfb5e2fa20eedcf3ef Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 30 Jul 2024 10:40:54 -0700 Subject: [PATCH 53/83] remove couple of remaining print statements --- amr-wind/projection/incflo_apply_nodal_projection.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 6a430f6803..5f5ab8e101 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -90,7 +90,7 @@ amr_wind::nodal_projection::enforce_inout_solvability ( vel_vec[lev][1] = new MultiFab(velocity(lev), amrex::make_alias, 1, 1); vel_vec[lev][2] = new MultiFab(velocity(lev), amrex::make_alias, 2, 1); } - Print() << "***** Calling enforceSolvability from AMR-Wind" << std::endl; + HydroUtils::enforceInOutSolvability(vel_vec, bc_type, geom, true); } @@ -177,7 +177,6 @@ void incflo::ApplyProjection( auto& grad_p = m_repo.get_field("gp"); auto& pressure = m_repo.get_field("p"); auto& velocity = icns().fields().field; -//amrex::WriteSingleLevelPlotfile("plt_vel_pre_nodalproj", velocity(0), {"u","v","w"}, geom[0], 0.0, 0); auto& velocity_old = icns().fields().field.state(amr_wind::FieldState::Old); amr_wind::Field const* mesh_fac = mesh_mapping From 79e4669a6f70fa3329ee24c3ede314aa98ae905e Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 30 Jul 2024 18:24:36 -0700 Subject: [PATCH 54/83] change logic to use IntVect instead of booleans --- .../MassInflowOutflowBC.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index f345d2a81f..73f0b00f8f 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -14,14 +14,14 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st { const auto& repo = m_field.repo(); const auto& velocity = repo.get_field("velocity"); - //const auto bcvals = m_field.bc_values_device(); + // const auto bcvals = m_field.bc_values_device(); const int ncomp = m_field.num_comp(); - const int idx = static_cast(m_ori); + // const int idx = static_cast(m_ori); const int idim = m_ori.coordDir(); const auto islow = m_ori.isLow(); const auto ishigh = m_ori.isHigh(); const int nlevels = m_field.repo().num_active_levels(); - const bool ib = (idim == 0), jb = (idim == 1), kb = (idim == 2); + const amrex::IntVect iv_dir = {idim == 0, idim == 1, idim == 2}; for (int lev = 0; lev < nlevels; ++lev) { const auto& domain = repo.mesh().Geom(lev).Domain(); @@ -35,7 +35,7 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st #endif for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { auto bx = mfi.validbox(); - bx.grow({!ib, !jb, !kb}); // filling 1 ghost cell is enough? – check + bx.grow({!iv_dir[0], !iv_dir[1], !iv_dir[2]}); const auto& bc_a = m_field(lev).array(mfi); const auto& vel = velocity(lev).array(mfi); @@ -43,9 +43,11 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st amrex::ParallelFor( amrex::bdryLo(bx, idim), [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if (vel(i-ib, j-jb, k-kb, idim) < 0) { + const amrex::IntVect iv = {i, j, k}; + const amrex::IntVect ivm = iv - iv_dir; + if (vel(ivm[0], ivm[1], ivm[2], idim) < 0) { for (int n = 0; n < ncomp; n++) { - bc_a(i-ib, j-jb, k-kb, n) = bc_a(i, j, k, n); + bc_a(ivm[0], ivm[1], ivm[2], n) = bc_a(i, j, k, n); } } }); @@ -55,9 +57,11 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st amrex::ParallelFor( amrex::bdryHi(bx, idim), [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { + const amrex::IntVect iv = {i, j, k}; + const amrex::IntVect ivm = iv - iv_dir; if (vel(i, j, k, idim) > 0) { for (int n = 0; n < ncomp; n++) { - bc_a(i, j, k, n) = bc_a(i-ib, j-jb, k-kb, n); + bc_a(i, j, k, n) = bc_a(ivm[0], ivm[1], ivm[2], n); } } }); From 4392755e10e1f07d0876a0a2fac23ac2b9912bfb Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 1 Aug 2024 16:58:43 -0700 Subject: [PATCH 55/83] attempting to add an internal flag indicating presence of inout bndry --- amr-wind/boundary_conditions/BCInterface.H | 5 +++++ amr-wind/boundary_conditions/BCInterface.cpp | 2 ++ amr-wind/equation_systems/AdvOp_Godunov.H | 3 ++- amr-wind/equation_systems/AdvOp_MOL.H | 3 ++- amr-wind/equation_systems/BCOps.H | 2 ++ amr-wind/equation_systems/PDE.H | 6 ++++-- amr-wind/equation_systems/icns/icns_advection.H | 16 +++++++++++----- .../equation_systems/icns/icns_advection.cpp | 6 ++++-- amr-wind/equation_systems/icns/icns_bcop.H | 2 ++ amr-wind/equation_systems/vof/vof_advection.H | 1 + amr-wind/equation_systems/vof/vof_bcop.H | 1 + amr-wind/incflo.H | 3 +++ .../projection/incflo_apply_nodal_projection.cpp | 5 +++-- 13 files changed, 42 insertions(+), 13 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.H b/amr-wind/boundary_conditions/BCInterface.H index 45c6e4e0ef..266e59e8e5 100644 --- a/amr-wind/boundary_conditions/BCInterface.H +++ b/amr-wind/boundary_conditions/BCInterface.H @@ -51,6 +51,9 @@ public: //! User-defined functions for Dirichlet-type boundaries amrex::Array get_dirichlet_udfs(); + //! Check if any of the boundaries is a mass-inflow-outflow + bool has_inout_bndry() const { return m_inout_bndry; } + protected: //! Setup AMReX mathematical BC types virtual void set_bcrec() = 0; @@ -87,6 +90,8 @@ protected: } } + bool m_inout_bndry{false}; + //! Field instance where BC is being set Field& m_field; }; diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index 67281c3f02..cd25822c1a 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -223,6 +223,7 @@ void BCVelocity::set_bcrec() break; case BC::mass_inflow_outflow: + m_inout_bndry = true; if (side == amrex::Orientation::low) { set_bcrec_lo(dir, amrex::BCType::direction_dependent); } else { @@ -324,6 +325,7 @@ void BCScalar::set_bcrec() break; case BC::mass_inflow_outflow: + m_inout_bndry = true; if (side == amrex::Orientation::low) { set_bcrec_lo(dir, amrex::BCType::direction_dependent); } else { diff --git a/amr-wind/equation_systems/AdvOp_Godunov.H b/amr-wind/equation_systems/AdvOp_Godunov.H index 09eaebadf8..4ef0e84457 100644 --- a/amr-wind/equation_systems/AdvOp_Godunov.H +++ b/amr-wind/equation_systems/AdvOp_Godunov.H @@ -30,7 +30,8 @@ struct AdvectionOp< bool /* has_overset */, bool /* variable density */, bool /* mesh mapping */, - bool /* is_anelastic */) + bool /* is_anelastic */, + bool /* has_inout_bndry */) : fields(fields_in) , density(fields_in.repo.get_field("density")) , u_mac(fields_in.repo.get_field("u_mac")) diff --git a/amr-wind/equation_systems/AdvOp_MOL.H b/amr-wind/equation_systems/AdvOp_MOL.H index b276e39d78..328fbe5aa7 100644 --- a/amr-wind/equation_systems/AdvOp_MOL.H +++ b/amr-wind/equation_systems/AdvOp_MOL.H @@ -29,7 +29,8 @@ struct AdvectionOp< bool /* has_overset */, bool /* variable density */, bool /* mesh mapping */, - bool /* is_anelastic */) + bool /* is_anelastic */, + bool /* has_inout_bndry */) : fields(fields_in) , density(fields_in.repo.get_field("density")) , u_mac(fields_in.repo.get_field("u_mac")) diff --git a/amr-wind/equation_systems/BCOps.H b/amr-wind/equation_systems/BCOps.H index 383cf4d515..50e91d8e68 100644 --- a/amr-wind/equation_systems/BCOps.H +++ b/amr-wind/equation_systems/BCOps.H @@ -27,6 +27,7 @@ struct BCOp>> const auto udfs = bc.get_dirichlet_udfs(); scalar_bc::register_scalar_dirichlet( m_fields.field, m_fields.repo.mesh(), m_time, udfs); + has_inout_bndry = bc.has_inout_bndry(); // Used for fillpatch operation on the source term BCSrcTerm bc_src(m_fields.src_term); @@ -54,6 +55,7 @@ struct BCOp>> PDEFields& m_fields; const SimTime& m_time; + bool has_inout_bndry{false}; }; } // namespace amr_wind::pde diff --git a/amr-wind/equation_systems/PDE.H b/amr-wind/equation_systems/PDE.H index 485616d680..3a075a2209 100644 --- a/amr-wind/equation_systems/PDE.H +++ b/amr-wind/equation_systems/PDE.H @@ -69,7 +69,8 @@ public: m_adv_op.reset(new AdvectionOp( m_sim, m_fields, m_sim.has_overset(), variable_density, - m_sim.has_mesh_mapping(), m_sim.is_anelastic())); + m_sim.has_mesh_mapping(), m_sim.is_anelastic(), + m_bc_op.has_inout_bndry)); m_src_op.init_source_terms(m_sim); // Post-solve operations should also apply after initialization @@ -92,7 +93,8 @@ public: m_adv_op.reset(new AdvectionOp( m_sim, m_fields, m_sim.has_overset(), variable_density, - m_sim.has_mesh_mapping(), m_sim.is_anelastic())); + m_sim.has_mesh_mapping(), m_sim.is_anelastic(), + m_bc_op.has_inout_bndry)); // Post-solve operations should also apply after a regrid m_post_solve_op(m_time.current_time()); diff --git a/amr-wind/equation_systems/icns/icns_advection.H b/amr-wind/equation_systems/icns/icns_advection.H index 68cf1b895e..4fbd5b81b4 100644 --- a/amr-wind/equation_systems/icns/icns_advection.H +++ b/amr-wind/equation_systems/icns/icns_advection.H @@ -28,7 +28,8 @@ public: bool /*has_overset*/, bool /*variable_density*/, bool /*mesh_mapping*/, - bool /*is_anelastic*/); + bool /*is_anelastic*/, + bool /*has_inout_bndry*/); void set_inflow_velocity(amrex::Real time); @@ -60,6 +61,7 @@ private: bool m_variable_density{false}; bool m_mesh_mapping{false}; bool m_is_anelastic{false}; + bool m_has_inout_bndry{false}; amrex::Real m_rho_0{1.0}; }; @@ -75,7 +77,8 @@ struct AdvectionOp bool has_overset, bool variable_density, bool mesh_mapping, - bool is_anelastic) + bool is_anelastic, + bool has_inout_bndry) : fields(fields_in) , u_mac(fields_in.repo.get_field("u_mac")) , v_mac(fields_in.repo.get_field("v_mac")) @@ -86,7 +89,8 @@ struct AdvectionOp has_overset, variable_density, mesh_mapping, - is_anelastic) + is_anelastic, + has_inout_bndry) { amrex::ParmParse pp("incflo"); @@ -452,7 +456,8 @@ struct AdvectionOp bool has_overset, bool variable_density, bool mesh_mapping, - bool is_anelastic) + bool is_anelastic, + bool has_inout_bndry) : fields(fields_in) , u_mac(fields_in.repo.get_field("u_mac")) , v_mac(fields_in.repo.get_field("v_mac")) @@ -464,7 +469,8 @@ struct AdvectionOp has_overset, variable_density, m_mesh_mapping, - is_anelastic) + is_anelastic, + has_inout_bndry) {} void preadvect( diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 2b04c5a84f..4690ef6aa2 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -47,7 +47,8 @@ MacProjOp::MacProjOp( bool has_overset, bool variable_density, bool mesh_mapping, - bool is_anelastic) + bool is_anelastic, + bool has_inout_bndry) : m_repo(repo) , m_phy_mgr(phy_mgr) , m_options("mac_proj") @@ -55,6 +56,7 @@ MacProjOp::MacProjOp( , m_variable_density(variable_density) , m_mesh_mapping(mesh_mapping) , m_is_anelastic(is_anelastic) + , m_has_inout_bndry(has_inout_bndry) { amrex::ParmParse pp("incflo"); pp.query("density", m_rho_0); @@ -277,7 +279,7 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } - enforce_inout_solvability(mac_vec); + if (m_has_inout_bndry) { enforce_inout_solvability(mac_vec); } m_mac_proj->setUMAC(mac_vec); diff --git a/amr-wind/equation_systems/icns/icns_bcop.H b/amr-wind/equation_systems/icns/icns_bcop.H index 83a02652cc..ee472585d6 100644 --- a/amr-wind/equation_systems/icns/icns_bcop.H +++ b/amr-wind/equation_systems/icns/icns_bcop.H @@ -26,6 +26,7 @@ struct BCOp const auto udfs = bc.get_dirichlet_udfs(); vel_bc::register_velocity_dirichlet( m_fields.field, m_fields.repo.mesh(), m_time, udfs); + has_inout_bndry = bc.has_inout_bndry(); auto& density = m_fields.repo.get_field("density"); const amrex::Real density_default = 1.0; @@ -66,6 +67,7 @@ struct BCOp PDEFields& m_fields; const SimTime& m_time; + bool has_inout_bndry{false}; }; } // namespace amr_wind::pde diff --git a/amr-wind/equation_systems/vof/vof_advection.H b/amr-wind/equation_systems/vof/vof_advection.H index 94eaab2af4..4f387fe90a 100644 --- a/amr-wind/equation_systems/vof/vof_advection.H +++ b/amr-wind/equation_systems/vof/vof_advection.H @@ -20,6 +20,7 @@ struct AdvectionOp bool /*unused*/, bool /*unused*/, bool /*unused*/, + bool /*unused*/, bool /*unused*/) : fields(fields_in) , u_mac(fields_in.repo.get_field("u_mac")) diff --git a/amr-wind/equation_systems/vof/vof_bcop.H b/amr-wind/equation_systems/vof/vof_bcop.H index 3987288c5a..3feb9b6d84 100644 --- a/amr-wind/equation_systems/vof/vof_bcop.H +++ b/amr-wind/equation_systems/vof/vof_bcop.H @@ -46,6 +46,7 @@ struct BCOp PDEFields& m_fields; const SimTime& m_time; + bool has_inout_bndry{false}; }; } // namespace amr_wind::pde diff --git a/amr-wind/incflo.H b/amr-wind/incflo.H index d9b4a70570..5e467ed6e5 100644 --- a/amr-wind/incflo.H +++ b/amr-wind/incflo.H @@ -172,6 +172,9 @@ private: // Prescribe advection velocity bool m_prescribe_vel = false; + // Account for mass-inflow-outflow boundary + bool has_inout_bndry = false; + //! number of cells on all levels including covered cells amrex::Long m_cell_count{-1}; diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 5f5ab8e101..8e0e8496d1 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -356,12 +356,13 @@ void incflo::ApplyProjection( // Need to apply custom Neumann funcs for inflow-outflow BC // after setting the inflow vels above. if (!proj_for_small_dt and !incremental) { - velocity.apply_bc_funcs(amr_wind::FieldState::New); + } + //if (has_inout_bndry) { amr_wind::nodal_projection::enforce_inout_solvability( velocity, m_repo.mesh().Geom(), m_repo.num_active_levels()); - } + //} if (is_anelastic) { for (int lev = 0; lev <= finest_level; ++lev) { From b63e6d61ac1422cd0422451fbe58cf503c2ff7d7 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 2 Aug 2024 11:16:09 -0700 Subject: [PATCH 56/83] define internal inout bndry flag in Field class instead of BCIface --- amr-wind/boundary_conditions/BCInterface.H | 5 ----- amr-wind/boundary_conditions/BCInterface.cpp | 4 ++-- amr-wind/core/Field.H | 9 +++++++++ amr-wind/equation_systems/BCOps.H | 2 -- amr-wind/equation_systems/PDE.H | 4 ++-- amr-wind/equation_systems/icns/icns_advection.cpp | 4 +++- amr-wind/equation_systems/icns/icns_bcop.H | 2 -- amr-wind/equation_systems/vof/vof_bcop.H | 1 - amr-wind/incflo.H | 3 --- amr-wind/projection/incflo_apply_nodal_projection.cpp | 6 +++--- unit_tests/equation_systems/test_icns_cstdens.cpp | 2 +- 11 files changed, 20 insertions(+), 22 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.H b/amr-wind/boundary_conditions/BCInterface.H index 266e59e8e5..45c6e4e0ef 100644 --- a/amr-wind/boundary_conditions/BCInterface.H +++ b/amr-wind/boundary_conditions/BCInterface.H @@ -51,9 +51,6 @@ public: //! User-defined functions for Dirichlet-type boundaries amrex::Array get_dirichlet_udfs(); - //! Check if any of the boundaries is a mass-inflow-outflow - bool has_inout_bndry() const { return m_inout_bndry; } - protected: //! Setup AMReX mathematical BC types virtual void set_bcrec() = 0; @@ -90,8 +87,6 @@ protected: } } - bool m_inout_bndry{false}; - //! Field instance where BC is being set Field& m_field; }; diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index cd25822c1a..a81ce1fe91 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -223,7 +223,7 @@ void BCVelocity::set_bcrec() break; case BC::mass_inflow_outflow: - m_inout_bndry = true; + m_field.set_inout_bndry(); if (side == amrex::Orientation::low) { set_bcrec_lo(dir, amrex::BCType::direction_dependent); } else { @@ -325,7 +325,7 @@ void BCScalar::set_bcrec() break; case BC::mass_inflow_outflow: - m_inout_bndry = true; + m_field.set_inout_bndry(); if (side == amrex::Orientation::low) { set_bcrec_lo(dir, amrex::BCType::direction_dependent); } else { diff --git a/amr-wind/core/Field.H b/amr-wind/core/Field.H index beaf535844..d00261ded6 100644 --- a/amr-wind/core/Field.H +++ b/amr-wind/core/Field.H @@ -396,6 +396,12 @@ public: inline bool& in_uniform_space() { return m_mesh_mapped; } inline bool in_uniform_space() const { return m_mesh_mapped; } + //! Check if any of the boundaries is a mass-inflow-outflow + inline bool has_inout_bndry() const { return m_inout_bndry; } + + //! Set the inout_bndry flag + void set_inout_bndry() { m_inout_bndry = true; } + protected: Field( FieldRepo& repo, @@ -425,6 +431,9 @@ protected: //! Flag to track mesh mapping (to uniform space) of field bool m_mesh_mapped{false}; + + //! Flag to indicate whether any of the boundaries is mass-inflow-outflow + bool m_inout_bndry{false}; }; } // namespace amr_wind diff --git a/amr-wind/equation_systems/BCOps.H b/amr-wind/equation_systems/BCOps.H index 50e91d8e68..383cf4d515 100644 --- a/amr-wind/equation_systems/BCOps.H +++ b/amr-wind/equation_systems/BCOps.H @@ -27,7 +27,6 @@ struct BCOp>> const auto udfs = bc.get_dirichlet_udfs(); scalar_bc::register_scalar_dirichlet( m_fields.field, m_fields.repo.mesh(), m_time, udfs); - has_inout_bndry = bc.has_inout_bndry(); // Used for fillpatch operation on the source term BCSrcTerm bc_src(m_fields.src_term); @@ -55,7 +54,6 @@ struct BCOp>> PDEFields& m_fields; const SimTime& m_time; - bool has_inout_bndry{false}; }; } // namespace amr_wind::pde diff --git a/amr-wind/equation_systems/PDE.H b/amr-wind/equation_systems/PDE.H index 3a075a2209..9601cf454f 100644 --- a/amr-wind/equation_systems/PDE.H +++ b/amr-wind/equation_systems/PDE.H @@ -70,7 +70,7 @@ public: m_adv_op.reset(new AdvectionOp( m_sim, m_fields, m_sim.has_overset(), variable_density, m_sim.has_mesh_mapping(), m_sim.is_anelastic(), - m_bc_op.has_inout_bndry)); + m_fields.field.has_inout_bndry())); m_src_op.init_source_terms(m_sim); // Post-solve operations should also apply after initialization @@ -94,7 +94,7 @@ public: m_adv_op.reset(new AdvectionOp( m_sim, m_fields, m_sim.has_overset(), variable_density, m_sim.has_mesh_mapping(), m_sim.is_anelastic(), - m_bc_op.has_inout_bndry)); + m_fields.field.has_inout_bndry())); // Post-solve operations should also apply after a regrid m_post_solve_op(m_time.current_time()); diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 4690ef6aa2..e2b15ceb3d 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -279,7 +279,9 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } - if (m_has_inout_bndry) { enforce_inout_solvability(mac_vec); } + if (m_has_inout_bndry) { + enforce_inout_solvability(mac_vec); + } m_mac_proj->setUMAC(mac_vec); diff --git a/amr-wind/equation_systems/icns/icns_bcop.H b/amr-wind/equation_systems/icns/icns_bcop.H index ee472585d6..83a02652cc 100644 --- a/amr-wind/equation_systems/icns/icns_bcop.H +++ b/amr-wind/equation_systems/icns/icns_bcop.H @@ -26,7 +26,6 @@ struct BCOp const auto udfs = bc.get_dirichlet_udfs(); vel_bc::register_velocity_dirichlet( m_fields.field, m_fields.repo.mesh(), m_time, udfs); - has_inout_bndry = bc.has_inout_bndry(); auto& density = m_fields.repo.get_field("density"); const amrex::Real density_default = 1.0; @@ -67,7 +66,6 @@ struct BCOp PDEFields& m_fields; const SimTime& m_time; - bool has_inout_bndry{false}; }; } // namespace amr_wind::pde diff --git a/amr-wind/equation_systems/vof/vof_bcop.H b/amr-wind/equation_systems/vof/vof_bcop.H index 3feb9b6d84..3987288c5a 100644 --- a/amr-wind/equation_systems/vof/vof_bcop.H +++ b/amr-wind/equation_systems/vof/vof_bcop.H @@ -46,7 +46,6 @@ struct BCOp PDEFields& m_fields; const SimTime& m_time; - bool has_inout_bndry{false}; }; } // namespace amr_wind::pde diff --git a/amr-wind/incflo.H b/amr-wind/incflo.H index 5e467ed6e5..d9b4a70570 100644 --- a/amr-wind/incflo.H +++ b/amr-wind/incflo.H @@ -172,9 +172,6 @@ private: // Prescribe advection velocity bool m_prescribe_vel = false; - // Account for mass-inflow-outflow boundary - bool has_inout_bndry = false; - //! number of cells on all levels including covered cells amrex::Long m_cell_count{-1}; diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index 8e0e8496d1..aa82e20ba3 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -355,14 +355,14 @@ void incflo::ApplyProjection( // Need to apply custom Neumann funcs for inflow-outflow BC // after setting the inflow vels above. - if (!proj_for_small_dt and !incremental) { + if (!proj_for_small_dt and !incremental and velocity.has_inout_bndry()) { velocity.apply_bc_funcs(amr_wind::FieldState::New); } - //if (has_inout_bndry) { + if (velocity.has_inout_bndry()) { amr_wind::nodal_projection::enforce_inout_solvability( velocity, m_repo.mesh().Geom(), m_repo.num_active_levels()); - //} + } if (is_anelastic) { for (int lev = 0; lev <= finest_level; ++lev) { diff --git a/unit_tests/equation_systems/test_icns_cstdens.cpp b/unit_tests/equation_systems/test_icns_cstdens.cpp index 806a9804e9..07f0921b7d 100644 --- a/unit_tests/equation_systems/test_icns_cstdens.cpp +++ b/unit_tests/equation_systems/test_icns_cstdens.cpp @@ -23,7 +23,7 @@ class ICNSConstDensTest : public MeshTest // Initialize MAC projection operator const auto& mco = amr_wind::pde::MacProjOp( - sim().repo(), sim().physics_manager(), false, false, false, false); + sim().repo(), sim().physics_manager(), false, false, false, false, false); // Get background density and check const amrex::Real rho0 = mco.rho0(); EXPECT_EQ(rho0, m_rho_0); From 31e8f8074553f7e3139fcf4996f9c35120e4b428 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Fri, 9 Aug 2024 13:32:33 -0700 Subject: [PATCH 57/83] add print statements to debug MAC-fill --- amr-wind/physics/udfs/TwoLayer.H | 2 ++ 1 file changed, 2 insertions(+) diff --git a/amr-wind/physics/udfs/TwoLayer.H b/amr-wind/physics/udfs/TwoLayer.H index b4db35bb98..e7d417984e 100644 --- a/amr-wind/physics/udfs/TwoLayer.H +++ b/amr-wind/physics/udfs/TwoLayer.H @@ -30,6 +30,8 @@ struct TwoLayer const int dcomp, const int orig_comp) const { +if (iv[0] == 0 && iv[1] == 1 && iv[2] == 1) { amrex::Print() << "time: " << time << std::endl; } +//if (iv[0] == -1 && iv[1] == 1 && iv[2] == 1) { amrex::Print() << "time: " << time << std::endl; } const auto* problo = geom.ProbLo(); const auto* dx = geom.CellSize(); const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; // not true for w-mac vels From fb704db8553409fe53e8075e4b277e0d46b1cc56 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 12 Aug 2024 14:48:11 -0700 Subject: [PATCH 58/83] add mass-inflow-outflow condition to routine that fills MAC vel valid cells before projection --- amr-wind/core/FieldFillPatchOps.H | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/amr-wind/core/FieldFillPatchOps.H b/amr-wind/core/FieldFillPatchOps.H index a733e2196e..296c84693e 100644 --- a/amr-wind/core/FieldFillPatchOps.H +++ b/amr-wind/core/FieldFillPatchOps.H @@ -415,7 +415,8 @@ public: for (amrex::OrientationIter oit; oit != nullptr; ++oit) { const auto ori = oit(); - if (bctype[ori] != BC::mass_inflow) { + if ((bctype[ori] != BC::mass_inflow) && + (bctype[ori] != BC::mass_inflow_outflow)) { continue; } From d9dddd253943d0b8099b708da6315209390ca750 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 12 Aug 2024 15:16:46 -0700 Subject: [PATCH 59/83] clean up and customize the TwoLayer UDF --- amr-wind/physics/udfs/TwoLayer.H | 24 ++++++++++---------- amr-wind/physics/udfs/TwoLayer.cpp | 35 ++++++++++++++++++------------ bctest/channel_godunov_wenoz_inout | 6 +++-- 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/amr-wind/physics/udfs/TwoLayer.H b/amr-wind/physics/udfs/TwoLayer.H index e7d417984e..06b1df933e 100644 --- a/amr-wind/physics/udfs/TwoLayer.H +++ b/amr-wind/physics/udfs/TwoLayer.H @@ -15,9 +15,10 @@ struct TwoLayer struct DeviceOp { // velocities of the top and bottom layer respectively - //amrex::Real pvel{0.0}; amrex::Real mvel{0.0}; - amrex::GpuArray pvel = {0.0}; - amrex::GpuArray mvel = {0.0}; + amrex::GpuArray top_vel = {0.0}; + amrex::GpuArray bottom_vel = {0.0}; + amrex::Real z_part{0.5}; // z-coordinate that divides top and bottom + amrex::Real init_perturb{1.0}; // initial perturbation for testing AMREX_GPU_DEVICE inline void operator()( @@ -25,28 +26,25 @@ struct TwoLayer amrex::Array4 const& field, amrex::GeometryData const& geom, const amrex::Real time, - amrex::Orientation ori, + amrex::Orientation /* ori */, const int comp, const int dcomp, const int orig_comp) const { -if (iv[0] == 0 && iv[1] == 1 && iv[2] == 1) { amrex::Print() << "time: " << time << std::endl; } -//if (iv[0] == -1 && iv[1] == 1 && iv[2] == 1) { amrex::Print() << "time: " << time << std::endl; } const auto* problo = geom.ProbLo(); const auto* dx = geom.CellSize(); - const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; // not true for w-mac vels + const auto z = problo[2] + (iv[2] + 0.5) * dx[2]; amrex::GpuArray vel; - if (z>=0.5) { - vel = {pvel[0], pvel[1], 0.0}; + if (z >= z_part) { + vel = {top_vel[0], top_vel[1], 0.0}; } else { - vel = {mvel[0], mvel[1], 0.0}; + vel = {bottom_vel[0], bottom_vel[1], 0.0}; } - // perturb by some factor at time 0 if (time == 0.0) { - vel[0] *= 0.9; - vel[1] *= 0.9; + vel[0] *= init_perturb; + vel[1] *= init_perturb; } field(iv[0], iv[1], iv[2], dcomp + comp) = vel[orig_comp + comp]; diff --git a/amr-wind/physics/udfs/TwoLayer.cpp b/amr-wind/physics/udfs/TwoLayer.cpp index a23964aaf9..050b7b590a 100644 --- a/amr-wind/physics/udfs/TwoLayer.cpp +++ b/amr-wind/physics/udfs/TwoLayer.cpp @@ -10,21 +10,28 @@ namespace amr_wind::udf { TwoLayer::TwoLayer(const Field& fld) { + // For a 2-layer flow with a top and a bottom layer + // divided at a specific z-coordinate (default: 0.5) + // and an optional initial perturbation (default: 1.0) + { - const int ncomp = fld.num_comp(); - amrex::ParmParse pp("TwoLayer"); - //pp.query("pvel", m_op.pvel); - //pp.query("mvel", m_op.mvel); - amrex::Vector pvel(0.0, ncomp); - amrex::Vector mvel(0.0, ncomp); - pp.getarr("pvel", pvel); - pp.getarr("mvel", mvel); - AMREX_ALWAYS_ASSERT(pvel.size() == ncomp); - AMREX_ALWAYS_ASSERT(mvel.size() == ncomp); - for (int i = 0; i < ncomp; ++i) { - m_op.pvel[i] = pvel[i]; - m_op.mvel[i] = mvel[i]; - } + const int ncomp = fld.num_comp(); + amrex::ParmParse pp("TwoLayer"); + + amrex::Vector top_vel(0.0, ncomp); + amrex::Vector bottom_vel(0.0, ncomp); + pp.getarr("top_vel", top_vel); + pp.getarr("bottom_vel", bottom_vel); + + AMREX_ALWAYS_ASSERT(top_vel.size() == ncomp); + AMREX_ALWAYS_ASSERT(bottom_vel.size() == ncomp); + for (int i = 0; i < ncomp; ++i) { + m_op.top_vel[i] = top_vel[i]; + m_op.bottom_vel[i] = bottom_vel[i]; + } + + pp.query("init_perturb", m_op.init_perturb); + pp.query("z_partition", m_op.z_part); } } diff --git a/bctest/channel_godunov_wenoz_inout b/bctest/channel_godunov_wenoz_inout index 29886c3c3d..9650bf43da 100644 --- a/bctest/channel_godunov_wenoz_inout +++ b/bctest/channel_godunov_wenoz_inout @@ -57,8 +57,10 @@ geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) # Boundary conditions -TwoLayer.mvel = -1.0 0.0 0.0 -TwoLayer.pvel = 1.0 0.0 0.0 +TwoLayer.bottom_vel = -1.0 0.0 0.0 +TwoLayer.top_vel = 1.0 0.0 0.0 +TwoLayer.init_perturb = 0.9 +TwoLayer.z_partition = 0.5 xlo.type = "mass_inflow_outflow" xlo.density = 1.0 From ab20628e1986ff32dd87f45f354a25aed6ebb1fc Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 12 Aug 2024 16:00:31 -0700 Subject: [PATCH 60/83] remove testing folder from index --- bctest/channel_godunov_wenoz_inout | 74 --------------------------- bctest/cmake_mpi.sh | 16 ------ bctest/cmake_mpi_debug.sh | 16 ------ bctest/input_inflow | 80 ------------------------------ bctest/input_inout | 80 ------------------------------ 5 files changed, 266 deletions(-) delete mode 100644 bctest/channel_godunov_wenoz_inout delete mode 100755 bctest/cmake_mpi.sh delete mode 100755 bctest/cmake_mpi_debug.sh delete mode 100644 bctest/input_inflow delete mode 100644 bctest/input_inout diff --git a/bctest/channel_godunov_wenoz_inout b/bctest/channel_godunov_wenoz_inout deleted file mode 100644 index 9650bf43da..0000000000 --- a/bctest/channel_godunov_wenoz_inout +++ /dev/null @@ -1,74 +0,0 @@ -# This is a 2D poiseuille flow when run to steady state - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# SIMULATION STOP # -#.......................................# -time.stop_time = 10.0 # Max (simulated) time to evolve -time.max_step = -10 #-1000 # Max number of time steps - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# TIME STEP COMPUTATION # -#.......................................# -time.fixed_dt = 0.02 # Use this constant dt if > 0 -time.cfl = 0.95 # CFL factor - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# INPUT AND OUTPUT # -#.......................................# -time.plot_interval = 10 # 100 # Steps between plot files -time.checkpoint_interval = -100 # Steps between checkpoint files - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# PHYSICS # -#.......................................# -incflo.density = 1.0 # Reference density -incflo.use_godunov = 1 -incflo.diffusion_type = 1 -incflo.godunov_type = "weno_z" - -#incflo.initial_iterations = 0 -#incflo.do_initial_proj = false -incflo.verbose = 3 - -transport.viscosity = 0.0 -turbulence.model = Laminar - -ICNS.source_terms = BodyForce -BodyForce.magnitude = 0 0 0 - -incflo.physics = FreeStream -FreeStream.velocity_type = TwoLayer - -io.output_default_variables = 1 - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# ADAPTIVE MESH REFINEMENT # -#.......................................# -amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel -amr.blocking_factor = 4 -amr.max_level = 0 # Max AMR level in hierarchy -fabarray.mfiter_tile_size = 1024 1024 1024 - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# GEOMETRY # -#.......................................# -geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates -geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates -geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) - -# Boundary conditions -TwoLayer.bottom_vel = -1.0 0.0 0.0 -TwoLayer.top_vel = 1.0 0.0 0.0 -TwoLayer.init_perturb = 0.9 -TwoLayer.z_partition = 0.5 - -xlo.type = "mass_inflow_outflow" -xlo.density = 1.0 -xlo.velocity.inflow_outflow_type = TwoLayer - -xhi.type = "mass_inflow_outflow" -xhi.density = 1.0 -xhi.velocity.inflow_outflow_type = TwoLayer - -zlo.type = "slip_wall" -zhi.type = "slip_wall" diff --git a/bctest/cmake_mpi.sh b/bctest/cmake_mpi.sh deleted file mode 100755 index 1cb287eb36..0000000000 --- a/bctest/cmake_mpi.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Example CMake config script for an OSX laptop with MPICH - -cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ - -DCMAKE_CXX_COMPILER:STRING=mpicxx \ - -DCMAKE_C_COMPILER:STRING=mpicc \ - -DCMAKE_Fortran_COMPILER:STRING=mpifort \ - -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DAMR_WIND_USE_INTERNAL_AMREX_HYDRO:BOOL=OFF \ - -DAMReX-Hydro_DIR:STRING=$HOME/AMReX-Hydro \ - -DAMR_WIND_ENABLE_MPI:BOOL=ON \ - -DAMR_WIND_ENABLE_TESTS:BOOL=OFF \ - -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ - .. && make -j8 diff --git a/bctest/cmake_mpi_debug.sh b/bctest/cmake_mpi_debug.sh deleted file mode 100755 index 57700d293b..0000000000 --- a/bctest/cmake_mpi_debug.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Example CMake config script for an OSX laptop with MPICH - -cmake -DCMAKE_INSTALL_PREFIX:PATH=./install \ - -DCMAKE_CXX_COMPILER:STRING=mpicxx \ - -DCMAKE_C_COMPILER:STRING=mpicc \ - -DCMAKE_Fortran_COMPILER:STRING=mpifort \ - -DMPIEXEC_PREFLAGS:STRING=--oversubscribe \ - -DCMAKE_BUILD_TYPE:STRING=Debug \ - -DAMR_WIND_USE_INTERNAL_AMREX_HYDRO:BOOL=OFF \ - -DAMReX-Hydro_DIR:STRING=$HOME/AMReX-Hydro \ - -DAMR_WIND_ENABLE_MPI:BOOL=ON \ - -DAMR_WIND_ENABLE_TESTS:BOOL=OFF \ - -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \ - .. && make -j8 diff --git a/bctest/input_inflow b/bctest/input_inflow deleted file mode 100644 index 2b8d73f19e..0000000000 --- a/bctest/input_inflow +++ /dev/null @@ -1,80 +0,0 @@ -# This is a 2D poiseuille flow when run to steady state - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# SIMULATION STOP # -#.......................................# -time.stop_time = 20.00 # Max (simulated) time to evolve -time.max_step = 2 #-1000 # Max number of time steps - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# TIME STEP COMPUTATION # -#.......................................# -time.fixed_dt = 0.01 # Use this constant dt if > 0 -time.cfl = 0.95 # CFL factor - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# INPUT AND OUTPUT # -#.......................................# -time.plot_interval = 10 # 100 # Steps between plot files -time.checkpoint_interval = -100 # Steps between checkpoint files - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# PHYSICS # -#.......................................# -incflo.density = 1.0 # Reference density -incflo.use_godunov = 1 -incflo.diffusion_type = 1 -incflo.godunov_type = "weno_z" - -#incflo.initial_iterations = 0 -#incflo.do_initial_proj = false -incflo.verbose = 3 - -transport.viscosity = 0.0 -turbulence.model = Laminar - -ICNS.source_terms = BodyForce -BodyForce.magnitude = 0 0 0 - -incflo.physics = FreeStream -FreeStream.velocity_type = TwoLayer - -io.output_default_variables = 1 - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# ADAPTIVE MESH REFINEMENT # -#.......................................# -amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel -amr.blocking_factor = 4 -amr.max_level = 0 # Max AMR level in hierarchy -fabarray.mfiter_tile_size = 1024 1024 1024 - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# GEOMETRY # -#.......................................# -geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates -geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates -geometry.is_periodic = 0 0 0 # Periodicity x y z (0/1) - -# Boundary conditions -TwoLayer.mvel = -1.0 0.0 0.0 -TwoLayer.pvel = 1.0 0.0 0.0 - -xlo.type = "mass_inflow" -xlo.density = 1.0 -xlo.velocity.inflow_type = TwoLayer - -xhi.type = "mass_inflow" -xhi.density = 1.0 -xhi.velocity.inflow_type = TwoLayer - -ylo.type = "mass_inflow" -ylo.density = 1.0 -ylo.velocity.inflow_type = TwoLayer - -yhi.type = "mass_inflow" -yhi.density = 1.0 -yhi.velocity.inflow_type = TwoLayer - -zlo.type = "slip_wall" -zhi.type = "slip_wall" diff --git a/bctest/input_inout b/bctest/input_inout deleted file mode 100644 index 20385c7b14..0000000000 --- a/bctest/input_inout +++ /dev/null @@ -1,80 +0,0 @@ -# This is a 2D poiseuille flow when run to steady state - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# SIMULATION STOP # -#.......................................# -time.stop_time = 20.00 # Max (simulated) time to evolve -time.max_step = -10 #-1000 # Max number of time steps - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# TIME STEP COMPUTATION # -#.......................................# -time.fixed_dt = 0.01 # Use this constant dt if > 0 -time.cfl = 0.95 # CFL factor - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# INPUT AND OUTPUT # -#.......................................# -time.plot_interval = 50 # 100 # Steps between plot files -time.checkpoint_interval = -100 # Steps between checkpoint files - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# PHYSICS # -#.......................................# -incflo.density = 1.0 # Reference density -incflo.use_godunov = 1 -incflo.diffusion_type = 1 -incflo.godunov_type = "weno_z" - -#incflo.initial_iterations = 0 -#incflo.do_initial_proj = false -incflo.verbose = 3 - -transport.viscosity = 0.0 -turbulence.model = Laminar - -ICNS.source_terms = BodyForce -BodyForce.magnitude = 0 0 0 - -incflo.physics = FreeStream -FreeStream.velocity_type = TwoLayer - -io.output_default_variables = 1 - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# ADAPTIVE MESH REFINEMENT # -#.......................................# -amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel -amr.blocking_factor = 4 -amr.max_level = 0 # Max AMR level in hierarchy -fabarray.mfiter_tile_size = 1024 1024 1024 - -#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# -# GEOMETRY # -#.......................................# -geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates -geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates -geometry.is_periodic = 0 0 0 # Periodicity x y z (0/1) - -# Boundary conditions -TwoLayer.mvel = -1.0 -2.0 0.0 -TwoLayer.pvel = 1.0 2.0 0.0 - -xlo.type = "mass_inflow_outflow" -xlo.density = 1.0 -xlo.velocity.inflow_outflow_type = TwoLayer - -xhi.type = "mass_inflow_outflow" -xhi.density = 1.0 -xhi.velocity.inflow_outflow_type = TwoLayer - -ylo.type = "mass_inflow_outflow" -ylo.density = 1.0 -ylo.velocity.inflow_outflow_type = TwoLayer - -yhi.type = "mass_inflow_outflow" -yhi.density = 1.0 -yhi.velocity.inflow_outflow_type = TwoLayer - -zlo.type = "slip_wall" -zhi.type = "slip_wall" From 2ae8fdb2c0113ca547df572b525e40aa8fd8bd76 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 12 Aug 2024 16:03:30 -0700 Subject: [PATCH 61/83] add inout test to cmake test suite --- test/CMakeLists.txt | 1 + .../freestream_godunov_inout.inp | 70 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 test/test_files/freestream_godunov_inout/freestream_godunov_inout.inp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 11fb5ae1c6..c28cbb4417 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -248,6 +248,7 @@ add_test_re(rankine) add_test_re(rankine-sym) add_test_re(box_refinement) add_test_re(cylinder_refinement) +add_test_re(freestream_godunov_inout) if(NOT AMR_WIND_ENABLE_CUDA) add_test_re(ctv_godunov_plm) diff --git a/test/test_files/freestream_godunov_inout/freestream_godunov_inout.inp b/test/test_files/freestream_godunov_inout/freestream_godunov_inout.inp new file mode 100644 index 0000000000..44fad0a075 --- /dev/null +++ b/test/test_files/freestream_godunov_inout/freestream_godunov_inout.inp @@ -0,0 +1,70 @@ +# This is a 2D poiseuille flow when run to steady state + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# SIMULATION STOP # +#.......................................# +time.stop_time = 10.0 # Max (simulated) time to evolve +time.max_step = 10 #-1000 # Max number of time steps + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# TIME STEP COMPUTATION # +#.......................................# +time.fixed_dt = 0.02 # Use this constant dt if > 0 +time.cfl = 0.95 # CFL factor + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# INPUT AND OUTPUT # +#.......................................# +time.plot_interval = 1 # 100 # Steps between plot files +time.checkpoint_interval = -100 # Steps between checkpoint files + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# PHYSICS # +#.......................................# +incflo.density = 1.0 # Reference density +incflo.use_godunov = 1 +incflo.diffusion_type = 1 +incflo.godunov_type = "weno_z" + +transport.viscosity = 0.0 +turbulence.model = Laminar + +ICNS.source_terms = BodyForce +BodyForce.magnitude = 0 0 0 + +incflo.physics = FreeStream +FreeStream.velocity_type = TwoLayer + +io.output_default_variables = 1 + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# ADAPTIVE MESH REFINEMENT # +#.......................................# +amr.n_cell = 16 8 16 # Grid cells at coarsest AMRlevel +amr.blocking_factor = 4 +amr.max_level = 0 # Max AMR level in hierarchy +fabarray.mfiter_tile_size = 1024 1024 1024 + +#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨# +# GEOMETRY # +#.......................................# +geometry.prob_lo = 0.0 0.0 0.0 # Lo corner coordinates +geometry.prob_hi = 1.0 0.5 1.0 # Hi corner coordinates +geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) + +# Boundary conditions +TwoLayer.bottom_vel = -1.0 0.0 0.0 +TwoLayer.top_vel = 1.0 0.0 0.0 +TwoLayer.init_perturb = 0.9 +TwoLayer.z_partition = 0.5 + +xlo.type = "mass_inflow_outflow" +xlo.density = 1.0 +xlo.velocity.inflow_outflow_type = TwoLayer + +xhi.type = "mass_inflow_outflow" +xhi.density = 1.0 +xhi.velocity.inflow_outflow_type = TwoLayer + +zlo.type = "slip_wall" +zhi.type = "slip_wall" From 7933313a03e3dd96e679560eb9c1499fa45c6a4e Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 12 Aug 2024 16:05:48 -0700 Subject: [PATCH 62/83] undo change for external hydro which is probably unnecessary --- cmake/amr-wind-utils.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/amr-wind-utils.cmake b/cmake/amr-wind-utils.cmake index bc95f9ea09..3168836304 100644 --- a/cmake/amr-wind-utils.cmake +++ b/cmake/amr-wind-utils.cmake @@ -79,7 +79,6 @@ macro(init_amrex_hydro) else() set(CMAKE_PREFIX_PATH ${AMReX-Hydro_DIR} ${CMAKE_PREFIX_PATH}) find_package(AMReX-Hydro CONFIG REQUIRED) - add_subdirectory(${AMReX-Hydro_DIR} build) message(STATUS "Found AMReX-Hydro = ${AMReX-Hydro_DIR}") endif() endmacro(init_amrex_hydro) From 07be53dce31803c18b24f2c07dd017d7e1ee4089 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 12 Aug 2024 16:43:32 -0700 Subject: [PATCH 63/83] apply clang-format --- amr-wind/boundary_conditions/BCInterface.cpp | 9 +++++---- amr-wind/boundary_conditions/MassInflowOutflowBC.H | 2 -- amr-wind/boundary_conditions/MassInflowOutflowBC.cpp | 12 ++++++------ amr-wind/core/FieldBCOps.H | 8 ++++---- amr-wind/equation_systems/icns/icns_advection.H | 4 +++- amr-wind/equation_systems/icns/icns_advection.cpp | 6 +++--- amr-wind/physics/udfs/TwoLayer.H | 6 +++--- .../projection/incflo_apply_nodal_projection.cpp | 7 ++++--- amr-wind/projection/nodal_projection_ops.H | 6 ++---- unit_tests/equation_systems/test_icns_cstdens.cpp | 3 ++- 10 files changed, 32 insertions(+), 31 deletions(-) diff --git a/amr-wind/boundary_conditions/BCInterface.cpp b/amr-wind/boundary_conditions/BCInterface.cpp index a81ce1fe91..b363f38d0f 100644 --- a/amr-wind/boundary_conditions/BCInterface.cpp +++ b/amr-wind/boundary_conditions/BCInterface.cpp @@ -104,12 +104,11 @@ void BCIface::set_bcfuncs() m_field.register_custom_bc(ori); } - if ((m_field.name() == "velocity") // only velocity for now + if ((m_field.name() == "velocity") // only velocity for now && (bct == BC::mass_inflow_outflow)) { m_field.register_custom_bc(ori); } - } } @@ -155,7 +154,8 @@ amrex::Array BCIface::get_dirichlet_udfs() if (has_inflow_outflow_udf && (inflow_outflow_udf != val)) { amrex::Abort( - "BC: Inflow-outflow UDF must be same for all inflow-outflow faces"); + "BC: Inflow-outflow UDF must be same for all " + "inflow-outflow faces"); } else { inflow_outflow_udf = val; has_inflow_outflow_udf = true; @@ -366,7 +366,8 @@ void BCScalar::read_values() amrex::ParmParse pp(bcid); if (((bct == BC::mass_inflow) && (const_dirichlet_inflow)) || - ((bct == BC::mass_inflow_outflow) && (const_dirichlet_inflow_outflow))) { + ((bct == BC::mass_inflow_outflow) && + (const_dirichlet_inflow_outflow))) { pp.getarr(fname.c_str(), bcval[ori], 0, ndim); } else { pp.queryarr(fname.c_str(), bcval[ori], 0, ndim); diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.H b/amr-wind/boundary_conditions/MassInflowOutflowBC.H index 46f33ba063..ba8d2c53bf 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.H +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.H @@ -28,6 +28,4 @@ private: } // namespace amr_wind - #endif /* MASSINFLOWOUTFLOWBC_H */ - diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index 73f0b00f8f..c3e6faa7ce 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -2,15 +2,13 @@ #include "amr-wind/boundary_conditions/MassInflowOutflowBC.H" namespace amr_wind { -namespace { - -} // namespace MassInflowOutflowBC::MassInflowOutflowBC(Field& field, amrex::Orientation ori) : m_field(field), m_ori(ori) {} -void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_state*/) +void MassInflowOutflowBC::operator()( + Field& /*field*/, const FieldState /*rho_state*/) { const auto& repo = m_field.repo(); const auto& velocity = repo.get_field("velocity"); @@ -47,7 +45,8 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st const amrex::IntVect ivm = iv - iv_dir; if (vel(ivm[0], ivm[1], ivm[2], idim) < 0) { for (int n = 0; n < ncomp; n++) { - bc_a(ivm[0], ivm[1], ivm[2], n) = bc_a(i, j, k, n); + bc_a(ivm[0], ivm[1], ivm[2], n) = + bc_a(i, j, k, n); } } }); @@ -61,7 +60,8 @@ void MassInflowOutflowBC::operator()(Field& /*field*/, const FieldState /*rho_st const amrex::IntVect ivm = iv - iv_dir; if (vel(i, j, k, idim) > 0) { for (int n = 0; n < ncomp; n++) { - bc_a(i, j, k, n) = bc_a(ivm[0], ivm[1], ivm[2], n); + bc_a(i, j, k, n) = + bc_a(ivm[0], ivm[1], ivm[2], n); } } }); diff --git a/amr-wind/core/FieldBCOps.H b/amr-wind/core/FieldBCOps.H index adcb23ecf0..4f0d2c3dc8 100644 --- a/amr-wind/core/FieldBCOps.H +++ b/amr-wind/core/FieldBCOps.H @@ -168,8 +168,8 @@ struct DirichletOp // Check if this boundary is a dirichlet or mixed type const auto bctyp = (ori.isLow() ? bc.lo(idir) : bc.hi(idir)); - if ((bctyp != amrex::BCType::ext_dir) - && (bctyp != amrex::BCType::direction_dependent)) { + if ((bctyp != amrex::BCType::ext_dir) && + (bctyp != amrex::BCType::direction_dependent)) { continue; } @@ -183,8 +183,8 @@ struct DirichletOp continue; } - if ((m_bc_type[ori] == BC::mass_inflow) - || (m_bc_type[ori] == BC::mass_inflow_outflow)) { + if ((m_bc_type[ori] == BC::mass_inflow) || + (m_bc_type[ori] == BC::mass_inflow_outflow)) { m_inflow_op( iv, field, geom, time, ori, n, dcomp, orig_comp); } else { diff --git a/amr-wind/equation_systems/icns/icns_advection.H b/amr-wind/equation_systems/icns/icns_advection.H index 4fbd5b81b4..130d680707 100644 --- a/amr-wind/equation_systems/icns/icns_advection.H +++ b/amr-wind/equation_systems/icns/icns_advection.H @@ -50,7 +50,9 @@ private: void init_projector(const FaceFabPtrVec& /*beta*/) noexcept; void init_projector(const amrex::Real /*beta*/) noexcept; - void enforce_inout_solvability(const amrex::Vector>& a_umac) noexcept; + void enforce_inout_solvability( + const amrex::Vector>& + a_umac) noexcept; FieldRepo& m_repo; PhysicsMgr& m_phy_mgr; diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index e2b15ceb3d..6cc502921d 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -66,9 +66,9 @@ MacProjOp::MacProjOp( m_has_overset = m_has_overset && !disable_ovst_mac; } -void MacProjOp::enforce_inout_solvability ( - const amrex::Vector>& a_umac -) noexcept +void MacProjOp::enforce_inout_solvability( + const amrex::Vector>& + a_umac) noexcept { auto& velocity = m_repo.get_field("velocity"); amrex::BCRec const* bc_type = velocity.bcrec_device().data(); diff --git a/amr-wind/physics/udfs/TwoLayer.H b/amr-wind/physics/udfs/TwoLayer.H index 06b1df933e..5de0418d5a 100644 --- a/amr-wind/physics/udfs/TwoLayer.H +++ b/amr-wind/physics/udfs/TwoLayer.H @@ -17,8 +17,8 @@ struct TwoLayer // velocities of the top and bottom layer respectively amrex::GpuArray top_vel = {0.0}; amrex::GpuArray bottom_vel = {0.0}; - amrex::Real z_part{0.5}; // z-coordinate that divides top and bottom - amrex::Real init_perturb{1.0}; // initial perturbation for testing + amrex::Real z_part{0.5}; // z-coordinate that divides top and bottom + amrex::Real init_perturb{1.0}; // initial perturbation for testing AMREX_GPU_DEVICE inline void operator()( @@ -42,7 +42,7 @@ struct TwoLayer vel = {bottom_vel[0], bottom_vel[1], 0.0}; } - if (time == 0.0) { + if (time == 0.0) { vel[0] *= init_perturb; vel[1] *= init_perturb; } diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index aa82e20ba3..e33efd3bd2 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -78,9 +78,10 @@ void amr_wind::nodal_projection::apply_dirichlet_vel( }); } -void -amr_wind::nodal_projection::enforce_inout_solvability ( - amr_wind::Field& velocity, const Vector& geom, const int num_levels) +void amr_wind::nodal_projection::enforce_inout_solvability( + amr_wind::Field& velocity, + const Vector& geom, + const int num_levels) { BCRec const* bc_type = velocity.bcrec_device().data(); Vector> vel_vec(num_levels); diff --git a/amr-wind/projection/nodal_projection_ops.H b/amr-wind/projection/nodal_projection_ops.H index e7f5df659d..c437146463 100644 --- a/amr-wind/projection/nodal_projection_ops.H +++ b/amr-wind/projection/nodal_projection_ops.H @@ -26,10 +26,8 @@ Array get_projection_bc( void apply_dirichlet_vel( amrex::MultiFab& mf_velocity, amrex::iMultiFab& mf_iblank); -void enforce_inout_solvability ( - amr_wind::Field& velocity, - const Vector& geom, - int num_levels); +void enforce_inout_solvability( + amr_wind::Field& velocity, const Vector& geom, int num_levels); } // namespace amr_wind::nodal_projection diff --git a/unit_tests/equation_systems/test_icns_cstdens.cpp b/unit_tests/equation_systems/test_icns_cstdens.cpp index 07f0921b7d..5995ef80ee 100644 --- a/unit_tests/equation_systems/test_icns_cstdens.cpp +++ b/unit_tests/equation_systems/test_icns_cstdens.cpp @@ -23,7 +23,8 @@ class ICNSConstDensTest : public MeshTest // Initialize MAC projection operator const auto& mco = amr_wind::pde::MacProjOp( - sim().repo(), sim().physics_manager(), false, false, false, false, false); + sim().repo(), sim().physics_manager(), false, false, false, false, + false); // Get background density and check const amrex::Real rho0 = mco.rho0(); EXPECT_EQ(rho0, m_rho_0); From 3da31bb0f4ddcbb205cace8d88cb80557013fd24 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 14 Aug 2024 12:08:01 -0700 Subject: [PATCH 64/83] more clang-format --- .../convection/incflo_godunov_advection.cpp | 33 +++++++------- amr-wind/convection/incflo_godunov_ppm.H | 5 ++- .../convection/incflo_godunov_predict.cpp | 43 +++++++++++-------- amr-wind/convection/incflo_godunov_weno.H | 37 ++++++++-------- 4 files changed, 63 insertions(+), 55 deletions(-) diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp index 24a2d47a97..9ec5f5e984 100644 --- a/amr-wind/convection/incflo_godunov_advection.cpp +++ b/amr-wind/convection/incflo_godunov_advection.cpp @@ -145,7 +145,8 @@ void godunov::compute_fluxes( auto bc = pbc[n]; Godunov_trans_xbc( - i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), + dlo.x, dhi.x); xlo(i, j, k, n) = lo; xhi(i, j, k, n) = hi; Real st = (uval) ? lo : hi; @@ -172,7 +173,8 @@ void godunov::compute_fluxes( auto bc = pbc[n]; Godunov_trans_ybc( - i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), + dlo.y, dhi.y); ylo(i, j, k, n) = lo; yhi(i, j, k, n) = hi; @@ -199,7 +201,8 @@ void godunov::compute_fluxes( } Godunov_trans_zbc( - i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), + dlo.z, dhi.z); zlo(i, j, k, n) = lo; zhi(i, j, k, n) = hi; @@ -232,8 +235,8 @@ void godunov::compute_fluxes( Real wad = wmac(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zylo, l_zyhi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); + i, j, k, n, q, l_zylo, l_zyhi, wad, wad, bc.lo(2), bc.hi(2), + dlo.z, dhi.z); constexpr Real small_vel = 1.e-8; @@ -251,8 +254,8 @@ void godunov::compute_fluxes( Real vad = vmac(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yzlo, l_yzhi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); + i, j, k, n, q, l_yzlo, l_yzhi, vad, vad, bc.lo(1), bc.hi(1), + dlo.y, dhi.y); constexpr Real small_vel = 1.e-8; @@ -344,8 +347,8 @@ void godunov::compute_fluxes( Real uad = umac(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xzlo, l_xzhi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); + i, j, k, n, q, l_xzlo, l_xzhi, uad, uad, bc.lo(0), bc.hi(0), + dlo.x, dhi.x); constexpr Real small_vel = 1.e-8; @@ -363,8 +366,8 @@ void godunov::compute_fluxes( Real wad = wmac(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zxlo, l_zxhi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); + i, j, k, n, q, l_zxlo, l_zxhi, wad, wad, bc.lo(2), bc.hi(2), + dlo.z, dhi.z); constexpr Real small_vel = 1.e-8; @@ -455,8 +458,8 @@ void godunov::compute_fluxes( Real uad = umac(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xylo, l_xyhi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); + i, j, k, n, q, l_xylo, l_xyhi, uad, uad, bc.lo(0), bc.hi(0), + dlo.x, dhi.x); constexpr Real small_vel = 1.e-8; @@ -474,8 +477,8 @@ void godunov::compute_fluxes( Real vad = vmac(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yxlo, l_yxhi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); + i, j, k, n, q, l_yxlo, l_yxhi, vad, vad, bc.lo(1), bc.hi(1), + dlo.y, dhi.y); constexpr Real small_vel = 1.e-8; diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index 3dce0fee8a..a9abf3bdd2 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -27,7 +27,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( { using namespace amrex; - // for MIO BC, use vel+ at bclo and vel- at bchi for determining ext_dir/foextrap treatment + // for MIO BC, use vel+ at bclo and vel- at bchi for determining + // ext_dir/foextrap treatment // Low X if (i <= domlo) { if (bclo == BCType::ext_dir || @@ -186,7 +187,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( // High Z if (k > domhi) { - if (bchi == BCType::ext_dir || + if (bchi == BCType::ext_dir || (bchi == BCType::direction_dependent && velm <= 0.0)) { // IAMR does this but it breaks lo/hi symmetry // Real st = (wedge >= -small_vel)? lo : s(i,j,domhi+1,n); diff --git a/amr-wind/convection/incflo_godunov_predict.cpp b/amr-wind/convection/incflo_godunov_predict.cpp index fe4670f57f..a679c0b948 100644 --- a/amr-wind/convection/incflo_godunov_predict.cpp +++ b/amr-wind/convection/incflo_godunov_predict.cpp @@ -53,7 +53,8 @@ void godunov::make_trans_velocities( auto bc = pbc[n]; Godunov_trans_xbc( - i, j, k, n, vel, lo, hi, lo, hi, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + i, j, k, n, vel, lo, hi, lo, hi, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); constexpr Real small_vel = 1e-8; @@ -77,7 +78,8 @@ void godunov::make_trans_velocities( auto bc = pbc[n]; Godunov_trans_ybc( - i, j, k, n, vel, lo, hi, lo, hi, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + i, j, k, n, vel, lo, hi, lo, hi, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); constexpr Real small_vel = 1e-8; @@ -101,7 +103,8 @@ void godunov::make_trans_velocities( auto bc = pbc[n]; Godunov_trans_zbc( - i, j, k, n, vel, lo, hi, lo, hi, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + i, j, k, n, vel, lo, hi, lo, hi, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); constexpr Real small_vel = 1e-8; @@ -184,7 +187,8 @@ void godunov::predict_godunov( auto bc = pbc[n]; Godunov_trans_xbc( - i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); + i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); xlo(i, j, k, n) = lo; xhi(i, j, k, n) = hi; @@ -211,7 +215,8 @@ void godunov::predict_godunov( auto bc = pbc[n]; Godunov_trans_ybc( - i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); + i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); ylo(i, j, k, n) = lo; yhi(i, j, k, n) = hi; @@ -238,7 +243,8 @@ void godunov::predict_godunov( auto bc = pbc[n]; Godunov_trans_zbc( - i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); + i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); zlo(i, j, k, n) = lo; zhi(i, j, k, n) = hi; @@ -281,8 +287,8 @@ void godunov::predict_godunov( Real wad = w_ad(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zylo, l_zyhi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); + i, j, k, n, q, l_zylo, l_zyhi, wad, wad, bc.lo(2), bc.hi(2), + dlo.z, dhi.z); constexpr Real small_vel = 1.e-8; @@ -300,8 +306,8 @@ void godunov::predict_godunov( Real vad = v_ad(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yzlo, l_yzhi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); + i, j, k, n, q, l_yzlo, l_yzhi, vad, vad, bc.lo(1), bc.hi(1), + dlo.y, dhi.y); constexpr Real small_vel = 1.e-8; @@ -365,8 +371,8 @@ void godunov::predict_godunov( Real uad = u_ad(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xzlo, l_xzhi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); + i, j, k, n, q, l_xzlo, l_xzhi, uad, uad, bc.lo(0), bc.hi(0), + dlo.x, dhi.x); constexpr Real small_vel = 1.e-8; @@ -384,8 +390,8 @@ void godunov::predict_godunov( Real wad = w_ad(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zxlo, l_zxhi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); + i, j, k, n, q, l_zxlo, l_zxhi, wad, wad, bc.lo(2), bc.hi(2), + dlo.z, dhi.z); constexpr Real small_vel = 1.e-8; @@ -449,8 +455,8 @@ void godunov::predict_godunov( Real uad = u_ad(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xylo, l_xyhi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); + i, j, k, n, q, l_xylo, l_xyhi, uad, uad, bc.lo(0), bc.hi(0), + dlo.x, dhi.x); constexpr Real small_vel = 1.e-8; @@ -472,8 +478,8 @@ void godunov::predict_godunov( Real vad = v_ad(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yxlo, l_yxhi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); + i, j, k, n, q, l_yxlo, l_yxhi, vad, vad, bc.lo(1), bc.hi(1), + dlo.y, dhi.y); constexpr Real small_vel = 1.e-8; @@ -513,4 +519,3 @@ void godunov::predict_godunov( qz(i, j, k) = ltm ? 0. : st; }); } - diff --git a/amr-wind/convection/incflo_godunov_weno.H b/amr-wind/convection/incflo_godunov_weno.H index 8887829459..22cbd5f915 100644 --- a/amr-wind/convection/incflo_godunov_weno.H +++ b/amr-wind/convection/incflo_godunov_weno.H @@ -71,8 +71,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( { using namespace amrex; - if (bclo == BCType::ext_dir || bclo == BCType::hoextrap - || (bclo == BCType::direction_dependent && velm >= 0.0)) { + if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || + (bclo == BCType::direction_dependent && velm >= 0.0)) { if (i == domlo) { sp = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); @@ -88,8 +88,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( } } - if (bchi == BCType::ext_dir || bchi == BCType::hoextrap - || (bchi == BCType::direction_dependent && velp <= 0.0)) { + if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || + (bchi == BCType::direction_dependent && velp <= 0.0)) { if (i == domhi) { sm = -0.2 * s(domhi + 1, j, k, n) + 0.75 * s(domhi, j, k, n) + 0.5 * s(domhi - 1, j, k, n) - 0.05 * s(domhi - 2, j, k, n); @@ -125,8 +125,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_ybc( { using namespace amrex; - if (bclo == BCType::ext_dir || bclo == BCType::hoextrap - || (bclo == BCType::direction_dependent && velm >= 0.0)) { + if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || + (bclo == BCType::direction_dependent && velm >= 0.0)) { if (j == domlo) { sp = -0.2 * s(i, domlo - 1, k, n) + 0.75 * s(i, domlo, k, n) + 0.5 * s(i, domlo + 1, k, n) - 0.05 * s(i, domlo + 2, k, n); @@ -142,8 +142,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_ybc( } } - if (bchi == BCType::ext_dir || bchi == BCType::hoextrap - || (bchi == BCType::direction_dependent && velp <= 0.0)) { + if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || + (bchi == BCType::direction_dependent && velp <= 0.0)) { if (j == domhi) { sm = -0.2 * s(i, domhi + 1, k, n) + 0.75 * s(i, domhi, k, n) + 0.5 * s(i, domhi - 1, k, n) - 0.05 * s(i, domhi - 2, k, n); @@ -179,8 +179,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_zbc( { using namespace amrex; - if (bclo == BCType::ext_dir || bclo == BCType::hoextrap - || (bclo == BCType::direction_dependent && velm >= 0.0)) { + if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || + (bclo == BCType::direction_dependent && velm >= 0.0)) { if (k == domlo) { sp = -0.2 * s(i, j, domlo - 1, n) + 0.75 * s(i, j, domlo, n) + 0.5 * s(i, j, domlo + 1, n) - 0.05 * s(i, j, domlo + 2, n); @@ -196,8 +196,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_zbc( } } - if (bchi == BCType::ext_dir || bchi == BCType::hoextrap - || (bchi == BCType::direction_dependent && velp <= 0.0)) { + if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || + (bchi == BCType::direction_dependent && velp <= 0.0)) { if (k == domhi) { sm = -0.2 * s(i, j, domhi + 1, n) + 0.75 * s(i, j, domhi, n) + 0.5 * s(i, j, domhi - 1, n) - 0.05 * s(i, j, domhi - 2, n); @@ -423,8 +423,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_x( amrex::Real sp = sedge2; Godunov_weno_xbc( - i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), vel_edge(i+1, j, k), - bc.lo(0), bc.hi(0), domlo, domhi); + i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), + vel_edge(i + 1, j, k), bc.lo(0), bc.hi(0), domlo, domhi); amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); @@ -482,8 +482,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_y( amrex::Real sp = sedge2; Godunov_weno_ybc( - i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), vel_edge(i, j+1, k), - bc.lo(1), bc.hi(1), domlo, domhi); + i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), + vel_edge(i, j + 1, k), bc.lo(1), bc.hi(1), domlo, domhi); amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); @@ -541,8 +541,8 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_z( amrex::Real sp = sedge2; Godunov_weno_zbc( - i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), vel_edge(i, j, k+1), - bc.lo(2), bc.hi(2), domlo, domhi); + i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), + vel_edge(i, j, k + 1), bc.lo(2), bc.hi(2), domlo, domhi); amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); @@ -567,4 +567,3 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_z( } // namespace #endif - From d7ba5ab3c327b3164ec746ea89a557cc3f3cef56 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 14 Aug 2024 12:11:06 -0700 Subject: [PATCH 65/83] removed files that are moved to hydro from the upstream repo --- .../convection/incflo_godunov_predict.cpp | 521 ---------------- amr-wind/convection/incflo_godunov_weno.H | 569 ------------------ 2 files changed, 1090 deletions(-) delete mode 100644 amr-wind/convection/incflo_godunov_predict.cpp delete mode 100644 amr-wind/convection/incflo_godunov_weno.H diff --git a/amr-wind/convection/incflo_godunov_predict.cpp b/amr-wind/convection/incflo_godunov_predict.cpp deleted file mode 100644 index a679c0b948..0000000000 --- a/amr-wind/convection/incflo_godunov_predict.cpp +++ /dev/null @@ -1,521 +0,0 @@ -#include -#include "amr-wind/convection/Godunov.H" -#include "amr-wind/convection/incflo_godunov_ppm.H" - -using namespace amrex; - -void godunov::make_trans_velocities( - int lev, - Box const& xbx, - Box const& ybx, - Box const& zbx, - Array4 const& u_ad, - Array4 const& v_ad, - Array4 const& w_ad, - Array4 const& Imx, - Array4 const& Ipx, - Array4 const& Imy, - Array4 const& Ipy, - Array4 const& Imz, - Array4 const& Ipz, - Array4 const& vel, - Array4 const& f, - Vector geom, - Real dt, - amrex::Gpu::DeviceVector& bcrec_device, - bool godunov_use_forces_in_trans) - -{ - BL_PROFILE("amr-wind::godunov::make_trans_velocities"); - Real l_dt = dt; - bool l_use_forces_in_trans = godunov_use_forces_in_trans; - - const Box& domain = geom[lev].Domain(); - const Dim3 dlo = amrex::lbound(domain); - const Dim3 dhi = amrex::ubound(domain); - - BCRec const* pbc = bcrec_device.data(); - - amrex::ParallelFor( - xbx, ybx, zbx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // We only care about x-velocity on x-faces here - constexpr int n = 0; - - Real lo, hi; - if (l_use_forces_in_trans) { - lo = Ipx(i - 1, j, k, n) + 0.5 * l_dt * f(i - 1, j, k, n); - hi = Imx(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); - } else { - lo = Ipx(i - 1, j, k, n); - hi = Imx(i, j, k, n); - } - - auto bc = pbc[n]; - Godunov_trans_xbc( - i, j, k, n, vel, lo, hi, lo, hi, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); - - constexpr Real small_vel = 1e-8; - - Real st = ((lo + hi) >= 0.) ? lo : hi; - bool ltm = - ((lo <= 0. && hi >= 0.) || (std::abs(lo + hi) < small_vel)); - u_ad(i, j, k) = ltm ? 0. : st; - }, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // We only care about y-velocity on y-faces here - constexpr int n = 1; - - Real lo, hi; - if (l_use_forces_in_trans) { - lo = Ipy(i, j - 1, k, n) + 0.5 * l_dt * f(i, j - 1, k, n); - hi = Imy(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); - } else { - lo = Ipy(i, j - 1, k, n); - hi = Imy(i, j, k, n); - } - - auto bc = pbc[n]; - Godunov_trans_ybc( - i, j, k, n, vel, lo, hi, lo, hi, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); - - constexpr Real small_vel = 1e-8; - - Real st = ((lo + hi) >= 0.) ? lo : hi; - bool ltm = - ((lo <= 0. && hi >= 0.) || (std::abs(lo + hi) < small_vel)); - v_ad(i, j, k) = ltm ? 0. : st; - }, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // We only care about z-velocity on z-faces here - constexpr int n = 2; - - Real lo, hi; - if (l_use_forces_in_trans) { - lo = Ipz(i, j, k - 1, n) + 0.5 * l_dt * f(i, j, k - 1, n); - hi = Imz(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); - } else { - lo = Ipz(i, j, k - 1, n); - hi = Imz(i, j, k, n); - } - - auto bc = pbc[n]; - Godunov_trans_zbc( - i, j, k, n, vel, lo, hi, lo, hi, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); - - constexpr Real small_vel = 1e-8; - - Real st = ((lo + hi) >= 0.) ? lo : hi; - bool ltm = - ((lo <= 0. && hi >= 0.) || (std::abs(lo + hi) < small_vel)); - w_ad(i, j, k) = ltm ? 0. : st; - }); -} - -void godunov::predict_godunov( - int lev, - Box const& bx, - int ncomp, - Box const& xbx, - Box const& ybx, - Box const& zbx, - Array4 const& qx, - Array4 const& qy, - Array4 const& qz, - Array4 const& q, - Array4 const& u_ad, - Array4 const& v_ad, - Array4 const& w_ad, - Array4 const& Imx, - Array4 const& Ipx, - Array4 const& Imy, - Array4 const& Ipy, - Array4 const& Imz, - Array4 const& Ipz, - Array4 const& f, - Real* p, - Vector geom, - Real dt, - amrex::Gpu::DeviceVector& bcrec_device, - bool godunov_use_forces_in_trans) -{ - BL_PROFILE("amr-wind::godunov::predict_godunov"); - Real l_dt = dt; - bool l_use_forces_in_trans = godunov_use_forces_in_trans; - - const Box& domain = geom[lev].Domain(); - const Dim3 dlo = amrex::lbound(domain); - const Dim3 dhi = amrex::ubound(domain); - Real dx = geom[lev].CellSize(0); - Real dy = geom[lev].CellSize(1); - Real dz = geom[lev].CellSize(2); - - BCRec const* pbc = bcrec_device.data(); - - Box xebox = Box(bx).grow(1, 1).grow(2, 1).surroundingNodes(0); - Box yebox = Box(bx).grow(0, 1).grow(2, 1).surroundingNodes(1); - Box zebox = Box(bx).grow(0, 1).grow(1, 1).surroundingNodes(2); - Array4 xlo = makeArray4(p, xebox, ncomp); - p += xlo.size(); - Array4 xhi = makeArray4(p, xebox, ncomp); - p += xhi.size(); - Array4 ylo = makeArray4(p, yebox, ncomp); - p += ylo.size(); - Array4 yhi = makeArray4(p, yebox, ncomp); - p += yhi.size(); - Array4 zlo = makeArray4(p, zebox, ncomp); - p += zlo.size(); - Array4 zhi = makeArray4(p, zebox, ncomp); - p += zhi.size(); // NOLINT: Value not read warning - - amrex::ParallelFor( - xebox, ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - Real lo, hi; - if (l_use_forces_in_trans) { - lo = Ipx(i - 1, j, k, n) + 0.5 * l_dt * f(i - 1, j, k, n); - hi = Imx(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); - } else { - lo = Ipx(i - 1, j, k, n); - hi = Imx(i, j, k, n); - } - - Real uad = u_ad(i, j, k); - auto bc = pbc[n]; - - Godunov_trans_xbc( - i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); - - xlo(i, j, k, n) = lo; - xhi(i, j, k, n) = hi; - - constexpr Real small_vel = 1e-8; - - Real st = (uad >= 0.) ? lo : hi; - Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; - Imx(i, j, k, n) = - fu * st + (1.0 - fu) * 0.5 * (hi + lo); // store xedge - }, - yebox, ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - Real lo, hi; - if (l_use_forces_in_trans) { - lo = Ipy(i, j - 1, k, n) + 0.5 * l_dt * f(i, j - 1, k, n); - hi = Imy(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); - } else { - lo = Ipy(i, j - 1, k, n); - hi = Imy(i, j, k, n); - } - - Real vad = v_ad(i, j, k); - auto bc = pbc[n]; - - Godunov_trans_ybc( - i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); - - ylo(i, j, k, n) = lo; - yhi(i, j, k, n) = hi; - - constexpr Real small_vel = 1e-8; - - Real st = (vad >= 0.) ? lo : hi; - Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; - Imy(i, j, k, n) = - fu * st + (1.0 - fu) * 0.5 * (hi + lo); // store yedge - }, - zebox, ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - Real lo, hi; - if (l_use_forces_in_trans) { - lo = Ipz(i, j, k - 1, n) + 0.5 * l_dt * f(i, j, k - 1, n); - hi = Imz(i, j, k, n) + 0.5 * l_dt * f(i, j, k, n); - } else { - lo = Ipz(i, j, k - 1, n); - hi = Imz(i, j, k, n); - } - - Real wad = w_ad(i, j, k); - auto bc = pbc[n]; - - Godunov_trans_zbc( - i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); - - zlo(i, j, k, n) = lo; - zhi(i, j, k, n) = hi; - - constexpr Real small_vel = 1e-8; - - Real st = (wad >= 0.) ? lo : hi; - Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; - Imz(i, j, k, n) = - fu * st + (1.0 - fu) * 0.5 * (hi + lo); // store zedge - }); - - Array4 xedge = Imx; - Array4 yedge = Imy; - Array4 zedge = Imz; - - // We can reuse the space in Ipy and Ipz. - - // - // X-Flux - // - Box const xbxtmp = Box(xbx).enclosedCells().grow(0, 1); - Array4 yzlo = - makeArray4(Ipy.dataPtr(), amrex::surroundingNodes(xbxtmp, 1), 1); - Array4 zylo = - makeArray4(Ipz.dataPtr(), amrex::surroundingNodes(xbxtmp, 2), 1); - // Add d/dy term to z-faces - // Start with {zlo,zhi} --> {zylo, zyhi} and upwind using w_ad to {zylo} - // Add d/dz to y-faces - // Start with {ylo,yhi} --> {yzlo, yzhi} and upwind using v_ad to {yzlo} - amrex::ParallelFor( - Box(zylo), Box(yzlo), - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 0; - const auto bc = pbc[n]; - Real l_zylo, l_zyhi; - Godunov_corner_couple_zy( - l_zylo, l_zyhi, i, j, k, n, l_dt, dy, false, zlo(i, j, k, n), - zhi(i, j, k, n), q, v_ad, yedge); - - Real wad = w_ad(i, j, k); - Godunov_trans_zbc( - i, j, k, n, q, l_zylo, l_zyhi, wad, wad, bc.lo(2), bc.hi(2), - dlo.z, dhi.z); - - constexpr Real small_vel = 1.e-8; - - Real st = (wad >= 0.) ? l_zylo : l_zyhi; - Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; - zylo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_zyhi + l_zylo); - }, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 0; - const auto bc = pbc[n]; - Real l_yzlo, l_yzhi; - Godunov_corner_couple_yz( - l_yzlo, l_yzhi, i, j, k, n, l_dt, dz, false, ylo(i, j, k, n), - yhi(i, j, k, n), q, w_ad, zedge); - - Real vad = v_ad(i, j, k); - Godunov_trans_ybc( - i, j, k, n, q, l_yzlo, l_yzhi, vad, vad, bc.lo(1), bc.hi(1), - dlo.y, dhi.y); - - constexpr Real small_vel = 1.e-8; - - Real st = (vad >= 0.) ? l_yzlo : l_yzhi; - Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; - yzlo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_yzhi + l_yzlo); - }); - // - amrex::ParallelFor(xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 0; - auto bc = pbc[n]; - Real stl = - xlo(i, j, k, n) - - (0.25 * l_dt / dy) * (v_ad(i - 1, j + 1, k) + v_ad(i - 1, j, k)) * - (yzlo(i - 1, j + 1, k) - yzlo(i - 1, j, k)) - - (0.25 * l_dt / dz) * (w_ad(i - 1, j, k + 1) + w_ad(i - 1, j, k)) * - (zylo(i - 1, j, k + 1) - zylo(i - 1, j, k)); - Real sth = xhi(i, j, k, n) - - (0.25 * l_dt / dy) * (v_ad(i, j + 1, k) + v_ad(i, j, k)) * - (yzlo(i, j + 1, k) - yzlo(i, j, k)) - - (0.25 * l_dt / dz) * (w_ad(i, j, k + 1) + w_ad(i, j, k)) * - (zylo(i, j, k + 1) - zylo(i, j, k)); - if (!l_use_forces_in_trans) { - stl += 0.5 * l_dt * f(i - 1, j, k, n); - sth += 0.5 * l_dt * f(i, j, k, n); - } - - Godunov_cc_xbc_lo(i, j, k, n, q, stl, sth, u_ad, bc.lo(0), dlo.x); - Godunov_cc_xbc_hi(i, j, k, n, q, stl, sth, u_ad, bc.hi(0), dhi.x); - - constexpr Real small_vel = 1.e-8; - - Real st = ((stl + sth) >= 0.) ? stl : sth; - bool ltm = - ((stl <= 0. && sth >= 0.) || (std::abs(stl + sth) < small_vel)); - qx(i, j, k) = ltm ? 0. : st; - }); - - // - // Y-Flux - // - Box const ybxtmp = Box(ybx).enclosedCells().grow(1, 1); - Array4 xzlo = - makeArray4(Ipy.dataPtr(), amrex::surroundingNodes(ybxtmp, 0), 1); - Array4 zxlo = - makeArray4(Ipz.dataPtr(), amrex::surroundingNodes(ybxtmp, 2), 1); - - // Add d/dz to x-faces - // Start with {xlo,xhi} --> {xzlo, xzhi} and upwind using u_ad to {xzlo} - // Add d/dx term to z-faces - // Start with {zlo,zhi} --> {zxlo, zxhi} and upwind using w_ad to {zxlo} - amrex::ParallelFor( - Box(xzlo), Box(zxlo), - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 1; - const auto bc = pbc[n]; - Real l_xzlo, l_xzhi; - Godunov_corner_couple_xz( - l_xzlo, l_xzhi, i, j, k, n, l_dt, dz, false, xlo(i, j, k, n), - xhi(i, j, k, n), q, w_ad, zedge); - - Real uad = u_ad(i, j, k); - Godunov_trans_xbc( - i, j, k, n, q, l_xzlo, l_xzhi, uad, uad, bc.lo(0), bc.hi(0), - dlo.x, dhi.x); - - constexpr Real small_vel = 1.e-8; - - Real st = (uad >= 0.) ? l_xzlo : l_xzhi; - Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; - xzlo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_xzhi + l_xzlo); - }, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 1; - const auto bc = pbc[n]; - Real l_zxlo, l_zxhi; - Godunov_corner_couple_zx( - l_zxlo, l_zxhi, i, j, k, n, l_dt, dx, false, zlo(i, j, k, n), - zhi(i, j, k, n), q, u_ad, xedge); - - Real wad = w_ad(i, j, k); - Godunov_trans_zbc( - i, j, k, n, q, l_zxlo, l_zxhi, wad, wad, bc.lo(2), bc.hi(2), - dlo.z, dhi.z); - - constexpr Real small_vel = 1.e-8; - - Real st = (wad >= 0.) ? l_zxlo : l_zxhi; - Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; - zxlo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_zxhi + l_zxlo); - }); - // - amrex::ParallelFor(ybx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 1; - auto bc = pbc[n]; - Real stl = - ylo(i, j, k, n) - - (0.25 * l_dt / dx) * (u_ad(i + 1, j - 1, k) + u_ad(i, j - 1, k)) * - (xzlo(i + 1, j - 1, k) - xzlo(i, j - 1, k)) - - (0.25 * l_dt / dz) * (w_ad(i, j - 1, k + 1) + w_ad(i, j - 1, k)) * - (zxlo(i, j - 1, k + 1) - zxlo(i, j - 1, k)); - Real sth = yhi(i, j, k, n) - - (0.25 * l_dt / dx) * (u_ad(i + 1, j, k) + u_ad(i, j, k)) * - (xzlo(i + 1, j, k) - xzlo(i, j, k)) - - (0.25 * l_dt / dz) * (w_ad(i, j, k + 1) + w_ad(i, j, k)) * - (zxlo(i, j, k + 1) - zxlo(i, j, k)); - if (!l_use_forces_in_trans) { - stl += 0.5 * l_dt * f(i, j - 1, k, n); - sth += 0.5 * l_dt * f(i, j, k, n); - } - - Godunov_cc_ybc_lo(i, j, k, n, q, stl, sth, v_ad, bc.lo(1), dlo.y); - Godunov_cc_ybc_hi(i, j, k, n, q, stl, sth, v_ad, bc.hi(1), dhi.y); - - constexpr Real small_vel = 1.e-8; - - Real st = ((stl + sth) >= 0.) ? stl : sth; - bool ltm = - ((stl <= 0. && sth >= 0.) || (std::abs(stl + sth) < small_vel)); - qy(i, j, k) = ltm ? 0. : st; - }); - - // - // Z-Flux - // - Box const zbxtmp = Box(zbx).enclosedCells().grow(2, 1); - Array4 xylo = - makeArray4(Ipy.dataPtr(), amrex::surroundingNodes(zbxtmp, 0), 1); - Array4 yxlo = - makeArray4(Ipz.dataPtr(), amrex::surroundingNodes(zbxtmp, 1), 1); - - amrex::ParallelFor( - Box(xylo), Box(yxlo), - // - // Add d/dy term to x-faces - // Start with {xlo,xhi} --> {xylo, xyhi} and upwind using u_ad to {xylo} - // - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 2; - const auto bc = pbc[n]; - Real l_xylo, l_xyhi; - Godunov_corner_couple_xy( - l_xylo, l_xyhi, i, j, k, n, l_dt, dy, false, xlo(i, j, k, n), - xhi(i, j, k, n), q, v_ad, yedge); - - Real uad = u_ad(i, j, k); - Godunov_trans_xbc( - i, j, k, n, q, l_xylo, l_xyhi, uad, uad, bc.lo(0), bc.hi(0), - dlo.x, dhi.x); - - constexpr Real small_vel = 1.e-8; - - Real st = (uad >= 0.) ? l_xylo : l_xyhi; - Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; - xylo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_xyhi + l_xylo); - }, - // - // Add d/dx term to y-faces - // Start with {ylo,yhi} --> {yxlo, yxhi} and upwind using v_ad to {yxlo} - // - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 2; - const auto bc = pbc[n]; - Real l_yxlo, l_yxhi; - Godunov_corner_couple_yx( - l_yxlo, l_yxhi, i, j, k, n, l_dt, dx, false, ylo(i, j, k, n), - yhi(i, j, k, n), q, u_ad, xedge); - - Real vad = v_ad(i, j, k); - Godunov_trans_ybc( - i, j, k, n, q, l_yxlo, l_yxhi, vad, vad, bc.lo(1), bc.hi(1), - dlo.y, dhi.y); - - constexpr Real small_vel = 1.e-8; - - Real st = (vad >= 0.) ? l_yxlo : l_yxhi; - Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; - yxlo(i, j, k) = fu * st + (1.0 - fu) * 0.5 * (l_yxhi + l_yxlo); - }); - // - amrex::ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - constexpr int n = 2; - auto bc = pbc[n]; - Real stl = - zlo(i, j, k, n) - - (0.25 * l_dt / dx) * (u_ad(i + 1, j, k - 1) + u_ad(i, j, k - 1)) * - (xylo(i + 1, j, k - 1) - xylo(i, j, k - 1)) - - (0.25 * l_dt / dy) * (v_ad(i, j + 1, k - 1) + v_ad(i, j, k - 1)) * - (yxlo(i, j + 1, k - 1) - yxlo(i, j, k - 1)); - Real sth = zhi(i, j, k, n) - - (0.25 * l_dt / dx) * (u_ad(i + 1, j, k) + u_ad(i, j, k)) * - (xylo(i + 1, j, k) - xylo(i, j, k)) - - (0.25 * l_dt / dy) * (v_ad(i, j + 1, k) + v_ad(i, j, k)) * - (yxlo(i, j + 1, k) - yxlo(i, j, k)); - - if (!l_use_forces_in_trans) { - stl += 0.5 * l_dt * f(i, j, k - 1, n); - sth += 0.5 * l_dt * f(i, j, k, n); - } - - Godunov_cc_zbc_lo(i, j, k, n, q, stl, sth, w_ad, bc.lo(2), dlo.z); - Godunov_cc_zbc_hi(i, j, k, n, q, stl, sth, w_ad, bc.hi(2), dhi.z); - - constexpr Real small_vel = 1.e-8; - - Real st = ((stl + sth) >= 0.) ? stl : sth; - bool ltm = - ((stl <= 0. && sth >= 0.) || (std::abs(stl + sth) < small_vel)); - qz(i, j, k) = ltm ? 0. : st; - }); -} diff --git a/amr-wind/convection/incflo_godunov_weno.H b/amr-wind/convection/incflo_godunov_weno.H deleted file mode 100644 index 22cbd5f915..0000000000 --- a/amr-wind/convection/incflo_godunov_weno.H +++ /dev/null @@ -1,569 +0,0 @@ -#ifndef GODUNOV_WENO_H -#define GODUNOV_WENO_H - -#include -#include - -/* This header file contains the inlined __host__ __device__ functions required - for the scalar advection routines for 3D Godunov. It also contains function - declarations for controlling host functions. */ - -namespace { - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real weno5( - const amrex::Real sm2, - const amrex::Real sm1, - const amrex::Real s, - const amrex::Real sp1, - const amrex::Real sp2, - bool weno_js) -{ - constexpr amrex::Real eps = 1.e-6; - - const amrex::Real beta1 = - 13.0 / 12.0 * (sm2 - 2.0 * sm1 + s) * (sm2 - 2.0 * sm1 + s) + - 0.25 * (sm2 - 4.0 * sm1 + 3.0 * s) * (sm2 - 4.0 * sm1 + 3.0 * s); - const amrex::Real beta2 = - 13.0 / 12.0 * (sm1 - 2.0 * s + sp1) * (sm1 - 2.0 * s + sp1) + - 0.25 * (sm1 - sp1) * (sm1 - sp1); - const amrex::Real beta3 = - 13.0 / 12.0 * (s - 2.0 * sp1 + sp2) * (s - 2.0 * sp1 + sp2) + - 0.25 * (3.0 * s - 4.0 * sp1 + sp2) * (3.0 * s - 4.0 * sp1 + sp2); - - amrex::Real omega1, omega2, omega3; - - if (weno_js) { - omega1 = 0.1 / (eps + beta1); - omega2 = 0.6 / (eps + beta2); - omega3 = 0.3 / (eps + beta3); - } else { - const amrex::Real t5 = std::abs(beta3 - beta1); - omega1 = 0.1 * (1.0 + t5 / (eps + beta1)); - omega2 = 0.6 * (1.0 + t5 / (eps + beta2)); - omega3 = 0.3 * (1.0 + t5 / (eps + beta3)); - } - - const amrex::Real omega = omega1 + omega2 + omega3; - - const amrex::Real v_1 = 2.0 * sm2 - 7.0 * sm1 + 11.0 * s; - const amrex::Real v_2 = -sm1 + 5.0 * s + 2.0 * sp1; - const amrex::Real v_3 = 2.0 * s + 5.0 * sp1 - sp2; - - return (omega1 * v_1 + omega2 * v_2 + omega3 * v_3) / (6.0 * omega); -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_xbc( - const int i, - const int j, - const int k, - const int n, - amrex::Real& sm, - amrex::Real& sp, - const amrex::Real& sedge1, - const amrex::Real& sedge2, - const amrex::Array4& s, - const amrex::Real velm, - const amrex::Real velp, - const int bclo, - const int bchi, - const int domlo, - const int domhi) -{ - using namespace amrex; - - if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || - (bclo == BCType::direction_dependent && velm >= 0.0)) { - if (i == domlo) { - sp = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + - 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); - - sm = s(domlo - 1, j, k, n); - - } else if (i == domlo + 1) { - - sm = -0.2 * s(domlo - 1, j, k, n) + 0.75 * s(domlo, j, k, n) + - 0.5 * s(domlo + 1, j, k, n) - 0.05 * s(domlo + 2, j, k, n); - - sp = sedge2; - } - } - - if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || - (bchi == BCType::direction_dependent && velp <= 0.0)) { - if (i == domhi) { - sm = -0.2 * s(domhi + 1, j, k, n) + 0.75 * s(domhi, j, k, n) + - 0.5 * s(domhi - 1, j, k, n) - 0.05 * s(domhi - 2, j, k, n); - - sp = s(domhi + 1, j, k, n); - - } else if (i == domhi - 1) { - - sp = -0.2 * s(domhi + 1, j, k, n) + 0.75 * s(domhi, j, k, n) + - 0.5 * s(domhi - 1, j, k, n) - 0.05 * s(domhi - 2, j, k, n); - - sm = sedge1; - } - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_ybc( - const int i, - const int j, - const int k, - const int n, - amrex::Real& sm, - amrex::Real& sp, - const amrex::Real& sedge1, - const amrex::Real& sedge2, - const amrex::Array4& s, - const amrex::Real velm, - const amrex::Real velp, - const int bclo, - const int bchi, - const int domlo, - const int domhi) -{ - using namespace amrex; - - if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || - (bclo == BCType::direction_dependent && velm >= 0.0)) { - if (j == domlo) { - sp = -0.2 * s(i, domlo - 1, k, n) + 0.75 * s(i, domlo, k, n) + - 0.5 * s(i, domlo + 1, k, n) - 0.05 * s(i, domlo + 2, k, n); - - sm = s(i, domlo - 1, k, n); - - } else if (j == domlo + 1) { - - sm = -0.2 * s(i, domlo - 1, k, n) + 0.75 * s(i, domlo, k, n) + - 0.5 * s(i, domlo + 1, k, n) - 0.05 * s(i, domlo + 2, k, n); - - sp = sedge2; - } - } - - if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || - (bchi == BCType::direction_dependent && velp <= 0.0)) { - if (j == domhi) { - sm = -0.2 * s(i, domhi + 1, k, n) + 0.75 * s(i, domhi, k, n) + - 0.5 * s(i, domhi - 1, k, n) - 0.05 * s(i, domhi - 2, k, n); - - sp = s(i, domhi + 1, k, n); - - } else if (j == domhi - 1) { - - sp = -0.2 * s(i, domhi + 1, k, n) + 0.75 * s(i, domhi, k, n) + - 0.5 * s(i, domhi - 1, k, n) - 0.05 * s(i, domhi - 2, k, n); - - sm = sedge1; - } - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_zbc( - const int i, - const int j, - const int k, - const int n, - amrex::Real& sm, - amrex::Real& sp, - const amrex::Real& sedge1, - const amrex::Real& sedge2, - const amrex::Array4& s, - const amrex::Real velm, - const amrex::Real velp, - const int bclo, - const int bchi, - const int domlo, - const int domhi) -{ - using namespace amrex; - - if (bclo == BCType::ext_dir || bclo == BCType::hoextrap || - (bclo == BCType::direction_dependent && velm >= 0.0)) { - if (k == domlo) { - sp = -0.2 * s(i, j, domlo - 1, n) + 0.75 * s(i, j, domlo, n) + - 0.5 * s(i, j, domlo + 1, n) - 0.05 * s(i, j, domlo + 2, n); - - sm = s(i, j, domlo - 1, n); - - } else if (k == domlo + 1) { - - sm = -0.2 * s(i, j, domlo - 1, n) + 0.75 * s(i, j, domlo, n) + - 0.5 * s(i, j, domlo + 1, n) - 0.05 * s(i, j, domlo + 2, n); - - sp = sedge2; - } - } - - if (bchi == BCType::ext_dir || bchi == BCType::hoextrap || - (bchi == BCType::direction_dependent && velp <= 0.0)) { - if (k == domhi) { - sm = -0.2 * s(i, j, domhi + 1, n) + 0.75 * s(i, j, domhi, n) + - 0.5 * s(i, j, domhi - 1, n) - 0.05 * s(i, j, domhi - 2, n); - - sp = s(i, j, domhi + 1, n); - - } else if (k == domhi - 1) { - - sp = -0.2 * s(i, j, domhi + 1, n) + 0.75 * s(i, j, domhi, n) + - 0.5 * s(i, j, domhi - 1, n) - 0.05 * s(i, j, domhi - 2, n); - - sm = sedge1; - } - } -} - -// This version is called before the MAC projection, when we use the -// cell-centered velocity -// for upwinding -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_x( - const int i, - const int j, - const int k, - const int n, - const amrex::Real dtdx, - const amrex::Real v_ad, - const amrex::Array4& S, - const amrex::Array4& Im, - const amrex::Array4& Ip, - const amrex::BCRec& bc, - const int domlo, - const int domhi, - const bool weno_js) -{ - - using namespace amrex; - - constexpr amrex::Real small_vel = 1e-8; - - amrex::Real sm2 = S(i - 2, j, k, n); - amrex::Real sm1 = S(i - 1, j, k, n); - amrex::Real s0 = S(i, j, k, n); - amrex::Real sp1 = S(i + 1, j, k, n); - amrex::Real sp2 = S(i + 2, j, k, n); - - // right of i-1/2 - amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT - // left of i+1/2 - amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT - - amrex::Real sm = sedge1; - amrex::Real sp = sedge2; - - Godunov_weno_xbc( - i, j, k, n, sm, sp, sedge1, sedge2, S, v_ad, v_ad, bc.lo(0), bc.hi(0), - domlo, domhi); - - amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); - - amrex::Real sigma = std::abs(v_ad) * dtdx; - - // upwind - if (v_ad > small_vel) { - Ip(i, j, k, n) = - sp - (0.5 * sigma) * ((sp - sm) - (1.0 - 2.0 / 3.0 * sigma) * s6); - Im(i, j, k, n) = S(i, j, k, n); - } else if (v_ad < -small_vel) { - Ip(i, j, k, n) = S(i, j, k, n); - Im(i, j, k, n) = - sm + (0.5 * sigma) * ((sp - sm) + (1.0 - 2.0 / 3.0 * sigma) * s6); - } else { - Ip(i, j, k, n) = S(i, j, k, n); - Im(i, j, k, n) = S(i, j, k, n); - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_y( - const int i, - const int j, - const int k, - const int n, - const amrex::Real dtdy, - const amrex::Real v_ad, - const amrex::Array4& S, - const amrex::Array4& Im, - const amrex::Array4& Ip, - const amrex::BCRec& bc, - const int domlo, - const int domhi, - const bool weno_js) -{ - using namespace amrex; - - constexpr amrex::Real small_vel = 1e-8; - - amrex::Real sm2 = S(i, j - 2, k, n); - amrex::Real sm1 = S(i, j - 1, k, n); - amrex::Real s0 = S(i, j, k, n); - amrex::Real sp1 = S(i, j + 1, k, n); - amrex::Real sp2 = S(i, j + 2, k, n); - - // right of j-1/2 - amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT - // left of j+1/2 - amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT - - amrex::Real sm = sedge1; - amrex::Real sp = sedge2; - - Godunov_weno_ybc( - i, j, k, n, sm, sp, sedge1, sedge2, S, v_ad, v_ad, bc.lo(1), bc.hi(1), - domlo, domhi); - - amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); - - amrex::Real sigma = std::abs(v_ad) * dtdy; - - if (v_ad > small_vel) { - Ip(i, j, k, n) = - sp - (0.5 * sigma) * ((sp - sm) - (1.0 - 2.0 / 3.0 * sigma) * s6); - Im(i, j, k, n) = S(i, j, k, n); - } else if (v_ad < -small_vel) { - Ip(i, j, k, n) = S(i, j, k, n); - Im(i, j, k, n) = - sm + (0.5 * sigma) * ((sp - sm) + (1.0 - 2.0 / 3.0 * sigma) * s6); - } else { - Ip(i, j, k, n) = S(i, j, k, n); - Im(i, j, k, n) = S(i, j, k, n); - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_pred_z( - const int i, - const int j, - const int k, - const int n, - const amrex::Real dtdz, - const amrex::Real v_ad, - const amrex::Array4& S, - const amrex::Array4& Im, - const amrex::Array4& Ip, - const amrex::BCRec& bc, - const int domlo, - const int domhi, - const bool weno_js) -{ - using namespace amrex; - - constexpr amrex::Real small_vel = 1e-8; - - amrex::Real sm2 = S(i, j, k - 2, n); - amrex::Real sm1 = S(i, j, k - 1, n); - amrex::Real s0 = S(i, j, k, n); - amrex::Real sp1 = S(i, j, k + 1, n); - amrex::Real sp2 = S(i, j, k + 2, n); - - // right of k-1/2 - amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT - // left of k+1/2 - amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT - - amrex::Real sm = sedge1; - amrex::Real sp = sedge2; - - Godunov_weno_zbc( - i, j, k, n, sm, sp, sedge1, sedge2, S, v_ad, v_ad, bc.lo(2), bc.hi(2), - domlo, domhi); - - amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); - - amrex::Real sigma = std::abs(v_ad) * dtdz; - - if (v_ad > small_vel) { - Ip(i, j, k, n) = - sp - (0.5 * sigma) * ((sp - sm) - (1.0 - 2.0 / 3.0 * sigma) * s6); - Im(i, j, k, n) = S(i, j, k, n); - } else if (v_ad < -small_vel) { - Ip(i, j, k, n) = S(i, j, k, n); - Im(i, j, k, n) = - sm + (0.5 * sigma) * ((sp - sm) + (1.0 - 2.0 / 3.0 * sigma) * s6); - } else { - Ip(i, j, k, n) = S(i, j, k, n); - Im(i, j, k, n) = S(i, j, k, n); - } -} - -// This version is called after the MAC projection, when we use the -// MAC-projected velocity -// for upwinding -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_x( - const int i, - const int j, - const int k, - const int n, - const amrex::Real dt, - const amrex::Real dx, - amrex::Real& Im, - amrex::Real& Ip, - const amrex::Array4& S, - const amrex::Array4& vel_edge, - const amrex::BCRec& bc, - const int domlo, - const int domhi, - const bool weno_js) -{ - - using namespace amrex; - - constexpr amrex::Real small_vel = 1e-8; - - amrex::Real sm2 = S(i - 2, j, k, n); - amrex::Real sm1 = S(i - 1, j, k, n); - amrex::Real s0 = S(i, j, k, n); - amrex::Real sp1 = S(i + 1, j, k, n); - amrex::Real sp2 = S(i + 2, j, k, n); - - // right of i-1/2 - amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT - // left of i+1/2 - amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT - - amrex::Real sm = sedge1; - amrex::Real sp = sedge2; - - Godunov_weno_xbc( - i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), - vel_edge(i + 1, j, k), bc.lo(0), bc.hi(0), domlo, domhi); - - amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); - - amrex::Real sigmap = std::abs(vel_edge(i + 1, j, k)) * dt / dx; - amrex::Real sigmam = std::abs(vel_edge(i, j, k)) * dt / dx; - - if (vel_edge(i + 1, j, k) > small_vel) { - Ip = sp - - (0.5 * sigmap) * ((sp - sm) - (1.e0 - 2.e0 / 3.e0 * sigmap) * s6); - } else { - Ip = S(i, j, k, n); - } - - if (vel_edge(i, j, k) < -small_vel) { - Im = sm + - (0.5 * sigmam) * ((sp - sm) + (1.e0 - 2.e0 / 3.e0 * sigmam) * s6); - } else { - Im = S(i, j, k, n); - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_y( - const int i, - const int j, - const int k, - const int n, - const amrex::Real dt, - const amrex::Real dx, - amrex::Real& Im, - amrex::Real& Ip, - const amrex::Array4& S, - const amrex::Array4& vel_edge, - const amrex::BCRec& bc, - const int domlo, - const int domhi, - const bool weno_js) -{ - - using namespace amrex; - - constexpr amrex::Real small_vel = 1e-8; - - amrex::Real sm2 = S(i, j - 2, k, n); - amrex::Real sm1 = S(i, j - 1, k, n); - amrex::Real s0 = S(i, j, k, n); - amrex::Real sp1 = S(i, j + 1, k, n); - amrex::Real sp2 = S(i, j + 2, k, n); - - // right of j-1/2 - amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT - // left of j+1/2 - amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT - - amrex::Real sm = sedge1; - amrex::Real sp = sedge2; - - Godunov_weno_ybc( - i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), - vel_edge(i, j + 1, k), bc.lo(1), bc.hi(1), domlo, domhi); - - amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); - - amrex::Real sigmap = std::abs(vel_edge(i, j + 1, k)) * dt / dx; - amrex::Real sigmam = std::abs(vel_edge(i, j, k)) * dt / dx; - - if (vel_edge(i, j + 1, k) > small_vel) { - Ip = sp - - (0.5 * sigmap) * ((sp - sm) - (1.e0 - 2.e0 / 3.e0 * sigmap) * s6); - } else { - Ip = S(i, j, k, n); - } - - if (vel_edge(i, j, k) < -small_vel) { - Im = sm + - (0.5 * sigmam) * ((sp - sm) + (1.e0 - 2.e0 / 3.e0 * sigmam) * s6); - } else { - Im = S(i, j, k, n); - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_weno_fpu_z( - const int i, - const int j, - const int k, - const int n, - const amrex::Real dt, - const amrex::Real dx, - amrex::Real& Im, - amrex::Real& Ip, - const amrex::Array4& S, - const amrex::Array4& vel_edge, - const amrex::BCRec& bc, - const int domlo, - const int domhi, - const bool weno_js) -{ - - using namespace amrex; - - constexpr amrex::Real small_vel = 1e-8; - - amrex::Real sm2 = S(i, j, k - 2, n); - amrex::Real sm1 = S(i, j, k - 1, n); - amrex::Real s0 = S(i, j, k, n); - amrex::Real sp1 = S(i, j, k + 1, n); - amrex::Real sp2 = S(i, j, k + 2, n); - - // right of k-1/2 - amrex::Real sedge1 = weno5(sp2, sp1, s0, sm1, sm2, weno_js); // NOLINT - // left of k+1/2 - amrex::Real sedge2 = weno5(sm2, sm1, s0, sp1, sp2, weno_js); // NOLINT - - amrex::Real sm = sedge1; - amrex::Real sp = sedge2; - - Godunov_weno_zbc( - i, j, k, n, sm, sp, sedge1, sedge2, S, vel_edge(i, j, k), - vel_edge(i, j, k + 1), bc.lo(2), bc.hi(2), domlo, domhi); - - amrex::Real s6 = 6.0 * s0 - 3.0 * (sm + sp); - - amrex::Real sigmap = std::abs(vel_edge(i, j, k + 1)) * dt / dx; - amrex::Real sigmam = std::abs(vel_edge(i, j, k)) * dt / dx; - - if (vel_edge(i, j, k + 1) > small_vel) { - Ip = sp - - (0.5 * sigmap) * ((sp - sm) - (1.e0 - 2.e0 / 3.e0 * sigmap) * s6); - } else { - Ip = S(i, j, k, n); - } - - if (vel_edge(i, j, k) < -small_vel) { - Im = sm + - (0.5 * sigmam) * ((sp - sm) + (1.e0 - 2.e0 / 3.e0 * sigmam) * s6); - } else { - Im = S(i, j, k, n); - } -} - -} // namespace - -#endif From 43c8298f8a4aa254782ac43b2d07953873113bd1 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 14 Aug 2024 12:14:38 -0700 Subject: [PATCH 66/83] minor clang-format --- amr-wind/convection/incflo_godunov_advection.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp index 9ec5f5e984..cca45ef0a7 100644 --- a/amr-wind/convection/incflo_godunov_advection.cpp +++ b/amr-wind/convection/incflo_godunov_advection.cpp @@ -145,8 +145,8 @@ void godunov::compute_fluxes( auto bc = pbc[n]; Godunov_trans_xbc( - i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), - dlo.x, dhi.x); + i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); xlo(i, j, k, n) = lo; xhi(i, j, k, n) = hi; Real st = (uval) ? lo : hi; @@ -173,8 +173,8 @@ void godunov::compute_fluxes( auto bc = pbc[n]; Godunov_trans_ybc( - i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), - dlo.y, dhi.y); + i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); ylo(i, j, k, n) = lo; yhi(i, j, k, n) = hi; @@ -201,8 +201,8 @@ void godunov::compute_fluxes( } Godunov_trans_zbc( - i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), - dlo.z, dhi.z); + i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); zlo(i, j, k, n) = lo; zhi(i, j, k, n) = hi; From 0fc4bdb49c77ed30170c9937aaa6605ba23e1a43 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 14 Aug 2024 12:17:36 -0700 Subject: [PATCH 67/83] one more formatting change --- amr-wind/utilities/MultiLevelVector.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amr-wind/utilities/MultiLevelVector.H b/amr-wind/utilities/MultiLevelVector.H index e4482f088d..3186ed4aff 100644 --- a/amr-wind/utilities/MultiLevelVector.H +++ b/amr-wind/utilities/MultiLevelVector.H @@ -22,7 +22,7 @@ namespace amr_wind { class MultiLevelVector { public: - MultiLevelVector(const FieldLoc floc = FieldLoc::CELL) : m_floc(floc) {}; + MultiLevelVector(const FieldLoc floc = FieldLoc::CELL) : m_floc(floc){}; void resize(const int axis, const amrex::Vector& geom); From b9c1a7dee38cafa0725b51beb0fed54490daee09 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 14 Aug 2024 13:37:20 -0700 Subject: [PATCH 68/83] pass by reference and make type conversions explicit --- amr-wind/boundary_conditions/MassInflowOutflowBC.cpp | 10 ++++++---- amr-wind/boundary_conditions/scalar_bcs.H | 2 +- amr-wind/boundary_conditions/scalar_bcs.cpp | 2 +- amr-wind/boundary_conditions/velocity_bcs.H | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index c3e6faa7ce..0a0cd0157a 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -12,14 +12,14 @@ void MassInflowOutflowBC::operator()( { const auto& repo = m_field.repo(); const auto& velocity = repo.get_field("velocity"); - // const auto bcvals = m_field.bc_values_device(); const int ncomp = m_field.num_comp(); - // const int idx = static_cast(m_ori); const int idim = m_ori.coordDir(); const auto islow = m_ori.isLow(); const auto ishigh = m_ori.isHigh(); const int nlevels = m_field.repo().num_active_levels(); - const amrex::IntVect iv_dir = {idim == 0, idim == 1, idim == 2}; + const amrex::IntVect iv_dir = {static_cast(idim == 0), + static_cast(idim == 1), + static_cast(idim == 2)}; for (int lev = 0; lev < nlevels; ++lev) { const auto& domain = repo.mesh().Geom(lev).Domain(); @@ -33,7 +33,9 @@ void MassInflowOutflowBC::operator()( #endif for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { auto bx = mfi.validbox(); - bx.grow({!iv_dir[0], !iv_dir[1], !iv_dir[2]}); + bx.grow({static_cast(idim != 0), + static_cast(idim != 1), + static_cast(idim != 2)}); const auto& bc_a = m_field(lev).array(mfi); const auto& vel = velocity(lev).array(mfi); diff --git a/amr-wind/boundary_conditions/scalar_bcs.H b/amr-wind/boundary_conditions/scalar_bcs.H index 8be799856c..1a4ebd2e10 100644 --- a/amr-wind/boundary_conditions/scalar_bcs.H +++ b/amr-wind/boundary_conditions/scalar_bcs.H @@ -29,7 +29,7 @@ void register_scalar_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const amrex::Array udfs); + const amrex::Array& udfs); } // namespace amr_wind::scalar_bc diff --git a/amr-wind/boundary_conditions/scalar_bcs.cpp b/amr-wind/boundary_conditions/scalar_bcs.cpp index b1f5988dca..de062d77e5 100644 --- a/amr-wind/boundary_conditions/scalar_bcs.cpp +++ b/amr-wind/boundary_conditions/scalar_bcs.cpp @@ -5,7 +5,7 @@ void register_scalar_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const amrex::Array udfs) + const amrex::Array& udfs) { const std::string inflow_udf = udfs[0]; const std::string inflow_outflow_udf = udfs[1]; diff --git a/amr-wind/boundary_conditions/velocity_bcs.H b/amr-wind/boundary_conditions/velocity_bcs.H index acc26a3919..359904be46 100644 --- a/amr-wind/boundary_conditions/velocity_bcs.H +++ b/amr-wind/boundary_conditions/velocity_bcs.H @@ -54,7 +54,7 @@ void register_velocity_dirichlet( Field& field, const amrex::AmrCore& mesh, const SimTime& time, - const amrex::Array udfs) + const amrex::Array& udfs) { const std::string inflow_udf = udfs[0]; const std::string inflow_outflow_udf = udfs[1]; From 1b5a15fa6481a20dac34f1bb692f8ca3a41db19e Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 14 Aug 2024 13:42:07 -0700 Subject: [PATCH 69/83] format new change --- amr-wind/boundary_conditions/MassInflowOutflowBC.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp index 0a0cd0157a..1262c096b5 100644 --- a/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp +++ b/amr-wind/boundary_conditions/MassInflowOutflowBC.cpp @@ -17,9 +17,9 @@ void MassInflowOutflowBC::operator()( const auto islow = m_ori.isLow(); const auto ishigh = m_ori.isHigh(); const int nlevels = m_field.repo().num_active_levels(); - const amrex::IntVect iv_dir = {static_cast(idim == 0), - static_cast(idim == 1), - static_cast(idim == 2)}; + const amrex::IntVect iv_dir = { + static_cast(idim == 0), static_cast(idim == 1), + static_cast(idim == 2)}; for (int lev = 0; lev < nlevels; ++lev) { const auto& domain = repo.mesh().Geom(lev).Domain(); @@ -33,9 +33,9 @@ void MassInflowOutflowBC::operator()( #endif for (amrex::MFIter mfi(m_field(lev), mfi_info); mfi.isValid(); ++mfi) { auto bx = mfi.validbox(); - bx.grow({static_cast(idim != 0), - static_cast(idim != 1), - static_cast(idim != 2)}); + bx.grow( + {static_cast(idim != 0), static_cast(idim != 1), + static_cast(idim != 2)}); const auto& bc_a = m_field(lev).array(mfi); const auto& vel = velocity(lev).array(mfi); From fdc1eb6083c446137c250f2fc4e6e5edb906e044 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 14 Aug 2024 14:11:32 -0700 Subject: [PATCH 70/83] use simpler path to use internal inout flag in advection --- amr-wind/equation_systems/AdvOp_Godunov.H | 3 +-- amr-wind/equation_systems/AdvOp_MOL.H | 3 +-- amr-wind/equation_systems/PDE.H | 6 ++---- amr-wind/equation_systems/icns/icns_advection.H | 16 +++++----------- .../equation_systems/icns/icns_advection.cpp | 8 ++++---- amr-wind/equation_systems/vof/vof_advection.H | 1 - .../equation_systems/test_icns_cstdens.cpp | 3 +-- 7 files changed, 14 insertions(+), 26 deletions(-) diff --git a/amr-wind/equation_systems/AdvOp_Godunov.H b/amr-wind/equation_systems/AdvOp_Godunov.H index 4ef0e84457..09eaebadf8 100644 --- a/amr-wind/equation_systems/AdvOp_Godunov.H +++ b/amr-wind/equation_systems/AdvOp_Godunov.H @@ -30,8 +30,7 @@ struct AdvectionOp< bool /* has_overset */, bool /* variable density */, bool /* mesh mapping */, - bool /* is_anelastic */, - bool /* has_inout_bndry */) + bool /* is_anelastic */) : fields(fields_in) , density(fields_in.repo.get_field("density")) , u_mac(fields_in.repo.get_field("u_mac")) diff --git a/amr-wind/equation_systems/AdvOp_MOL.H b/amr-wind/equation_systems/AdvOp_MOL.H index 328fbe5aa7..b276e39d78 100644 --- a/amr-wind/equation_systems/AdvOp_MOL.H +++ b/amr-wind/equation_systems/AdvOp_MOL.H @@ -29,8 +29,7 @@ struct AdvectionOp< bool /* has_overset */, bool /* variable density */, bool /* mesh mapping */, - bool /* is_anelastic */, - bool /* has_inout_bndry */) + bool /* is_anelastic */) : fields(fields_in) , density(fields_in.repo.get_field("density")) , u_mac(fields_in.repo.get_field("u_mac")) diff --git a/amr-wind/equation_systems/PDE.H b/amr-wind/equation_systems/PDE.H index 9601cf454f..485616d680 100644 --- a/amr-wind/equation_systems/PDE.H +++ b/amr-wind/equation_systems/PDE.H @@ -69,8 +69,7 @@ public: m_adv_op.reset(new AdvectionOp( m_sim, m_fields, m_sim.has_overset(), variable_density, - m_sim.has_mesh_mapping(), m_sim.is_anelastic(), - m_fields.field.has_inout_bndry())); + m_sim.has_mesh_mapping(), m_sim.is_anelastic())); m_src_op.init_source_terms(m_sim); // Post-solve operations should also apply after initialization @@ -93,8 +92,7 @@ public: m_adv_op.reset(new AdvectionOp( m_sim, m_fields, m_sim.has_overset(), variable_density, - m_sim.has_mesh_mapping(), m_sim.is_anelastic(), - m_fields.field.has_inout_bndry())); + m_sim.has_mesh_mapping(), m_sim.is_anelastic())); // Post-solve operations should also apply after a regrid m_post_solve_op(m_time.current_time()); diff --git a/amr-wind/equation_systems/icns/icns_advection.H b/amr-wind/equation_systems/icns/icns_advection.H index 130d680707..934ae092fd 100644 --- a/amr-wind/equation_systems/icns/icns_advection.H +++ b/amr-wind/equation_systems/icns/icns_advection.H @@ -28,8 +28,7 @@ public: bool /*has_overset*/, bool /*variable_density*/, bool /*mesh_mapping*/, - bool /*is_anelastic*/, - bool /*has_inout_bndry*/); + bool /*is_anelastic*/); void set_inflow_velocity(amrex::Real time); @@ -63,7 +62,6 @@ private: bool m_variable_density{false}; bool m_mesh_mapping{false}; bool m_is_anelastic{false}; - bool m_has_inout_bndry{false}; amrex::Real m_rho_0{1.0}; }; @@ -79,8 +77,7 @@ struct AdvectionOp bool has_overset, bool variable_density, bool mesh_mapping, - bool is_anelastic, - bool has_inout_bndry) + bool is_anelastic) : fields(fields_in) , u_mac(fields_in.repo.get_field("u_mac")) , v_mac(fields_in.repo.get_field("v_mac")) @@ -91,8 +88,7 @@ struct AdvectionOp has_overset, variable_density, mesh_mapping, - is_anelastic, - has_inout_bndry) + is_anelastic) { amrex::ParmParse pp("incflo"); @@ -458,8 +454,7 @@ struct AdvectionOp bool has_overset, bool variable_density, bool mesh_mapping, - bool is_anelastic, - bool has_inout_bndry) + bool is_anelastic) : fields(fields_in) , u_mac(fields_in.repo.get_field("u_mac")) , v_mac(fields_in.repo.get_field("v_mac")) @@ -471,8 +466,7 @@ struct AdvectionOp has_overset, variable_density, m_mesh_mapping, - is_anelastic, - has_inout_bndry) + is_anelastic) {} void preadvect( diff --git a/amr-wind/equation_systems/icns/icns_advection.cpp b/amr-wind/equation_systems/icns/icns_advection.cpp index 6cc502921d..e191801e3c 100644 --- a/amr-wind/equation_systems/icns/icns_advection.cpp +++ b/amr-wind/equation_systems/icns/icns_advection.cpp @@ -47,8 +47,7 @@ MacProjOp::MacProjOp( bool has_overset, bool variable_density, bool mesh_mapping, - bool is_anelastic, - bool has_inout_bndry) + bool is_anelastic) : m_repo(repo) , m_phy_mgr(phy_mgr) , m_options("mac_proj") @@ -56,7 +55,6 @@ MacProjOp::MacProjOp( , m_variable_density(variable_density) , m_mesh_mapping(mesh_mapping) , m_is_anelastic(is_anelastic) - , m_has_inout_bndry(has_inout_bndry) { amrex::ParmParse pp("incflo"); pp.query("density", m_rho_0); @@ -279,7 +277,9 @@ void MacProjOp::operator()(const FieldState fstate, const amrex::Real dt) } } - if (m_has_inout_bndry) { + const bool has_inout_bndry = + (m_repo.get_field("velocity")).has_inout_bndry(); + if (has_inout_bndry) { enforce_inout_solvability(mac_vec); } diff --git a/amr-wind/equation_systems/vof/vof_advection.H b/amr-wind/equation_systems/vof/vof_advection.H index 4f387fe90a..94eaab2af4 100644 --- a/amr-wind/equation_systems/vof/vof_advection.H +++ b/amr-wind/equation_systems/vof/vof_advection.H @@ -20,7 +20,6 @@ struct AdvectionOp bool /*unused*/, bool /*unused*/, bool /*unused*/, - bool /*unused*/, bool /*unused*/) : fields(fields_in) , u_mac(fields_in.repo.get_field("u_mac")) diff --git a/unit_tests/equation_systems/test_icns_cstdens.cpp b/unit_tests/equation_systems/test_icns_cstdens.cpp index 5995ef80ee..806a9804e9 100644 --- a/unit_tests/equation_systems/test_icns_cstdens.cpp +++ b/unit_tests/equation_systems/test_icns_cstdens.cpp @@ -23,8 +23,7 @@ class ICNSConstDensTest : public MeshTest // Initialize MAC projection operator const auto& mco = amr_wind::pde::MacProjOp( - sim().repo(), sim().physics_manager(), false, false, false, false, - false); + sim().repo(), sim().physics_manager(), false, false, false, false); // Get background density and check const amrex::Real rho0 = mco.rho0(); EXPECT_EQ(rho0, m_rho_0); From e587ab2d8ff486a6b83867c3f2d21f09d76e9efd Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 14 Aug 2024 16:35:46 -0700 Subject: [PATCH 71/83] use hydro routines for edge BCs --- .../convection/incflo_godunov_advection.cpp | 33 ++-- amr-wind/convection/incflo_godunov_ppm.H | 166 ++---------------- 2 files changed, 34 insertions(+), 165 deletions(-) diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp index cca45ef0a7..e78f88d02a 100644 --- a/amr-wind/convection/incflo_godunov_advection.cpp +++ b/amr-wind/convection/incflo_godunov_advection.cpp @@ -145,8 +145,7 @@ void godunov::compute_fluxes( auto bc = pbc[n]; Godunov_trans_xbc( - i, j, k, n, q, lo, hi, uad, uad, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); + i, j, k, n, q, lo, hi, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); xlo(i, j, k, n) = lo; xhi(i, j, k, n) = hi; Real st = (uval) ? lo : hi; @@ -173,8 +172,7 @@ void godunov::compute_fluxes( auto bc = pbc[n]; Godunov_trans_ybc( - i, j, k, n, q, lo, hi, vad, vad, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); + i, j, k, n, q, lo, hi, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); ylo(i, j, k, n) = lo; yhi(i, j, k, n) = hi; @@ -201,8 +199,7 @@ void godunov::compute_fluxes( } Godunov_trans_zbc( - i, j, k, n, q, lo, hi, wad, wad, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); + i, j, k, n, q, lo, hi, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); zlo(i, j, k, n) = lo; zhi(i, j, k, n) = hi; @@ -235,8 +232,8 @@ void godunov::compute_fluxes( Real wad = wmac(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zylo, l_zyhi, wad, wad, bc.lo(2), bc.hi(2), - dlo.z, dhi.z); + i, j, k, n, q, l_zylo, l_zyhi, wad, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); constexpr Real small_vel = 1.e-8; @@ -254,8 +251,8 @@ void godunov::compute_fluxes( Real vad = vmac(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yzlo, l_yzhi, vad, vad, bc.lo(1), bc.hi(1), - dlo.y, dhi.y); + i, j, k, n, q, l_yzlo, l_yzhi, vad, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); constexpr Real small_vel = 1.e-8; @@ -347,8 +344,8 @@ void godunov::compute_fluxes( Real uad = umac(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xzlo, l_xzhi, uad, uad, bc.lo(0), bc.hi(0), - dlo.x, dhi.x); + i, j, k, n, q, l_xzlo, l_xzhi, uad, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); constexpr Real small_vel = 1.e-8; @@ -366,8 +363,8 @@ void godunov::compute_fluxes( Real wad = wmac(i, j, k); Godunov_trans_zbc( - i, j, k, n, q, l_zxlo, l_zxhi, wad, wad, bc.lo(2), bc.hi(2), - dlo.z, dhi.z); + i, j, k, n, q, l_zxlo, l_zxhi, wad, bc.lo(2), bc.hi(2), dlo.z, + dhi.z); constexpr Real small_vel = 1.e-8; @@ -458,8 +455,8 @@ void godunov::compute_fluxes( Real uad = umac(i, j, k); Godunov_trans_xbc( - i, j, k, n, q, l_xylo, l_xyhi, uad, uad, bc.lo(0), bc.hi(0), - dlo.x, dhi.x); + i, j, k, n, q, l_xylo, l_xyhi, uad, bc.lo(0), bc.hi(0), dlo.x, + dhi.x); constexpr Real small_vel = 1.e-8; @@ -477,8 +474,8 @@ void godunov::compute_fluxes( Real vad = vmac(i, j, k); Godunov_trans_ybc( - i, j, k, n, q, l_yxlo, l_yxhi, vad, vad, bc.lo(1), bc.hi(1), - dlo.y, dhi.y); + i, j, k, n, q, l_yxlo, l_yxhi, vad, bc.lo(1), bc.hi(1), dlo.y, + dhi.y); constexpr Real small_vel = 1.e-8; diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H index a9abf3bdd2..f57f9ce779 100644 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ b/amr-wind/convection/incflo_godunov_ppm.H @@ -3,6 +3,7 @@ #include #include +#include /* This header file contains the inlined __host__ __device__ functions required for the scalar advection routines for 3D Godunov. It also contains function @@ -18,63 +19,18 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( const amrex::Array4& s, amrex::Real& lo, amrex::Real& hi, - amrex::Real& velm, - amrex::Real& velp, + amrex::Real macvel, const int bclo, const int bchi, const int domlo, const int domhi) { - using namespace amrex; - - // for MIO BC, use vel+ at bclo and vel- at bchi for determining - // ext_dir/foextrap treatment - // Low X if (i <= domlo) { - if (bclo == BCType::ext_dir || - (bclo == BCType::direction_dependent && velp >= 0.0)) { - // IAMR does this but it breaks lo/hi symmetry - // Real st = (uedge <= small_vel) ? hi : s(domlo-1,j,k,n); - // So here we do something simpler... - Real st = s(domlo - 1, j, k, n); - lo = st; - hi = st; - } - - else if ( - bclo == BCType::foextrap || bclo == BCType::hoextrap || - bclo == BCType::reflect_even || - (bclo == BCType::direction_dependent && velp < 0.0)) { - lo = hi; - - } else if (bclo == BCType::reflect_odd) { - hi = 0.; - lo = 0.; - } - } - - // High X - if (i > domhi) { - if (bchi == BCType::ext_dir || - (bchi == BCType::direction_dependent && velm <= 0.0)) { - // IAMR does this but it breaks lo/hi symmetry - // Real st = (uedge >= -small_vel)? lo : s(domhi+1,j,k,n); - // So here we do something simpler... - Real st = s(domhi + 1, j, k, n); - lo = st; - hi = st; - } - - else if ( - bchi == BCType::foextrap || bchi == BCType::hoextrap || - bchi == BCType::reflect_even || - (bchi == BCType::direction_dependent && velm > 0.0)) { - hi = lo; - - } else if (bchi == BCType::reflect_odd) { - hi = 0.; - lo = 0.; - } + HydroBC::SetEdgeBCsLo( + 0, i, j, k, n, s, lo, hi, macvel, bclo, domlo, true); + } else if (i > domhi) { + HydroBC::SetEdgeBCsHi( + 0, i, j, k, n, s, lo, hi, macvel, bchi, domhi, true); } } @@ -86,61 +42,18 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_ybc( const amrex::Array4& s, amrex::Real& lo, amrex::Real& hi, - amrex::Real& velm, - amrex::Real& velp, + amrex::Real macvel, const int bclo, const int bchi, const int domlo, const int domhi) { - using namespace amrex; - - // Low Y if (j <= domlo) { - if (bclo == BCType::ext_dir || - (bclo == BCType::direction_dependent && velp >= 0.0)) { - // IAMR does this but it breaks lo/hi symmetry - // Real st = (vedge <= small_vel) ? hi : s(i,domlo-1,k,n); - // So here we do something simpler... - Real st = s(i, domlo - 1, k, n); - lo = st; - hi = st; - } - - else if ( - bclo == BCType::foextrap || bclo == BCType::hoextrap || - bclo == BCType::reflect_even || - (bclo == BCType::direction_dependent && velp < 0.0)) { - lo = hi; - - } else if (bclo == BCType::reflect_odd) { - hi = 0.; - lo = 0.; - } - } - - // High Y - if (j > domhi) { - if (bchi == BCType::ext_dir || - (bchi == BCType::direction_dependent && velm <= 0.0)) { - // IAMR does this but it breaks lo/hi symmetry - // Real st = (vedge >= -small_vel)? lo : s(i,domhi+1,k,n); - // So here we do something simpler... - Real st = s(i, domhi + 1, k, n); - lo = st; - hi = st; - } - - else if ( - bchi == BCType::foextrap || bchi == BCType::hoextrap || - bchi == BCType::reflect_even || - (bchi == BCType::direction_dependent && velm > 0.0)) { - hi = lo; - - } else if (bchi == BCType::reflect_odd) { - hi = 0.; - lo = 0.; - } + HydroBC::SetEdgeBCsLo( + 1, i, j, k, n, s, lo, hi, macvel, bclo, domlo, true); + } else if (j > domhi) { + HydroBC::SetEdgeBCsHi( + 1, i, j, k, n, s, lo, hi, macvel, bchi, domhi, true); } } @@ -152,59 +65,18 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( const amrex::Array4& s, amrex::Real& lo, amrex::Real& hi, - amrex::Real& velm, - amrex::Real& velp, + amrex::Real macvel, const int bclo, const int bchi, const int domlo, const int domhi) { - using namespace amrex; - - // Low Z if (k <= domlo) { - if (bclo == BCType::ext_dir || - (bclo == BCType::direction_dependent && velp >= 0.0)) { - // IAMR does this but it breaks lo/hi symmetry - // Real st = (wedge <= small_vel) ? hi : s(i,j,domlo-1,n); - // So here we do something simpler... - Real st = s(i, j, domlo - 1, n); - lo = st; - hi = st; - } - - else if ( - bclo == BCType::foextrap || bclo == BCType::hoextrap || - bclo == BCType::reflect_even || - (bclo == BCType::direction_dependent && velp < 0.0)) { - lo = hi; - - } else if (bclo == BCType::reflect_odd) { - hi = 0.; - lo = 0.; - } - } - - // High Z - if (k > domhi) { - if (bchi == BCType::ext_dir || - (bchi == BCType::direction_dependent && velm <= 0.0)) { - // IAMR does this but it breaks lo/hi symmetry - // Real st = (wedge >= -small_vel)? lo : s(i,j,domhi+1,n); - // So here we do something simpler... - Real st = s(i, j, domhi + 1, n); - lo = st; - hi = st; - } else if ( - bchi == BCType::foextrap || bchi == BCType::hoextrap || - bchi == BCType::reflect_even || - (bchi == BCType::direction_dependent && velm > 0.0)) { - hi = lo; - - } else if (bchi == BCType::reflect_odd) { - hi = 0.; - lo = 0.; - } + HydroBC::SetEdgeBCsLo( + 2, i, j, k, n, s, lo, hi, macvel, bclo, domlo, true); + } else if (k > domhi) { + HydroBC::SetEdgeBCsHi( + 2, i, j, k, n, s, lo, hi, macvel, bchi, domhi, true); } } From 4e0c6eb938fba31714e7dc64f53f43722eb45088 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 15 Aug 2024 14:27:06 -0700 Subject: [PATCH 72/83] use Dirichlet for in-out boundary diffusion --- amr-wind/diffusion/incflo_diffusion.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amr-wind/diffusion/incflo_diffusion.cpp b/amr-wind/diffusion/incflo_diffusion.cpp index 20886e85db..da951dfa4c 100644 --- a/amr-wind/diffusion/incflo_diffusion.cpp +++ b/amr-wind/diffusion/incflo_diffusion.cpp @@ -20,7 +20,6 @@ Vector> get_diffuse_tensor_bc( switch (bc) { case BC::pressure_inflow: case BC::pressure_outflow: - case BC::mass_inflow_outflow: case BC::zero_gradient: { // All three components are Neumann r[0][dir] = LinOpBCType::Neumann; @@ -30,6 +29,7 @@ Vector> get_diffuse_tensor_bc( } case BC::wave_generation: case BC::mass_inflow: + case BC::mass_inflow_outflow: case BC::no_slip_wall: { // All three components are Dirichlet r[0][dir] = LinOpBCType::Dirichlet; @@ -78,7 +78,6 @@ get_diffuse_scalar_bc(amr_wind::Field& scalar, Orientation::Side side) noexcept switch (bc) { case BC::pressure_inflow: case BC::pressure_outflow: - case BC::mass_inflow_outflow: case BC::zero_gradient: case BC::symmetric_wall: case BC::slip_wall: { @@ -92,6 +91,7 @@ get_diffuse_scalar_bc(amr_wind::Field& scalar, Orientation::Side side) noexcept } case BC::wave_generation: case BC::mass_inflow: + case BC::mass_inflow_outflow: case BC::no_slip_wall: { r[dir] = LinOpBCType::Dirichlet; break; From 9644abcc51c181b0778a3d69cef4d62089cb47ae Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 15 Aug 2024 15:18:18 -0700 Subject: [PATCH 73/83] do not enforce in-out solvability before nodal projection in initial iterations --- amr-wind/projection/incflo_apply_nodal_projection.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/amr-wind/projection/incflo_apply_nodal_projection.cpp b/amr-wind/projection/incflo_apply_nodal_projection.cpp index e33efd3bd2..ad12059c8c 100644 --- a/amr-wind/projection/incflo_apply_nodal_projection.cpp +++ b/amr-wind/projection/incflo_apply_nodal_projection.cpp @@ -355,12 +355,11 @@ void incflo::ApplyProjection( } // Need to apply custom Neumann funcs for inflow-outflow BC - // after setting the inflow vels above. + // after setting the inflow vels above + // and then enforce solvability by matching outflow to inflow. if (!proj_for_small_dt and !incremental and velocity.has_inout_bndry()) { velocity.apply_bc_funcs(amr_wind::FieldState::New); - } - if (velocity.has_inout_bndry()) { amr_wind::nodal_projection::enforce_inout_solvability( velocity, m_repo.mesh().Geom(), m_repo.num_active_levels()); } From 33f3400d752c0971df8400745a7e33c73bf6166f Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Thu, 15 Aug 2024 15:22:41 -0700 Subject: [PATCH 74/83] fix format issue --- amr-wind/utilities/MultiLevelVector.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amr-wind/utilities/MultiLevelVector.H b/amr-wind/utilities/MultiLevelVector.H index 3186ed4aff..e4482f088d 100644 --- a/amr-wind/utilities/MultiLevelVector.H +++ b/amr-wind/utilities/MultiLevelVector.H @@ -22,7 +22,7 @@ namespace amr_wind { class MultiLevelVector { public: - MultiLevelVector(const FieldLoc floc = FieldLoc::CELL) : m_floc(floc){}; + MultiLevelVector(const FieldLoc floc = FieldLoc::CELL) : m_floc(floc) {}; void resize(const int axis, const amrex::Vector& geom); From a0f0d18c572cd19b66d33accd1df0cd7c99536fc Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 19 Aug 2024 15:48:30 -0700 Subject: [PATCH 75/83] change copy to const references to satisfy clang-tidy --- amr-wind/boundary_conditions/scalar_bcs.cpp | 6 +++--- amr-wind/boundary_conditions/velocity_bcs.H | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/amr-wind/boundary_conditions/scalar_bcs.cpp b/amr-wind/boundary_conditions/scalar_bcs.cpp index de062d77e5..3158e4fa8e 100644 --- a/amr-wind/boundary_conditions/scalar_bcs.cpp +++ b/amr-wind/boundary_conditions/scalar_bcs.cpp @@ -7,9 +7,9 @@ void register_scalar_dirichlet( const SimTime& time, const amrex::Array& udfs) { - const std::string inflow_udf = udfs[0]; - const std::string inflow_outflow_udf = udfs[1]; - const std::string wall_udf = udfs[2]; + const std::string& inflow_udf = udfs[0]; + const std::string& inflow_outflow_udf = udfs[1]; + const std::string& wall_udf = udfs[2]; if ((inflow_udf == "ConstDirichlet") && (inflow_outflow_udf == "ConstDirichlet") && diff --git a/amr-wind/boundary_conditions/velocity_bcs.H b/amr-wind/boundary_conditions/velocity_bcs.H index 359904be46..a2a901e8c8 100644 --- a/amr-wind/boundary_conditions/velocity_bcs.H +++ b/amr-wind/boundary_conditions/velocity_bcs.H @@ -56,9 +56,9 @@ void register_velocity_dirichlet( const SimTime& time, const amrex::Array& udfs) { - const std::string inflow_udf = udfs[0]; - const std::string inflow_outflow_udf = udfs[1]; - const std::string wall_udf = udfs[2]; + const std::string& inflow_udf = udfs[0]; + const std::string& inflow_outflow_udf = udfs[1]; + const std::string& wall_udf = udfs[2]; if ((inflow_udf == "ConstDirichlet") && (inflow_outflow_udf == "ConstDirichlet") && From 0e7f21b7d2183c6505ee2ff9f52e8a5818808378 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 19 Aug 2024 15:49:55 -0700 Subject: [PATCH 76/83] do not fill outflow velocities to MAC valid cells --- amr-wind/core/FieldFillPatchOps.H | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/amr-wind/core/FieldFillPatchOps.H b/amr-wind/core/FieldFillPatchOps.H index 296c84693e..c7fe17516f 100644 --- a/amr-wind/core/FieldFillPatchOps.H +++ b/amr-wind/core/FieldFillPatchOps.H @@ -443,12 +443,25 @@ public: } const auto& marr = mfab[mfi].array(); + amrex::FArrayBox tmp_fab(bx, mfab.nComp()); + amrex::Elixir tmp_eli = tmp_fab.elixir(); + amrex::Array4 const& tmp_marr = + tmp_fab.array(); + amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE( const int i, const int j, const int k) noexcept { bcfunc.set_inflow( - {i, j, k}, marr, gdata, time, ori, 0, 0, fdir); + {i, j, k}, tmp_marr, gdata, time, ori, 0, 0, + fdir); + + if ((bctype[ori] == BC::mass_inflow_outflow) && + ((ori.isLow() && tmp_marr(i, j, k) >= 0) || + (ori.isHigh() && tmp_marr(i, j, k) <= 0))) { + + marr(i, j, k) = tmp_marr(i, j, k); + } }); } } From b6075cab94f1da92e41f884144946b0487d4f459 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 19 Aug 2024 18:40:18 -0700 Subject: [PATCH 77/83] correct sibling fill bug --- amr-wind/core/FieldFillPatchOps.H | 74 +++++++++++++++++-------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/amr-wind/core/FieldFillPatchOps.H b/amr-wind/core/FieldFillPatchOps.H index c7fe17516f..ba25e8ec56 100644 --- a/amr-wind/core/FieldFillPatchOps.H +++ b/amr-wind/core/FieldFillPatchOps.H @@ -452,52 +452,60 @@ public: bx, [=] AMREX_GPU_DEVICE( const int i, const int j, const int k) noexcept { - bcfunc.set_inflow( - {i, j, k}, tmp_marr, gdata, time, ori, 0, 0, - fdir); - - if ((bctype[ori] == BC::mass_inflow_outflow) && - ((ori.isLow() && tmp_marr(i, j, k) >= 0) || - (ori.isHigh() && tmp_marr(i, j, k) <= 0))) { + bcfunc.set_inflow( + {i, j, k}, tmp_marr, gdata, time, ori, 0, 0, fdir); + + const bool is_outflow = + (ori.isLow() && tmp_marr(i, j, k) < 0) || + (ori.isHigh() && tmp_marr(i, j, k) > 0); + if ((bctype[ori] == BC::mass_inflow_outflow) && + is_outflow) { + marr(i, j, k) = marr(i, j, k); + } else { + marr(i, j, k) = tmp_marr(i, j, k); + } - marr(i, j, k) = tmp_marr(i, j, k); - } - }); + }); } } } } -protected: - Functor bc_functor() { return m_op(); } - Functor bc_functor_face(const int face_dir) { return m_op(face_dir); } +protected + : Functor + bc_functor() +{ + return m_op(); +} - void check_face_mapper() - { - if (std::any_of( - m_mesh.refRatio().begin(), m_mesh.refRatio().end(), - [](amrex::IntVect iv) { return iv != amrex::IntVect(2); })) { - amrex::Print() - << "WARNING: Changing the face interpolator to FaceLinear " - "because a refinement ratio is different than 2" - << std::endl; - m_face_mapper = field_impl::get_interpolation_operator( - FieldInterpolator::FaceLinear); - } +Functor bc_functor_face(const int face_dir) { return m_op(face_dir); } + +void check_face_mapper() +{ + if (std::any_of( + m_mesh.refRatio().begin(), m_mesh.refRatio().end(), + [](amrex::IntVect iv) { return iv != amrex::IntVect(2); })) { + amrex::Print() + << "WARNING: Changing the face interpolator to FaceLinear " + "because a refinement ratio is different than 2" + << std::endl; + m_face_mapper = field_impl::get_interpolation_operator( + FieldInterpolator::FaceLinear); } +} - const SimTime& m_time; - const amrex::AmrCore& m_mesh; - Field& m_field; +const SimTime& m_time; +const amrex::AmrCore& m_mesh; +Field & m_field; - const BCOpCreator m_op; +const BCOpCreator m_op; - //! Function that handles interpolation from coarse to fine level - amrex::Interpolater* m_mapper; +//! Function that handles interpolation from coarse to fine level +amrex::Interpolater* m_mapper; - //! Function that handles interpolation from coarse to fine level for faces - amrex::Interpolater* m_face_mapper; +//! Function that handles interpolation from coarse to fine level for faces +amrex::Interpolater* m_face_mapper; }; } // namespace amr_wind From 04a99532659e8654ae14d6542474dd94fb90d249 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 20 Aug 2024 10:15:45 -0700 Subject: [PATCH 78/83] clang-format FieldFillPatchOps --- amr-wind/core/FieldFillPatchOps.H | 79 +++++++++++++++---------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/amr-wind/core/FieldFillPatchOps.H b/amr-wind/core/FieldFillPatchOps.H index ba25e8ec56..0796ce0332 100644 --- a/amr-wind/core/FieldFillPatchOps.H +++ b/amr-wind/core/FieldFillPatchOps.H @@ -452,60 +452,55 @@ public: bx, [=] AMREX_GPU_DEVICE( const int i, const int j, const int k) noexcept { - bcfunc.set_inflow( - {i, j, k}, tmp_marr, gdata, time, ori, 0, 0, fdir); - - const bool is_outflow = - (ori.isLow() && tmp_marr(i, j, k) < 0) || - (ori.isHigh() && tmp_marr(i, j, k) > 0); - if ((bctype[ori] == BC::mass_inflow_outflow) && - is_outflow) { - marr(i, j, k) = marr(i, j, k); - } else { - marr(i, j, k) = tmp_marr(i, j, k); - } - - }); + bcfunc.set_inflow( + {i, j, k}, tmp_marr, gdata, time, ori, 0, 0, + fdir); + + const bool is_outflow = + (ori.isLow() && tmp_marr(i, j, k) < 0) || + (ori.isHigh() && tmp_marr(i, j, k) > 0); + if ((bctype[ori] == BC::mass_inflow_outflow) && + is_outflow) { + marr(i, j, k) = marr(i, j, k); + } else { + marr(i, j, k) = tmp_marr(i, j, k); + } + }); } } } } +protected: + Functor bc_functor() { return m_op(); } -protected - : Functor - bc_functor() -{ - return m_op(); -} + Functor bc_functor_face(const int face_dir) { return m_op(face_dir); } -Functor bc_functor_face(const int face_dir) { return m_op(face_dir); } - -void check_face_mapper() -{ - if (std::any_of( - m_mesh.refRatio().begin(), m_mesh.refRatio().end(), - [](amrex::IntVect iv) { return iv != amrex::IntVect(2); })) { - amrex::Print() - << "WARNING: Changing the face interpolator to FaceLinear " - "because a refinement ratio is different than 2" - << std::endl; - m_face_mapper = field_impl::get_interpolation_operator( - FieldInterpolator::FaceLinear); + void check_face_mapper() + { + if (std::any_of( + m_mesh.refRatio().begin(), m_mesh.refRatio().end(), + [](amrex::IntVect iv) { return iv != amrex::IntVect(2); })) { + amrex::Print() + << "WARNING: Changing the face interpolator to FaceLinear " + "because a refinement ratio is different than 2" + << std::endl; + m_face_mapper = field_impl::get_interpolation_operator( + FieldInterpolator::FaceLinear); + } } -} -const SimTime& m_time; -const amrex::AmrCore& m_mesh; -Field & m_field; + const SimTime& m_time; + const amrex::AmrCore& m_mesh; + Field& m_field; -const BCOpCreator m_op; + const BCOpCreator m_op; -//! Function that handles interpolation from coarse to fine level -amrex::Interpolater* m_mapper; + //! Function that handles interpolation from coarse to fine level + amrex::Interpolater* m_mapper; -//! Function that handles interpolation from coarse to fine level for faces -amrex::Interpolater* m_face_mapper; + //! Function that handles interpolation from coarse to fine level for faces + amrex::Interpolater* m_face_mapper; }; } // namespace amr_wind From 1366af05a766aa2bb84726d0c86dde3f3fb199ba Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 20 Aug 2024 10:26:36 -0700 Subject: [PATCH 79/83] remove files from index --- .../convection/incflo_godunov_advection.cpp | 549 ------------------ amr-wind/convection/incflo_godunov_ppm.H | 477 --------------- 2 files changed, 1026 deletions(-) delete mode 100644 amr-wind/convection/incflo_godunov_advection.cpp delete mode 100644 amr-wind/convection/incflo_godunov_ppm.H diff --git a/amr-wind/convection/incflo_godunov_advection.cpp b/amr-wind/convection/incflo_godunov_advection.cpp deleted file mode 100644 index e78f88d02a..0000000000 --- a/amr-wind/convection/incflo_godunov_advection.cpp +++ /dev/null @@ -1,549 +0,0 @@ -#include "amr-wind/convection/incflo_godunov_ppm.H" -#include "amr-wind/convection/incflo_godunov_minmod.H" -#include "amr-wind/convection/incflo_godunov_upwind.H" -#include "amr-wind/convection/Godunov.H" -#include - -using namespace amrex; - -void godunov::compute_fluxes( - int lev, - Box const& bx, - int ncomp, - Array4 const& fx, - Array4 const& fy, - Array4 const& fz, - Array4 const& q, - Array4 const& umac, - Array4 const& vmac, - Array4 const& wmac, - Array4 const& fq, - BCRec const* pbc, - int const* iconserv, - Real* p, - Vector geom, - Real dt, - godunov::scheme godunov_scheme, - bool use_forces_in_trans) -{ - - /* iconserv functionality: (would be better served by two flags) - * ------------------------------------------------------------------------ - * == 0 : non-conservative formulation of interpolation, fluxes not - * multiplied by MAC velocity - * == 1 : conservative formulation of interpolation, fluxes include factor - * of MAC velocity - * (!= 1 && != 0) : conservative formulation of interpolation, fluxes not - * multiplied by MAC velocity */ - - BL_PROFILE("amr-wind::godunov::compute_fluxes"); - Box const& xbx = amrex::surroundingNodes(bx, 0); - Box const& ybx = amrex::surroundingNodes(bx, 1); - Box const& zbx = amrex::surroundingNodes(bx, 2); - Box const& bxg1 = amrex::grow(bx, 1); - Box xebox = Box(xbx).grow(1, 1).grow(2, 1); - Box yebox = Box(ybx).grow(0, 1).grow(2, 1); - Box zebox = Box(zbx).grow(0, 1).grow(1, 1); - - const Real dx = geom[lev].CellSize(0); - const Real dy = geom[lev].CellSize(1); - const Real dz = geom[lev].CellSize(2); - Real l_dt = dt; - Real dtdx = l_dt / dx; - Real dtdy = l_dt / dy; - Real dtdz = l_dt / dz; - - Box const& domain = geom[lev].Domain(); - const auto dlo = amrex::lbound(domain); - const auto dhi = amrex::ubound(domain); - - Array4 Imx = makeArray4(p, bxg1, ncomp); - p += Imx.size(); - Array4 Ipx = makeArray4(p, bxg1, ncomp); - p += Ipx.size(); - Array4 Imy = makeArray4(p, bxg1, ncomp); - p += Imy.size(); - Array4 Ipy = makeArray4(p, bxg1, ncomp); - p += Ipy.size(); - Array4 Imz = makeArray4(p, bxg1, ncomp); - p += Imz.size(); - Array4 Ipz = makeArray4(p, bxg1, ncomp); - p += Ipz.size(); - Array4 xlo = makeArray4(p, xebox, ncomp); - p += xlo.size(); - Array4 xhi = makeArray4(p, xebox, ncomp); - p += xhi.size(); - Array4 ylo = makeArray4(p, yebox, ncomp); - p += ylo.size(); - Array4 yhi = makeArray4(p, yebox, ncomp); - p += yhi.size(); - Array4 zlo = makeArray4(p, zebox, ncomp); - p += zlo.size(); - Array4 zhi = makeArray4(p, zebox, ncomp); - p += zhi.size(); - Array4 xyzlo = makeArray4(p, bxg1, ncomp); - p += xyzlo.size(); - Array4 xyzhi = makeArray4(p, bxg1, ncomp); - - // Use PPM to generate Im and Ip */ - switch (godunov_scheme) { - case godunov::scheme::MINMOD: { - amrex::ParallelFor( - bxg1, ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - Godunov_minmod_fpu_x( - i, j, k, n, l_dt, dx, Imx(i, j, k, n), Ipx(i, j, k, n), q, - umac, pbc[n], dlo.x, dhi.x); - Godunov_minmod_fpu_y( - i, j, k, n, l_dt, dy, Imy(i, j, k, n), Ipy(i, j, k, n), q, - vmac, pbc[n], dlo.y, dhi.y); - Godunov_minmod_fpu_z( - i, j, k, n, l_dt, dz, Imz(i, j, k, n), Ipz(i, j, k, n), q, - wmac, pbc[n], dlo.z, dhi.z); - }); - break; - } - case godunov::scheme::UPWIND: { - amrex::ParallelFor( - bxg1, ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - Godunov_upwind_fpu( - i, j, k, n, Imx(i, j, k, n), Ipx(i, j, k, n), q); - Godunov_upwind_fpu( - i, j, k, n, Imy(i, j, k, n), Ipy(i, j, k, n), q); - Godunov_upwind_fpu( - i, j, k, n, Imz(i, j, k, n), Ipz(i, j, k, n), q); - }); - break; - } - default: { - amrex::Abort( - "This code path only used in multiphase simulations with MINMOD " - "and UPWIND"); - } - } - - amrex::ParallelFor( - xebox, ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - constexpr Real small_vel = 1.e-8; - - Real uad = umac(i, j, k); - Real fux = (std::abs(uad) < small_vel) ? 0. : 1.; - bool uval = uad >= 0.; - // divu = 0 - // Real cons1 = (iconserv[n]) ? -0.5*l_dt*q(i-1,j,k,n)*divu(i-1,j,k) - // : 0.; Real cons2 = (iconserv[n]) ? -0.5*l_dt*q(i ,j,k,n)*divu(i - // ,j,k) : 0.; - Real lo = Ipx(i - 1, j, k, n); // + cons1; - Real hi = Imx(i, j, k, n); // + cons2; - if (use_forces_in_trans && fq) { - lo += 0.5 * l_dt * fq(i - 1, j, k, n); - hi += 0.5 * l_dt * fq(i, j, k, n); - } - - auto bc = pbc[n]; - - Godunov_trans_xbc( - i, j, k, n, q, lo, hi, uad, bc.lo(0), bc.hi(0), dlo.x, dhi.x); - xlo(i, j, k, n) = lo; - xhi(i, j, k, n) = hi; - Real st = (uval) ? lo : hi; - Imx(i, j, k, n) = fux * st + (1. - fux) * 0.5 * (hi + lo); - }, - yebox, ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - constexpr Real small_vel = 1.e-8; - - Real vad = vmac(i, j, k); - Real fuy = (std::abs(vad) < small_vel) ? 0. : 1.; - bool vval = vad >= 0.; - // divu = 0 - // Real cons1 = (iconserv[n]) ? -0.5*l_dt*q(i,j-1,k,n)*divu(i,j-1,k) - // : 0.; Real cons2 = (iconserv[n]) ? -0.5*l_dt*q(i,j ,k,n)*divu(i,j - // ,k) : 0.; - Real lo = Ipy(i, j - 1, k, n); // + cons1; - Real hi = Imy(i, j, k, n); // + cons2; - if (use_forces_in_trans && fq) { - lo += 0.5 * l_dt * fq(i, j - 1, k, n); - hi += 0.5 * l_dt * fq(i, j, k, n); - } - - auto bc = pbc[n]; - - Godunov_trans_ybc( - i, j, k, n, q, lo, hi, vad, bc.lo(1), bc.hi(1), dlo.y, dhi.y); - - ylo(i, j, k, n) = lo; - yhi(i, j, k, n) = hi; - Real st = (vval) ? lo : hi; - Imy(i, j, k, n) = fuy * st + (1. - fuy) * 0.5 * (hi + lo); - }, - zebox, ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - constexpr Real small_vel = 1.e-8; - - Real wad = wmac(i, j, k); - Real fuz = (std::abs(wad) < small_vel) ? 0. : 1.; - bool wval = wad >= 0.; - auto bc = pbc[n]; - // divu = 0 - // Real cons1 = (iconserv[n]) ? -0.5*l_dt*q(i,j,k-1,n)*divu(i,j,k-1) - // : 0.; Real cons2 = (iconserv[n]) ? -0.5*l_dt*q(i,j,k - // ,n)*divu(i,j,k ) : 0.; - Real lo = Ipz(i, j, k - 1, n); // + cons1; - Real hi = Imz(i, j, k, n); // + cons2; - if (use_forces_in_trans && fq) { - lo += 0.5 * l_dt * fq(i, j, k - 1, n); - hi += 0.5 * l_dt * fq(i, j, k, n); - } - - Godunov_trans_zbc( - i, j, k, n, q, lo, hi, wad, bc.lo(2), bc.hi(2), dlo.z, dhi.z); - - zlo(i, j, k, n) = lo; - zhi(i, j, k, n) = hi; - Real st = (wval) ? lo : hi; - Imz(i, j, k, n) = fuz * st + (1. - fuz) * 0.5 * (hi + lo); - }); - - Array4 xedge = Imx; - Array4 yedge = Imy; - Array4 zedge = Imz; - - // We can reuse the space in Ipx, Ipy and Ipz. - - // - // x-direction - // - Box const& xbxtmp = amrex::grow(bx, 0, 1); - Array4 yzlo = - makeArray4(xyzlo.dataPtr(), amrex::surroundingNodes(xbxtmp, 1), ncomp); - Array4 zylo = - makeArray4(xyzhi.dataPtr(), amrex::surroundingNodes(xbxtmp, 2), ncomp); - amrex::ParallelFor( - Box(zylo), ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - const auto bc = pbc[n]; - Real l_zylo, l_zyhi; - Godunov_corner_couple_zy( - l_zylo, l_zyhi, i, j, k, n, l_dt, dy, iconserv[n] != 0, - zlo(i, j, k, n), zhi(i, j, k, n), q, vmac, yedge); - - Real wad = wmac(i, j, k); - Godunov_trans_zbc( - i, j, k, n, q, l_zylo, l_zyhi, wad, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); - - constexpr Real small_vel = 1.e-8; - - Real st = (wad >= 0.) ? l_zylo : l_zyhi; - Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; - zylo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_zyhi + l_zylo); - }, - Box(yzlo), ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - const auto bc = pbc[n]; - Real l_yzlo, l_yzhi; - Godunov_corner_couple_yz( - l_yzlo, l_yzhi, i, j, k, n, l_dt, dz, iconserv[n] != 0, - ylo(i, j, k, n), yhi(i, j, k, n), q, wmac, zedge); - - Real vad = vmac(i, j, k); - Godunov_trans_ybc( - i, j, k, n, q, l_yzlo, l_yzhi, vad, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); - - constexpr Real small_vel = 1.e-8; - - Real st = (vad >= 0.) ? l_yzlo : l_yzhi; - Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; - yzlo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_yzhi + l_yzlo); - }); - // - - amrex::ParallelFor( - xbx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - Real stl, sth; - constexpr Real small_vel = 1.e-8; - if (iconserv[n] != 0) { - stl = xlo(i, j, k, n) - - (0.5 * dtdy) * - (yzlo(i - 1, j + 1, k, n) * vmac(i - 1, j + 1, k) - - yzlo(i - 1, j, k, n) * vmac(i - 1, j, k)) - - (0.5 * dtdz) * - (zylo(i - 1, j, k + 1, n) * wmac(i - 1, j, k + 1) - - zylo(i - 1, j, k, n) * wmac(i - 1, j, k)) + - (0.5 * dtdy) * q(i - 1, j, k, n) * - (vmac(i - 1, j + 1, k) - vmac(i - 1, j, k)) + - (0.5 * dtdz) * q(i - 1, j, k, n) * - (wmac(i - 1, j, k + 1) - wmac(i - 1, j, k)); - - sth = xhi(i, j, k, n) - - (0.5 * dtdy) * (yzlo(i, j + 1, k, n) * vmac(i, j + 1, k) - - yzlo(i, j, k, n) * vmac(i, j, k)) - - (0.5 * dtdz) * (zylo(i, j, k + 1, n) * wmac(i, j, k + 1) - - zylo(i, j, k, n) * wmac(i, j, k)) + - (0.5 * dtdy) * q(i, j, k, n) * - (vmac(i, j + 1, k) - vmac(i, j, k)) + - (0.5 * dtdz) * q(i, j, k, n) * - (wmac(i, j, k + 1) - wmac(i, j, k)); - - } else { - stl = xlo(i, j, k, n) - - (0.25 * dtdy) * - (vmac(i - 1, j + 1, k) + vmac(i - 1, j, k)) * - (yzlo(i - 1, j + 1, k, n) - yzlo(i - 1, j, k, n)) - - (0.25 * dtdz) * - (wmac(i - 1, j, k + 1) + wmac(i - 1, j, k)) * - (zylo(i - 1, j, k + 1, n) - zylo(i - 1, j, k, n)); - - sth = xhi(i, j, k, n) - - (0.25 * dtdy) * (vmac(i, j + 1, k) + vmac(i, j, k)) * - (yzlo(i, j + 1, k, n) - yzlo(i, j, k, n)) - - (0.25 * dtdz) * (wmac(i, j, k + 1) + wmac(i, j, k)) * - (zylo(i, j, k + 1, n) - zylo(i, j, k, n)); - } - - stl += (!use_forces_in_trans && fq) - ? 0.5 * l_dt * fq(i - 1, j, k, n) - : 0.; - sth += - (!use_forces_in_trans && fq) ? 0.5 * l_dt * fq(i, j, k, n) : 0.; - - auto bc = pbc[n]; - Godunov_cc_xbc_lo(i, j, k, n, q, stl, sth, umac, bc.lo(0), dlo.x); - Godunov_cc_xbc_hi(i, j, k, n, q, stl, sth, umac, bc.hi(0), dhi.x); - - Real qx = (umac(i, j, k) >= 0.) ? stl : sth; - qx = (std::abs(umac(i, j, k)) < small_vel) ? 0.5 * (stl + sth) : qx; - - if (iconserv[n] == 1) { - fx(i, j, k, n) = umac(i, j, k) * qx; - } else { - fx(i, j, k, n) = qx; - } - }); - - // - // y-direction - // - Box const& ybxtmp = amrex::grow(bx, 1, 1); - Array4 xzlo = - makeArray4(xyzlo.dataPtr(), amrex::surroundingNodes(ybxtmp, 0), ncomp); - Array4 zxlo = - makeArray4(xyzhi.dataPtr(), amrex::surroundingNodes(ybxtmp, 2), ncomp); - amrex::ParallelFor( - Box(xzlo), ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - const auto bc = pbc[n]; - Real l_xzlo, l_xzhi; - Godunov_corner_couple_xz( - l_xzlo, l_xzhi, i, j, k, n, l_dt, dz, iconserv[n] != 0, - xlo(i, j, k, n), xhi(i, j, k, n), q, wmac, zedge); - - Real uad = umac(i, j, k); - Godunov_trans_xbc( - i, j, k, n, q, l_xzlo, l_xzhi, uad, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); - - constexpr Real small_vel = 1.e-8; - - Real st = (uad >= 0.) ? l_xzlo : l_xzhi; - Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; - xzlo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_xzhi + l_xzlo); - }, - Box(zxlo), ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - const auto bc = pbc[n]; - Real l_zxlo, l_zxhi; - Godunov_corner_couple_zx( - l_zxlo, l_zxhi, i, j, k, n, l_dt, dx, iconserv[n] != 0, - zlo(i, j, k, n), zhi(i, j, k, n), q, umac, xedge); - - Real wad = wmac(i, j, k); - Godunov_trans_zbc( - i, j, k, n, q, l_zxlo, l_zxhi, wad, bc.lo(2), bc.hi(2), dlo.z, - dhi.z); - - constexpr Real small_vel = 1.e-8; - - Real st = (wad >= 0.) ? l_zxlo : l_zxhi; - Real fu = (std::abs(wad) < small_vel) ? 0.0 : 1.0; - zxlo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_zxhi + l_zxlo); - }); - - amrex::ParallelFor( - ybx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - Real stl, sth; - constexpr Real small_vel = 1.e-8; - - if (iconserv[n] != 0) { - stl = ylo(i, j, k, n) - - (0.5 * dtdx) * - (xzlo(i + 1, j - 1, k, n) * umac(i + 1, j - 1, k) - - xzlo(i, j - 1, k, n) * umac(i, j - 1, k)) - - (0.5 * dtdz) * - (zxlo(i, j - 1, k + 1, n) * wmac(i, j - 1, k + 1) - - zxlo(i, j - 1, k, n) * wmac(i, j - 1, k)) + - (0.5 * dtdx) * q(i, j - 1, k, n) * - (umac(i + 1, j - 1, k) - umac(i, j - 1, k)) + - (0.5 * dtdz) * q(i, j - 1, k, n) * - (wmac(i, j - 1, k + 1) - wmac(i, j - 1, k)); - - sth = yhi(i, j, k, n) - - (0.5 * dtdx) * (xzlo(i + 1, j, k, n) * umac(i + 1, j, k) - - xzlo(i, j, k, n) * umac(i, j, k)) - - (0.5 * dtdz) * (zxlo(i, j, k + 1, n) * wmac(i, j, k + 1) - - zxlo(i, j, k, n) * wmac(i, j, k)) + - (0.5 * dtdx) * q(i, j, k, n) * - (umac(i + 1, j, k) - umac(i, j, k)) + - (0.5 * dtdz) * q(i, j, k, n) * - (wmac(i, j, k + 1) - wmac(i, j, k)); - } else { - stl = ylo(i, j, k, n) - - (0.25 * dtdx) * - (umac(i + 1, j - 1, k) + umac(i, j - 1, k)) * - (xzlo(i + 1, j - 1, k, n) - xzlo(i, j - 1, k, n)) - - (0.25 * dtdz) * - (wmac(i, j - 1, k + 1) + wmac(i, j - 1, k)) * - (zxlo(i, j - 1, k + 1, n) - zxlo(i, j - 1, k, n)); - - sth = yhi(i, j, k, n) - - (0.25 * dtdx) * (umac(i + 1, j, k) + umac(i, j, k)) * - (xzlo(i + 1, j, k, n) - xzlo(i, j, k, n)) - - (0.25 * dtdz) * (wmac(i, j, k + 1) + wmac(i, j, k)) * - (zxlo(i, j, k + 1, n) - zxlo(i, j, k, n)); - } - - stl += (!use_forces_in_trans && fq) - ? 0.5 * l_dt * fq(i, j - 1, k, n) - : 0.; - sth += - (!use_forces_in_trans && fq) ? 0.5 * l_dt * fq(i, j, k, n) : 0.; - - auto bc = pbc[n]; - Godunov_cc_ybc_lo(i, j, k, n, q, stl, sth, vmac, bc.lo(1), dlo.y); - Godunov_cc_ybc_hi(i, j, k, n, q, stl, sth, vmac, bc.hi(1), dhi.y); - - Real qy = (vmac(i, j, k) >= 0.) ? stl : sth; - qy = (std::abs(vmac(i, j, k)) < small_vel) ? 0.5 * (stl + sth) : qy; - - if (iconserv[n] == 1) { - fy(i, j, k, n) = vmac(i, j, k) * qy; - } else { - fy(i, j, k, n) = qy; - } - }); - - // - // z-direction - // - Box const& zbxtmp = amrex::grow(bx, 2, 1); - Array4 xylo = - makeArray4(xyzlo.dataPtr(), amrex::surroundingNodes(zbxtmp, 0), ncomp); - Array4 yxlo = - makeArray4(xyzhi.dataPtr(), amrex::surroundingNodes(zbxtmp, 1), ncomp); - amrex::ParallelFor( - Box(xylo), ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - const auto bc = pbc[n]; - Real l_xylo, l_xyhi; - Godunov_corner_couple_xy( - l_xylo, l_xyhi, i, j, k, n, l_dt, dy, iconserv[n] != 0, - xlo(i, j, k, n), xhi(i, j, k, n), q, vmac, yedge); - - Real uad = umac(i, j, k); - Godunov_trans_xbc( - i, j, k, n, q, l_xylo, l_xyhi, uad, bc.lo(0), bc.hi(0), dlo.x, - dhi.x); - - constexpr Real small_vel = 1.e-8; - - Real st = (uad >= 0.) ? l_xylo : l_xyhi; - Real fu = (std::abs(uad) < small_vel) ? 0.0 : 1.0; - xylo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_xyhi + l_xylo); - }, - Box(yxlo), ncomp, - [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - const auto bc = pbc[n]; - Real l_yxlo, l_yxhi; - Godunov_corner_couple_yx( - l_yxlo, l_yxhi, i, j, k, n, l_dt, dx, iconserv[n] != 0, - ylo(i, j, k, n), yhi(i, j, k, n), q, umac, xedge); - - Real vad = vmac(i, j, k); - Godunov_trans_ybc( - i, j, k, n, q, l_yxlo, l_yxhi, vad, bc.lo(1), bc.hi(1), dlo.y, - dhi.y); - - constexpr Real small_vel = 1.e-8; - - Real st = (vad >= 0.) ? l_yxlo : l_yxhi; - Real fu = (std::abs(vad) < small_vel) ? 0.0 : 1.0; - yxlo(i, j, k, n) = fu * st + (1.0 - fu) * 0.5 * (l_yxhi + l_yxlo); - }); - - amrex::ParallelFor( - zbx, ncomp, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) noexcept { - Real stl, sth; - constexpr Real small_vel = 1.e-8; - if (iconserv[n] != 0) { - stl = zlo(i, j, k, n) - - (0.5 * dtdx) * - (xylo(i + 1, j, k - 1, n) * umac(i + 1, j, k - 1) - - xylo(i, j, k - 1, n) * umac(i, j, k - 1)) - - (0.5 * dtdy) * - (yxlo(i, j + 1, k - 1, n) * vmac(i, j + 1, k - 1) - - yxlo(i, j, k - 1, n) * vmac(i, j, k - 1)) + - (0.5 * dtdx) * q(i, j, k - 1, n) * - (umac(i + 1, j, k - 1) - umac(i, j, k - 1)) + - (0.5 * dtdy) * q(i, j, k - 1, n) * - (vmac(i, j + 1, k - 1) - vmac(i, j, k - 1)); - - sth = zhi(i, j, k, n) - - (0.5 * dtdx) * (xylo(i + 1, j, k, n) * umac(i + 1, j, k) - - xylo(i, j, k, n) * umac(i, j, k)) - - (0.5 * dtdy) * (yxlo(i, j + 1, k, n) * vmac(i, j + 1, k) - - yxlo(i, j, k, n) * vmac(i, j, k)) + - (0.5 * dtdx) * q(i, j, k, n) * - (umac(i + 1, j, k) - umac(i, j, k)) + - (0.5 * dtdy) * q(i, j, k, n) * - (vmac(i, j + 1, k) - vmac(i, j, k)); - } else { - stl = zlo(i, j, k, n) - - (0.25 * dtdx) * - (umac(i + 1, j, k - 1) + umac(i, j, k - 1)) * - (xylo(i + 1, j, k - 1, n) - xylo(i, j, k - 1, n)) - - (0.25 * dtdy) * - (vmac(i, j + 1, k - 1) + vmac(i, j, k - 1)) * - (yxlo(i, j + 1, k - 1, n) - yxlo(i, j, k - 1, n)); - - sth = zhi(i, j, k, n) - - (0.25 * dtdx) * (umac(i + 1, j, k) + umac(i, j, k)) * - (xylo(i + 1, j, k, n) - xylo(i, j, k, n)) - - (0.25 * dtdy) * (vmac(i, j + 1, k) + vmac(i, j, k)) * - (yxlo(i, j + 1, k, n) - yxlo(i, j, k, n)); - } - - stl += (!use_forces_in_trans && fq) - ? 0.5 * l_dt * fq(i, j, k - 1, n) - : 0.; - sth += - (!use_forces_in_trans && fq) ? 0.5 * l_dt * fq(i, j, k, n) : 0.; - - auto bc = pbc[n]; - Godunov_cc_zbc_lo(i, j, k, n, q, stl, sth, wmac, bc.lo(2), dlo.z); - Godunov_cc_zbc_hi(i, j, k, n, q, stl, sth, wmac, bc.hi(2), dhi.z); - - Real qz = (wmac(i, j, k) >= 0.) ? stl : sth; - qz = (std::abs(wmac(i, j, k)) < small_vel) ? 0.5 * (stl + sth) : qz; - - if (iconserv[n] == 1) { - fz(i, j, k, n) = wmac(i, j, k) * qz; - } else { - fz(i, j, k, n) = qz; - } - }); -} - diff --git a/amr-wind/convection/incflo_godunov_ppm.H b/amr-wind/convection/incflo_godunov_ppm.H deleted file mode 100644 index f57f9ce779..0000000000 --- a/amr-wind/convection/incflo_godunov_ppm.H +++ /dev/null @@ -1,477 +0,0 @@ -#ifndef GODUNOV_PPM_H -#define GODUNOV_PPM_H - -#include -#include -#include - -/* This header file contains the inlined __host__ __device__ functions required - for the scalar advection routines for 3D Godunov. It also contains function - declarations for controlling host functions. */ - -namespace { - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_xbc( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - amrex::Real macvel, - const int bclo, - const int bchi, - const int domlo, - const int domhi) -{ - if (i <= domlo) { - HydroBC::SetEdgeBCsLo( - 0, i, j, k, n, s, lo, hi, macvel, bclo, domlo, true); - } else if (i > domhi) { - HydroBC::SetEdgeBCsHi( - 0, i, j, k, n, s, lo, hi, macvel, bchi, domhi, true); - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_ybc( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - amrex::Real macvel, - const int bclo, - const int bchi, - const int domlo, - const int domhi) -{ - if (j <= domlo) { - HydroBC::SetEdgeBCsLo( - 1, i, j, k, n, s, lo, hi, macvel, bclo, domlo, true); - } else if (j > domhi) { - HydroBC::SetEdgeBCsHi( - 1, i, j, k, n, s, lo, hi, macvel, bchi, domhi, true); - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_trans_zbc( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - amrex::Real macvel, - const int bclo, - const int bchi, - const int domlo, - const int domhi) -{ - if (k <= domlo) { - HydroBC::SetEdgeBCsLo( - 2, i, j, k, n, s, lo, hi, macvel, bclo, domlo, true); - } else if (k > domhi) { - HydroBC::SetEdgeBCsHi( - 2, i, j, k, n, s, lo, hi, macvel, bchi, domhi, true); - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_lo( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - const amrex::Array4& uedge, - const int bclo, - const int domlo) -{ - using namespace amrex; - - if (i == domlo) { - if ((bclo == BCType::ext_dir && uedge(i, j, k) >= 0.) || - (bclo == BCType::direction_dependent && uedge(i, j, k) >= 0.)) { - hi = s(domlo - 1, j, k, n); - lo = hi; - } else if ( - bclo == BCType::foextrap || bclo == BCType::hoextrap || - bclo == BCType::reflect_even || - (bclo == BCType::ext_dir && uedge(i, j, k) < 0) || - (bclo == BCType::direction_dependent && uedge(i, j, k) < 0)) { - lo = hi; - - } else if (bclo == BCType::reflect_odd) { - hi = 0.; - lo = hi; - } - } else { - return; - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_xbc_hi( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - const amrex::Array4& uedge, - const int bchi, - const int domhi) -{ - using namespace amrex; - - if (i == domhi + 1) { - if ((bchi == BCType::ext_dir && uedge(i, j, k) <= 0.) || - (bchi == BCType::direction_dependent && uedge(i, j, k) <= 0.)) { - lo = s(domhi + 1, j, k, n); - hi = lo; - } else if ( - bchi == BCType::foextrap || bchi == BCType::hoextrap || - bchi == BCType::reflect_even || - (bchi == BCType::ext_dir && uedge(i, j, k) > 0) || - (bchi == BCType::direction_dependent && uedge(i, j, k) > 0)) { - hi = lo; - - } else if (bchi == BCType::reflect_odd) { - lo = 0.; - hi = lo; - } - } else { - return; - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_ybc_lo( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - const amrex::Array4& vedge, - const int bclo, - const int domlo) -{ - using namespace amrex; - - if (j == domlo) { - if ((bclo == BCType::ext_dir && vedge(i, j, k) >= 0.) || - (bclo == BCType::direction_dependent && vedge(i, j, k) >= 0.)) { - hi = s(i, domlo - 1, k, n); - lo = hi; - } else if ( - bclo == BCType::foextrap || bclo == BCType::hoextrap || - bclo == BCType::reflect_even || - (bclo == BCType::ext_dir && vedge(i, j, k) < 0) || - (bclo == BCType::direction_dependent && vedge(i, j, k) < 0)) { - lo = hi; - - } else if (bclo == BCType::reflect_odd) { - hi = 0.; - lo = hi; - } - } else { - return; - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_ybc_hi( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - const amrex::Array4& vedge, - const int bchi, - const int domhi) -{ - using namespace amrex; - - if (j == domhi + 1) { - if ((bchi == BCType::ext_dir && vedge(i, j, k) <= 0.) || - (bchi == BCType::direction_dependent && vedge(i, j, k) <= 0.)) { - lo = s(i, domhi + 1, k, n); - hi = lo; - } else if ( - bchi == BCType::foextrap || bchi == BCType::hoextrap || - bchi == BCType::reflect_even || - (bchi == BCType::ext_dir && vedge(i, j, k) > 0) || - (bchi == BCType::direction_dependent && vedge(i, j, k) > 0)) { - hi = lo; - - } else if (bchi == BCType::reflect_odd) { - lo = 0.; - hi = lo; - } - } else { - return; - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_zbc_lo( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - const amrex::Array4& wedge, - const int bclo, - const int domlo) -{ - using namespace amrex; - - if (k == domlo) { - if ((bclo == BCType::ext_dir && wedge(i, j, k) >= 0.) || - (bclo == BCType::direction_dependent && wedge(i, j, k) >= 0.)) { - hi = s(i, j, domlo - 1, n); - lo = hi; - } - - else if ( - bclo == BCType::foextrap || bclo == BCType::hoextrap || - bclo == BCType::reflect_even || - (bclo == BCType::ext_dir && wedge(i, j, k) < 0) || - (bclo == BCType::direction_dependent && wedge(i, j, k) < 0)) { - lo = hi; - - } else if (bclo == BCType::reflect_odd) { - hi = 0.; - lo = hi; - } - } else { - return; - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_cc_zbc_hi( - const int i, - const int j, - const int k, - const int n, - const amrex::Array4& s, - amrex::Real& lo, - amrex::Real& hi, - const amrex::Array4& wedge, - const int bchi, - const int domhi) -{ - using namespace amrex; - - if (k == domhi + 1) { - if ((bchi == BCType::ext_dir && wedge(i, j, k) <= 0.) || - (bchi == BCType::direction_dependent && wedge(i, j, k) <= 0.)) { - lo = s(i, j, domhi + 1, n); - hi = lo; - } else if ( - bchi == BCType::foextrap || bchi == BCType::hoextrap || - bchi == BCType::reflect_even || - (bchi == BCType::ext_dir && wedge(i, j, k) > 0) || - (bchi == BCType::direction_dependent && wedge(i, j, k) > 0)) { - hi = lo; - - } else if (bchi == BCType::reflect_odd) { - lo = 0.; - hi = lo; - } - } else { - return; - } -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_yx( - amrex::Real& lo1, - amrex::Real& hi1, - int i, - int j, - int k, - int n, - amrex::Real dt, - amrex::Real dx, - bool /*iconserv*/, - amrex::Real lo, - amrex::Real hi, - amrex::Array4 const& s, - amrex::Array4 const& mac, - amrex::Array4 const& state) -{ - lo1 = lo - - dt / (amrex::Real(3.0) * dx) * - (state(i + 1, j - 1, k, n) * mac(i + 1, j - 1, k) - - state(i, j - 1, k, n) * mac(i, j - 1, k)) + - dt / (amrex::Real(3.0) * dx) * s(i, j - 1, k, n) * - (mac(i + 1, j - 1, k) - mac(i, j - 1, k)); - hi1 = hi - - dt / (amrex::Real(3.0) * dx) * - (state(i + 1, j, k, n) * mac(i + 1, j, k) - - state(i, j, k, n) * mac(i, j, k)) + - dt / (amrex::Real(3.0) * dx) * s(i, j, k, n) * - (mac(i + 1, j, k) - mac(i, j, k)); -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_zx( - amrex::Real& lo1, - amrex::Real& hi1, - int i, - int j, - int k, - int n, - amrex::Real dt, - amrex::Real dx, - bool /*iconserv*/, - amrex::Real lo, - amrex::Real hi, - amrex::Array4 const& s, - amrex::Array4 const& mac, - amrex::Array4 const& state) -{ - lo1 = lo - - dt / (amrex::Real(3.0) * dx) * - (state(i + 1, j, k - 1, n) * mac(i + 1, j, k - 1) - - state(i, j, k - 1, n) * mac(i, j, k - 1)) + - dt / (amrex::Real(3.0) * dx) * s(i, j, k - 1, n) * - (mac(i + 1, j, k - 1) - mac(i, j, k - 1)); - hi1 = hi - - dt / (amrex::Real(3.0) * dx) * - (state(i + 1, j, k, n) * mac(i + 1, j, k) - - state(i, j, k, n) * mac(i, j, k)) + - dt / (amrex::Real(3.0) * dx) * s(i, j, k, n) * - (mac(i + 1, j, k) - mac(i, j, k)); -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_xy( - amrex::Real& lo1, - amrex::Real& hi1, - int i, - int j, - int k, - int n, - amrex::Real dt, - amrex::Real dy, - bool /*iconserv*/, - amrex::Real lo, - amrex::Real hi, - amrex::Array4 const& s, - amrex::Array4 const& mac, - amrex::Array4 const& state) -{ - lo1 = lo - - dt / (amrex::Real(3.0) * dy) * - (state(i - 1, j + 1, k, n) * mac(i - 1, j + 1, k) - - state(i - 1, j, k, n) * mac(i - 1, j, k)) + - dt / (amrex::Real(3.0) * dy) * s(i - 1, j, k, n) * - (mac(i - 1, j + 1, k) - mac(i - 1, j, k)); - hi1 = hi - - dt / (amrex::Real(3.0) * dy) * - (state(i, j + 1, k, n) * mac(i, j + 1, k) - - state(i, j, k, n) * mac(i, j, k)) + - dt / (amrex::Real(3.0) * dy) * s(i, j, k, n) * - (mac(i, j + 1, k) - mac(i, j, k)); -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_zy( - amrex::Real& lo1, - amrex::Real& hi1, - int i, - int j, - int k, - int n, - amrex::Real dt, - amrex::Real dy, - bool /*iconserv*/, - amrex::Real lo, - amrex::Real hi, - amrex::Array4 const& s, - amrex::Array4 const& mac, - amrex::Array4 const& state) -{ - lo1 = lo - - dt / (amrex::Real(3.0) * dy) * - (state(i, j + 1, k - 1, n) * mac(i, j + 1, k - 1) - - state(i, j, k - 1, n) * mac(i, j, k - 1)) + - dt / (amrex::Real(3.0) * dy) * s(i, j, k - 1, n) * - (mac(i, j + 1, k - 1) - mac(i, j, k - 1)); - hi1 = hi - - dt / (amrex::Real(3.0) * dy) * - (state(i, j + 1, k, n) * mac(i, j + 1, k) - - state(i, j, k, n) * mac(i, j, k)) + - dt / (amrex::Real(3.0) * dy) * s(i, j, k, n) * - (mac(i, j + 1, k) - mac(i, j, k)); -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_xz( - amrex::Real& lo1, - amrex::Real& hi1, - int i, - int j, - int k, - int n, - amrex::Real dt, - amrex::Real dz, - bool /*iconserv*/, - amrex::Real lo, - amrex::Real hi, - amrex::Array4 const& s, - amrex::Array4 const& mac, - amrex::Array4 const& state) -{ - lo1 = lo - - dt / (amrex::Real(3.0) * dz) * - (state(i - 1, j, k + 1, n) * mac(i - 1, j, k + 1) - - state(i - 1, j, k, n) * mac(i - 1, j, k)) + - dt / (amrex::Real(3.0) * dz) * s(i - 1, j, k, n) * - (mac(i - 1, j, k + 1) - mac(i - 1, j, k)); - hi1 = hi - - dt / (amrex::Real(3.0) * dz) * - (state(i, j, k + 1, n) * mac(i, j, k + 1) - - state(i, j, k, n) * mac(i, j, k)) + - dt / (amrex::Real(3.0) * dz) * s(i, j, k, n) * - (mac(i, j, k + 1) - mac(i, j, k)); -} - -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void Godunov_corner_couple_yz( - amrex::Real& lo1, - amrex::Real& hi1, - int i, - int j, - int k, - int n, - amrex::Real dt, - amrex::Real dz, - bool /*iconserv*/, - amrex::Real lo, - amrex::Real hi, - amrex::Array4 const& s, - amrex::Array4 const& mac, - amrex::Array4 const& state) -{ - lo1 = lo - - dt / (amrex::Real(3.0) * dz) * - (state(i, j - 1, k + 1, n) * mac(i, j - 1, k + 1) - - state(i, j - 1, k, n) * mac(i, j - 1, k)) + - dt / (amrex::Real(3.0) * dz) * s(i, j - 1, k, n) * - (mac(i, j - 1, k + 1) - mac(i, j - 1, k)); - hi1 = hi - - dt / (amrex::Real(3.0) * dz) * - (state(i, j, k + 1, n) * mac(i, j, k + 1) - - state(i, j, k, n) * mac(i, j, k)) + - dt / (amrex::Real(3.0) * dz) * s(i, j, k, n) * - (mac(i, j, k + 1) - mac(i, j, k)); -} - -} // namespace -#endif - From adc70227c63f943c6e9b2584eb6c99b69ccf4804 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 20 Aug 2024 12:02:40 -0700 Subject: [PATCH 80/83] Use Async Arena instead of Elixir Co-authored-by: Marc T. Henry de Frahan --- amr-wind/core/FieldFillPatchOps.H | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/amr-wind/core/FieldFillPatchOps.H b/amr-wind/core/FieldFillPatchOps.H index 0796ce0332..6b0cd8aa2f 100644 --- a/amr-wind/core/FieldFillPatchOps.H +++ b/amr-wind/core/FieldFillPatchOps.H @@ -443,8 +443,7 @@ public: } const auto& marr = mfab[mfi].array(); - amrex::FArrayBox tmp_fab(bx, mfab.nComp()); - amrex::Elixir tmp_eli = tmp_fab.elixir(); + amrex::FArrayBox tmp_fab(bx, mfab.nComp(), amrex::The_Async_Arena()); amrex::Array4 const& tmp_marr = tmp_fab.array(); From cf6ff0341d644a5508fd1f7f26ae8b721f2f58fa Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Tue, 20 Aug 2024 18:00:56 -0700 Subject: [PATCH 81/83] add documentation about inout BC --- .../user/inputs_Boundary_conditions.rst | 57 ++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/docs/sphinx/user/inputs_Boundary_conditions.rst b/docs/sphinx/user/inputs_Boundary_conditions.rst index eaae52bb51..8ebf26e6df 100644 --- a/docs/sphinx/user/inputs_Boundary_conditions.rst +++ b/docs/sphinx/user/inputs_Boundary_conditions.rst @@ -1,21 +1,21 @@ Section: Boundary conditions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - + This section controls the boundary conditions. Only non-periodic BC's need to be defined here. .. input_param:: xlo.type (or ylo.type, zlo.type, xhi.type, yhi.type, zhi.type) **type:** String, mandatory if non-periodic - - Boundary condition type on the lo (or hi) side of the domain. - Current options are: periodic, pressure_inflow, pressure_outflow, mass_inflow, - no_slip_wall, slip_wall, symmetric_wall and wall_model. + + Boundary condition type on the lo (or hi) side of the domain. + Current options are: periodic, pressure_inflow, pressure_outflow, mass_inflow, + mass_inflow_outflow, no_slip_wall, slip_wall, symmetric_wall and wall_model. .. input_param:: xlo.temperature (or ylo.temperature, etc) **type:** Real, optional, default = 0.0 - - Specifies a temperature gradient at the wall, only activated for slip_wall and wall_model BC types. + + Specifies a temperature gradient at the wall, only activated for slip_wall and wall_model BC types. Mass inflow boundary conditions ``````````````````````````````` @@ -42,3 +42,46 @@ If the user wants to define their own boundary conditions, this is done by editi CustomVelocity.foo = 1.0 They do not both need to be defined at the same time. It is the user's responsibility to ensure that the source files are appropriately edited for their use case. Examples of how these files can be edited are found through comparison of the other mass_inflow functions in the `udfs` folder. + +Mass inflow-outflow boundary conditions +``````````````````````````````````````` + +The mass_inflow_outflow boundary condition is designed to handle both inflow and outflow at the same boundary. +For the advection schemes, it implements a Neumann type behavior at the outflow cells and a Dirichlet behavior at the inflow cells. +It uses Neumann conditions for the MAC and nodal projections and +enforces solvability before the projections +by correcting the outflow to match with the inflow within the specified mass_inflow_outflow boundaries. +It uses a Dirichlet condition for the diffusion solver. + +Both the approaches mentioned above for the mass inflow condition, +constant values and UDFs, can be used to specify the boundary values. +The outflow values will be automatically replaced by a value from the interior cell +to enforce the Neumann type behavior. +See the ``freestream_godunov_inout`` test for an example that uses the TwoLayer UDF. +This test involves two "layers" of the flow in the z-direction, with opposite directions. +The input file options are copied here:: + + geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) + + # Boundary conditions + TwoLayer.bottom_vel = -1.0 0.0 0.0 + TwoLayer.top_vel = 1.0 0.0 0.0 + TwoLayer.init_perturb = 0.9 + TwoLayer.z_partition = 0.5 + + xlo.type = "mass_inflow_outflow" + xlo.density = 1.0 + xlo.velocity.inflow_outflow_type = TwoLayer + + xhi.type = "mass_inflow_outflow" + xhi.density = 1.0 + xhi.velocity.inflow_outflow_type = TwoLayer + + zlo.type = "slip_wall" + zhi.type = "slip_wall" + + +The most applicable use case for this boundary condition is with the +:ref:`amrwind-abl-bndry-io` for flows that change directions +across the vertical coorinate or with time. +The work to integrate this condition with the ABL class is under progress. From d12f4e2f71e91d722d086c4a29ebcbd2e3a9a48b Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 21 Aug 2024 09:19:52 -0700 Subject: [PATCH 82/83] clang-format change --- amr-wind/core/FieldFillPatchOps.H | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/amr-wind/core/FieldFillPatchOps.H b/amr-wind/core/FieldFillPatchOps.H index 6b0cd8aa2f..90561875fa 100644 --- a/amr-wind/core/FieldFillPatchOps.H +++ b/amr-wind/core/FieldFillPatchOps.H @@ -443,7 +443,8 @@ public: } const auto& marr = mfab[mfi].array(); - amrex::FArrayBox tmp_fab(bx, mfab.nComp(), amrex::The_Async_Arena()); + amrex::FArrayBox tmp_fab( + bx, mfab.nComp(), amrex::The_Async_Arena()); amrex::Array4 const& tmp_marr = tmp_fab.array(); From 285db734d748ed6d9a7b9d12fb6f5b84d021bdf6 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Wed, 21 Aug 2024 09:28:35 -0700 Subject: [PATCH 83/83] correct spelling and minor change in docs --- docs/sphinx/user/inputs_Boundary_conditions.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sphinx/user/inputs_Boundary_conditions.rst b/docs/sphinx/user/inputs_Boundary_conditions.rst index 8ebf26e6df..326714c2e5 100644 --- a/docs/sphinx/user/inputs_Boundary_conditions.rst +++ b/docs/sphinx/user/inputs_Boundary_conditions.rst @@ -58,10 +58,10 @@ constant values and UDFs, can be used to specify the boundary values. The outflow values will be automatically replaced by a value from the interior cell to enforce the Neumann type behavior. See the ``freestream_godunov_inout`` test for an example that uses the TwoLayer UDF. -This test involves two "layers" of the flow in the z-direction, with opposite directions. +This test involves two z-layers of the flow along opposite x-directions. The input file options are copied here:: - geometry.is_periodic = 0 1 0 # Periodicity x y z (0/1) + geometry.is_periodic = 0 1 0 # Periodic in y # Boundary conditions TwoLayer.bottom_vel = -1.0 0.0 0.0 @@ -77,11 +77,11 @@ The input file options are copied here:: xhi.density = 1.0 xhi.velocity.inflow_outflow_type = TwoLayer - zlo.type = "slip_wall" - zhi.type = "slip_wall" + zlo.type = "slip_wall" + zhi.type = "slip_wall" The most applicable use case for this boundary condition is with the :ref:`amrwind-abl-bndry-io` for flows that change directions -across the vertical coorinate or with time. +across the vertical coordinate or with time. The work to integrate this condition with the ABL class is under progress.