From de9b8047c021b786965d7d05aa97bdd06997806b Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 22 Nov 2024 08:07:05 -0700 Subject: [PATCH 1/4] Fixes for Intel "Erroneous arithmetic operation" with compiler optimization (#1976) * Fix for arithmetic error with intel when using compiler opt This allows a canonical dry abl test case to run with -O1 Tested with Intel(R) oneAPI DPC++/C++ Compiler 2023.2.0 (2023.2.0.20230622) * Fix for erroneous arithmetic operation with intel -O2 --- Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp | 2 +- Source/SourceTerms/ERF_ApplySpongeZoneBCs.cpp | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp b/Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp index 9e23f43d4..c07c7ab3e 100644 --- a/Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp +++ b/Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp @@ -189,7 +189,7 @@ void ComputeTurbulentViscosityLES (const MultiFab& Tau11, const MultiFab& Tau22, - cell_data(i,j,k-1,RhoTheta_comp)/cell_data(i,j,k-1,Rho_comp) )*dzInv; } Real E = amrex::max(cell_data(i,j,k,RhoKE_comp)/cell_data(i,j,k,Rho_comp),Real(0.0)); - Real stratification = l_abs_g * dtheta_dz * l_inv_theta0; // stratification + volatile Real stratification = l_abs_g * dtheta_dz * l_inv_theta0; Real length; if (stratification <= eps) { length = DeltaMsf; diff --git a/Source/SourceTerms/ERF_ApplySpongeZoneBCs.cpp b/Source/SourceTerms/ERF_ApplySpongeZoneBCs.cpp index bbfa5de02..8b9d927a1 100644 --- a/Source/SourceTerms/ERF_ApplySpongeZoneBCs.cpp +++ b/Source/SourceTerms/ERF_ApplySpongeZoneBCs.cpp @@ -27,6 +27,13 @@ ApplySpongeZoneBCsForCC ( const int use_yhi_sponge_damping = spongeChoice.use_yhi_sponge_damping; const int use_zlo_sponge_damping = spongeChoice.use_zlo_sponge_damping; const int use_zhi_sponge_damping = spongeChoice.use_zhi_sponge_damping; + if (!use_xlo_sponge_damping && + !use_xhi_sponge_damping && + !use_ylo_sponge_damping && + !use_yhi_sponge_damping && + !use_zlo_sponge_damping && + !use_zhi_sponge_damping) + return; const Real xlo_sponge_end = spongeChoice.xlo_sponge_end; const Real xhi_sponge_start = spongeChoice.xhi_sponge_start; @@ -136,6 +143,13 @@ ApplySpongeZoneBCsForMom ( const int use_yhi_sponge_damping = spongeChoice.use_yhi_sponge_damping; const int use_zlo_sponge_damping = spongeChoice.use_zlo_sponge_damping; const int use_zhi_sponge_damping = spongeChoice.use_zhi_sponge_damping; + if (!use_xlo_sponge_damping && + !use_xhi_sponge_damping && + !use_ylo_sponge_damping && + !use_yhi_sponge_damping && + !use_zlo_sponge_damping && + !use_zhi_sponge_damping) + return; const Real xlo_sponge_end = spongeChoice.xlo_sponge_end; const Real xhi_sponge_start = spongeChoice.xhi_sponge_start; From 567baa9cd8ab1cab2c131f7217e8dab01f0a5334 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Fri, 22 Nov 2024 09:55:48 -0700 Subject: [PATCH 2/4] Make intel opt fix SIMD friendly (#1977) --- Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp b/Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp index c07c7ab3e..a18337a49 100644 --- a/Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp +++ b/Source/Diffusion/ERF_ComputeTurbulentViscosity.cpp @@ -189,12 +189,12 @@ void ComputeTurbulentViscosityLES (const MultiFab& Tau11, const MultiFab& Tau22, - cell_data(i,j,k-1,RhoTheta_comp)/cell_data(i,j,k-1,Rho_comp) )*dzInv; } Real E = amrex::max(cell_data(i,j,k,RhoKE_comp)/cell_data(i,j,k,Rho_comp),Real(0.0)); - volatile Real stratification = l_abs_g * dtheta_dz * l_inv_theta0; + Real stratification = l_abs_g * dtheta_dz * l_inv_theta0; Real length; if (stratification <= eps) { length = DeltaMsf; } else { - length = 0.76 * std::sqrt(E / stratification); + length = 0.76 * std::sqrt(E / amrex::max(stratification,eps)); // mixing length should be _reduced_ for stable stratification length = amrex::min(length, DeltaMsf); // following WRF, make sure the mixing length isn't too small From e12bec0400d3ca52dfb13c0d326778fce58d4325 Mon Sep 17 00:00:00 2001 From: Ann Almgren Date: Fri, 22 Nov 2024 09:37:46 -0800 Subject: [PATCH 3/4] fix Make.ERF to find netcdf (#1978) --- Exec/Make.ERF | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Exec/Make.ERF b/Exec/Make.ERF index b03ef317a..779e68b93 100644 --- a/Exec/Make.ERF +++ b/Exec/Make.ERF @@ -251,8 +251,14 @@ endif #turn on NetCDF macro define ifeq ($(USE_NETCDF), TRUE) DEFINES += -DERF_USE_NETCDF - includes += $(shell pkg-config --cflags netcdf) - LIBRARIES += $(shell pkg-config --libs netcdf) + has_netcdf_mpi := $(shell pkg-config --cflags netcdf-mpi > /dev/null 2>&1; echo $$?) + ifeq ($(has_netcdf_mpi),0) + includes += $(shell pkg-config --cflags netcdf-mpi) + LIBRARIES += $(shell pkg-config --libs netcdf-mpi) + else + includes += $(shell pkg-config --cflags netcdf) + LIBRARIES += $(shell pkg-config --libs netcdf) + endif endif ifeq ($(USE_HDF5), TRUE) From 17bb541ebfecf6b6ed02a6fa40edbe0b6516a207 Mon Sep 17 00:00:00 2001 From: Ann Almgren Date: Sat, 23 Nov 2024 08:03:06 -0800 Subject: [PATCH 4/4] impose Dirichlet bcs everwhere we can (#1979) --- .../ERF_BoundaryConditions_cons.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Source/BoundaryConditions/ERF_BoundaryConditions_cons.cpp b/Source/BoundaryConditions/ERF_BoundaryConditions_cons.cpp index 5932ee22b..698aef412 100644 --- a/Source/BoundaryConditions/ERF_BoundaryConditions_cons.cpp +++ b/Source/BoundaryConditions/ERF_BoundaryConditions_cons.cpp @@ -70,7 +70,12 @@ void ERFPhysBCFunct_cons::impose_lateral_cons_bcs (const Array4& dest_arr, { Box bx_xlo(bx); bx_xlo.setBig (0,dom_lo.x-1); Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1); - + // + // If we are setting Dirichlet values, set them in all ghost cells on an inflow face + // "bx" is already grown in the x- and y-directions so here we just grow it in z + // + bx_xlo.grow(2,ng[2]); + bx_xhi.grow(2,ng[2]); ParallelFor( bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { @@ -109,6 +114,12 @@ void ERFPhysBCFunct_cons::impose_lateral_cons_bcs (const Array4& dest_arr, { Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-1); Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1); + // + // If we are setting Dirichlet values, set them in all ghost cells on an inflow face + // "bx" is already grown in the x- and y-directions so here we just grow it in z + // + bx_ylo.grow(2,ng[2]); + bx_yhi.grow(2,ng[2]); ParallelFor( bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)