WIP merging and fixing merge errors from recent commits from developm…

…ent.
wiersema1 · Oct 12, 2023 · 2640976 · 2640976
2 parents c87d821 + 0fd0e07
commit 2640976
Show file tree

Hide file tree

Showing 289 changed files with 11,561 additions and 11,085 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,7 @@
 
 # Log files
 datlog
+log.*
 
 # Ignore everything in Build except the cmake script
 /Build/*
@@ -21,6 +22,9 @@ datlog
 !/Build/cmake_with_poisson.sh
 !/Build/cmake_with_warm_no_precip.sh
 
+# Temporary files
+*/Testing/Temporary
+.*.swp
 
 /Exec/WORK/*
 ic.txt
@@ -35,3 +39,4 @@ cmake_build*
 cmake*ser*
 cmake*par*
 .idea
+*.ipynb
diff --git a/CHANGES b/CHANGES
@@ -0,0 +1,5 @@
+# 23.10
+
+  -- First release since JOSS paper. Too many key ERF PRs to summarize.
+
+  -- AMReX submodule set to 23.10 release hash (388738d)
diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake
@@ -37,6 +37,9 @@ function(build_erf_lib erf_lib_name)
   endif()
 
   if(ERF_ENABLE_PARTICLES)
+    target_sources(${erf_lib_name} PRIVATE
+                   ${SRC_DIR}/Particles/TerrainFittedPC.cpp)
+    target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Particles)
     target_compile_definitions(${erf_lib_name} PUBLIC ERF_USE_PARTICLES)
   endif()
 
@@ -125,9 +128,11 @@ function(build_erf_lib erf_lib_name)
        ${SRC_DIR}/Diffusion/NumericalDiffusion.cpp
        ${SRC_DIR}/Diffusion/PBLModels.cpp
        ${SRC_DIR}/Initialization/ERF_init_custom.cpp
+       ${SRC_DIR}/Initialization/ERF_init_from_hse.cpp
        ${SRC_DIR}/Initialization/ERF_init_from_input_sounding.cpp
        ${SRC_DIR}/Initialization/ERF_init_from_wrfinput.cpp
        ${SRC_DIR}/Initialization/ERF_init_from_metgrid.cpp
+       ${SRC_DIR}/Initialization/ERF_init_uniform.cpp
        ${SRC_DIR}/Initialization/ERF_init1d.cpp
        ${SRC_DIR}/IO/Checkpoint.cpp
        ${SRC_DIR}/IO/ERF_ReadBndryPlanes.cpp
@@ -191,6 +196,7 @@ function(build_erf_lib erf_lib_name)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR})
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Advection)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/BoundaryConditions)
+  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/DataStructs)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Diffusion)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Initialization)
   target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/IO)

diff --git a/Docs/source/conf.py b/Docs/source/conf.py
diff --git a/Docs/source/index.rst b/Docs/source/index.rst
diff --git a/Docs/sphinx_doc/BoundaryConditions.rst b/Docs/sphinx_doc/BoundaryConditions.rst
@@ -311,6 +311,14 @@ In ERF, when the MOST boundary condition is applied, velocity and temperature in
 
 MOST Inputs
 ~~~~~~~~~~~~~~~~~~~
+To evaluate the fluxes with MOST, the surface rougness parameter :math:`z_{0}` must be specified. This quantity may be considered a constant or may be parameterized through the friction velocity :math:`u_{\star}`. ERF supports three methods for parameterizing the surface roughness: ``constant``, ``charnock``, and ``modified_charnock``. The latter two methods parameterize :math:`z_{0} = f(u_{\star})` and are described in `Jimenez & Dudhia, American Meteorological Society, 2018 <https://doi.org/10.1175/JAMC-D-17-0137.1>`_. The rougness calculation method may be specified with
+
+::
+
+   erf.most.roughness_type    = STRING    #Z_0 type (constant, charnock, modified_charnock)
+
+If the ``charnock`` method is employed, the :math:`a` constant may be specified with ``erf.most.charnock_constant`` (defaults to 0.0185). If the ``modified_charnock`` method is employed, the depth :math:`d` may be specified with ``erf.most.modified_charnock_depth`` (defaults to 30 m).
+
 When computing an average :math:`\overline{\phi}` for the MOST boundary, where :math:`\phi` denotes a generic variable, ERF supports a variety of approaches. Specifically, ``planar averages`` and ``local region averages`` may be computed with or without ``time averaging``. With each averaging methodology, the query point :math:`z` may be determined from the following procedures: specified vertical distance :math:`z_{ref}` from the bottom surface, specified :math:`k_{index}`, or (when employing terrain-fit coordinates) specified normal vector length :math:`z_{ref}`. The available inputs to the MOST boundary and their associated data types are
 
 ::

diff --git a/Docs/sphinx_doc/E3D_mol_ppm.png b/Docs/sphinx_doc/E3D_mol_ppm.png
diff --git a/Docs/sphinx_doc/GettingStarted.rst b/Docs/sphinx_doc/GettingStarted.rst
@@ -24,6 +24,9 @@ type:
 
              git clone --recursive https://github.com/erf-model/ERF.git
 
+.. include:: submodule.rst
+
+
 .. include:: building.rst
 
 

diff --git a/Docs/sphinx_doc/KE_mol_ppm.png b/Docs/sphinx_doc/KE_mol_ppm.png
diff --git a/Docs/sphinx_doc/Particles.rst b/Docs/sphinx_doc/Particles.rst
@@ -0,0 +1,52 @@
+
+ .. role:: cpp(code)
+    :language: c++
+
+ .. _Particles:
+
+Particles
+=========
+
+ERF has the option to include Lagrangian particles in addition to the mesh-based solution.  Currently the
+particle functionality is very simple -- the particles are initialized one per mesh cell
+in a particular plane, and are advected by the velocity field.
+
+However, the AMReX particle data structure is very general and particles may take on a number of
+different roles in future.
+
+To enable the use of particles, one must set
+
+::
+
+   USE_PARTICLES = TRUE
+
+in the GNUmakefile if using gmake, or add
+
+::
+
+   -DERF_ENABLE_PARTICLES:BOOL=ON \
+
+to the cmake command if using cmake.  (See, e.g., ``Build/cmake_with_particles.sh``)
+
+One must also set
+
+::
+
+   erf.use_tracer_particles = true
+
+in the inputs file or on the command line at runtime.
+
+Currently, by default, the particles are initialized at cell centers, one per cell when the cell index is
+(3,j,k), with zero initial velocity.  They are advanced in time every time step.
+
+Caveat: the particle information is currently output when using the AMReX-native plotfile format, but not
+when using netcdf.  Writing particles into the netcdf files is a WIP.
+
+To see an example of using the particle functionality, build the executable using gmake in Exec/DevTests/ParticlesOverWoA.
+
+To visualize the number of particles per cell as a mesh-based variable, add ``particle_count`` to the line in the inputs file
+
+::
+
+   erf.plot_vars_1 =
+
diff --git a/Docs/sphinx_doc/Visualization.rst b/Docs/sphinx_doc/Visualization.rst
@@ -77,8 +77,8 @@ first `heat equation`_ tutorial code.
 
 .. _`heat equation`: https://github.com/AMReX-Codes/amrex-tutorials/tree/main/GuidedTutorials/HeatEquation
 
-Next, download and install VisIt from
-https://wci.llnl.gov/simulation/computer-codes/visit.  To open a single
+Next, download VisIt from
+https://visit-dav.github.io/visit-website/ and install.  To open a single
 plotfile, run VisIt, then select "File" :math:`\rightarrow` "Open file ...",
 then select the Header file associated the the plotfile of interest (e.g.,
 plt00000/Header).  Assuming you ran the simulation in 2D, here are instructions
@@ -309,5 +309,3 @@ The output of this is :numref:`fig:yt_Nyx_vol_rend`.
    Volume rendering of :math:`128^3` Nyx simulation using yt. This corresponds
    to the same plot file used to generate the slice plot in
    :numref:`fig:yt_Nyx_slice_plot`.
-
-
diff --git a/Docs/sphinx_doc/dissipation_mol_ppm.png b/Docs/sphinx_doc/dissipation_mol_ppm.png
diff --git a/Docs/sphinx_doc/index.rst b/Docs/sphinx_doc/index.rst
@@ -54,6 +54,7 @@ In addition to this documentation, there is API documentation for ERF generated
    theory/DNSvsLES.rst
    theory/PBLschemes.rst
    theory/Forcings.rst
+   Particles.rst
    theory/UnitsAndConstants.rst
 
 .. toctree::

diff --git a/Docs/sphinx_doc/submodule.rst b/Docs/sphinx_doc/submodule.rst
@@ -0,0 +1,22 @@
+.. _Submodule:
+
+Git Submodules
+--------------
+
+When using the submodule to build, it is ideal to properly update and match what is in the repository. Depending on Git version, different commands and options to ensure these match. An example workflow is to run ``git pull`` to get the latest commit on your current branch, and then run ``git submodule update`` to explicitly update the submodule. This should work for all versions of ``git`` which support submodules.
+
+The following example demonstrates a shorter form that combines both commands and requires Git 2.14 or newer:
+
+    .. code:: shell
+              # Replaces your git pull to use both the updated code and the updated submodule
+              git pull --recurse-submodules
+The following example demonstrates setting defaults in the config file for the repository and requires Git 2.15 or newer:
+
+    .. code:: shell
+              # Set this once
+              git config submodule.recurse true
+              # When configured as true, this will use both the updated code and the updated submodule
+              git pull
+              # This option will override any configuration and not update the submodule
+              git pull --no-recurse-submodules
+These example also apply to ``git checkout``. For more details see _`Git Tools Submodules`: https://git-scm.com/book/en/v2/Git-Tools-Submodules
diff --git a/Exec/ABL/inputs.read b/Exec/ABL/inputs.read
@@ -53,6 +53,8 @@ erf.molec_diff_type = "None"
 erf.les_type        = "Smagorinsky"
 erf.Cs              = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs.write b/Exec/ABL/inputs.write
@@ -46,6 +46,8 @@ erf.molec_diff_type = "None"
 erf.les_type        = "Smagorinsky"
 erf.Cs              = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs_deardorff b/Exec/ABL/inputs_deardorff
@@ -45,6 +45,8 @@ erf.Ck       = 0.1
 erf.sigma_k  = 1.0
 erf.Ce       = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs_deardorff_msf b/Exec/ABL/inputs_deardorff_msf
@@ -50,6 +50,8 @@ erf.Ck       = 0.1
 erf.sigma_k  = 1.0
 erf.Ce       = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs_deardorff_no_msf b/Exec/ABL/inputs_deardorff_no_msf
@@ -49,6 +49,8 @@ erf.Ck       = 0.1
 erf.sigma_k  = 1.0
 erf.Ce       = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs_ml_most b/Exec/ABL/inputs_ml_most
@@ -24,7 +24,7 @@ erf.fixed_dt           = 1.0  # fixed time step depending on grid resolution
 # DIAGNOSTICS & VERBOSITY
 erf.sum_interval   = 1       # timesteps between computing mass
 erf.v              = 1       # verbosity in ERF.cpp
-amr.v                = 1       # verbosity in Amr.cpp
+amr.v              = 1       # verbosity in Amr.cpp
 
 # REFINEMENT / REGRIDDING
 amr.max_level       = 0       # maximum level number allowed
@@ -35,7 +35,7 @@ erf.check_int       = 100        # number of timesteps between checkpoints
 
 # PLOTFILES
 erf.plot_file_1     = plt       # prefix of plotfile name
-erf.plot_int_1      = 4         # number of timesteps between plotfiles
+erf.plot_int_1      = 1         # number of timesteps between plotfiles
 erf.plot_vars_1     = density rhoadv_0 x_velocity y_velocity z_velocity pressure temp theta
 
 # SOLVER CHOICE
@@ -46,10 +46,12 @@ erf.use_gravity = false
 erf.les_type = "Smagorinsky"
 erf.Cs       = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
-prob.A_0 = 1.0
-prob.T_0 = 1.0
+prob.A_0   = 1.0
+prob.T_0   = 1.0
 
 prob.U_0 = 10.0
 prob.V_0 = 0.0
@@ -62,11 +64,11 @@ prob.V_0_Pert_Mag = 0.08 #
 prob.W_0_Pert_Mag = 0.0
 
 amr.max_level = 1
-amr.ref_ratio_vect = 2 2 1
+amr.ref_ratio_vect = 20 20 1
 erf.refinement_indicators = box1
 
 erf.box1.max_level = 1
-erf.box1.in_box_lo =  256 256
-erf.box1.in_box_hi =  768 768
+erf.box1.in_box_lo =  400 400
+erf.box1.in_box_hi =  480 480
 
-erf.coupling_type = "TwoWay"
+erf.coupling_type = "OneWay"
diff --git a/Exec/ABL/inputs_most b/Exec/ABL/inputs_most
@@ -47,6 +47,8 @@ erf.molec_diff_type = "None"
 erf.les_type        = "Smagorinsky"
 erf.Cs              = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs_most_msf b/Exec/ABL/inputs_most_msf
@@ -60,6 +60,8 @@ erf.dycore_vert_adv_type     = "Upwind_3rd"
 erf.dryscal_horiz_adv_type   = "Upwind_3rd"
 erf.dryscal_vert_adv_type    = "Upwind_3rd"
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs_most_no_msf b/Exec/ABL/inputs_most_no_msf
@@ -58,6 +58,8 @@ erf.dycore_vert_adv_type     = "Upwind_3rd"
 erf.dryscal_horiz_adv_type   = "Upwind_3rd"
 erf.dryscal_vert_adv_type    = "Upwind_3rd"
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs_most_test b/Exec/ABL/inputs_most_test
@@ -61,6 +61,8 @@ erf.les_type = "Smagorinsky"
 #erf.pbl_type = "MYNN2.5"
 erf.Cs       = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0   = 1.0

diff --git a/Exec/ABL/inputs_numdiff b/Exec/ABL/inputs_numdiff
@@ -48,6 +48,8 @@ erf.Ce       = 0.1
 erf.use_NumDiff  = true
 erf.NumDiffCoeff = 0.5
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0   = 1.0

diff --git a/Exec/ABL/inputs_omp b/Exec/ABL/inputs_omp
@@ -51,6 +51,8 @@ erf.Ck       = 0.1
 erf.sigma_k  = 1.0
 erf.Ce       = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0

diff --git a/Exec/ABL/inputs_sample b/Exec/ABL/inputs_sample
@@ -51,6 +51,8 @@ erf.molec_diff_type = "None"
 erf.les_type        = "Smagorinsky"
 erf.Cs              = 0.1
 
+erf.init_type = "uniform"
+
 # PROBLEM PARAMETERS
 prob.rho_0 = 1.0
 prob.A_0 = 1.0
Original file line number	Diff line number	Diff line change
Expand Up		@@ -24,6 +24,9 @@ type:

		git clone --recursive https://github.com/erf-model/ERF.git

		.. include:: submodule.rst


		.. include:: building.rst


Expand Down