Merge branch 'development' into MYNNfix

.github/workflows/draft-pdf.yml

@@ -14,7 +14,7 @@ jobs:
           # This should be the path to the paper within your repo.
           paper-path: paper/paper.md
       - name: Upload
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: paper
           # This is the output path where Pandoc will write the compiled

Docs/sphinx_doc/building.rst

@@ -214,7 +214,7 @@ HDF5 (tested with v1.14.3)
 #. Configure for your system ``../configure --prefix=/usr/local --enable-parallel``
 #. Build ``make -j8`` and ``sudo make install``
 
-.. _hdfgroup.org: https://www.hdfgroup.org/downloads/hdf5/source-code/
+.. _hdfgroup.org: https://www.hdfgroup.org/download-hdf5/source-code/
 
 NetCDF (tested with v4.9.2)
 

Source/TimeIntegration/ERF_fast_rhs_MT.cpp

@@ -615,6 +615,12 @@ void erf_fast_rhs_MT (int step, int nrk,
                     {{AMREX_D_DECL(&(flux[0]), &(flux[1]), &(flux[2]))}},
                     dx, dtau, strt_comp_reflux, strt_comp_reflux, num_comp_reflux, RunOn::Device);
             }
+
+            // This is necessary here so we don't go on to the next FArrayBox without
+            // having finished copying the fluxes into the FluxRegisters (since the fluxes
+            // are stored in temporary FArrayBox's)
+            Gpu::streamSynchronize();
+
         } // two-way coupling
 
     } // mfi

Source/TimeIntegration/ERF_fast_rhs_N.cpp

@@ -529,6 +529,12 @@ void erf_fast_rhs_N (int step, int nrk,
                     {{AMREX_D_DECL(&(flux[0]), &(flux[1]), &(flux[2]))}},
                     dx, dtau, strt_comp_reflux, strt_comp_reflux, num_comp_reflux, RunOn::Device);
             }
+
+            // This is necessary here so we don't go on to the next FArrayBox without
+            // having finished copying the fluxes into the FluxRegisters (since the fluxes
+            // are stored in temporary FArrayBox's)
+            Gpu::streamSynchronize();
+
         } // two-way coupling
     } // mfi
     } // OMP

Source/TimeIntegration/ERF_fast_rhs_T.cpp

@@ -689,6 +689,12 @@ void erf_fast_rhs_T (int step, int nrk,
                     {{AMREX_D_DECL(&(flux[0]), &(flux[1]), &(flux[2]))}},
                     dx, dtau, strt_comp_reflux, strt_comp_reflux, num_comp_reflux, RunOn::Device);
             }
+
+            // This is necessary here so we don't go on to the next FArrayBox without
+            // having finished copying the fluxes into the FluxRegisters (since the fluxes
+            // are stored in temporary FArrayBox's)
+            Gpu::streamSynchronize();
+
         } // two-way coupling
     } // mfi
     } // OMP

Source/TimeIntegration/ERF_slow_rhs_post.cpp

@@ -545,6 +545,12 @@ void erf_slow_rhs_post (int level, int finest_level,
                     {{AMREX_D_DECL(&(flux[0]), &(flux[1]), &(flux[2]))}},
                     dx, dt, strt_comp_reflux, strt_comp_reflux, num_comp_reflux, RunOn::Device);
             }
+
+            // This is necessary here so we don't go on to the next FArrayBox without
+            // having finished copying the fluxes into the FluxRegisters (since the fluxes
+            // are stored in temporary FArrayBox's)
+            Gpu::streamSynchronize();
+
         } // two-way coupling
         } // end profile
       } // mfi

Source/TimeIntegration/ERF_slow_rhs_pre.cpp

@@ -764,6 +764,12 @@ if (cell_data(i,j,k,RhoTheta_comp) < 0.) printf("BAD THETA AT %d %d %d %e %e \n"
                     {{AMREX_D_DECL(&(flux[0]), &(flux[1]), &(flux[2]))}},
                     dx, dt, strt_comp_reflux, strt_comp_reflux, num_comp_reflux, RunOn::Device);
             }
+
+            // This is necessary here so we don't go on to the next FArrayBox without
+            // having finished copying the fluxes into the FluxRegisters (since the fluxes
+            // are stored in temporary FArrayBox's)
+            Gpu::streamSynchronize();
+
         } // two-way coupling
         } // end profile
     } // mfi