From 8e6a7aa2e10a9d006d09bc37e614709edfca0a42 Mon Sep 17 00:00:00 2001
From: Michael Zingale <michael.zingale@stonybrook.edu>
Date: Wed, 21 Aug 2024 13:32:10 -0400
Subject: [PATCH] update Frontier

---
 sphinx_docs/source/olcf-compilers.rst | 11 ++++++-----
 sphinx_docs/source/olcf-workflow.rst  | 17 +++++++++++++++++
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/sphinx_docs/source/olcf-compilers.rst b/sphinx_docs/source/olcf-compilers.rst
index e04c47a..a233083 100644
--- a/sphinx_docs/source/olcf-compilers.rst
+++ b/sphinx_docs/source/olcf-compilers.rst
@@ -67,17 +67,18 @@ load modules:
 .. prompt:: bash
 
    module load PrgEnv-gnu
-   module load cray-mpich/8.1.27
+   module load cray-mpich/8.1.28
    module load craype-accel-amd-gfx90a
-   module load amd-mixed/6.0.0
+   module load amd-mixed/6.1.3
    module unload darshan-runtime
 
-this will load ROCm 6.0.0
+this will load ROCm 6.1.3
 
 .. note::
 
-   Tabulate rates seem to exhibit a strange slow down on Frontier, so
-   it is best to run without rate tabulation.
+   In the past, tabulated rates seem to exhibit a strange slow down on
+   Frontier, so it is best to run test with and without rate
+   tabulation to see if there is a performance issue.
 
 build via:
 
diff --git a/sphinx_docs/source/olcf-workflow.rst b/sphinx_docs/source/olcf-workflow.rst
index b929ddf..cc18d32 100644
--- a/sphinx_docs/source/olcf-workflow.rst
+++ b/sphinx_docs/source/olcf-workflow.rst
@@ -502,6 +502,23 @@ https://github.com/AMReX-Astro/workflow/blob/main/job_scripts/frontier/frontier.
 Also see the WarpX docs: https://warpx.readthedocs.io/en/latest/install/hpc/frontier.html
 
 
+GPU-aware MPI
+^^^^^^^^^^^^^
+
+Some codes run better with GPU-aware MPI.  To enable this add the following to your
+submission script:
+
+.. code:: bash
+
+   export MPICH_GPU_SUPPORT_ENABLED=1
+   export FI_MR_CACHE_MONITOR=memhooks
+
+and set the runtime parameter:
+
+.. code::
+
+   amrex.use_gpu_aware_mpi=1
+
 Job Status
 ^^^^^^^^^^