From b76ec280c41fa7ceb463fd76451f91ae9ba28d4d Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 26 Mar 2024 15:35:32 -0700
Subject: [PATCH 01/80] initial commit build-and-test.yml

---
 .github/workflows/build-and-test.yml | 146 +++++++++++++++++++++++++++
 1 file changed, 146 insertions(+)
 create mode 100644 .github/workflows/build-and-test.yml

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
new file mode 100644
index 00000000..8529adce
--- /dev/null
+++ b/.github/workflows/build-and-test.yml
@@ -0,0 +1,146 @@
+name: build-and-test
+
+on:
+  pull_request:
+    types:
+      - labeled
+
+jobs:
+  build:
+    if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7", "3.8", "3.9"]
+        #python-version: ["3.10"]
+        #os: [ubuntu-latest]
+        os: [ubuntu-20.04]
+        # USE_FLAGS :  cuda, parallel, libceed
+        env:
+           - { USE_FLAGS: "000"}
+           - { USE_FLAGS: "010"}
+
+        include:
+           - os: macos-latest
+             python-version: 3.9
+             env: {USE_FLAGS: "000"}
+
+    runs-on: ${{ matrix.os }}	   
+    #env: ${{ matrix.env }}
+    env: 
+       USE_FLAGS: ${{ matrix.env.USE_FLAGS }}
+       CUDA: "11.5"
+       SANDBOX: ~/sandbox
+
+    steps:
+    - uses: actions/checkout@v3
+ 
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -c "import setuptools;print(setuptools.__version__)"
+
+        PYTHONLIB=~/sandbox/lib/python${{ matrix.python-version }}/site-packages
+        mkdir -p $PYTHONLIB
+
+        export PYTHONPATH=$PYTHONLIB:$PYTHONPATH
+        echo "PYTHONPATH:"$PYTHONPATH
+
+        pip install six  --verbose	
+        if [ "${{matrix.python-version}}" = "3.10" ] ; then
+            pip install numba numba-scipy --verbose
+        else
+            pip install numba numba-scipy --verbose
+        fi
+
+        if [ -f requirements.txt ]; then
+           pip install -r requirements.txt --prefix=~/sandbox --verbose
+        fi
+
+        python -c "import sys;print(sys.path)"
+        python -c "import numpy;print(numpy.__file__)"	
+
+        which swig  # this default is 4.0.1 
+        pip install swig --prefix=~/sandbox
+        export PATH=/usr/local/cuda-${CUDA}/bin:~/sandbox/bin:$PATH	
+        which swig
+        swig -version
+
+        if [ "${USE_FLAGS:0:1}" = "1" ] ; then
+            echo $cuda
+            source ./ci_scripts/add_cuda_11_5.sh;
+        fi
+        if [ "${USE_FLAGS:1:1}" = "1" ] ; then
+            sudo apt-get install mpich;
+            sudo apt-get install libmpich-dev;
+            pip install mpi4py --prefix=~/sandbox
+            python -c "import mpi4py;print(mpi4py.get_include())";	    
+        fi
+        ls -l $PYTHONLIB	
+        
+    - name: Build
+      run: |
+        export PATH=/usr/local/cuda-${CUDA}/bin:~/sandbox/bin:$PATH
+        PYTHONLIB=~/sandbox/lib/python${{ matrix.python-version }}/site-packages	
+        export PYTHONPATH=$PYTHONLIB:$PYTHONPATH
+        echo $PATH
+        echo $PYTHONPATH
+        echo $SANDBOX
+        echo "SWIG exe"$(which swig)
+
+        if [ "${USE_FLAGS}" = "000" ]; then
+           # test workflow to manually run swig
+           python setup.py clean --swig
+           python setup.py install --ext-only --with-gslib --verbose
+           python setup.py install --swig --with-gslib --verbose
+           python setup.py install --skip-ext --skip-swig --with-gslib --verbose
+        fi
+        if [ "${USE_FLAGS}" = "010" ]; then
+           python setup.py install --with-gslib --with-parallel --prefix=$SANDBOX
+        fi
+
+    - name: RUN_EXAMPLES
+      run: |
+
+        export PATH=/usr/local/cuda-${CUDA}/bin:$PATH
+        PYTHONLIB=~/sandbox/lib/python${{ matrix.python-version }}/site-packages	
+        export PYTHONPATH=$PYTHONLIB:$PYTHONPATH
+        echo $PATH
+        echo $PYTHONPATH
+        echo $SANDBOX
+        cd test
+        echo $PATH
+        echo $PYTHONPATH
+        if [ "${USE_FLAGS}" = "000" ]; then
+           python run_examples.py -serial -verbose
+        fi
+        if [ "${USE_FLAGS}" = "010" ]; then
+           which mpicc
+           python run_examples.py -parallel -verbose -np 2
+        fi
+
+    - name: Generate Artifact
+      if: always()
+      run: |
+        tar -cvzf sandbox.tar.gz test/sandbox      
+
+    - name: Generate artifact name
+      if: always()    
+      id: generate-name
+      run: |
+          txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
+          name="test_result_"${txt}"_"${{ github.run_id }}".tar.gz"
+          echo $name
+          echo "artifact=${name}" >> $GITHUB_OUTPUT
+
+    - name: Upload Artifact
+      uses: actions/upload-artifact@v3
+      if: failure()      
+      with:
+          name: ${{ steps.generate-name.outputs.artifact }}      
+          path: sandbox.tar.gz
+          retention-days: 1 

From ac953d6cde5a22199205af948b52d141eb2ec335 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 26 Mar 2024 15:36:50 -0700
Subject: [PATCH 02/80] update action versions

---
 .github/workflows/build-and-test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 8529adce..49465fb8 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -33,10 +33,10 @@ jobs:
        SANDBOX: ~/sandbox
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
  
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -138,7 +138,7 @@ jobs:
           echo "artifact=${name}" >> $GITHUB_OUTPUT
 
     - name: Upload Artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure()      
       with:
           name: ${{ steps.generate-name.outputs.artifact }}      

From 682a30194a324e9fbe71396f9a5b279dfd5243a3 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 26 Mar 2024 17:06:24 -0700
Subject: [PATCH 03/80] move PATH/PYTHONPATH to separate action and export via
 GITHUB_ENV and GITHUB_PATH

---
 .github/workflows/build-and-test.yml | 44 ++++++++--------------------
 1 file changed, 13 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 49465fb8..a69f2a4a 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -1,3 +1,4 @@
+# This is a clean-up/refactor of test_with_MFEM_release.yml
 name: build-and-test
 
 on:
@@ -40,22 +41,22 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install dependencies
+    - name: Set PYTHONPATH
       run: |
-        python -c "import setuptools;print(setuptools.__version__)"
-
-        PYTHONLIB=~/sandbox/lib/python${{ matrix.python-version }}/site-packages
+        echo "PYTHONLIB=$HOME/sandbox/lib/python${{ matrix.python-version }}/site-packages" >> $GITHUB_ENV
         mkdir -p $PYTHONLIB
+        echo "PYTHONPATH=$PYTHONLIB:$PYTHONPATH" >> $GITHUB_ENV
 
-        export PYTHONPATH=$PYTHONLIB:$PYTHONPATH
-        echo "PYTHONPATH:"$PYTHONPATH
+    - name: Set PATH
+      run: |
+        echo "/usr/local/cuda-${CUDA}/bin" >> $GITHUB_PATH
+        echo "$HOME/sandbox/bin" >> $GITHUB_PATH
 
-        pip install six  --verbose	
-        if [ "${{matrix.python-version}}" = "3.10" ] ; then
-            pip install numba numba-scipy --verbose
-        else
-            pip install numba numba-scipy --verbose
-        fi
+    - name: Install dependencies
+      run: |
+        python -c "import setuptools;print(setuptools.__version__)"
+
+        pip install six numba numba-scipy --verbose
 
         if [ -f requirements.txt ]; then
            pip install -r requirements.txt --prefix=~/sandbox --verbose
@@ -64,9 +65,7 @@ jobs:
         python -c "import sys;print(sys.path)"
         python -c "import numpy;print(numpy.__file__)"	
 
-        which swig  # this default is 4.0.1 
         pip install swig --prefix=~/sandbox
-        export PATH=/usr/local/cuda-${CUDA}/bin:~/sandbox/bin:$PATH	
         which swig
         swig -version
 
@@ -84,14 +83,6 @@ jobs:
         
     - name: Build
       run: |
-        export PATH=/usr/local/cuda-${CUDA}/bin:~/sandbox/bin:$PATH
-        PYTHONLIB=~/sandbox/lib/python${{ matrix.python-version }}/site-packages	
-        export PYTHONPATH=$PYTHONLIB:$PYTHONPATH
-        echo $PATH
-        echo $PYTHONPATH
-        echo $SANDBOX
-        echo "SWIG exe"$(which swig)
-
         if [ "${USE_FLAGS}" = "000" ]; then
            # test workflow to manually run swig
            python setup.py clean --swig
@@ -105,16 +96,7 @@ jobs:
 
     - name: RUN_EXAMPLES
       run: |
-
-        export PATH=/usr/local/cuda-${CUDA}/bin:$PATH
-        PYTHONLIB=~/sandbox/lib/python${{ matrix.python-version }}/site-packages	
-        export PYTHONPATH=$PYTHONLIB:$PYTHONPATH
-        echo $PATH
-        echo $PYTHONPATH
-        echo $SANDBOX
         cd test
-        echo $PATH
-        echo $PYTHONPATH
         if [ "${USE_FLAGS}" = "000" ]; then
            python run_examples.py -serial -verbose
         fi

From 002c72656ea6f9bd0b3aee3191974e20805147f4 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 4 Apr 2024 12:11:22 -0700
Subject: [PATCH 04/80] lots of changes to build-and-test.yml

---
 .github/workflows/build-and-test.yml | 103 +++++++++++++--------------
 1 file changed, 50 insertions(+), 53 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a69f2a4a..16a22f93 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -12,26 +12,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
-        #python-version: ["3.10"]
-        #os: [ubuntu-latest]
-        os: [ubuntu-20.04]
-        # USE_FLAGS :  cuda, parallel, libceed
-        env:
-           - { USE_FLAGS: "000"}
-           - { USE_FLAGS: "010"}
-
-        include:
-           - os: macos-latest
-             python-version: 3.9
-             env: {USE_FLAGS: "000"}
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        os: [ubuntu-latest, macos-latest]
+        # os: [ubuntu-20.04]
+
+        cuda: [false]
+        parallel: [true, false]
+        libceed: [false]
 
     runs-on: ${{ matrix.os }}	   
-    #env: ${{ matrix.env }}
     env: 
-       USE_FLAGS: ${{ matrix.env.USE_FLAGS }}
        CUDA: "11.5"
-       SANDBOX: ~/sandbox
+       SANDBOX: $HOME/sandbox
 
     steps:
     - uses: actions/checkout@v4
@@ -49,61 +41,66 @@ jobs:
 
     - name: Set PATH
       run: |
-        echo "/usr/local/cuda-${CUDA}/bin" >> $GITHUB_PATH
         echo "$HOME/sandbox/bin" >> $GITHUB_PATH
 
-    - name: Install dependencies
+    - name: Install core dependencies
       run: |
         python -c "import setuptools;print(setuptools.__version__)"
 
         pip install six numba numba-scipy --verbose
 
         if [ -f requirements.txt ]; then
-           pip install -r requirements.txt --prefix=~/sandbox --verbose
+           pip install -r requirements.txt --prefix=$HOME/sandbox --verbose
         fi
 
         python -c "import sys;print(sys.path)"
         python -c "import numpy;print(numpy.__file__)"	
 
-        pip install swig --prefix=~/sandbox
+        pip install swig --prefix=$HOME/sandbox
         which swig
         swig -version
+    
+    - if: ${{ matrix.cuda == 'true' }}
+      name: Install CUDA
+      run: |
+        echo $cuda
+        source ./ci_scripts/add_cuda_11_5.sh;
+        echo "/usr/local/cuda-${CUDA}/bin" >> $GITHUB_PATH
 
-        if [ "${USE_FLAGS:0:1}" = "1" ] ; then
-            echo $cuda
-            source ./ci_scripts/add_cuda_11_5.sh;
-        fi
-        if [ "${USE_FLAGS:1:1}" = "1" ] ; then
-            sudo apt-get install mpich;
-            sudo apt-get install libmpich-dev;
-            pip install mpi4py --prefix=~/sandbox
-            python -c "import mpi4py;print(mpi4py.get_include())";	    
-        fi
+    - if: ${{ matrix.parallel == 'true' }}
+      name: Install MPI
+      run: |
+        sudo apt-get install mpich;
+        sudo apt-get install libmpich-dev;
+        pip install mpi4py --prefix=$HOME/sandbox
+        python -c "import mpi4py;print(mpi4py.get_include())";	    
         ls -l $PYTHONLIB	
         
-    - name: Build
+    - if: ${{ matrix.parallel != 'true' }}
+      name: Build serial
       run: |
-        if [ "${USE_FLAGS}" = "000" ]; then
-           # test workflow to manually run swig
-           python setup.py clean --swig
-           python setup.py install --ext-only --with-gslib --verbose
-           python setup.py install --swig --with-gslib --verbose
-           python setup.py install --skip-ext --skip-swig --with-gslib --verbose
-        fi
-        if [ "${USE_FLAGS}" = "010" ]; then
-           python setup.py install --with-gslib --with-parallel --prefix=$SANDBOX
-        fi
+        # test workflow to manually run swig
+        python setup.py clean --swig
+        python setup.py install --ext-only --with-gslib --verbose
+        python setup.py install --swig --with-gslib --verbose
+        python setup.py install --skip-ext --skip-swig --with-gslib --verbose
+
+    - if: ${{ matrix.parallel == 'true' }}
+      name: Build parallel
+      run: |
+        python setup.py install --with-gslib --with-parallel --prefix=$SANDBOX
 
-    - name: RUN_EXAMPLES
+    - if: ${{ matrix.parallel != 'true' }}
+      name: Run serial
       run: |
         cd test
-        if [ "${USE_FLAGS}" = "000" ]; then
-           python run_examples.py -serial -verbose
-        fi
-        if [ "${USE_FLAGS}" = "010" ]; then
-           which mpicc
-           python run_examples.py -parallel -verbose -np 2
-        fi
+        python run_examples.py -serial -verbose
+    - if: ${{ matrix.parallel == 'true' }}
+      name: Run parallel
+      run: |
+        which mpicc
+        cd test
+        python run_examples.py -parallel -verbose -np 2
 
     - name: Generate Artifact
       if: always()
@@ -114,10 +111,10 @@ jobs:
       if: always()    
       id: generate-name
       run: |
-          txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
-          name="test_result_"${txt}"_"${{ github.run_id }}".tar.gz"
-          echo $name
-          echo "artifact=${name}" >> $GITHUB_OUTPUT
+        txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
+        name="test_result_"${txt}"_"${{ github.run_id }}".tar.gz"
+        echo $name
+        echo "artifact=${name}" >> $GITHUB_OUTPUT
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4

From 406912e1f918648d952f31a4cdaa168bf679ed12 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Fri, 5 Apr 2024 13:09:21 -0700
Subject: [PATCH 05/80] attempting to test; adding workflow_dispatch to 'on'

---
 .github/workflows/build-and-test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 16a22f93..902818c0 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -2,9 +2,8 @@
 name: build-and-test
 
 on:
+  workflow_dispatch:
   pull_request:
-    types:
-      - labeled
 
 jobs:
   build:

From a935d149ee253f4c9341c1f3ebd06612518f1987 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Fri, 5 Apr 2024 13:21:12 -0700
Subject: [PATCH 06/80] remove conditional for trigger

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 902818c0..94f817fb 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
+    # if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
     strategy:
       fail-fast: false
       matrix:

From fa2eb0b16875a1486739e84a01c06d42765285f6 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Fri, 5 Apr 2024 13:25:54 -0700
Subject: [PATCH 07/80] remove matrix options for testing, setting fail fast to
 true, fixing github_env call

---
 .github/workflows/build-and-test.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 94f817fb..e762d0e4 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -9,10 +9,12 @@ jobs:
   build:
     # if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.9"]
+        #"3.7", "3.8", "3.9", "3.10"]
+        os: [ubuntu-latest]
+        #, macos-latest]
         # os: [ubuntu-20.04]
 
         cuda: [false]
@@ -35,8 +37,8 @@ jobs:
     - name: Set PYTHONPATH
       run: |
         echo "PYTHONLIB=$HOME/sandbox/lib/python${{ matrix.python-version }}/site-packages" >> $GITHUB_ENV
-        mkdir -p $PYTHONLIB
-        echo "PYTHONPATH=$PYTHONLIB:$PYTHONPATH" >> $GITHUB_ENV
+        mkdir -p ${{ env.PYTHONLIB }}
+        echo "PYTHONPATH=${{ env.PYTHONLIB }}:$PYTHONPATH" >> $GITHUB_ENV
 
     - name: Set PATH
       run: |

From adc2636e2de751e3f8aaa8eb8c6ea2256f247e9f Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Fri, 5 Apr 2024 13:31:12 -0700
Subject: [PATCH 08/80] GITHUB_ENV cant be accessed until next step

---
 .github/workflows/build-and-test.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index e762d0e4..de6722d9 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -34,11 +34,11 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Set PYTHONPATH
+    - name: Create PYTHONLIB and set PYTHONPATH
       run: |
-        echo "PYTHONLIB=$HOME/sandbox/lib/python${{ matrix.python-version }}/site-packages" >> $GITHUB_ENV
-        mkdir -p ${{ env.PYTHONLIB }}
-        echo "PYTHONPATH=${{ env.PYTHONLIB }}:$PYTHONPATH" >> $GITHUB_ENV
+        PYTHONLIB=${HOME}/sandbox/lib/python${{ matrix.python-version }}/site-packages
+        mkdir -p $PYTHONLIB
+        echo "PYTHONPATH=$PYTHONLIB:$PYTHONPATH" >> $GITHUB_ENV
 
     - name: Set PATH
       run: |
@@ -75,7 +75,6 @@ jobs:
         sudo apt-get install libmpich-dev;
         pip install mpi4py --prefix=$HOME/sandbox
         python -c "import mpi4py;print(mpi4py.get_include())";	    
-        ls -l $PYTHONLIB	
         
     - if: ${{ matrix.parallel != 'true' }}
       name: Build serial

From 7e9041c181adb721a0115b5e1fd8f8265eed2b65 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Fri, 5 Apr 2024 13:40:57 -0700
Subject: [PATCH 09/80] fix bools

---
 .github/workflows/build-and-test.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index de6722d9..493f6e4e 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -61,14 +61,14 @@ jobs:
         which swig
         swig -version
     
-    - if: ${{ matrix.cuda == 'true' }}
+    - if: ${{ matrix.cuda == true }}
       name: Install CUDA
       run: |
         echo $cuda
         source ./ci_scripts/add_cuda_11_5.sh;
         echo "/usr/local/cuda-${CUDA}/bin" >> $GITHUB_PATH
 
-    - if: ${{ matrix.parallel == 'true' }}
+    - if: ${{ matrix.parallel == true }}
       name: Install MPI
       run: |
         sudo apt-get install mpich;
@@ -76,7 +76,7 @@ jobs:
         pip install mpi4py --prefix=$HOME/sandbox
         python -c "import mpi4py;print(mpi4py.get_include())";	    
         
-    - if: ${{ matrix.parallel != 'true' }}
+    - if: ${{ matrix.parallel == false }}
       name: Build serial
       run: |
         # test workflow to manually run swig
@@ -85,17 +85,17 @@ jobs:
         python setup.py install --swig --with-gslib --verbose
         python setup.py install --skip-ext --skip-swig --with-gslib --verbose
 
-    - if: ${{ matrix.parallel == 'true' }}
+    - if: ${{ matrix.parallel == true }}
       name: Build parallel
       run: |
         python setup.py install --with-gslib --with-parallel --prefix=$SANDBOX
 
-    - if: ${{ matrix.parallel != 'true' }}
+    - if: ${{ matrix.parallel == false }}
       name: Run serial
       run: |
         cd test
         python run_examples.py -serial -verbose
-    - if: ${{ matrix.parallel == 'true' }}
+    - if: ${{ matrix.parallel == true }}
       name: Run parallel
       run: |
         which mpicc

From f48877e497380de6e24af76bc634df72fdef5381 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Fri, 5 Apr 2024 17:56:31 -0700
Subject: [PATCH 10/80] removing unnecessary dependencies?

---
 .github/workflows/build-and-test.yml | 40 +++++++++++++++-------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 493f6e4e..5fc48f2f 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -46,14 +46,17 @@ jobs:
 
     - name: Install core dependencies
       run: |
-        python -c "import setuptools;print(setuptools.__version__)"
 
-        pip install six numba numba-scipy --verbose
+        # pip install six numba numba-scipy --verbose
+        # jgl: are six and numba-scipy used? I don't see them imported anywhere.
+        # jgl: should numba just go in requirements.txt?
+        pip install numba --verbose
 
         if [ -f requirements.txt ]; then
            pip install -r requirements.txt --prefix=$HOME/sandbox --verbose
         fi
 
+        python -c "import setuptools;print(setuptools.__version__)"
         python -c "import sys;print(sys.path)"
         python -c "import numpy;print(numpy.__file__)"	
 
@@ -61,23 +64,23 @@ jobs:
         which swig
         swig -version
     
-    - if: ${{ matrix.cuda == true }}
-      name: Install CUDA
+    - name: Install CUDA
+      if: ${{ matrix.cuda == true }}
       run: |
         echo $cuda
         source ./ci_scripts/add_cuda_11_5.sh;
         echo "/usr/local/cuda-${CUDA}/bin" >> $GITHUB_PATH
 
-    - if: ${{ matrix.parallel == true }}
-      name: Install MPI
+    - name: Install MPI
+      if: ${{ matrix.parallel == true }}
       run: |
         sudo apt-get install mpich;
         sudo apt-get install libmpich-dev;
         pip install mpi4py --prefix=$HOME/sandbox
         python -c "import mpi4py;print(mpi4py.get_include())";	    
         
-    - if: ${{ matrix.parallel == false }}
-      name: Build serial
+    - name: Build serial
+      if: ${{ matrix.parallel == false }}
       run: |
         # test workflow to manually run swig
         python setup.py clean --swig
@@ -85,18 +88,18 @@ jobs:
         python setup.py install --swig --with-gslib --verbose
         python setup.py install --skip-ext --skip-swig --with-gslib --verbose
 
-    - if: ${{ matrix.parallel == true }}
-      name: Build parallel
+    - name: Build parallel
+      if: ${{ matrix.parallel == true }}
       run: |
         python setup.py install --with-gslib --with-parallel --prefix=$SANDBOX
 
-    - if: ${{ matrix.parallel == false }}
-      name: Run serial
+    - name: Run serial
+      if: ${{ matrix.parallel == false }}
       run: |
         cd test
         python run_examples.py -serial -verbose
-    - if: ${{ matrix.parallel == true }}
-      name: Run parallel
+    - name: Run parallel
+      if: ${{ matrix.parallel == true }}
       run: |
         which mpicc
         cd test
@@ -109,17 +112,16 @@ jobs:
 
     - name: Generate artifact name
       if: always()    
-      id: generate-name
+      id: generate-artifact-name
       run: |
         txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
         name="test_result_"${txt}"_"${{ github.run_id }}".tar.gz"
-        echo $name
-        echo "artifact=${name}" >> $GITHUB_OUTPUT
+        echo "name=${name}" >> $GITHUB_OUTPUT
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4
-      if: failure()      
+      # if: failure()
       with:
-          name: ${{ steps.generate-name.outputs.artifact }}      
+          name: ${{ steps.generate-artifact-name.outputs.name }}      
           path: sandbox.tar.gz
           retention-days: 1 

From c5237451400968f16cb93bf7df25747db1956c33 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 10:57:22 -0700
Subject: [PATCH 11/80] removing trailing whitespace

---
 .github/workflows/build-and-test.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 5fc48f2f..1ddec65e 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -21,14 +21,14 @@ jobs:
         parallel: [true, false]
         libceed: [false]
 
-    runs-on: ${{ matrix.os }}	   
-    env: 
+    runs-on: ${{ matrix.os }}
+    env:
        CUDA: "11.5"
        SANDBOX: $HOME/sandbox
 
     steps:
     - uses: actions/checkout@v4
- 
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
@@ -58,12 +58,12 @@ jobs:
 
         python -c "import setuptools;print(setuptools.__version__)"
         python -c "import sys;print(sys.path)"
-        python -c "import numpy;print(numpy.__file__)"	
+        python -c "import numpy;print(numpy.__file__)"
 
         pip install swig --prefix=$HOME/sandbox
         which swig
         swig -version
-    
+
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
@@ -77,8 +77,8 @@ jobs:
         sudo apt-get install mpich;
         sudo apt-get install libmpich-dev;
         pip install mpi4py --prefix=$HOME/sandbox
-        python -c "import mpi4py;print(mpi4py.get_include())";	    
-        
+        python -c "import mpi4py;print(mpi4py.get_include())";
+
     - name: Build serial
       if: ${{ matrix.parallel == false }}
       run: |
@@ -108,10 +108,10 @@ jobs:
     - name: Generate Artifact
       if: always()
       run: |
-        tar -cvzf sandbox.tar.gz test/sandbox      
+        tar -cvzf sandbox.tar.gz test/sandbox
 
     - name: Generate artifact name
-      if: always()    
+      if: always()
       id: generate-artifact-name
       run: |
         txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
@@ -122,6 +122,6 @@ jobs:
       uses: actions/upload-artifact@v4
       # if: failure()
       with:
-          name: ${{ steps.generate-artifact-name.outputs.name }}      
+          name: ${{ steps.generate-artifact-name.outputs.name }}
           path: sandbox.tar.gz
-          retention-days: 1 
+          retention-days: 1

From 4394196f187d95794c6f9939e5b348801a1034cb Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 11:01:12 -0700
Subject: [PATCH 12/80] remove numba_scipy dependency from ex25 and ex25p

---
 examples/ex25.py  | 2 --
 examples/ex25p.py | 2 --
 2 files changed, 4 deletions(-)

diff --git a/examples/ex25.py b/examples/ex25.py
index eadbe5b8..104f02fb 100644
--- a/examples/ex25.py
+++ b/examples/ex25.py
@@ -13,8 +13,6 @@
 
 '''
 from numba import jit, types, carray
-import numba
-import numba_scipy
 import os
 import mfem.ser as mfem
 from mfem.ser import intArray
diff --git a/examples/ex25p.py b/examples/ex25p.py
index a91972b1..b7843e7a 100644
--- a/examples/ex25p.py
+++ b/examples/ex25p.py
@@ -13,8 +13,6 @@
      
 '''
 from numba import jit, types, carray
-import numba
-import numba_scipy
 import os
 import mfem.par as mfem
 from mfem.par import intArray

From 3c4d14af1d1a9cc6fc48694f85f5e4da52ea279b Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 11:10:06 -0700
Subject: [PATCH 13/80] add numba to requirements.txt and removed from CI

---
 .github/workflows/build-and-test.yml | 6 +-----
 requirements.txt                     | 4 ++--
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 1ddec65e..3385aa7c 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -50,11 +50,7 @@ jobs:
         # pip install six numba numba-scipy --verbose
         # jgl: are six and numba-scipy used? I don't see them imported anywhere.
         # jgl: should numba just go in requirements.txt?
-        pip install numba --verbose
-
-        if [ -f requirements.txt ]; then
-           pip install -r requirements.txt --prefix=$HOME/sandbox --verbose
-        fi
+        pip install -r requirements.txt --prefix=$HOME/sandbox --verbose
 
         python -c "import setuptools;print(setuptools.__version__)"
         python -c "import sys;print(sys.path)"
diff --git a/requirements.txt b/requirements.txt
index 5d422a6c..c8d05440 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 numpy>=1.19.4; python_version<"3.7"
 numpy>=1.20.0; python_version>="3.7"
+numba
 scipy
 swig >= 4.1.1
-cmake
-
+cmake
\ No newline at end of file

From 6984965a42a02335b734d751e787b0f84c1c5db1 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 11:10:23 -0700
Subject: [PATCH 14/80] remove swig install from CI becuase it is in
 requirements.txt

---
 .github/workflows/build-and-test.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 3385aa7c..947edef4 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -56,10 +56,6 @@ jobs:
         python -c "import sys;print(sys.path)"
         python -c "import numpy;print(numpy.__file__)"
 
-        pip install swig --prefix=$HOME/sandbox
-        which swig
-        swig -version
-
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |

From b7ab2ebb851a32c20cf22404291c25f0e6e46daa Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 11:58:20 -0700
Subject: [PATCH 15/80] clean up CI script

---
 .github/workflows/build-and-test.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 947edef4..420f1d9f 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -44,17 +44,18 @@ jobs:
       run: |
         echo "$HOME/sandbox/bin" >> $GITHUB_PATH
 
-    - name: Install core dependencies
+    - name: Install core dependencies via requirements.txt
       run: |
-
-        # pip install six numba numba-scipy --verbose
-        # jgl: are six and numba-scipy used? I don't see them imported anywhere.
-        # jgl: should numba just go in requirements.txt?
         pip install -r requirements.txt --prefix=$HOME/sandbox --verbose
 
-        python -c "import setuptools;print(setuptools.__version__)"
-        python -c "import sys;print(sys.path)"
-        python -c "import numpy;print(numpy.__file__)"
+    - name: Print versions of installed python packages
+      run: |
+        packages=("setuptools", "numpy", "numba", "scipy", "swig", "cmake", "mpi4py")
+        for package in "${packages[@]}"
+        do
+            echo -n "Checking for $package... "
+            pip freeze | grep "$package"
+        done
 
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
@@ -66,8 +67,7 @@ jobs:
     - name: Install MPI
       if: ${{ matrix.parallel == true }}
       run: |
-        sudo apt-get install mpich;
-        sudo apt-get install libmpich-dev;
+        sudo apt-get install mpich libmpich-dev
         pip install mpi4py --prefix=$HOME/sandbox
         python -c "import mpi4py;print(mpi4py.get_include())";
 

From d37646fb6526b14cda4f1a82f70fe63f9e9e4d8f Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 11:58:32 -0700
Subject: [PATCH 16/80] trying to add numba-scipy to requirements

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index c8d05440..c48fc7eb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 numpy>=1.19.4; python_version<"3.7"
 numpy>=1.20.0; python_version>="3.7"
 numba
+numba-scipy
 scipy
 swig >= 4.1.1
 cmake
\ No newline at end of file

From bb8025b890281afe28c4008a5a525c5dc83cb666 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 12:04:21 -0700
Subject: [PATCH 17/80] typo

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 420f1d9f..096db772 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -50,7 +50,7 @@ jobs:
 
     - name: Print versions of installed python packages
       run: |
-        packages=("setuptools", "numpy", "numba", "scipy", "swig", "cmake", "mpi4py")
+        packages=("setuptools" "numpy" "numba" "scipy" "swig" "cmake" "mpi4py")
         for package in "${packages[@]}"
         do
             echo -n "Checking for $package... "

From 7c3e0caed39feea269a67de2199158a8718707db Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 12:16:08 -0700
Subject: [PATCH 18/80] comment out print for now

---
 .github/workflows/build-and-test.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 096db772..9f961fe3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -48,14 +48,14 @@ jobs:
       run: |
         pip install -r requirements.txt --prefix=$HOME/sandbox --verbose
 
-    - name: Print versions of installed python packages
-      run: |
-        packages=("setuptools" "numpy" "numba" "scipy" "swig" "cmake" "mpi4py")
-        for package in "${packages[@]}"
-        do
-            echo -n "Checking for $package... "
-            pip freeze | grep "$package"
-        done
+    # - name: Print versions of installed python packages
+    #   run: |
+    #     packages=("setuptools" "numpy" "numba" "scipy" "swig" "cmake" "mpi4py")
+    #     for package in "${packages[@]}"
+    #     do
+    #         echo -n "Checking for $package... "
+    #         pip freeze | grep "$package"
+    #     done
 
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}

From 3db7c846fae0cb5ca9b56baa117f87f46d441f17 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 16:03:16 -0700
Subject: [PATCH 19/80] remove env.SANDBOX

---
 .github/workflows/build-and-test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 9f961fe3..8c1162ed 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -24,7 +24,6 @@ jobs:
     runs-on: ${{ matrix.os }}
     env:
        CUDA: "11.5"
-       SANDBOX: $HOME/sandbox
 
     steps:
     - uses: actions/checkout@v4
@@ -83,7 +82,7 @@ jobs:
     - name: Build parallel
       if: ${{ matrix.parallel == true }}
       run: |
-        python setup.py install --with-gslib --with-parallel --prefix=$SANDBOX
+        python setup.py install --with-gslib --with-parallel --prefix=$HOME/sandbox
 
     - name: Run serial
       if: ${{ matrix.parallel == false }}

From bf14b27da04ddd179c9c77d7f07887c8cd52cdd7 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 16:32:40 -0700
Subject: [PATCH 20/80] hopefully fixed python package print out

---
 .github/workflows/build-and-test.yml | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 8c1162ed..fdedb847 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -47,15 +47,6 @@ jobs:
       run: |
         pip install -r requirements.txt --prefix=$HOME/sandbox --verbose
 
-    # - name: Print versions of installed python packages
-    #   run: |
-    #     packages=("setuptools" "numpy" "numba" "scipy" "swig" "cmake" "mpi4py")
-    #     for package in "${packages[@]}"
-    #     do
-    #         echo -n "Checking for $package... "
-    #         pip freeze | grep "$package"
-    #     done
-
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
@@ -70,6 +61,19 @@ jobs:
         pip install mpi4py --prefix=$HOME/sandbox
         python -c "import mpi4py;print(mpi4py.get_include())";
 
+    - name: Print versions of installed python packages
+      run: |
+        packages=("setuptools" "wheel" "numpy" "numba" "scipy" "swig" "cmake" "mpi4py")
+        for package in "${packages[@]}"
+        do
+            output=$(pip list | grep "$package")
+            if [[ -n $output ]]; then
+                echo $output
+            else
+                echo "$package not installed"
+            fi
+        done
+
     - name: Build serial
       if: ${{ matrix.parallel == false }}
       run: |

From 9eae6ea2aed177331e2f24a68d7c1162d2292feb Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 16:37:37 -0700
Subject: [PATCH 21/80] remove numba-scipy

---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index c48fc7eb..c8d05440 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,6 @@
 numpy>=1.19.4; python_version<"3.7"
 numpy>=1.20.0; python_version>="3.7"
 numba
-numba-scipy
 scipy
 swig >= 4.1.1
 cmake
\ No newline at end of file

From 71329ea4d60bed2779da42e60abc330fe9560b2a Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 17:04:35 -0700
Subject: [PATCH 22/80] pip list

---
 .github/workflows/build-and-test.yml | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index fdedb847..8df999d6 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -63,16 +63,18 @@ jobs:
 
     - name: Print versions of installed python packages
       run: |
-        packages=("setuptools" "wheel" "numpy" "numba" "scipy" "swig" "cmake" "mpi4py")
-        for package in "${packages[@]}"
-        do
-            output=$(pip list | grep "$package")
-            if [[ -n $output ]]; then
-                echo $output
-            else
-                echo "$package not installed"
-            fi
-        done
+        pip list
+
+      # packages=("setuptools" "wheel" "numpy" "numba" "scipy" "swig" "cmake" "mpi4py")
+      # for package in "${packages[@]}"
+      # do
+      #     output=$(pip list | grep "$package")
+      #     if [[ -n $output ]]; then
+      #         echo $output
+      #     else
+      #         echo "$package not installed"
+      #     fi
+      # done
 
     - name: Build serial
       if: ${{ matrix.parallel == false }}

From dba224a4fbb2bc46946cf7a2164d0ec4ca9d2399 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 17:06:32 -0700
Subject: [PATCH 23/80] cleanup

---
 .github/workflows/build-and-test.yml | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 8df999d6..7c35c144 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -65,17 +65,6 @@ jobs:
       run: |
         pip list
 
-      # packages=("setuptools" "wheel" "numpy" "numba" "scipy" "swig" "cmake" "mpi4py")
-      # for package in "${packages[@]}"
-      # do
-      #     output=$(pip list | grep "$package")
-      #     if [[ -n $output ]]; then
-      #         echo $output
-      #     else
-      #         echo "$package not installed"
-      #     fi
-      # done
-
     - name: Build serial
       if: ${{ matrix.parallel == false }}
       run: |

From fc441ea2d66cdafc7de72b5d568a1686d73fb683 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 8 Apr 2024 17:59:59 -0700
Subject: [PATCH 24/80] little bit more cleanup and re-add numba-scipy (even
 though it is not imported, it does seem necessary)

---
 .github/workflows/build-and-test.yml | 7 +------
 requirements.txt                     | 1 +
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 7c35c144..3cbd37f2 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -50,7 +50,6 @@ jobs:
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
-        echo $cuda
         source ./ci_scripts/add_cuda_11_5.sh;
         echo "/usr/local/cuda-${CUDA}/bin" >> $GITHUB_PATH
 
@@ -59,9 +58,8 @@ jobs:
       run: |
         sudo apt-get install mpich libmpich-dev
         pip install mpi4py --prefix=$HOME/sandbox
-        python -c "import mpi4py;print(mpi4py.get_include())";
 
-    - name: Print versions of installed python packages
+    - name: Print python package versions
       run: |
         pip list
 
@@ -87,17 +85,14 @@ jobs:
     - name: Run parallel
       if: ${{ matrix.parallel == true }}
       run: |
-        which mpicc
         cd test
         python run_examples.py -parallel -verbose -np 2
 
     - name: Generate Artifact
-      if: always()
       run: |
         tar -cvzf sandbox.tar.gz test/sandbox
 
     - name: Generate artifact name
-      if: always()
       id: generate-artifact-name
       run: |
         txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
diff --git a/requirements.txt b/requirements.txt
index c8d05440..c48fc7eb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 numpy>=1.19.4; python_version<"3.7"
 numpy>=1.20.0; python_version>="3.7"
 numba
+numba-scipy
 scipy
 swig >= 4.1.1
 cmake
\ No newline at end of file

From 5b738f1ee2b774b354eb8778677dd04c5d634cc0 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 9 Apr 2024 11:10:12 -0700
Subject: [PATCH 25/80] remove sandbox

---
 .github/workflows/build-and-test.yml | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 3cbd37f2..d806e194 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -33,19 +33,20 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Create PYTHONLIB and set PYTHONPATH
-      run: |
-        PYTHONLIB=${HOME}/sandbox/lib/python${{ matrix.python-version }}/site-packages
-        mkdir -p $PYTHONLIB
-        echo "PYTHONPATH=$PYTHONLIB:$PYTHONPATH" >> $GITHUB_ENV
+    # - name: Create PYTHONLIB and set PYTHONPATH
+    #   run: |
+    #     PYTHONLIB=${HOME}/sandbox/lib/python${{ matrix.python-version }}/site-packages
+    #     mkdir -p $PYTHONLIB
+    #     echo "PYTHONPATH=$PYTHONLIB:$PYTHONPATH" >> $GITHUB_ENV
 
-    - name: Set PATH
-      run: |
-        echo "$HOME/sandbox/bin" >> $GITHUB_PATH
+    # - name: Set PATH
+    #   run: |
+    #     echo "$HOME/sandbox/bin" >> $GITHUB_PATH
 
     - name: Install core dependencies via requirements.txt
       run: |
-        pip install -r requirements.txt --prefix=$HOME/sandbox --verbose
+        pip install -r requirements.txt --verbose
+    #--prefix=$HOME/sandbox 
 
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
@@ -57,7 +58,8 @@ jobs:
       if: ${{ matrix.parallel == true }}
       run: |
         sudo apt-get install mpich libmpich-dev
-        pip install mpi4py --prefix=$HOME/sandbox
+        pip install mpi4py
+      #--prefix=$HOME/sandbox
 
     - name: Print python package versions
       run: |
@@ -75,7 +77,8 @@ jobs:
     - name: Build parallel
       if: ${{ matrix.parallel == true }}
       run: |
-        python setup.py install --with-gslib --with-parallel --prefix=$HOME/sandbox
+        python setup.py install --with-gslib --with-parallel
+      # --prefix=$HOME/sandbox
 
     - name: Run serial
       if: ${{ matrix.parallel == false }}

From 0bd3beb30d38813c0adab319abbe6c43710cea6e Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 9 Apr 2024 12:36:58 -0700
Subject: [PATCH 26/80] more cleanup

---
 .github/workflows/build-and-test.yml | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index d806e194..cb856953 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -26,27 +26,17 @@ jobs:
        CUDA: "11.5"
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Checkout repo
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
-    # - name: Create PYTHONLIB and set PYTHONPATH
-    #   run: |
-    #     PYTHONLIB=${HOME}/sandbox/lib/python${{ matrix.python-version }}/site-packages
-    #     mkdir -p $PYTHONLIB
-    #     echo "PYTHONPATH=$PYTHONLIB:$PYTHONPATH" >> $GITHUB_ENV
-
-    # - name: Set PATH
-    #   run: |
-    #     echo "$HOME/sandbox/bin" >> $GITHUB_PATH
-
     - name: Install core dependencies via requirements.txt
       run: |
         pip install -r requirements.txt --verbose
-    #--prefix=$HOME/sandbox 
 
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
@@ -59,7 +49,6 @@ jobs:
       run: |
         sudo apt-get install mpich libmpich-dev
         pip install mpi4py
-      #--prefix=$HOME/sandbox
 
     - name: Print python package versions
       run: |
@@ -78,7 +67,6 @@ jobs:
       if: ${{ matrix.parallel == true }}
       run: |
         python setup.py install --with-gslib --with-parallel
-      # --prefix=$HOME/sandbox
 
     - name: Run serial
       if: ${{ matrix.parallel == false }}

From 39783013acc382b68252a4a1da6f939c9140210a Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 9 Apr 2024 17:26:29 -0700
Subject: [PATCH 27/80] trying flag for phased build

---
 .github/workflows/build-and-test.yml | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index cb856953..8cbe25dc 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -19,11 +19,16 @@ jobs:
 
         cuda: [false]
         parallel: [true, false]
+        # When phased==true, run each individual build step explicitly: mfem -> swig -> pymfem
+        phased: [true, false]
         libceed: [false]
+        gslib: [true]
 
     runs-on: ${{ matrix.os }}
     env:
        CUDA: "11.5"
+       build-flags: ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}
+                    ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
 
     steps:
     - name: Checkout repo
@@ -54,19 +59,19 @@ jobs:
       run: |
         pip list
 
-    - name: Build serial
-      if: ${{ matrix.parallel == false }}
+    - name: Build (phased)
+      if: ${{ matrix.phased == true }}
       run: |
         # test workflow to manually run swig
         python setup.py clean --swig
-        python setup.py install --ext-only --with-gslib --verbose
-        python setup.py install --swig --with-gslib --verbose
-        python setup.py install --skip-ext --skip-swig --with-gslib --verbose
+        python setup.py install --ext-only --verbose ${{ env.build-flags }}
+        python setup.py install --swig --verbose ${{ env.build-flags }}
+        python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
 
-    - name: Build parallel
-      if: ${{ matrix.parallel == true }}
+    - name: Build
+      if: ${{ matrix.phased == false }}
       run: |
-        python setup.py install --with-gslib --with-parallel
+        python setup.py install ${{ env.build-flags }}
 
     - name: Run serial
       if: ${{ matrix.parallel == false }}

From 11c0a24e200bc588d02ab55c5ff671c9d838e768 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 9 Apr 2024 17:47:26 -0700
Subject: [PATCH 28/80] Adding in rest of flags (WIP)

---
 .github/workflows/build-and-test.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 8cbe25dc..1413f165 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -1,12 +1,12 @@
 # This is a clean-up/refactor of test_with_MFEM_release.yml
-name: build-and-test
+name: Build and Test
 
 on:
   workflow_dispatch:
   pull_request:
 
 jobs:
-  build:
+  build-and-test:
     # if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
     strategy:
       fail-fast: true
@@ -17,18 +17,25 @@ jobs:
         #, macos-latest]
         # os: [ubuntu-20.04]
 
+        # right now, default is a hardcoded sha defined in setup.py:repos_sha
+        mfem-branch: [master, default]
         cuda: [false]
         parallel: [true, false]
         # When phased==true, run each individual build step explicitly: mfem -> swig -> pymfem
         phased: [true, false]
         libceed: [false]
         gslib: [true]
+    name: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+
 
     runs-on: ${{ matrix.os }}
     env:
        CUDA: "11.5"
        build-flags: ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
+                    ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
+                    ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
+                    ${{ ((matrix.mfem-branch == default) && '') || '--mfem-branch='${{ matrix.mfem-branch }}'' }}
 
     steps:
     - name: Checkout repo

From 8667ef5b94551ff459f69d8167813f868fdb2720 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 13:20:55 -0700
Subject: [PATCH 29/80] change matrix so that 'phases' build is only tested
 once

---
 .github/workflows/build-and-test.yml | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 1413f165..22a7b997 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -15,16 +15,23 @@ jobs:
         #"3.7", "3.8", "3.9", "3.10"]
         os: [ubuntu-latest]
         #, macos-latest]
-        # os: [ubuntu-20.04]
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
         mfem-branch: [master, default]
+        parallel: [false, true]
         cuda: [false]
-        parallel: [true, false]
-        # When phased==true, run each individual build step explicitly: mfem -> swig -> pymfem
-        phased: [true, false]
         libceed: [false]
         gslib: [true]
+        # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem
+        phases: [false]
+        include:
+          # Include a single example where the build is executed in phases
+          - parallel: false
+            cuda: false
+            libceed: false
+            gslib: true
+            phases: true
+
     name: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
 
 
@@ -66,8 +73,8 @@ jobs:
       run: |
         pip list
 
-    - name: Build (phased)
-      if: ${{ matrix.phased == true }}
+    - name: Build (phases)
+      if: ${{ matrix.phases == true }}
       run: |
         # test workflow to manually run swig
         python setup.py clean --swig
@@ -76,7 +83,7 @@ jobs:
         python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
 
     - name: Build
-      if: ${{ matrix.phased == false }}
+      if: ${{ matrix.phases == false }}
       run: |
         python setup.py install ${{ env.build-flags }}
 

From 13135265fb3f2874990e90ca130158b783f8cbf0 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 15:47:11 -0700
Subject: [PATCH 30/80] fix job name, exclude macos+cuda, add cuda run file

---
 .github/workflows/build-and-test.yml | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 22a7b997..b0f30bf5 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -19,7 +19,7 @@ jobs:
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
         mfem-branch: [master, default]
         parallel: [false, true]
-        cuda: [false]
+        cuda: [false, true]
         libceed: [false]
         gslib: [true]
         # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem
@@ -31,17 +31,26 @@ jobs:
             libceed: false
             gslib: true
             phases: true
-
-    name: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-
+        exclude:
+          # CUDA does not support MacOS
+          - os: macos-latest
+            cuda: true
+
+    name: ${{ matrix.os }}-
+          ${{ matrix.python-version }}-
+          ${{ ((matrix.parallel == true) && 'par-') || '' }}
+          ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
+          ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
+          ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
+          ${{ ((matrix.mfem-branch == default) && '') || 'mfem='${{ matrix.mfem-branch }}'' }}
 
     runs-on: ${{ matrix.os }}
     env:
        CUDA: "11.5"
        build-flags: ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}
-                    ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
+                    ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
+                    ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
                     ${{ ((matrix.mfem-branch == default) && '') || '--mfem-branch='${{ matrix.mfem-branch }}'' }}
 
     steps:
@@ -60,8 +69,8 @@ jobs:
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
-        source ./ci_scripts/add_cuda_11_5.sh;
-        echo "/usr/local/cuda-${CUDA}/bin" >> $GITHUB_PATH
+        wget https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
+        sudo sh cuda_12.4.1_550.54.15_linux.run
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}

From 20123cf720f71597164c884952588828d1c7b53d Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 15:49:49 -0700
Subject: [PATCH 31/80] fix typo

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index b0f30bf5..f85bd2b7 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -42,7 +42,7 @@ jobs:
           ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
           ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
           ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-          ${{ ((matrix.mfem-branch == default) && '') || 'mfem='${{ matrix.mfem-branch }}'' }}
+          ${{ ((matrix.mfem-branch == 'default') && '') || 'mfem='${{ matrix.mfem-branch }}'' }}
 
     runs-on: ${{ matrix.os }}
     env:
@@ -51,7 +51,7 @@ jobs:
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
                     ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    ${{ ((matrix.mfem-branch == default) && '') || '--mfem-branch='${{ matrix.mfem-branch }}'' }}
+                    ${{ ((matrix.mfem-branch == 'default') && '') || '--mfem-branch='${{ matrix.mfem-branch }}'' }}
 
     steps:
     - name: Checkout repo

From 84befb81d6e1eeef80328eaecd8e042d7ced4e54 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 15:51:27 -0700
Subject: [PATCH 32/80] fix formatting

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f85bd2b7..95fe81fe 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -42,7 +42,7 @@ jobs:
           ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
           ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
           ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-          ${{ ((matrix.mfem-branch == 'default') && '') || 'mfem='${{ matrix.mfem-branch }}'' }}
+          ${{ ((matrix.mfem-branch == 'default') && '') || 'mfem='${{ matrix.mfem-branch }} }}
 
     runs-on: ${{ matrix.os }}
     env:
@@ -51,7 +51,7 @@ jobs:
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
                     ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    ${{ ((matrix.mfem-branch == 'default') && '') || '--mfem-branch='${{ matrix.mfem-branch }}'' }}
+                    ${{ ((matrix.mfem-branch == 'default') && '') || '--mfem-branch='${{ matrix.mfem-branch }} }}
 
     steps:
     - name: Checkout repo

From 8a8ce354a3c48351ff177aa80e9e6d21d6044775 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 16:05:04 -0700
Subject: [PATCH 33/80] vscode linter doesn't like it, but is it the right
 syntax?

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 95fe81fe..21f15aa0 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -42,7 +42,7 @@ jobs:
           ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
           ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
           ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-          ${{ ((matrix.mfem-branch == 'default') && '') || 'mfem='${{ matrix.mfem-branch }} }}
+          ${{ ((matrix.mfem-branch == 'default') && '') || 'mfem=${{ matrix.mfem-branch }}' }}
 
     runs-on: ${{ matrix.os }}
     env:
@@ -51,7 +51,7 @@ jobs:
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
                     ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    ${{ ((matrix.mfem-branch == 'default') && '') || '--mfem-branch='${{ matrix.mfem-branch }} }}
+                    ${{ ((matrix.mfem-branch == 'default') && '') || '--mfem-branch=${{ matrix.mfem-branch }}' }}
 
     steps:
     - name: Checkout repo

From 926d52cdb134a042b2f063b7ae973559cf79aba8 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 16:08:38 -0700
Subject: [PATCH 34/80] trying gh action format() instead (I don't think nested
 variables works)

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 21f15aa0..7af79b08 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -42,7 +42,7 @@ jobs:
           ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
           ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
           ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-          ${{ ((matrix.mfem-branch == 'default') && '') || 'mfem=${{ matrix.mfem-branch }}' }}
+          ${{ ((matrix.mfem-branch == 'default') && '') || format('mfem={0}', matrix.mfem-branch) }}
 
     runs-on: ${{ matrix.os }}
     env:
@@ -51,7 +51,7 @@ jobs:
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
                     ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    ${{ ((matrix.mfem-branch == 'default') && '') || '--mfem-branch=${{ matrix.mfem-branch }}' }}
+                    ${{ ((matrix.mfem-branch == 'default') && '') || format('--mfem-branch={0}', matrix.mfem-branch) }}
 
     steps:
     - name: Checkout repo

From 81734d566a61ace92da3a060e0ce01ff22ce8af0 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 16:40:29 -0700
Subject: [PATCH 35/80] another formatting attempt

---
 .github/workflows/build-and-test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 7af79b08..b2a94753 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -36,13 +36,14 @@ jobs:
           - os: macos-latest
             cuda: true
 
+    # https://7tonshark.com/posts/github-actions-ternary-operator/
     name: ${{ matrix.os }}-
           ${{ matrix.python-version }}-
           ${{ ((matrix.parallel == true) && 'par-') || '' }}
           ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
           ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
           ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-          ${{ ((matrix.mfem-branch == 'default') && '') || format('mfem={0}', matrix.mfem-branch) }}
+          ${{ (!(matrix.mfem-branch == 'default') && format('mfem={0}', matrix.mfem-branch)) || '' }}
 
     runs-on: ${{ matrix.os }}
     env:
@@ -51,7 +52,7 @@ jobs:
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
                     ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    ${{ ((matrix.mfem-branch == 'default') && '') || format('--mfem-branch={0}', matrix.mfem-branch) }}
+                    ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
 
     steps:
     - name: Checkout repo

From 9c3f999868510180ce794f2bc0e631006277c94a Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 16:43:44 -0700
Subject: [PATCH 36/80] another formatting attempt

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index b2a94753..fb5728b3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -43,7 +43,7 @@ jobs:
           ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
           ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
           ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-          ${{ (!(matrix.mfem-branch == 'default') && format('mfem={0}', matrix.mfem-branch)) || '' }}
+          # ${{ (!(matrix.mfem-branch == 'default') && format('mfem={0}', matrix.mfem-branch)) || '' }}
 
     runs-on: ${{ matrix.os }}
     env:
@@ -52,7 +52,7 @@ jobs:
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
                     ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
+                    # ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
 
     steps:
     - name: Checkout repo

From e411cd492a8b2046205d082c64473c2b47e159f8 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 16:46:40 -0700
Subject: [PATCH 37/80] another formatting attempt

---
 .github/workflows/build-and-test.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index fb5728b3..5b13bc81 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -24,6 +24,11 @@ jobs:
         gslib: [true]
         # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem
         phases: [false]
+
+        exclude:
+          # CUDA does not support MacOS
+          - os: macos-latest
+            cuda: true
         include:
           # Include a single example where the build is executed in phases
           - parallel: false
@@ -31,10 +36,6 @@ jobs:
             libceed: false
             gslib: true
             phases: true
-        exclude:
-          # CUDA does not support MacOS
-          - os: macos-latest
-            cuda: true
 
     # https://7tonshark.com/posts/github-actions-ternary-operator/
     name: ${{ matrix.os }}-
@@ -43,7 +44,6 @@ jobs:
           ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
           ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
           ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-          # ${{ (!(matrix.mfem-branch == 'default') && format('mfem={0}', matrix.mfem-branch)) || '' }}
 
     runs-on: ${{ matrix.os }}
     env:
@@ -52,7 +52,6 @@ jobs:
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
                     ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    # ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
 
     steps:
     - name: Checkout repo

From f7fba223387bd8d243a79186e21c6e1be430fa49 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 16:47:30 -0700
Subject: [PATCH 38/80] another formatting attempt

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 5b13bc81..cbffe126 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -37,6 +37,8 @@ jobs:
             gslib: true
             phases: true
 
+    runs-on: ${{ matrix.os }}
+
     # https://7tonshark.com/posts/github-actions-ternary-operator/
     name: ${{ matrix.os }}-
           ${{ matrix.python-version }}-
@@ -44,8 +46,6 @@ jobs:
           ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
           ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
           ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-
-    runs-on: ${{ matrix.os }}
     env:
        CUDA: "11.5"
        build-flags: ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}

From 467ea002554b58e5c885faf837e43aec15dd5ef8 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 16:51:07 -0700
Subject: [PATCH 39/80] another formatting attempt

---
 .github/workflows/build-and-test.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index cbffe126..f4529166 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -39,13 +39,6 @@ jobs:
 
     runs-on: ${{ matrix.os }}
 
-    # https://7tonshark.com/posts/github-actions-ternary-operator/
-    name: ${{ matrix.os }}-
-          ${{ matrix.python-version }}-
-          ${{ ((matrix.parallel == true) && 'par-') || '' }}
-          ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
-          ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
-          ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
     env:
        CUDA: "11.5"
        build-flags: ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}

From c3452003bd32f9995b4107fad2fe9de0b9ec9eeb Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 16:55:30 -0700
Subject: [PATCH 40/80] another formatting attempt; os needed to be in
 'include'

---
 .github/workflows/build-and-test.yml | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f4529166..fb712ff1 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -31,7 +31,9 @@ jobs:
             cuda: true
         include:
           # Include a single example where the build is executed in phases
-          - parallel: false
+          - os: ubuntu-latest
+            mfem-branch: default
+            parallel: false
             cuda: false
             libceed: false
             gslib: true
@@ -39,12 +41,22 @@ jobs:
 
     runs-on: ${{ matrix.os }}
 
+    # https://7tonshark.com/posts/github-actions-ternary-operator/
+    name: ${{ matrix.os }}-
+          ${{ matrix.python-version }}-
+          ${{ ((matrix.parallel == true) && 'par-') || '' }}
+          ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
+          ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
+          ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
+          ${{ (!(matrix.mfem-branch == 'default') && format('mfem={0}', matrix.mfem-branch)) || '' }}
+
     env:
        CUDA: "11.5"
        build-flags: ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}
                     ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
                     ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
                     ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
+                    ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
 
     steps:
     - name: Checkout repo

From 0424edb8d6584ae3f0dd738907902a25de8e0403 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 17:01:18 -0700
Subject: [PATCH 41/80] use yaml > syntax

---
 .github/workflows/build-and-test.yml | 31 +++++++++++++++-------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index fb712ff1..6e5542b9 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -11,10 +11,10 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.9"]
-        #"3.7", "3.8", "3.9", "3.10"]
         os: [ubuntu-latest]
         #, macos-latest]
+        python-version: [3.9]
+        #"3.7", "3.8", "3.9", "3.10"]
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
         mfem-branch: [master, default]
@@ -32,6 +32,7 @@ jobs:
         include:
           # Include a single example where the build is executed in phases
           - os: ubuntu-latest
+            python-version: 3.9
             mfem-branch: default
             parallel: false
             cuda: false
@@ -42,21 +43,23 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     # https://7tonshark.com/posts/github-actions-ternary-operator/
-    name: ${{ matrix.os }}-
-          ${{ matrix.python-version }}-
-          ${{ ((matrix.parallel == true) && 'par-') || '' }}
-          ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
-          ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
-          ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-          ${{ (!(matrix.mfem-branch == 'default') && format('mfem={0}', matrix.mfem-branch)) || '' }}
+    name: >
+      ${{ matrix.os }}-
+      ${{ matrix.python-version }}-
+      ${{ ((matrix.parallel == true) && 'par-') || '' }}
+      ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
+      ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
+      ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
+      ${{ (!(matrix.mfem-branch == 'default') && format('mfem={0}', matrix.mfem-branch)) || '' }}
 
     env:
        CUDA: "11.5"
-       build-flags: ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}
-                    ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
-                    ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
-                    ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
-                    ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
+       build-flags: >
+        ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}
+        ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
+        ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
+        ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
+        ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
 
     steps:
     - name: Checkout repo

From 1764ccaf79b0e27ffbc2bb6f269bfaab8cbd3993 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 17:30:00 -0700
Subject: [PATCH 42/80] update name

---
 .github/workflows/build-and-test.yml | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 6e5542b9..67cd62c0 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -43,14 +43,12 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     # https://7tonshark.com/posts/github-actions-ternary-operator/
-    name: >
-      ${{ matrix.os }}-
-      ${{ matrix.python-version }}-
-      ${{ ((matrix.parallel == true) && 'par-') || '' }}
-      ${{ ((matrix.cuda == true) && 'cuda-') || '' }}
-      ${{ ((matrix.libceed == true) && 'libceed-') || '' }}
-      ${{ ((matrix.gslib == true) && 'gslib-') || '' }}
-      ${{ (!(matrix.mfem-branch == 'default') && format('mfem={0}', matrix.mfem-branch)) || '' }}
+    name: >-
+      ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.mfem-branch }}
+      (${{ matrix.parallel && 'parallel' || '' }},
+       ${{ matrix.cuda && 'cuda' || '' }},
+       ${{ matrix.libceed && 'libceed' || '' }},
+       ${{ matrix.gslib && 'gslib' || '' }})
 
     env:
        CUDA: "11.5"
@@ -78,7 +76,7 @@ jobs:
       if: ${{ matrix.cuda == true }}
       run: |
         wget https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        sudo sh cuda_12.4.1_550.54.15_linux.run
+        ./cuda_12.4.1_550.54.15_linux.run
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}

From 7dbb88f54173ed8ca6a529b121a9155e119b81fd Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 17:43:05 -0700
Subject: [PATCH 43/80] change cuda install to sudo and update name/flag
 formatting

---
 .github/workflows/build-and-test.yml | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 67cd62c0..40f72ade 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -44,19 +44,18 @@ jobs:
 
     # https://7tonshark.com/posts/github-actions-ternary-operator/
     name: >-
-      ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.mfem-branch }}
-      (${{ matrix.parallel && 'parallel' || '' }},
-       ${{ matrix.cuda && 'cuda' || '' }},
-       ${{ matrix.libceed && 'libceed' || '' }},
-       ${{ matrix.gslib && 'gslib' || '' }})
+      ${{ matrix.os }}
+      py${{ matrix.python-version }}
+      mfem=${{ matrix.mfem-branch }}
+      (${{ matrix.parallel && 'par' || 'ser' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + ceed' || '' }}${{ matrix.gslib && ' + gs' || '' }})
 
     env:
        CUDA: "11.5"
-       build-flags: >
-        ${{ ((matrix.parallel == true) && '--with-parallel ') || '' }}
-        ${{ ((matrix.cuda == true) && '--with-cuda ') || '' }}
-        ${{ ((matrix.libceed == true) && '--with-libceed ') || '' }}
-        ${{ ((matrix.gslib == true) && '--with-gslib ') || '' }}
+       build-flags: >-
+        ${{ matrix.parallel && '--with-parallel' || '' }}
+        ${{ matrix.cuda && '--with-cuda' || '' }}
+        ${{ matrix.libceed && '--with-libceed' || '' }}
+        ${{ matrix.gslib && '--with-gslib' || '' }}
         ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
 
     steps:
@@ -76,7 +75,7 @@ jobs:
       if: ${{ matrix.cuda == true }}
       run: |
         wget https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        ./cuda_12.4.1_550.54.15_linux.run
+        sudo bash cuda_12.4.1_550.54.15_linux.run
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}

From a19a326ea272f4bd325dec9d3d917943e2dd685e Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 18:03:56 -0700
Subject: [PATCH 44/80] update cuda install

---
 .github/workflows/build-and-test.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 40f72ade..b4cb2422 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -74,8 +74,9 @@ jobs:
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
-        wget https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        sudo bash cuda_12.4.1_550.54.15_linux.run
+        wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
+        sudo chmod +x cuda_12.4.1_550.54.15_linux.run
+        sudo ./cuda_12.4.1_550.54.15_linux.run -silent
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}
@@ -112,7 +113,7 @@ jobs:
         cd test
         python run_examples.py -parallel -verbose -np 2
 
-    - name: Generate Artifact
+    - name: Generate test results artifact
       run: |
         tar -cvzf sandbox.tar.gz test/sandbox
 
@@ -120,12 +121,12 @@ jobs:
       id: generate-artifact-name
       run: |
         txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
-        name="test_result_"${txt}"_"${{ github.run_id }}".tar.gz"
+        name="test_results_"${txt}"_"${{ github.run_id }}".tar.gz"
         echo "name=${name}" >> $GITHUB_OUTPUT
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4
-      # if: failure()
+      if: failure()
       with:
           name: ${{ steps.generate-artifact-name.outputs.name }}
           path: sandbox.tar.gz

From 234a33ebfa5947c9ecf8b914640e4124c8178433 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 18:08:02 -0700
Subject: [PATCH 45/80] update cuda install

---
 .github/workflows/build-and-test.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index b4cb2422..7a29005c 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -44,10 +44,10 @@ jobs:
 
     # https://7tonshark.com/posts/github-actions-ternary-operator/
     name: >-
-      ${{ matrix.os }}
-      py${{ matrix.python-version }}
-      mfem=${{ matrix.mfem-branch }}
-      (${{ matrix.parallel && 'par' || 'ser' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + ceed' || '' }}${{ matrix.gslib && ' + gs' || '' }})
+      ${{ matrix.os }},
+      ${{ matrix.python-version }},
+      ${{ matrix.mfem-branch }},
+      (${{ matrix.parallel && 'parallel' || 'serial' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + libceed' || '' }}${{ matrix.gslib && ' + gslib' || '' }})
 
     env:
        CUDA: "11.5"
@@ -76,7 +76,7 @@ jobs:
       run: |
         wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
         sudo chmod +x cuda_12.4.1_550.54.15_linux.run
-        sudo ./cuda_12.4.1_550.54.15_linux.run -silent
+        sudo ./cuda_12.4.1_550.54.15_linux.run --silent
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}

From 8d00a874ee2f4d003416a19f764eb2ea560f93f8 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 18:14:45 -0700
Subject: [PATCH 46/80] need to escape single quote for mfem-branch flag

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 7a29005c..2c4469af 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -56,7 +56,7 @@ jobs:
         ${{ matrix.cuda && '--with-cuda' || '' }}
         ${{ matrix.libceed && '--with-libceed' || '' }}
         ${{ matrix.gslib && '--with-gslib' || '' }}
-        ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch={0}', matrix.mfem-branch)) || '' }}
+        ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch=''{0}''', matrix.mfem-branch)) || '' }}
 
     steps:
     - name: Checkout repo

From 69397bdac0ef5e9423b9a132d9861359490fc1d8 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Wed, 10 Apr 2024 18:15:11 -0700
Subject: [PATCH 47/80] remove CUDA env var

---
 .github/workflows/build-and-test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 2c4469af..0af675a6 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -50,7 +50,6 @@ jobs:
       (${{ matrix.parallel && 'parallel' || 'serial' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + libceed' || '' }}${{ matrix.gslib && ' + gslib' || '' }})
 
     env:
-       CUDA: "11.5"
        build-flags: >-
         ${{ matrix.parallel && '--with-parallel' || '' }}
         ${{ matrix.cuda && '--with-cuda' || '' }}

From 0096669ded51f35b5a8f920a808a945dc1987e1a Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 14:58:39 -0700
Subject: [PATCH 48/80] change run to source for cuda

---
 .github/workflows/build-and-test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 0af675a6..a5ef5ad3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -70,12 +70,13 @@ jobs:
       run: |
         pip install -r requirements.txt --verbose
 
+    # The --silent flag will bypass user-input, e.g. implicitly accepting the EULA
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
         wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
         sudo chmod +x cuda_12.4.1_550.54.15_linux.run
-        sudo ./cuda_12.4.1_550.54.15_linux.run --silent
+        source cuda_12.4.1_550.54.15_linux.run --silent
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}

From ddfa4b5ed4a37d055cf001169b8674b33848993e Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 15:13:05 -0700
Subject: [PATCH 49/80] use sudo sh for cuda run script

---
 .github/workflows/build-and-test.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a5ef5ad3..14fcb503 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -17,9 +17,13 @@ jobs:
         #"3.7", "3.8", "3.9", "3.10"]
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
-        mfem-branch: [master, default]
-        parallel: [false, true]
-        cuda: [false, true]
+        # mfem-branch: [master, default]
+        # parallel: [false, true]
+        # cuda: [false, true]
+        # temporary: only test cuda
+        mfem-branch: [default]
+        parallel: [true]
+        cuda: [true]
         libceed: [false]
         gslib: [true]
         # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem
@@ -75,8 +79,7 @@ jobs:
       if: ${{ matrix.cuda == true }}
       run: |
         wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        sudo chmod +x cuda_12.4.1_550.54.15_linux.run
-        source cuda_12.4.1_550.54.15_linux.run --silent
+        sudo sh cuda_12.4.1_550.54.15_linux.run --silent
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}

From 022c7f3a6855219eb9ab676286cfe42a0ed542ea Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 15:23:24 -0700
Subject: [PATCH 50/80] debugging

---
 .github/workflows/build-and-test.yml | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 14fcb503..3daf492a 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -22,7 +22,7 @@ jobs:
         # cuda: [false, true]
         # temporary: only test cuda
         mfem-branch: [default]
-        parallel: [true]
+        parallel: [false]
         cuda: [true]
         libceed: [false]
         gslib: [true]
@@ -33,16 +33,16 @@ jobs:
           # CUDA does not support MacOS
           - os: macos-latest
             cuda: true
-        include:
-          # Include a single example where the build is executed in phases
-          - os: ubuntu-latest
-            python-version: 3.9
-            mfem-branch: default
-            parallel: false
-            cuda: false
-            libceed: false
-            gslib: true
-            phases: true
+        # include:
+        #   # Include a single example where the build is executed in phases
+        #   - os: ubuntu-latest
+        #     python-version: 3.9
+        #     mfem-branch: default
+        #     parallel: false
+        #     cuda: false
+        #     libceed: false
+        #     gslib: true
+        #     phases: true
 
     runs-on: ${{ matrix.os }}
 
@@ -80,6 +80,8 @@ jobs:
       run: |
         wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
         sudo sh cuda_12.4.1_550.54.15_linux.run --silent
+        which sh
+        which nvcc
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}

From 80983ed3fef43b9f8b5e8879a4ab10744abbefc0 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 16:22:49 -0700
Subject: [PATCH 51/80] trying apt

---
 .github/workflows/build-and-test.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 3daf492a..90c39577 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -78,10 +78,9 @@ jobs:
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
-        wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        sudo sh cuda_12.4.1_550.54.15_linux.run --silent
-        which sh
+        sudo apt install nvidia-cuda-toolkit -y
         which nvcc
+        nvcc --version
 
     - name: Install MPI
       if: ${{ matrix.parallel == true }}

From db1fdcef4724a7b458339e59df7806e3c0a9e8b9 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 16:26:09 -0700
Subject: [PATCH 52/80] add update

---
 .github/workflows/build-and-test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 90c39577..7e9af2f5 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -78,7 +78,8 @@ jobs:
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
-        sudo apt install nvidia-cuda-toolkit -y
+        sudo apt-get update
+        sudo apt-get install nvidia-cuda-toolkit -y
         which nvcc
         nvcc --version
 

From 663762553c9a5c29b4d48ecbfcb52b2f07e0d17e Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 16:42:29 -0700
Subject: [PATCH 53/80] break up build into phases. test cuda and non-cuda

---
 .github/workflows/build-and-test.yml | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 7e9af2f5..2f76db0d 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -9,7 +9,7 @@ jobs:
   build-and-test:
     # if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         os: [ubuntu-latest]
         #, macos-latest]
@@ -23,7 +23,7 @@ jobs:
         # temporary: only test cuda
         mfem-branch: [default]
         parallel: [false]
-        cuda: [true]
+        cuda: [false, true]
         libceed: [false]
         gslib: [true]
         # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem
@@ -80,7 +80,6 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install nvidia-cuda-toolkit -y
-        which nvcc
         nvcc --version
 
     - name: Install MPI
@@ -93,26 +92,33 @@ jobs:
       run: |
         pip list
 
-    - name: Build (phases)
+    - name: Build MFEM (step 1)
       if: ${{ matrix.phases == true }}
       run: |
-        # test workflow to manually run swig
         python setup.py clean --swig
         python setup.py install --ext-only --verbose ${{ env.build-flags }}
+
+    - name: Build SWIG wrappers (step 2)
+      if: ${{ matrix.phases == true }}
+      run: |
         python setup.py install --swig --verbose ${{ env.build-flags }}
+
+    - name: Build PyMFEM (step 3)
+      if: ${{ matrix.phases == true }}
+      run: |
         python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
 
-    - name: Build
+    - name: Build all (steps 1-3)
       if: ${{ matrix.phases == false }}
       run: |
         python setup.py install ${{ env.build-flags }}
 
-    - name: Run serial
+    - name: Run tests (serial)
       if: ${{ matrix.parallel == false }}
       run: |
         cd test
         python run_examples.py -serial -verbose
-    - name: Run parallel
+    - name: Run tests (parallel)
       if: ${{ matrix.parallel == true }}
       run: |
         cd test

From 4a631a39b7c7da19a963112fbe00a531acf272da Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 17:16:02 -0700
Subject: [PATCH 54/80] toolkit flag

---
 .github/workflows/build-and-test.yml | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 2f76db0d..837c1780 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -78,8 +78,22 @@ jobs:
     - name: Install CUDA
       if: ${{ matrix.cuda == true }}
       run: |
-        sudo apt-get update
-        sudo apt-get install nvidia-cuda-toolkit -y
+        # apt
+        # sudo apt-get update
+        # sudo apt-get install nvidia-cuda-toolkit -y
+
+        # script
+        # source ./ci_scripts/add_cuda_11_5.sh;
+
+        # run file
+        wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
+        sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit --driver
+
+        # paths
+        export "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH
+
+        # check
+        which nvcc
         nvcc --version
 
     - name: Install MPI

From b6610d0c88e3b464c620326ec7046d6802019703 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 17:19:26 -0700
Subject: [PATCH 55/80] test only cuda

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 837c1780..d35d1d19 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -9,7 +9,7 @@ jobs:
   build-and-test:
     # if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os: [ubuntu-latest]
         #, macos-latest]
@@ -23,7 +23,7 @@ jobs:
         # temporary: only test cuda
         mfem-branch: [default]
         parallel: [false]
-        cuda: [false, true]
+        cuda: [true]
         libceed: [false]
         gslib: [true]
         # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem

From f01f7ad4d3304739a1b2544501b2f16e7f03692f Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 17:24:28 -0700
Subject: [PATCH 56/80] use cuda script

---
 .github/workflows/build-and-test.yml | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index d35d1d19..78d7ca56 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -83,11 +83,11 @@ jobs:
         # sudo apt-get install nvidia-cuda-toolkit -y
 
         # script
-        # source ./ci_scripts/add_cuda_11_5.sh;
+        source ./ci_scripts/add_cuda_11_5.sh;
 
         # run file
-        wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit --driver
+        # wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
+        # sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit --driver
 
         # paths
         export "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH
@@ -139,20 +139,17 @@ jobs:
         python run_examples.py -parallel -verbose -np 2
 
     - name: Generate test results artifact
+      id: generate-artifact
       run: |
         tar -cvzf sandbox.tar.gz test/sandbox
-
-    - name: Generate artifact name
-      id: generate-artifact-name
-      run: |
+        # generate a name for the artifact
         txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
-        name="test_results_"${txt}"_"${{ github.run_id }}".tar.gz"
-        echo "name=${name}" >> $GITHUB_OUTPUT
+        echo name="test_results_"${txt}"_"${{ github.run_id }}".tar.gz" >> $GITHUB_OUTPUT
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4
       if: failure()
       with:
-          name: ${{ steps.generate-artifact-name.outputs.name }}
+          name: ${{ steps.generate-artifact.outputs.name }}
           path: sandbox.tar.gz
           retention-days: 1

From 07ea801eb1981ca453c9c9eaa11ba739071fed4e Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 17:26:11 -0700
Subject: [PATCH 57/80] typo

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 78d7ca56..196fc9a8 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -90,7 +90,7 @@ jobs:
         # sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit --driver
 
         # paths
-        export "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH
+        echo "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH
 
         # check
         which nvcc

From d3c33fb9346d75b9e4d747499e2d1ebbb422b201 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 17:26:29 -0700
Subject: [PATCH 58/80] compare with run file

---
 .github/workflows/build-and-test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 196fc9a8..e38060b0 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -83,11 +83,11 @@ jobs:
         # sudo apt-get install nvidia-cuda-toolkit -y
 
         # script
-        source ./ci_scripts/add_cuda_11_5.sh;
+        # source ./ci_scripts/add_cuda_11_5.sh;
 
         # run file
-        # wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        # sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit --driver
+        wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
+        sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit --driver
 
         # paths
         echo "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH

From d80b4d6fe66637ac4967087e1d0bece57e3adfe8 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Mon, 15 Apr 2024 17:40:39 -0700
Subject: [PATCH 59/80] remove driver flag

---
 .github/workflows/build-and-test.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index e38060b0..dbdffe40 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -87,12 +87,15 @@ jobs:
 
         # run file
         wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit --driver
+        sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit
+        echo "/usr/local/cuda/bin" >> $GITHUB_PATH
 
         # paths
-        echo "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH
+        # echo "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH
 
-        # check
+    - name: (debugging) Check CUDA
+      if: ${{ matrix.cuda == true }}
+      run: |
         which nvcc
         nvcc --version
 

From 6b45cdbe86b0dcfb9544356aa3370962d2bfb0ae Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 10:45:45 -0700
Subject: [PATCH 60/80] cleanup

---
 .github/workflows/build-and-test.yml | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index dbdffe40..fddcc4f0 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -14,7 +14,7 @@ jobs:
         os: [ubuntu-latest]
         #, macos-latest]
         python-version: [3.9]
-        #"3.7", "3.8", "3.9", "3.10"]
+        #3.7, 3.8, 3.9, 3.10, 3.11
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
         # mfem-branch: [master, default]
@@ -54,6 +54,7 @@ jobs:
       (${{ matrix.parallel && 'parallel' || 'serial' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + libceed' || '' }}${{ matrix.gslib && ' + gslib' || '' }})
 
     env:
+       cuda-installer-url: https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
        build-flags: >-
         ${{ matrix.parallel && '--with-parallel' || '' }}
         ${{ matrix.cuda && '--with-cuda' || '' }}
@@ -76,18 +77,10 @@ jobs:
 
     # The --silent flag will bypass user-input, e.g. implicitly accepting the EULA
     - name: Install CUDA
-      if: ${{ matrix.cuda == true }}
+      if: matrix.cuda
       run: |
-        # apt
-        # sudo apt-get update
-        # sudo apt-get install nvidia-cuda-toolkit -y
-
-        # script
-        # source ./ci_scripts/add_cuda_11_5.sh;
-
-        # run file
-        wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-        sudo sh cuda_12.4.1_550.54.15_linux.run --silent --toolkit
+        wget -q -O cuda.run ${{ env.cuda-installer-url }}
+        sudo sh cuda.run --silent --toolkit
         echo "/usr/local/cuda/bin" >> $GITHUB_PATH
 
         # paths

From 28ae5a77709f66beb04f945ac2ed808bb6647676 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 11:20:52 -0700
Subject: [PATCH 61/80] a lot more cleanup and testing actions/cache@v4 with
 CUDA since it is large

---
 .github/workflows/build-and-test.yml | 58 ++++++++++++++--------------
 1 file changed, 28 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index fddcc4f0..78ef1c43 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -13,15 +13,15 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         #, macos-latest]
-        python-version: [3.9]
+        python-version: [3.9, 3.11]
         #3.7, 3.8, 3.9, 3.10, 3.11
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
-        # mfem-branch: [master, default]
+        mfem-branch: [master, default]
         # parallel: [false, true]
         # cuda: [false, true]
         # temporary: only test cuda
-        mfem-branch: [default]
+        # mfem-branch: [default]
         parallel: [false]
         cuda: [true]
         libceed: [false]
@@ -72,64 +72,62 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install core dependencies via requirements.txt
-      run: |
-        pip install -r requirements.txt --verbose
+      run: pip install -r requirements.txt --verbose
+
+    - name: Cache CUDA
+      if: matrix.cuda
+      id: cache-cuda
+      uses: actions/cache@v4
+      with:
+        path: cuda.run
+        key: cuda-cache
+
+    - name: Download CUDA
+      if: matrix.cuda && steps.cache-cuda.outputs.cache-hit != 'true'
+      run: wget -q -O cuda.run ${{ env.cuda-installer-url }}
 
-    # The --silent flag will bypass user-input, e.g. implicitly accepting the EULA
     - name: Install CUDA
       if: matrix.cuda
       run: |
-        wget -q -O cuda.run ${{ env.cuda-installer-url }}
+        # The --silent flag is necessary to bypass user-input, e.g. accepting the EULA
         sudo sh cuda.run --silent --toolkit
         echo "/usr/local/cuda/bin" >> $GITHUB_PATH
-
-        # paths
-        # echo "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH
-
-    - name: (debugging) Check CUDA
-      if: ${{ matrix.cuda == true }}
-      run: |
-        which nvcc
         nvcc --version
 
     - name: Install MPI
-      if: ${{ matrix.parallel == true }}
+      if: matrix.parallel
       run: |
         sudo apt-get install mpich libmpich-dev
         pip install mpi4py
 
     - name: Print python package versions
-      run: |
-        pip list
+      run: pip list
 
     - name: Build MFEM (step 1)
-      if: ${{ matrix.phases == true }}
+      if: matrix.phases
       run: |
         python setup.py clean --swig
         python setup.py install --ext-only --verbose ${{ env.build-flags }}
 
     - name: Build SWIG wrappers (step 2)
-      if: ${{ matrix.phases == true }}
-      run: |
-        python setup.py install --swig --verbose ${{ env.build-flags }}
+      if: matrix.phases
+      run: python setup.py install --swig --verbose ${{ env.build-flags }}
 
     - name: Build PyMFEM (step 3)
-      if: ${{ matrix.phases == true }}
-      run: |
-        python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
+      if: matrix.phases
+      run: python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
 
     - name: Build all (steps 1-3)
-      if: ${{ matrix.phases == false }}
-      run: |
-        python setup.py install ${{ env.build-flags }}
+      if: !matrix.phases
+      run: python setup.py install ${{ env.build-flags }}
 
     - name: Run tests (serial)
-      if: ${{ matrix.parallel == false }}
+      if: !matrix.parallel
       run: |
         cd test
         python run_examples.py -serial -verbose
     - name: Run tests (parallel)
-      if: ${{ matrix.parallel == true }}
+      if: matrix.parallel
       run: |
         cd test
         python run_examples.py -parallel -verbose -np 2

From c6f251e7df086afe618f162c3eabdbf072356eae Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 11:21:46 -0700
Subject: [PATCH 62/80] syntax fix

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 78ef1c43..9a22c6ca 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -118,11 +118,11 @@ jobs:
       run: python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
 
     - name: Build all (steps 1-3)
-      if: !matrix.phases
+      if: !(matrix.phases)
       run: python setup.py install ${{ env.build-flags }}
 
     - name: Run tests (serial)
-      if: !matrix.parallel
+      if: !(matrix.parallel)
       run: |
         cd test
         python run_examples.py -serial -verbose

From 3be2363af3b0ae01a4e3f521ced6eb9470599d4d Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 11:23:09 -0700
Subject: [PATCH 63/80] another attempt at syntax fix

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 9a22c6ca..a03a9b7d 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -118,11 +118,11 @@ jobs:
       run: python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
 
     - name: Build all (steps 1-3)
-      if: !(matrix.phases)
+      if: matrix.phases == false
       run: python setup.py install ${{ env.build-flags }}
 
     - name: Run tests (serial)
-      if: !(matrix.parallel)
+      if: matrix.parallel == false
       run: |
         cd test
         python run_examples.py -serial -verbose

From 0f320b89359eff022122d10bef5d0307d2cb5bff Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 11:42:35 -0700
Subject: [PATCH 64/80] debugging cache

---
 .github/workflows/build-and-test.yml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a03a9b7d..fef5b1f3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -79,19 +79,24 @@ jobs:
       id: cache-cuda
       uses: actions/cache@v4
       with:
-        path: cuda.run
-        key: cuda-cache
+        path: ~/cache
+        key: ${{ matrix.os }}-${{ env.cuda-installer-url }}
 
     - name: Download CUDA
       if: matrix.cuda && steps.cache-cuda.outputs.cache-hit != 'true'
-      run: wget -q -O cuda.run ${{ env.cuda-installer-url }}
+      run: wget -q -O ~/cache/cuda.run ${{ env.cuda-installer-url }}
 
     - name: Install CUDA
       if: matrix.cuda
       run: |
         # The --silent flag is necessary to bypass user-input, e.g. accepting the EULA
-        sudo sh cuda.run --silent --toolkit
+        sudo sh ~/cache/cuda.run --silent --toolkit
         echo "/usr/local/cuda/bin" >> $GITHUB_PATH
+
+    - name: Print CUDA
+      if: matrix.cuda
+      run: |
+        which nvcc
         nvcc --version
 
     - name: Install MPI

From db228b53c3d25d750d5c523e1c0b3a7b70e8bc72 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 11:50:16 -0700
Subject: [PATCH 65/80] more debugging cache

---
 .github/workflows/build-and-test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index fef5b1f3..1d054e92 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -79,18 +79,18 @@ jobs:
       id: cache-cuda
       uses: actions/cache@v4
       with:
-        path: ~/cache
+        path: ~/cuda.run
         key: ${{ matrix.os }}-${{ env.cuda-installer-url }}
 
     - name: Download CUDA
       if: matrix.cuda && steps.cache-cuda.outputs.cache-hit != 'true'
-      run: wget -q -O ~/cache/cuda.run ${{ env.cuda-installer-url }}
+      run: wget -q -O ~/cuda.run ${{ env.cuda-installer-url }}
 
     - name: Install CUDA
       if: matrix.cuda
       run: |
         # The --silent flag is necessary to bypass user-input, e.g. accepting the EULA
-        sudo sh ~/cache/cuda.run --silent --toolkit
+        sudo sh ~/cuda.run --silent --toolkit
         echo "/usr/local/cuda/bin" >> $GITHUB_PATH
 
     - name: Print CUDA

From 8ac4efdd7aae185b3d63d1560e67d632a8938532 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 12:04:12 -0700
Subject: [PATCH 66/80] debug cache

---
 .github/workflows/build-and-test.yml | 117 ++++++++++++++-------------
 1 file changed, 60 insertions(+), 57 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 1d054e92..6419cf8c 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -13,7 +13,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         #, macos-latest]
-        python-version: [3.9, 3.11]
+        python-version: [3.9]
         #3.7, 3.8, 3.9, 3.10, 3.11
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
@@ -29,6 +29,9 @@ jobs:
         # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem
         phases: [false]
 
+        cuda-version: [12.4.1]
+        cuda-driver-version: [550.54.15]
+
         exclude:
           # CUDA does not support MacOS
           - os: macos-latest
@@ -54,8 +57,8 @@ jobs:
       (${{ matrix.parallel && 'parallel' || 'serial' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + libceed' || '' }}${{ matrix.gslib && ' + gslib' || '' }})
 
     env:
-       cuda-installer-url: https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
-       build-flags: >-
+      cuda-installer-url: https://developer.download.nvidia.com/compute/cuda/${{ matrix.cuda-version }}/local_installers/cuda_${{ matrix.cuda-version }}_${{ matrix.cuda-driver-version }}_linux.run
+      build-flags: >-
         ${{ matrix.parallel && '--with-parallel' || '' }}
         ${{ matrix.cuda && '--with-cuda' || '' }}
         ${{ matrix.libceed && '--with-libceed' || '' }}
@@ -80,7 +83,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ~/cuda.run
-        key: ${{ matrix.os }}-${{ env.cuda-installer-url }}
+        key: ${{ matrix.cuda-version }}_${{ matrix.cuda-driver-version }}
 
     - name: Download CUDA
       if: matrix.cuda && steps.cache-cuda.outputs.cache-hit != 'true'
@@ -99,56 +102,56 @@ jobs:
         which nvcc
         nvcc --version
 
-    - name: Install MPI
-      if: matrix.parallel
-      run: |
-        sudo apt-get install mpich libmpich-dev
-        pip install mpi4py
-
-    - name: Print python package versions
-      run: pip list
-
-    - name: Build MFEM (step 1)
-      if: matrix.phases
-      run: |
-        python setup.py clean --swig
-        python setup.py install --ext-only --verbose ${{ env.build-flags }}
-
-    - name: Build SWIG wrappers (step 2)
-      if: matrix.phases
-      run: python setup.py install --swig --verbose ${{ env.build-flags }}
-
-    - name: Build PyMFEM (step 3)
-      if: matrix.phases
-      run: python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
-
-    - name: Build all (steps 1-3)
-      if: matrix.phases == false
-      run: python setup.py install ${{ env.build-flags }}
-
-    - name: Run tests (serial)
-      if: matrix.parallel == false
-      run: |
-        cd test
-        python run_examples.py -serial -verbose
-    - name: Run tests (parallel)
-      if: matrix.parallel
-      run: |
-        cd test
-        python run_examples.py -parallel -verbose -np 2
-
-    - name: Generate test results artifact
-      id: generate-artifact
-      run: |
-        tar -cvzf sandbox.tar.gz test/sandbox
-        # generate a name for the artifact
-        txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
-        echo name="test_results_"${txt}"_"${{ github.run_id }}".tar.gz" >> $GITHUB_OUTPUT
-
-    - name: Upload Artifact
-      uses: actions/upload-artifact@v4
-      if: failure()
-      with:
-          name: ${{ steps.generate-artifact.outputs.name }}
-          path: sandbox.tar.gz
-          retention-days: 1
+    # - name: Install MPI
+    #   if: matrix.parallel
+    #   run: |
+    #     sudo apt-get install mpich libmpich-dev
+    #     pip install mpi4py
+
+    # - name: Print python package versions
+    #   run: pip list
+
+    # - name: Build MFEM (step 1)
+    #   if: matrix.phases
+    #   run: |
+    #     python setup.py clean --swig
+    #     python setup.py install --ext-only --verbose ${{ env.build-flags }}
+
+    # - name: Build SWIG wrappers (step 2)
+    #   if: matrix.phases
+    #   run: python setup.py install --swig --verbose ${{ env.build-flags }}
+
+    # - name: Build PyMFEM (step 3)
+    #   if: matrix.phases
+    #   run: python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
+
+    # - name: Build all (steps 1-3)
+    #   if: matrix.phases == false
+    #   run: python setup.py install ${{ env.build-flags }}
+
+    # - name: Run tests (serial)
+    #   if: matrix.parallel == false
+    #   run: |
+    #     cd test
+    #     python run_examples.py -serial -verbose
+    # - name: Run tests (parallel)
+    #   if: matrix.parallel
+    #   run: |
+    #     cd test
+    #     python run_examples.py -parallel -verbose -np 2
+
+    # - name: Generate test results artifact
+    #   id: generate-artifact
+    #   run: |
+    #     tar -cvzf sandbox.tar.gz test/sandbox
+    #     # generate a name for the artifact
+    #     txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
+    #     echo name="test_results_"${txt}"_"${{ github.run_id }}".tar.gz" >> $GITHUB_OUTPUT
+
+    # - name: Upload Artifact
+    #   uses: actions/upload-artifact@v4
+    #   if: failure()
+    #   with:
+    #       name: ${{ steps.generate-artifact.outputs.name }}
+    #       path: sandbox.tar.gz
+    #       retention-days: 1

From 61a1f6f6f3661461b42702944fe5d0c23a544a2c Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 12:15:04 -0700
Subject: [PATCH 67/80] testing cuda ache

---
 .github/workflows/build-and-test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 6419cf8c..cd22b4be 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -82,18 +82,18 @@ jobs:
       id: cache-cuda
       uses: actions/cache@v4
       with:
-        path: ~/cuda.run
-        key: ${{ matrix.cuda-version }}_${{ matrix.cuda-driver-version }}
+        path: ~/cache
+        key: cuda-${{ matrix.cuda-version }}-${{ matrix.cuda-driver-version }}
 
     - name: Download CUDA
       if: matrix.cuda && steps.cache-cuda.outputs.cache-hit != 'true'
-      run: wget -q -O ~/cuda.run ${{ env.cuda-installer-url }}
+      run: curl -o ~/cache/cuda.run --create-dirs ${{ env.cuda-installer-url }}
 
     - name: Install CUDA
       if: matrix.cuda
       run: |
         # The --silent flag is necessary to bypass user-input, e.g. accepting the EULA
-        sudo sh ~/cuda.run --silent --toolkit
+        sudo sh ~/cache/cuda.run --silent --toolkit
         echo "/usr/local/cuda/bin" >> $GITHUB_PATH
 
     - name: Print CUDA

From 6b8fa84e5b84e473213624db4dc24a4cbc1abfcf Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 12:20:45 -0700
Subject: [PATCH 68/80] rerun and see if cache works

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index cd22b4be..27f46ad3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -76,7 +76,7 @@ jobs:
 
     - name: Install core dependencies via requirements.txt
       run: pip install -r requirements.txt --verbose
-
+# rerun
     - name: Cache CUDA
       if: matrix.cuda
       id: cache-cuda

From 1b3aaa490f7255db3db75f9b69216bb6ee16755c Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 12:43:12 -0700
Subject: [PATCH 69/80] cleanup; run matrix

---
 .github/workflows/build-and-test.yml | 142 +++++++++++++++------------
 1 file changed, 77 insertions(+), 65 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 27f46ad3..f3018605 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -11,9 +11,8 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-latest]
-        #, macos-latest]
-        python-version: [3.9]
+        os: [ubuntu-latest, macos-latest]
+        python-version: [3.9, 3.10, 3.11, 3.12]
         #3.7, 3.8, 3.9, 3.10, 3.11
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
@@ -27,7 +26,7 @@ jobs:
         libceed: [false]
         gslib: [true]
         # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem
-        phases: [false]
+        phases: [true]
 
         cuda-version: [12.4.1]
         cuda-driver-version: [550.54.15]
@@ -43,6 +42,8 @@ jobs:
         #     mfem-branch: default
         #     parallel: false
         #     cuda: false
+        #     cuda-version: 12.4.1
+        #     cuda-driver-version: 550.54.15
         #     libceed: false
         #     gslib: true
         #     phases: true
@@ -74,19 +75,28 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    # -------------------------------------------------------------------------------------------------
+    # Setup dependencies
+    # -------------------------------------------------------------------------------------------------
     - name: Install core dependencies via requirements.txt
       run: pip install -r requirements.txt --verbose
-# rerun
+
+    - name: Install MPI
+      if: matrix.parallel
+      run: |
+        sudo apt-get install mpich libmpich-dev
+        pip install mpi4py
+
     - name: Cache CUDA
       if: matrix.cuda
       id: cache-cuda
       uses: actions/cache@v4
       with:
         path: ~/cache
-        key: cuda-${{ matrix.cuda-version }}-${{ matrix.cuda-driver-version }}
+        key: cuda-installer-${{ matrix.cuda-version }}-${{ matrix.cuda-driver-version }}
 
     - name: Download CUDA
-      if: matrix.cuda && steps.cache-cuda.outputs.cache-hit != 'true'
+      if: matrix.cuda && steps.cache-cuda.outputs.cache-hit == false
       run: curl -o ~/cache/cuda.run --create-dirs ${{ env.cuda-installer-url }}
 
     - name: Install CUDA
@@ -96,62 +106,64 @@ jobs:
         sudo sh ~/cache/cuda.run --silent --toolkit
         echo "/usr/local/cuda/bin" >> $GITHUB_PATH
 
-    - name: Print CUDA
-      if: matrix.cuda
+    - name: Print dependency information
+      run: |
+        pip list
+        printf "\n\n---------- MPI ----------\n"
+        mpiexec --version || printf "MPI not installed"
+        printf "\n\n---------- CUDA ----------\n"
+        nvcc --version || printf "CUDA not installed"
+
+    # -------------------------------------------------------------------------------------------------
+    # Build
+    # -------------------------------------------------------------------------------------------------
+    - name: Build MFEM (step 1)
+      if: matrix.phases
       run: |
-        which nvcc
-        nvcc --version
-
-    # - name: Install MPI
-    #   if: matrix.parallel
-    #   run: |
-    #     sudo apt-get install mpich libmpich-dev
-    #     pip install mpi4py
-
-    # - name: Print python package versions
-    #   run: pip list
-
-    # - name: Build MFEM (step 1)
-    #   if: matrix.phases
-    #   run: |
-    #     python setup.py clean --swig
-    #     python setup.py install --ext-only --verbose ${{ env.build-flags }}
-
-    # - name: Build SWIG wrappers (step 2)
-    #   if: matrix.phases
-    #   run: python setup.py install --swig --verbose ${{ env.build-flags }}
-
-    # - name: Build PyMFEM (step 3)
-    #   if: matrix.phases
-    #   run: python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
-
-    # - name: Build all (steps 1-3)
-    #   if: matrix.phases == false
-    #   run: python setup.py install ${{ env.build-flags }}
-
-    # - name: Run tests (serial)
-    #   if: matrix.parallel == false
-    #   run: |
-    #     cd test
-    #     python run_examples.py -serial -verbose
-    # - name: Run tests (parallel)
-    #   if: matrix.parallel
-    #   run: |
-    #     cd test
-    #     python run_examples.py -parallel -verbose -np 2
-
-    # - name: Generate test results artifact
-    #   id: generate-artifact
-    #   run: |
-    #     tar -cvzf sandbox.tar.gz test/sandbox
-    #     # generate a name for the artifact
-    #     txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
-    #     echo name="test_results_"${txt}"_"${{ github.run_id }}".tar.gz" >> $GITHUB_OUTPUT
-
-    # - name: Upload Artifact
-    #   uses: actions/upload-artifact@v4
-    #   if: failure()
-    #   with:
-    #       name: ${{ steps.generate-artifact.outputs.name }}
-    #       path: sandbox.tar.gz
-    #       retention-days: 1
+        python setup.py clean --swig
+        python setup.py install --ext-only --verbose ${{ env.build-flags }}
+
+    - name: Build SWIG wrappers (step 2)
+      if: matrix.phases
+      run: python setup.py install --swig --verbose ${{ env.build-flags }}
+
+    - name: Build PyMFEM (step 3)
+      if: matrix.phases
+      run: python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
+
+    - name: Build all (steps 1-3)
+      if: matrix.phases == false
+      run: python setup.py install ${{ env.build-flags }}
+
+    # -------------------------------------------------------------------------------------------------
+    # Run tests
+    # -------------------------------------------------------------------------------------------------
+    - name: Run tests (serial)
+      if: matrix.parallel == false
+      run: |
+        cd test
+        python run_examples.py -serial -verbose
+    - name: Run tests (parallel)
+      if: matrix.parallel
+      run: |
+        cd test
+        python run_examples.py -parallel -verbose -np 2
+
+    # -------------------------------------------------------------------------------------------------
+    # Artifact
+    # -------------------------------------------------------------------------------------------------
+    - name: Generate test results artifact
+      id: generate-artifact
+      run: |
+        tar -cvzf sandbox.tar.gz test/sandbox
+        # generate a name for the artifact
+        txt=$(python -c "import datetime;print(datetime.datetime.now().strftime('%H_%M_%S_%f'))")
+        echo name="test_results_"${txt}"_"${{ github.run_id }}".tar.gz" >> $GITHUB_OUTPUT
+
+    - name: Upload Artifact
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+          name: ${{ steps.generate-artifact.outputs.name }}
+          path: sandbox.tar.gz
+          retention-days: 1

From 5eba2cb441a9936ce4ed6181a0d00a32f70b345c Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 12:45:44 -0700
Subject: [PATCH 70/80] python versions must be strings because of 3.10

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f3018605..e56b00db 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -12,7 +12,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.9, 3.10, 3.11, 3.12]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
         #3.7, 3.8, 3.9, 3.10, 3.11
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha

From 110a0daf61598ee8cea358e254909a494d51a293 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 16 Apr 2024 12:47:12 -0700
Subject: [PATCH 71/80] add more to matrix, turn off fail fast, update
 requirements

---
 .github/workflows/build-and-test.yml | 6 +++---
 requirements.txt                     | 3 +--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index e56b00db..dce964e8 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -9,7 +9,7 @@ jobs:
   build-and-test:
     # if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: ['3.9', '3.10', '3.11', '3.12']
@@ -17,11 +17,11 @@ jobs:
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
         mfem-branch: [master, default]
-        # parallel: [false, true]
+        parallel: [false, true]
         # cuda: [false, true]
         # temporary: only test cuda
         # mfem-branch: [default]
-        parallel: [false]
+        # parallel: [false]
         cuda: [true]
         libceed: [false]
         gslib: [true]
diff --git a/requirements.txt b/requirements.txt
index c48fc7eb..67ab34f6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,4 @@
-numpy>=1.19.4; python_version<"3.7"
-numpy>=1.20.0; python_version>="3.7"
+numpy >= 1.20.0
 numba
 numba-scipy
 scipy

From a8166ab8a658a9ae0cc09f5affba8dd11007e114 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 13:57:19 -0700
Subject: [PATCH 72/80] add back in a single case where build is run all at
 once

---
 .github/workflows/build-and-test.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index dce964e8..20a0d6e1 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -35,18 +35,18 @@ jobs:
           # CUDA does not support MacOS
           - os: macos-latest
             cuda: true
-        # include:
-        #   # Include a single example where the build is executed in phases
-        #   - os: ubuntu-latest
-        #     python-version: 3.9
-        #     mfem-branch: default
-        #     parallel: false
-        #     cuda: false
-        #     cuda-version: 12.4.1
-        #     cuda-driver-version: 550.54.15
-        #     libceed: false
-        #     gslib: true
-        #     phases: true
+        include:
+          # Include a single example where the build is executed all at once
+          - os: ubuntu-latest
+            python-version: 3.9
+            mfem-branch: default
+            parallel: false
+            cuda: false
+            cuda-version: 12.4.1
+            cuda-driver-version: 550.54.15
+            libceed: false
+            gslib: true
+            phases: false
 
     runs-on: ${{ matrix.os }}
 

From 8b2b103b1d4a1278fda5fac1f1a200bf2a0139dd Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 13:58:02 -0700
Subject: [PATCH 73/80] change python versions to 3.7-3.11 (3.12 is not
 supported by libraries like scipy)

---
 .github/workflows/build-and-test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 20a0d6e1..f669d960 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -12,8 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.9', '3.10', '3.11', '3.12']
-        #3.7, 3.8, 3.9, 3.10, 3.11
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
         # right now, default is a hardcoded sha defined in setup.py:repos_sha
         mfem-branch: [master, default]

From 02fc931485977b3f305ce47bcb57d1abaca26a30 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 14:20:39 -0700
Subject: [PATCH 74/80] cleanup and add comments

---
 .github/workflows/build-and-test.yml | 48 ++++++++++++++++++----------
 1 file changed, 31 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f669d960..bb5fd6e9 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -1,39 +1,47 @@
-# This is a clean-up/refactor of test_with_MFEM_release.yml
+# build-and-test.yml
+#
+#  - Builds PyMFEM in three stages:
+#    - MFEM dependencies + MFEM
+#    - SWIG bindings
+#    - PyMFEM
+#  - Runs tests under `run_examples.py`
+#  - If there is a failure, uploads test outputs as an artifact
+
 name: Build and Test
 
 on:
   workflow_dispatch:
   pull_request:
+  # TODO: setup a trigger to run on MFEM releases
 
 jobs:
   build-and-test:
-    # if: contains(github.event.pull_request.labels.*.name, 'in-test-with-mfem-release')
     strategy:
       fail-fast: false
       matrix:
+        # ---------- Main ----------
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
-
-        # right now, default is a hardcoded sha defined in setup.py:repos_sha
-        mfem-branch: [master, default]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] # 3.12 is not supported by scipy
+        # NOTE: If setup.py could accept a commit hash as an argument, that could give us more flexibility here
+        mfem-branch: [master, default] # 'default' uses a specific commit hash defined in setup.py:repos_sha
         parallel: [false, true]
-        # cuda: [false, true]
-        # temporary: only test cuda
-        # mfem-branch: [default]
-        # parallel: [false]
+
+        # ---------- Dependencies ----------
         cuda: [true]
+        cuda-version: [12.4.1]
+        cuda-driver-version: [550.54.15]
         libceed: [false]
         gslib: [true]
+
+        # ---------- Build process ----------
         # When phases==true, run each individual build step explicitly: mfem -> swig -> pymfem
         phases: [true]
 
-        cuda-version: [12.4.1]
-        cuda-driver-version: [550.54.15]
-
         exclude:
           # CUDA does not support MacOS
           - os: macos-latest
             cuda: true
+
         include:
           # Include a single example where the build is executed all at once
           - os: ubuntu-latest
@@ -49,7 +57,7 @@ jobs:
 
     runs-on: ${{ matrix.os }}
 
-    # https://7tonshark.com/posts/github-actions-ternary-operator/
+    # Reference for $${{ x && y || z }} syntax: https://7tonshark.com/posts/github-actions-ternary-operator/
     name: >-
       ${{ matrix.os }},
       ${{ matrix.python-version }},
@@ -57,7 +65,9 @@ jobs:
       (${{ matrix.parallel && 'parallel' || 'serial' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + libceed' || '' }}${{ matrix.gslib && ' + gslib' || '' }})
 
     env:
+      # If we wanted to add windows support we could add another url
       cuda-installer-url: https://developer.download.nvidia.com/compute/cuda/${{ matrix.cuda-version }}/local_installers/cuda_${{ matrix.cuda-version }}_${{ matrix.cuda-driver-version }}_linux.run
+      # These are all passed to setup.py as one concatenated string
       build-flags: >-
         ${{ matrix.parallel && '--with-parallel' || '' }}
         ${{ matrix.cuda && '--with-cuda' || '' }}
@@ -65,6 +75,9 @@ jobs:
         ${{ matrix.gslib && '--with-gslib' || '' }}
         ${{ (!(matrix.mfem-branch == 'default') && format('--mfem-branch=''{0}''', matrix.mfem-branch)) || '' }}
 
+    # -------------------------------------------------------------------------------------------------
+    # Begin workflow
+    # -------------------------------------------------------------------------------------------------
     steps:
     - name: Checkout repo
       uses: actions/checkout@v4
@@ -75,7 +88,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     # -------------------------------------------------------------------------------------------------
-    # Setup dependencies
+    # Download/install dependencies
     # -------------------------------------------------------------------------------------------------
     - name: Install core dependencies via requirements.txt
       run: pip install -r requirements.txt --verbose
@@ -114,7 +127,7 @@ jobs:
         nvcc --version || printf "CUDA not installed"
 
     # -------------------------------------------------------------------------------------------------
-    # Build
+    # Build MFEM + SWIG Bindings + PyMFEM
     # -------------------------------------------------------------------------------------------------
     - name: Build MFEM (step 1)
       if: matrix.phases
@@ -142,6 +155,7 @@ jobs:
       run: |
         cd test
         python run_examples.py -serial -verbose
+
     - name: Run tests (parallel)
       if: matrix.parallel
       run: |
@@ -149,7 +163,7 @@ jobs:
         python run_examples.py -parallel -verbose -np 2
 
     # -------------------------------------------------------------------------------------------------
-    # Artifact
+    # Generate an artifact (output of tests) on failure
     # -------------------------------------------------------------------------------------------------
     - name: Generate test results artifact
       id: generate-artifact

From c27e9df64c5a7405ac8db11f28db2c8b57eb615f Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 14:21:23 -0700
Subject: [PATCH 75/80] flag for setup.py should be --vv, not --verbose

---
 .github/workflows/build-and-test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index bb5fd6e9..00134507 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -133,19 +133,19 @@ jobs:
       if: matrix.phases
       run: |
         python setup.py clean --swig
-        python setup.py install --ext-only --verbose ${{ env.build-flags }}
+        python setup.py install --ext-only --vv ${{ env.build-flags }}
 
     - name: Build SWIG wrappers (step 2)
       if: matrix.phases
-      run: python setup.py install --swig --verbose ${{ env.build-flags }}
+      run: python setup.py install --swig --vv ${{ env.build-flags }}
 
     - name: Build PyMFEM (step 3)
       if: matrix.phases
-      run: python setup.py install --skip-ext --skip-swig --verbose ${{ env.build-flags }}
+      run: python setup.py install --skip-ext --skip-swig --vv ${{ env.build-flags }}
 
     - name: Build all (steps 1-3)
       if: matrix.phases == false
-      run: python setup.py install ${{ env.build-flags }}
+      run: python setup.py install --vv ${{ env.build-flags }}
 
     # -------------------------------------------------------------------------------------------------
     # Run tests

From dbc524e4ec1a06e9e131bceb8dccc358965a16a1 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 14:36:15 -0700
Subject: [PATCH 76/80] move cuda-version to env and change to dict

---
 .github/workflows/build-and-test.yml | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 00134507..e9c984a9 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -28,8 +28,6 @@ jobs:
 
         # ---------- Dependencies ----------
         cuda: [true]
-        cuda-version: [12.4.1]
-        cuda-driver-version: [550.54.15]
         libceed: [false]
         gslib: [true]
 
@@ -49,8 +47,6 @@ jobs:
             mfem-branch: default
             parallel: false
             cuda: false
-            cuda-version: 12.4.1
-            cuda-driver-version: 550.54.15
             libceed: false
             gslib: true
             phases: false
@@ -65,8 +61,7 @@ jobs:
       (${{ matrix.parallel && 'parallel' || 'serial' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + libceed' || '' }}${{ matrix.gslib && ' + gslib' || '' }})
 
     env:
-      # If we wanted to add windows support we could add another url
-      cuda-installer-url: https://developer.download.nvidia.com/compute/cuda/${{ matrix.cuda-version }}/local_installers/cuda_${{ matrix.cuda-version }}_${{ matrix.cuda-driver-version }}_linux.run
+      cuda-version: {toolkit: 12.4.1, driver: 550.54.15}
       # These are all passed to setup.py as one concatenated string
       build-flags: >-
         ${{ matrix.parallel && '--with-parallel' || '' }}
@@ -105,11 +100,13 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ~/cache
-        key: cuda-installer-${{ matrix.cuda-version }}-${{ matrix.cuda-driver-version }}
+        key: cuda-installer-${{ env.cuda-version.toolkit }}-${{ env.cuda-version.driver }}
 
     - name: Download CUDA
       if: matrix.cuda && steps.cache-cuda.outputs.cache-hit == false
-      run: curl -o ~/cache/cuda.run --create-dirs ${{ env.cuda-installer-url }}
+      run: |
+      echo "CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${{ env.cuda-version.toolkit }}/local_installers/cuda_${{ env.cuda-version.toolkit }}_${{ env.cuda-version.driver }}_linux.run" >> $GITHUB_ENV
+      curl -o ~/cache/cuda.run --create-dirs $CUDA_URL
 
     - name: Install CUDA
       if: matrix.cuda

From ea0f04f23333a5ccba9f951c6b88654b81965faf Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 14:44:06 -0700
Subject: [PATCH 77/80] fix env var

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index e9c984a9..4c9b27fe 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -105,7 +105,7 @@ jobs:
     - name: Download CUDA
       if: matrix.cuda && steps.cache-cuda.outputs.cache-hit == false
       run: |
-      echo "CUDA_URL=https://developer.download.nvidia.com/compute/cuda/${{ env.cuda-version.toolkit }}/local_installers/cuda_${{ env.cuda-version.toolkit }}_${{ env.cuda-version.driver }}_linux.run" >> $GITHUB_ENV
+      CUDA_URL="https://developer.download.nvidia.com/compute/cuda/${{ env.cuda-version.toolkit }}/local_installers/cuda_${{ env.cuda-version.toolkit }}_${{ env.cuda-version.driver }}_linux.run"
       curl -o ~/cache/cuda.run --create-dirs $CUDA_URL
 
     - name: Install CUDA

From bdf20c42d4bd6002462e991d58dc849dddd6e7b2 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 14:44:48 -0700
Subject: [PATCH 78/80] remove swig clean line

---
 .github/workflows/build-and-test.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 4c9b27fe..9d08ec1b 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -128,9 +128,7 @@ jobs:
     # -------------------------------------------------------------------------------------------------
     - name: Build MFEM (step 1)
       if: matrix.phases
-      run: |
-        python setup.py clean --swig
-        python setup.py install --ext-only --vv ${{ env.build-flags }}
+      run: python setup.py install --ext-only --vv ${{ env.build-flags }}
 
     - name: Build SWIG wrappers (step 2)
       if: matrix.phases

From 838e8ffe17013f8721973c10106bb65ce35ec87f Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 14:46:56 -0700
Subject: [PATCH 79/80] fixed indentation

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 9d08ec1b..c7f9ce29 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -105,8 +105,8 @@ jobs:
     - name: Download CUDA
       if: matrix.cuda && steps.cache-cuda.outputs.cache-hit == false
       run: |
-      CUDA_URL="https://developer.download.nvidia.com/compute/cuda/${{ env.cuda-version.toolkit }}/local_installers/cuda_${{ env.cuda-version.toolkit }}_${{ env.cuda-version.driver }}_linux.run"
-      curl -o ~/cache/cuda.run --create-dirs $CUDA_URL
+        CUDA_URL="https://developer.download.nvidia.com/compute/cuda/${{ env.cuda-version.toolkit }}/local_installers/cuda_${{ env.cuda-version.toolkit }}_${{ env.cuda-version.driver }}_linux.run"
+        curl -o ~/cache/cuda.run --create-dirs $CUDA_URL
 
     - name: Install CUDA
       if: matrix.cuda

From 4876868ccdd6adf473ecb2b05deef276c3084fce Mon Sep 17 00:00:00 2001
From: Justin Laughlin <justinglaughlin@gmail.com>
Date: Fri, 19 Apr 2024 14:57:25 -0700
Subject: [PATCH 80/80] hopefully this fixes the yaml error

---
 .github/workflows/build-and-test.yml | 7 ++++---
 .vscode/settings.json                | 2 ++
 2 files changed, 6 insertions(+), 3 deletions(-)
 create mode 100644 .vscode/settings.json

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index c7f9ce29..99644efd 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -61,7 +61,8 @@ jobs:
       (${{ matrix.parallel && 'parallel' || 'serial' }}${{ matrix.cuda && ' + cuda' || '' }}${{ matrix.libceed && ' + libceed' || '' }}${{ matrix.gslib && ' + gslib' || '' }})
 
     env:
-      cuda-version: {toolkit: 12.4.1, driver: 550.54.15}
+      cuda-toolkit-version: '12.4.1'
+      cuda-driver-version: '550.54.15'
       # These are all passed to setup.py as one concatenated string
       build-flags: >-
         ${{ matrix.parallel && '--with-parallel' || '' }}
@@ -100,12 +101,12 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ~/cache
-        key: cuda-installer-${{ env.cuda-version.toolkit }}-${{ env.cuda-version.driver }}
+        key: cuda-installer-${{ env.cuda-toolkit-version }}-${{ env.cuda-driver-version }}
 
     - name: Download CUDA
       if: matrix.cuda && steps.cache-cuda.outputs.cache-hit == false
       run: |
-        CUDA_URL="https://developer.download.nvidia.com/compute/cuda/${{ env.cuda-version.toolkit }}/local_installers/cuda_${{ env.cuda-version.toolkit }}_${{ env.cuda-version.driver }}_linux.run"
+        CUDA_URL="https://developer.download.nvidia.com/compute/cuda/${{ env.cuda-toolkit-version }}/local_installers/cuda_${{ env.cuda-toolkit-version }}_${{ env.cuda-driver-version }}_linux.run"
         curl -o ~/cache/cuda.run --create-dirs $CUDA_URL
 
     - name: Install CUDA
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 00000000..7a73a41b
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,2 @@
+{
+}
\ No newline at end of file