Adapt to PeleRad in PelePhysics

.github/workflows/ci.yml

@@ -86,6 +86,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
         build_type: [Release, Debug]
+        enable_eb: [EB-OFF, EB-ON]
         include:
           - os: macos-latest
             install_deps: brew install mpich automake ccache
@@ -101,8 +102,12 @@ jobs:
             ctest_args: -LE no-ci
             ccache_size: 100M
           - build_type: Debug
-            ctest_args: -R eb-c12
+            ctest_args: -LE no-ci
             ccache_size: 500M
+          - enable_eb: EB-OFF
+            use_eb: "OFF"
+          - enable_eb: EB-ON
+            use_eb: "ON"
         exclude:
           - os: macos-latest
             build_type: Debug
@@ -144,22 +149,22 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ${{matrix.ccache_cache}}
-          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.build_type}}-git-${{github.sha}}
+          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}}-git-${{github.sha}}
           restore-keys: |
-               ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.build_type}}-git-
+               ccache-${{github.workflow}}-${{github.job}}-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}}-git-
       - name: Configure
         run: |
           (for DIM in 2 3; do \
             printf "\n-------- Configuring ${DIM}D --------\n"; \
-            cmake -B${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}} \
-            -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install-${DIM}d-${{matrix.os}}-${{matrix.build_type}} \
+            cmake -B${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
+            -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install-${DIM}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
             -DCMAKE_BUILD_TYPE:STRING=${{matrix.build_type}} \
             -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
             -DPELELMEX_DIM:STRING=${DIM} \
             -DPELELMEX_ENABLE_MPI:BOOL=ON \
+            -DPELELMEX_ENABLE_EB:BOOL=${{matrix.use_eb}} \
             -DPELELMEX_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
             -DPELELMEX_ENABLE_MASA:BOOL=OFF \
-            -DMASA_ROOT:PATH=${{runner.workspace}}/deps/install/MASA \
             -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
             ${{github.workspace}}; \
             if [ $? -ne 0 ]; then exit 1; fi \
@@ -169,7 +174,7 @@ jobs:
           ccache -z
           (for DIM in 2 3; do \
             printf "\n-------- Building ${DIM}D --------\n"; \
-            cmake --build ${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}} \
+            cmake --build ${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}} \
               --parallel ${{env.NPROCS}} 2>&1 | tee -a ${{runner.workspace}}/build-output.txt; \
             if [ ${PIPESTATUS[0]} -ne 0 ]; then exit 1; fi \
           done)
@@ -191,7 +196,7 @@ jobs:
         run: |
           (for DIM in 2 3; do \
             printf "\n-------- Testing ${DIM}D --------\n"; \
-            cd ${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}}; \
+            cd ${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}}-${{matrix.enable_eb}}; \
             ctest -j ${{env.NPROCS}} ${{matrix.ctest_args}} --output-on-failure; \
             if [ $? -ne 0 ]; then exit 1; fi \
           done)
@@ -402,20 +407,20 @@ jobs:
           du -hs ~/.cache/ccache
   Lint-clang-tidy:
     needs: Formatting
-    runs-on: ubuntu-latest
+    runs-on: macos-13
     strategy:
       matrix:
-        enableeb: [EB-OFF, EB-ON]
+        enable_eb: [EB-OFF, EB-ON]
         include:
-          - enableeb: EB-OFF
-            useeb: "OFF"
-          - enableeb: EB-ON
-            useeb: "ON"
+          - enable_eb: EB-OFF
+            use_eb: "OFF"
+          - enable_eb: EB-ON
+            use_eb: "ON"
       fail-fast: false
     steps:
       - name: Setup
         run: |
-          echo "NPROCS=$(nproc)" >> $GITHUB_ENV
+          echo "NPROCS=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV
           echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV
           echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV
           echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
@@ -426,25 +431,23 @@ jobs:
           submodules: recursive
       - name: Install Ccache
         run: |
-          wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
-          tar xvf ccache-4.8-linux-x86_64.tar.xz
-          sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/
+          brew install ccache
       - name: Set Up Ccache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/ccache
-          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.useeb}}-git-${{github.sha}}
+          path: /Users/runner/Library/Caches/ccache
+          key: ccache-${{github.workflow}}-${{github.job}}-${{matrix.enable_eb}}-git-${{github.sha}}
           restore-keys: |
-               ccache-${{github.workflow}}-${{github.job}}-${{matrix.useeb}}-git-
+               ccache-${{github.workflow}}-${{github.job}}-${{matrix.enable_eb}}-git-
       - name: Configure
         run: |
-          echo "NPROCS=$(nproc)" >> $GITHUB_ENV
+          export PATH=$(brew --prefix llvm@15)/bin:${PATH}
           cmake -B${{runner.workspace}}/build-clang-tidy \
           -DCMAKE_BUILD_TYPE:STRING=Release \
           -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
           -DCMAKE_CXX_COMPILER:STRING=clang++ \
           -DCMAKE_C_COMPILER:STRING=clang \
-          -DPELELMEX_ENABLE_EB:BOOL=${{matrix.useeb}} \
+          -DPELELMEX_ENABLE_EB:BOOL=${{matrix.use_eb}} \
           -DPELELMEX_ENABLE_MPI:BOOL=OFF \
           -DPELELMEX_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
           -DPELELMEX_ENABLE_MASA:BOOL=OFF \
@@ -455,15 +458,16 @@ jobs:
         working-directory: ${{runner.workspace}}/build-clang-tidy
         run: |
           #set -eu -o pipefail
-          cmake --build . --parallel 2 2>&1 | tee -a clang-tidy-full-report.txt
+          export PATH=$(brew --prefix llvm@15)/bin:${PATH}
+          cmake --build . --parallel ${{env.NPROCS}} 2>&1 | tee -a clang-tidy-full-report.txt
           egrep "warning:|error:" clang-tidy-full-report.txt \
             | egrep -v "Submodules/amrex|Submodules/sundials|Submodules/AMReX-Hydro" \
             | egrep -v "ld: warning:" | sort | uniq \
             | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > clang-tidy-warnings.txt
       - name: Ccache Report
         run: |
           ccache -s
-          du -hs ~/.cache/ccache
+          du -hs /Users/runner/Library/Caches/ccache
       - name: Full report
         working-directory: ${{runner.workspace}}/build-clang-tidy
         run: cat clang-tidy-full-report.txt

.github/workflows/cleanup-cache-postpr.yml

@@ -31,7 +31,10 @@ jobs:
           set +e
 
           keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1)
+          # $keys might contain spaces. Thus we set IFS to \n.
+          IFS=$'\n'
           for k in $keys
           do
-            gh actions-cache delete $k -R $REPO -B $BRANCH --confirm
+            gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
           done
+          unset IFS

.github/workflows/cleanup-cache.yml

@@ -27,7 +27,7 @@ jobs:
           EVENT=${{ github.event.workflow_run.event }}
 
           # Triggering workflow run name (e.g., LinuxClang)
-          WORKFLOW_NAME=${{ github.event.workflow_run.name }}
+          WORKFLOW_NAME="${{ github.event.workflow_run.name }}"
 
           if [[ $EVENT == "pull_request" ]]; then
             gh run download ${{ github.event.workflow_run.id }} -n pr_number
@@ -45,16 +45,19 @@ jobs:
           # The goal is to keep the last used key of each job and delete all others.
 
           # something like ccache-LinuxClang-
-          keyprefix=ccache-${WORKFLOW_NAME}-
+          keyprefix="ccache-${WORKFLOW_NAME}-"
 
-          cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key $keyprefix | awk -F '-git-' '{print $1}' | sort | uniq)
+          cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq)
 
           # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d".
+          # It might also contain spaces. Thus we set IFS to \n.
+          IFS=$'\n'
           for j in $cached_jobs
           do
-            old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key ${j}-git- --sort last-used | cut -f 1 | tail -n +2)
+            old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2)
             for k in $old_keys
             do
-              gh actions-cache delete $k -R $REPO -B $BRANCH --confirm
+              gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
             done
           done
+          unset IFS

.github/workflows/linux.yml

@@ -85,40 +85,29 @@ jobs:
       {CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
     steps:
     - uses: actions/checkout@v3
-    - name: System Dependencies
-      run: .github/workflows/dependencies/dependencies_gcc10.sh
     - name: Repo Dependencies
       run: |
         Utils/CloneDeps.sh
-        git clone --branch main https://github.com/AMReX-Combustion/PeleRad.git Submodules/PeleRad
+    - name: System Dependencies
+      run: .github/workflows/dependencies/dependencies_gcc10.sh
     - name: Build Release
-      env:
-         PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
       working-directory: ./Exec/RegTests/SootRadTest/
       run: |
         make TPL COMP=gnu USE_MPI=TRUE TINY_PROFILE=FALSE
         make -j 2 COMP=gnu USE_MPI=TRUE TINY_PROFILE=FALSE
     - name: Run Release
-      env:
-         PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
       working-directory: ./Exec/RegTests/SootRadTest/
       run: |
-        eval DATA_PATH=$PELERAD_HOME
-        mpirun -n 2 ./PeleLMeX2d.gnu.MPI.ex first-input.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2 pelerad.kppath="$DATA_PATH/data/kpDB/"
+        mpirun -n 2 ./PeleLMeX2d.gnu.MPI.ex soot-rad-2d.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2
     - name: Build Debug
-      env:
-         PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
       working-directory: ./Exec/RegTests/SootRadTest/
       run: |
         make TPL COMP=gnu USE_MPI=TRUE DEBUG=TRUE TINY_PROFILE=FALSE
         make -j 2 COMP=gnu USE_MPI=TRUE DEBUG=TRUE TINY_PROFILE=FALSE
     - name: Run Debug
-      env:
-         PELERAD_HOME: ${GITHUB_WORKSPACE}/Submodules/PeleRad
       working-directory: ./Exec/RegTests/SootRadTest/
       run: |
-        eval DATA_PATH=$PELERAD_HOME
-        mpirun -n 2 ./PeleLMeX2d.gnu.DEBUG.MPI.ex first-input.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2 pelerad.kppath="$DATA_PATH/data/kpDB/"
+        mpirun -n 2 ./PeleLMeX2d.gnu.DEBUG.MPI.ex soot-rad-2d.inp amr.max_step=2 amr.plot_int=2 amr.check_int=2
 
   # Build and Run the EB_BackwardStepFlame RegTest with GNU9.3 and MPI support
   EBBFS:

CMake/BuildExeAndLib.cmake

@@ -1,6 +1,6 @@
 get_filename_component(DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 set(pelelmex_lib_name PelePhysics-Lib-${PELELMEX_EOS_MODEL}-${PELELMEX_CHEMISTRY_MODEL}-${PELELMEX_TRANSPORT_MODEL})
-set(pelelmex_exe_name PeleLMeX-${DIR_NAME})
+set(pelelmex_exe_name ${PROJECT_NAME}-${DIR_NAME})
 include(BuildPeleLMeXLib)
 include(BuildPeleLMeXExe)
 build_pelelmex_lib(${pelelmex_lib_name})

CMake/BuildPeleLMeXExe.cmake

@@ -106,7 +106,12 @@ function(build_pelelmex_exe pelelmex_exe_name pelelmex_lib_name)
   if(PELELMEX_ENABLE_SOOT)
     target_sources(${pelelmex_exe_name}
       PRIVATE
-        ${SRC_DIR}/Soot.cpp)
+        ${SRC_DIR}/PeleLMeX_Soot.cpp)
+    if(PELELMEX_ENABLE_RADIATION)
+      target_sources(${pelelmex_exe_name}
+        PRIVATE
+          ${SRC_DIR}/PeleLMeX_Radiation.cpp)
+    endif()
   endif()
 
   #if(PELELMEX_ENABLE_ASCENT)

CMake/BuildPeleLMeXLib.cmake

@@ -2,14 +2,15 @@ function(build_pelelmex_lib pelelmex_lib_name)
   if (NOT (TARGET ${pelelmex_lib_name}))
     add_library(${pelelmex_lib_name} OBJECT)
 
-    set(PELE_PHYSICS_SRC_DIR ${CMAKE_SOURCE_DIR}/Submodules/PelePhysics)
+    set(PELE_PHYSICS_SRC_DIR "${CMAKE_SOURCE_DIR}/Submodules/PelePhysics")
     set(PELE_PHYSICS_TRANSPORT_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Transport")
     set(PELE_PHYSICS_EOS_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Eos")
     set(PELE_PHYSICS_MECHANISM_DIR "${PELE_PHYSICS_SRC_DIR}/Mechanisms/${PELELMEX_CHEMISTRY_MODEL}")
     set(PELE_PHYSICS_UTILITY_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Utility")
     set(PELE_PHYSICS_REACTIONS_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Reactions")
     set(PELE_PHYSICS_SOOT_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Soot")
-    set(AMREX_SUNDIALS_DIR ${AMREX_SUBMOD_LOCATION}/Src/Extern/SUNDIALS)
+    set(PELE_PHYSICS_RADIATION_DIR "${PELE_PHYSICS_SRC_DIR}/Source/Radiation")
+    set(AMREX_SUNDIALS_DIR "${AMREX_SUBMOD_LOCATION}/Src/Extern/SUNDIALS")
 
     if(CLANG_TIDY_EXE)
       set_target_properties(${pelelmex_lib_name} PROPERTIES CXX_CLANG_TIDY
@@ -144,12 +145,12 @@ function(build_pelelmex_lib pelelmex_lib_name)
     target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_REACTIONS_DIR})
 
     if(PELELMEX_ENABLE_SOOT)
-       target_compile_definitions(${pelelmex_lib_name} PUBLIC PELELMEX_USE_SOOT)
-       target_compile_definitions(${pelelmex_lib_name} PUBLIC NUM_SOOT_MOMENTS=${PELE_NUM_SOOT_MOMENTS})
        set(SOOT_MOMENTS_VALUES 3 6)
        if(NOT PELE_NUM_SOOT_MOMENTS IN_LIST SOOT_MOMENTS_VALUES)
          message(FATAL_ERROR "NUM_SOOT_MOMENTS must be either 3 or 6")
        endif()
+       target_compile_definitions(${pelelmex_lib_name} PUBLIC PELELM_USE_SOOT)
+       target_compile_definitions(${pelelmex_lib_name} PUBLIC NUM_SOOT_MOMENTS=${PELE_NUM_SOOT_MOMENTS})
        target_sources(${pelelmex_lib_name} PRIVATE
                       ${PELE_PHYSICS_SOOT_DIR}/SootModel.H
                       ${PELE_PHYSICS_SOOT_DIR}/SootModel.cpp
@@ -159,7 +160,24 @@ function(build_pelelmex_lib pelelmex_lib_name)
                       ${PELE_PHYSICS_SOOT_DIR}/Constants_Soot.H
                       ${PELE_PHYSICS_SOOT_DIR}/SootData.H
                       ${PELE_PHYSICS_SOOT_DIR}/SootReactions.H)
-       target_include_directories(${pele=lmex_lib_name} PUBLIC ${PELE_PHYSICS_SOOT_DIR})
+       target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_SOOT_DIR})
+       if(PELELMEX_ENABLE_RADIATION)
+         target_sources(${pelelmex_lib_name} PRIVATE
+                        ${PELE_PHYSICS_RADIATION_DIR}/AMRParam.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/Constants.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/MLMGParam.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/POneMulti.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/POneMultiEB.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/POneMultiLevbyLev.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/POneSingle.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/POneSingleEB.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/PeleCRad.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/PeleLMRad.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/PlanckMean.H
+                        ${PELE_PHYSICS_RADIATION_DIR}/SpectralModels.H)
+         target_include_directories(${pelelmex_lib_name} PUBLIC ${PELE_PHYSICS_RADIATION_DIR})
+         target_compile_definitions(${pelelmex_lib_name} PUBLIC PELELM_USE_RADIATION)
+       endif()
     endif()
 
     include(AMReXBuildInfo)

CMakeLists.txt

@@ -29,6 +29,8 @@ option(PELELMEX_ENABLE_EB "Enable Embedded Boundary" OFF)
 option(PELELMEX_ENABLE_EFIELD "Enable Electric field module" OFF)
 option(PELELMEX_ENABLE_SOOT "Enable Soot module" OFF)
 option(PELELMEX_ENABLE_PARTICLES "Enable particles" OFF)
+option(PELELMEX_ENABLE_RADIATION "Enable radiation" OFF)
+set(PELE_NUM_SOOT_MOMENTS "3" CACHE STRING "Number of soot moments: 3 or 6")
 
 # HPC options
 option(PELELMEX_ENABLE_MPI "Enable MPI" OFF)
@@ -113,3 +115,4 @@ add_subdirectory(Exec)
 enable_testing()
 include(CTest)
 add_subdirectory(Tests)
+add_subdirectory(${CMAKE_SOURCE_DIR}/Submodules/PelePhysics/Testing/Exec/Radiation)

Exec/Make.PeleLMeX

@@ -53,11 +53,8 @@ ifeq ($(USE_SOOT), TRUE)
    DEFINES += -DNUM_SOOT_MOMENTS=$(NUM_SOOT_MOMENTS)
 endif
 
-ifeq ($(USE_PELERAD), TRUE)
-   DEFINES += -DPELELM_USE_RAD
-   ifeq ($(USE_HIP), TRUE)
-       DEFINES += -DPELERAD_USE_HIP
-   endif
+ifeq ($(USE_RADIATION), TRUE)
+   DEFINES += -DPELELM_USE_RADIATION
 endif
 
 Bpack += $(foreach dir, $(LMdirs), $(PELELMEX_HOME)/$(dir)/Make.package)
@@ -126,8 +123,10 @@ ifeq ($(USE_SOOT), TRUE)
    Bpack += $(SOOT_HOME)/Make.package
    Blocs += $(SOOT_HOME)/Source/Soot_Models
 endif
-ifeq ($(USE_PELERAD), TRUE)
-   Blocs += $(PELERAD_HOME)/src
+ifeq ($(USE_RADIATION), TRUE)
+   RADIATION_HOME = $(PELE_PHYSICS_HOME)/Source/Radiation
+   Bpack += $(RADIATION_HOME)/Make.package
+   Blocs += $(RADIATION_HOME)
 endif
 
 #---------------

Exec/RegTests/CMakeLists.txt

@@ -3,8 +3,13 @@ if(PELELMEX_ENABLE_EB)
   add_subdirectory(EB_EnclosedFlame)
   add_subdirectory(EB_EnclosedVortex)
   add_subdirectory(EB_FlowPastCylinder)
-  add_subdirectory(EB_PipeFlow)
+  if(PELELMEX_DIM GREATER 2)
+    add_subdirectory(EB_PipeFlow)
+  endif()
 else()
+  if(PELELMEX_ENABLE_RADIATION)
+    add_subdirectory(SootRadTest)
+  endif()
   add_subdirectory(EnclosedFlame)
   add_subdirectory(EnclosedInjection)
   add_subdirectory(FlameSheet)

Exec/RegTests/EB_EnclosedFlame/pelelmex_prob.H

@@ -40,8 +40,6 @@ pelelmex_initdata(
   AMREX_D_TERM(const amrex::Real xc = prob_lo[0] + 0.5 * Lx;
                , const amrex::Real yc = prob_lo[1] + 0.5 * Ly;, );
 
-  constexpr amrex::Real Pi = 3.14159265358979323846264338327950288;
-
   auto eos = pele::physics::PhysicsType::eos();
   amrex::GpuArray<amrex::Real, NUM_SPECIES + 4> pmf_vals = {0.0};
   amrex::Real molefrac[NUM_SPECIES] = {0.0};
@@ -65,6 +63,7 @@ pelelmex_initdata(
   //   3.6 - (radius - prob_parm.standoff + 1.42 * dx[1] + pert) * 100;
 
 #elif (AMREX_SPACEDIM == 3)
+    constexpr amrex::Real Pi = 3.14159265358979323846264338327950288;
     amrex::Real pertLz = (prob_parm.pertLz > 0.0) ? prob_parm.pertLz : Lz;
     amrex::Real angle = std::atan2((y - yc), (x - xc));
     pert = prob_parm.pertmag *