Skip to content

Commit

Permalink
Fix HIP CI (#256)
Browse files Browse the repository at this point in the history
- Also convert some CI tests to CTest.
  • Loading branch information
jrood-nrel authored Sep 19, 2023
1 parent 98b08f6 commit 12f1d1f
Show file tree
Hide file tree
Showing 11 changed files with 611 additions and 233 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -394,12 +394,12 @@ jobs:
run: |
curl -O https://repo.radeon.com/rocm/rocm.gpg.key
sudo apt-key add rocm.gpg.key
echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \
echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/5.6.1 ubuntu main' \
| sudo tee /etc/apt/sources.list.d/rocm.list
echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
| sudo tee -a /etc/profile.d/rocm.sh
sudo apt-get update
export TMPDIR=${{runner.workspace}}/.tmp && mkdir -p $TMPDIR && sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev rocm-libs
sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev
- name: Install Ccache
run: |
wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz
Expand Down
232 changes: 4 additions & 228 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,209 +12,8 @@ concurrency:
cancel-in-progress: true

jobs:
#Build and run the 3D FlameSheet RegTest with GNU9.3 and no MPI support
F32D_NoMPI:
name: [email protected] NOMPI Comp [FS3D]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: Utils/CloneDeps.sh
- name: Build
env:
AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
PELELMEX_HOME: ${GITHUB_WORKSPACE}
AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
SUNDIALS_HOME : ${GITHUB_WORKSPACE}/build/sundials
run: |
cd Exec/RegTests/FlameSheet/
make help
make TPL DIM=3 COMP=gnu USE_MPI=FALSE
make -j 2 DIM=3 COMP=gnu USE_MPI=FALSE
- name: Run
working-directory: ./Exec/RegTests/FlameSheet/
run: |
./PeleLMeX3d.gnu.ex input.3d-regt peleLM.num_init_iter=1 amr.max_step=1 amr.plot_int=-1 amr.check_int=-1
# Build and Run the 2D FlameSheet RegTest with GNU9.3 and MPI support
FS2D_MPI:
name: [email protected] MPI Comp [FS2D]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: Utils/CloneDeps.sh
- name: Build
env:
AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
PELELMEX_HOME: ${GITHUB_WORKSPACE}
AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
SUNDIALS_HOME : ${GITHUB_WORKSPACE}/build/sundials
run: |
cd Exec/RegTests/FlameSheet/
make TPL COMP=gnu
make -j 2 COMP=gnu
- name: RunBase
working-directory: ./Exec/RegTests/FlameSheet/
run: |
./PeleLMeX2d.gnu.MPI.ex input.2d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
- name: RunSoret
working-directory: ./Exec/RegTests/FlameSheet/
run: |
./PeleLMeX2d.gnu.MPI.ex input.2d-regt transport.use_soret=1 amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
- name: RunUnityLe
working-directory: ./Exec/RegTests/FlameSheet/
run: |
./PeleLMeX2d.gnu.MPI.ex input.2d-regt peleLM.unity_Le=1 peleLM.Prandtl=0.75 amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
# Build the 2D FlameSheet RegTest with GNU9.3 and MPI+OMP support
FS2D_MPIOMP:
name: [email protected] MPI OMP Comp [FS2D]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: Utils/CloneDeps.sh
- name: Build
env:
AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
PELELMEX_HOME: ${GITHUB_WORKSPACE}
AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
SUNDIALS_HOME : ${GITHUB_WORKSPACE}/build/sundials
run: |
cd Exec/RegTests/FlameSheet/
make TPL COMP=gnu USE_OMP=TRUE
make -j 2 COMP=gnu USE_OMP=TRUE
# Build and Run the CoVo RegTest with GNU9.3 and no MPI support
COVO2D_NoMPIRun:
name: [email protected] NOMPI Run [COVO2D]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: Utils/CloneDeps.sh
- name: Build
env:
AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
PELELMEX_HOME: ${GITHUB_WORKSPACE}
AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
SUNDIALS_HOME : ${GITHUB_WORKSPACE}/build/sundials
working-directory: ./Exec/RegTests/PeriodicCases/
run: |
make TPL COMP=gnu USE_MPI=FALSE
make -j 2 COMP=gnu USE_MPI=FALSE
- name: Run
working-directory: ./Exec/RegTests/PeriodicCases/
run: |
./PeleLMeX2d.gnu.ex input.2d_CoVo amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
# Build and Run the HoBubble RegTest with GNU9.3 and MPI support
HB2D_MPIRun:
name: [email protected] MPI Run [HB2D]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: Utils/CloneDeps.sh
- name: Build
env:
AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
PELELMEX_HOME: ${GITHUB_WORKSPACE}
AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
SUNDIALS_HOME : ${GITHUB_WORKSPACE}/build/sundials
working-directory: ./Exec/RegTests/HotBubble/
run: |
make TPL COMP=gnu USE_MPI=TRUE
make -j 2 COMP=gnu USE_MPI=TRUE
- name: Run
working-directory: ./Exec/RegTests/HotBubble/
run: |
./PeleLMeX2d.gnu.MPI.ex input.2d-regt amr.max_step=20 amr.plot_int=-1 amr.check_int=-1
# Build and Run the HotBubble RegTest with GNU9.3 in DEBUG
HB2D_DBGRun:
name: [email protected] DEBUG Run [HB2DDBG]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: Utils/CloneDeps.sh
- name: Build
env:
AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
PELELMEX_HOME: ${GITHUB_WORKSPACE}
AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
SUNDIALS_HOME : ${GITHUB_WORKSPACE}/build/sundials
working-directory: ./Exec/RegTests/HotBubble/
run: |
make TPL COMP=gnu DEBUG=TRUE USE_MPI=FALSE
make -j 2 COMP=gnu DEBUG=TRUE USE_MPI=FALSE
- name: Run
working-directory: ./Exec/RegTests/HotBubble/
run: |
./PeleLMeX2d.gnu.DEBUG.ex input.2d-regt amr.max_step=20 amr.plot_int=-1 amr.check_int=-1
# Build and Run the EnclosedFlame RegTest with GNU9.3 and MPI support
EF2D_MPIRun:
name: [email protected] MPI Run [EF2D]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: Utils/CloneDeps.sh
- name: Build
env:
AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
PELELMEX_HOME: ${GITHUB_WORKSPACE}
AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
SUNDIALS_HOME : ${GITHUB_WORKSPACE}/build/sundials
working-directory: ./Exec/RegTests/EnclosedFlame/
run: |
make TPL COMP=gnu USE_MPI=TRUE
make -j 2 COMP=gnu USE_MPI=TRUE
- name: Run
working-directory: ./Exec/RegTests/EnclosedFlame/
run: |
./PeleLMeX2d.gnu.MPI.ex input.2d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
# Build and Run the EB_FlowPastCylinder RegTest with GNU9.3 and MPI support
FPC2D_MPIRun:
FPC2D:
name: [email protected] MPI Run [FPC2DDBG]
runs-on: ubuntu-latest
env:
Expand Down Expand Up @@ -242,28 +41,6 @@ jobs:
./PeleLMeX2d.gnu.DEBUG.MPI.ex input.2d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1 amrex.fpe_trap_invalid=1 amrex.fpe_trap_zero=1 amrex.fpe_trap_overflow=1
./PeleLMeX2d.gnu.DEBUG.MPI.ex input.2d-regt_WallBump amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
# Build and Run the HIT RegTest with GNU9.3 and MPI support
HIT3D_MPIRun:
name: [email protected] MPI Run [HIT3D]
runs-on: ubuntu-latest
env:
{CXXFLAGS: "-Werror -Wshadow -Woverloaded-virtual -Wunreachable-code"}
steps:
- uses: actions/checkout@v3
- name: System Dependencies
run: .github/workflows/dependencies/dependencies_gcc10.sh
- name: Repo Dependencies
run: Utils/CloneDeps.sh
- name: Build
env:
AMREX_HOME: ${GITHUB_WORKSPACE}/build/amrex
PELE_PHYSICS_HOME: ${GITHUB_WORKSPACE}/build/PelePhysics
PELELMEX_HOME: ${GITHUB_WORKSPACE}
AMREX_HYDRO_HOME: ${GITHUB_WORKSPACE}/build/AMReX-Hydro
SUNDIALS_HOME : ${GITHUB_WORKSPACE}/build/sundials
working-directory: ./Exec/RegTests/HITDecay/
run: |
make TPL COMP=gnu USE_MPI=TRUE
make -j 2 COMP=gnu USE_MPI=TRUE
- name: RunDNS
working-directory: ./Exec/RegTests/HITDecay/
Expand All @@ -278,9 +55,8 @@ jobs:
run: |
./PeleLMeX3d.gnu.MPI.ex input.3d_les peleLM.les_model="WALE" amr.max_step=10 amr.plot_int=-1 amr.check_int=-1
# Build and Run the EB_PipeFlow RegTest with GNU9.3 and MPI support
EBPF3D_MPIRun:
EBPF3D:
name: [email protected] MPI Run [EBPF3D]
runs-on: ubuntu-latest
env:
Expand Down Expand Up @@ -308,7 +84,7 @@ jobs:
mpirun -n 2 ./PeleLMeX3d.gnu.MPI.ex input.3d-regt amr.max_step=2 amr.plot_int=-1 amr.check_int=-1 amr.n_cell=128 32 32
# Build and Run the TurbInflow RegTest in DEBUG with GNU9.3 and MPI support
TI3D_MPIRun:
TI3D:
name: [email protected] MPI Run [TI3DDBG]
runs-on: ubuntu-latest
env:
Expand Down Expand Up @@ -350,7 +126,7 @@ jobs:
mpirun -n 2 ./PeleLMeX3d.gnu.DEBUG.MPI.ex input.3d_BoxLoZ peleLM.num_init_iter=1 amr.n_cell=32 32 32 amr.max_step=2 amr.plot_int=-1 amr.check_int=-1
# Build and Run the SprayTest RegTest with GNU9.3 and MPI support
SPRAY_MPIRun:
SPRAY:
name: [email protected] MPI Run [SPRAY]
runs-on: ubuntu-latest
env:
Expand Down
86 changes: 86 additions & 0 deletions Exec/RegTests/EnclosedFlame/enclosedflame-2d.inp
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
#----------------------DOMAIN DEFINITION------------------------
geometry.is_periodic = 0 0 # For each dir, 0: non-perio, 1: periodic
geometry.coord_sys = 0 # 0 => cart, 1 => RZ
geometry.prob_lo = -0.02 -0.02 0.0 # x_lo y_lo (z_lo)
geometry.prob_hi = 0.02 0.02 0.016 # x_hi y_hi (z_hi)

# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<<
# Interior, Inflow, Outflow, Symmetry,
# SlipWallAdiab, NoSlipWallAdiab, SlipWallIsotherm, NoSlipWallIsotherm
peleLM.lo_bc = NoSlipWallAdiab NoSlipWallAdiab
peleLM.hi_bc = NoSlipWallAdiab NoSlipWallAdiab


#-------------------------AMR CONTROL----------------------------
amr.n_cell = 128 128 32 # Level 0 number of cells in each direction
amr.v = 1 # AMR verbose
amr.max_level = 2 # maximum level number allowed
amr.ref_ratio = 2 2 2 2 2 # refinement ratio
amr.regrid_int = 5 # how often to regrid
amr.n_error_buf = 2 2 2 2 # number of buffer cells in error est
amr.grid_eff = 0.7 # what constitutes an efficient grid
amr.blocking_factor = 16 # block factor in grid generation (min box size)
amr.max_grid_size = 128 # max box size


#--------------------------- Problem -------------------------------
prob.P_mean = 101325.0
prob.standoff = -0.002
prob.pertmag = 0.0002
pmf.datafile = "drm19_pmf.dat"
pmf.do_cellAverage = 0

#-------------------------PeleLM CONTROL----------------------------
peleLM.v = 1
peleLM.incompressible = 0
peleLM.rho = 1.17
peleLM.mu = 0.0
peleLM.use_wbar = 1
peleLM.sdc_iterMax = 2
peleLM.floor_species = 0
peleLM.num_divu_iter = 1
peleLM.num_init_iter = 3

peleLM.do_temporals = 1
peleLM.do_mass_balance = 1

#amr.restart = chk00005
amr.check_int = 100
amr.plot_int = 100
amr.max_step = 10
amr.dt_shrink = 0.0001
amr.stop_time = 1.0
#amr.stop_time = 1.00
amr.cfl = 0.15
amr.derive_plot_vars = avg_pressure mag_vort mass_fractions

# --------------- INPUTS TO CHEMISTRY REACTOR ---------------
peleLM.chem_integrator = "ReactorCvode"
peleLM.use_typ_vals_chem = 1 # Use species/temp typical values in CVODE
ode.rtol = 1.0e-6 # Relative tolerance of the chemical solve
ode.atol = 1.0e-5 # Absolute tolerance factor applied on typical values
cvode.solve_type = denseAJ_direct # CVODE Linear solve type (for Newton direction)
cvode.max_order = 4 # CVODE max BDF order.

#--------------------REFINEMENT CONTROL------------------------
#amr.refinement_indicators = temp
#amr.temp.max_level = 1
#amr.temp.value_greater = 305
#amr.temp.field_name = temp

#amr.refinement_indicators = magVort
#amr.magVort.max_level = 1
#amr.magVort.value_greater = 500.0
#amr.magVort.field_name = mag_vort

amr.refinement_indicators = yH
amr.yH.max_level = 5
amr.yH.value_greater = 3.0e-7
amr.yH.field_name = Y(H)

#--------------------LINEAR SOLVER CONTROL------------------------
nodal_proj.verbose = 2
nodal_proj.rtol = 2.0e-11
amrex.fpe_trap_invalid = 1
amrex.fpe_trap_zero = 1
amrex.fpe_trap_overflow = 1
Loading

0 comments on commit 12f1d1f

Please sign in to comment.