From 2174730a50a9318918e243e2341a618ed95a72ab Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 26 Sep 2024 12:46:21 +0100 Subject: [PATCH] Use `--oversubscribe` for parallel CI jobs The option previously used only works for OpenMPI v4 and earlier - the options changed in v5 because they changed the backend (? see https://github.com/open-mpi/ompi/issues/8955). `--oversubscribe` seems more generic, so hopefully keeps working. --- .github/workflows/debug_checks.yml | 10 +++------- .github/workflows/parallel_test.yml | 20 ++++++-------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/.github/workflows/debug_checks.yml b/.github/workflows/debug_checks.yml index 7cb78a9d0..fc1e7a4a0 100644 --- a/.github/workflows/debug_checks.yml +++ b/.github/workflows/debug_checks.yml @@ -33,11 +33,7 @@ jobs: julia --project -O3 --check-bounds=yes -e 'using Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.precompile()' julia --project -O3 --check-bounds=yes precompile.jl --debug 2 - # Need to use openmpi so that the following arguments work: - # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes - # than physical cores). - # * `--mca mpi_yield_when_idle 1` changes a setting to prevent excessively - # terrible performance when oversubscribing. + # Need to use openmpi so that we can use `--oversubscribe` to allow using more MPI ranks than physical cores ## Don't use --compiled-modules=no for now, as it currently breaks Symbolics.jl - #mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project --check-bounds=yes --compiled-modules=no moment_kinetics/debug_test/sound_wave_tests.jl --debug 2 - ./mpiexecjl -np 4 --mca rmaps_base_oversubscribe 1 julia --project -Jmoment_kinetics.so -O3 --check-bounds=yes moment_kinetics/debug_test/runtests.jl --debug 2 + #mpiexec -np 4 --oversubscribe 1 julia --project --check-bounds=yes --compiled-modules=no moment_kinetics/debug_test/sound_wave_tests.jl --debug 2 + ./mpiexecjl -np 4 --oversubscribe julia --project -Jmoment_kinetics.so -O3 --check-bounds=yes moment_kinetics/debug_test/runtests.jl --debug 2 diff --git a/.github/workflows/parallel_test.yml b/.github/workflows/parallel_test.yml index 8ce2ed470..df1ccf121 100644 --- a/.github/workflows/parallel_test.yml +++ b/.github/workflows/parallel_test.yml @@ -25,14 +25,10 @@ jobs: julia --project -O3 --check-bounds=no -e 'using MPI; MPI.install_mpiexecjl(; destdir=".")' julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["Random", "SpecialFunctions", "Test"]); Pkg.develop(path="moment_kinetics/")' julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.precompile()' - # Need to use openmpi so that the following arguments work: - # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes - # than physical cores). - # * `--mca mpi_yield_when_idle 1` changes a setting to prevent excessively - # terrible performance when oversubscribing. - ./mpiexecjl -np 3 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 - ./mpiexecjl -np 4 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 - ./mpiexecjl -np 2 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --long + # Need to use openmpi so that we can use `--oversubscribe` to allow using more MPI ranks than physical cores + ./mpiexecjl -np 3 --oversubscribe julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 + ./mpiexecjl -np 4 --oversubscribe julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 + ./mpiexecjl -np 2 --oversubscribe julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --long # Note: MPI.jl's default implementation is mpich, which has a similar option # `--with-device=ch3:sock`, but that needs to be set when compiling mpich. shell: bash @@ -54,12 +50,8 @@ jobs: julia --project -O3 --check-bounds=no -e 'using MPI; MPI.install_mpiexecjl(; destdir=".")' julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["Random", "SpecialFunctions", "Test"]); Pkg.develop(path="moment_kinetics/")' julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.precompile()' - # Need to use openmpi so that the following arguments work: - # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes - # than physical cores). - # * `--mca mpi_yield_when_idle 1` changes a setting to prevent excessively - # terrible performance when oversubscribing. - ./mpiexecjl -np 4 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 + # Need to use openmpi so that we can use `--oversubscribe` to allow using more MPI ranks than physical cores + ./mpiexecjl -np 4 --oversubscribe julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 # Note: MPI.jl's default implementation is mpich, which has a similar option # `--with-device=ch3:sock`, but that needs to be set when compiling mpich. shell: bash