Changes for ats2 system (#42)

Co-authored-by: pearce8 <[email protected]>
LLNL · Nov 1, 2023 · 0a1d24d · 0a1d24d
1 parent f072ea2
commit 0a1d24d
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 80 deletions.
diff --git a/configs/ats2/spack.yaml b/configs/ats2/spack.yaml
@@ -1,16 +1,16 @@
 spack:
   packages:
     default-compiler:
-      spack_spec: [email protected]{xl_version}-cuda{default_cuda}
+      spack_spec: [email protected]{default_cuda_version}
+    default-mpi:
+      spack_spec: [email protected]
     compiler-gcc:
       spack_spec: [email protected]
-    default-mpi:
-      spack_spec: spectrum-mpi@{mpi_version}
-    compiler-cuda{cuda_version}:
-      spack_spec: [email protected]{xl_version}-cuda{cuda_version}
-    cublas{cuda_version}:
-      spack_spec: cublas@{cuda_version}
+    compiler-cuda10:
+      spack_spec: [email protected]{cuda10_version}
+    cublas-cuda-10:
+      spack_spec: cublas@{cuda10_version}
+    cublas-cuda-default:
+      spack_spec: cublas@{default_cuda_version} 
     lapack:
       spack_spec: [email protected]
-    mpi-{compilers}:
-      spack_spec: spectrum-mpi@{mpi_version}
diff --git a/configs/ats2/variables.yaml b/configs/ats2/variables.yaml
@@ -1,13 +1,11 @@
 variables:
   gtl_flag: ''  # to be overwritten by tests that need GTL
+  batch_time: '02:00'
   mpi_command: '/usr/tcetmp/bin/lrun -n {n_ranks} -T {processes_per_node} {gtl_flag}'
   batch_submit: 'bsub -q pdebug {execute_experiment}'
   batch_nodes: '#BSUB -nnodes {n_nodes}'
   batch_ranks: '#BSUB -nnodes {n_nodes}'
   batch_timeout: '#BSUB -W {batch_time}'
-  cuda_version: ['10.1.243', '11.8.0']
-  default_cuda: '11.8.0'
+  cuda10_version: '10.1.243'
+  default_cuda_version: '11.8.0'
   cuda_arch: '70'
-  xl_version: '2022.08.19'
-  mpi_version: '2022.08.19'
-  compilers: [compiler-cuda10.1.243, compiler-cuda11.8.0]
diff --git a/experiments/amg2023/cuda/ramble.yaml b/experiments/amg2023/cuda/ramble.yaml
@@ -10,7 +10,6 @@ ramble:
         problem1:
           variables:
             n_ranks: '{processes_per_node} * {n_nodes}'
-            batch_time: '02:00'
             p: 2
             px: '{p}'
             py: '{p}'
@@ -20,81 +19,59 @@ ramble:
             ny: '{n}'
             nz: '{n}'
             experiment_setup: ''
+            gtl: ['gtl', 'nogtl']
+            gtlflag: ['-M"-gpu"', '']
           experiments:
-            amg2023_cuda{cuda_version}_problem1_{gtl}_{n_nodes}_{px}_{py}_{pz}_{nx}_{ny}_{nz}:
+            # because MPS only works for cuda<=v10
+            amg2023_cuda_10_problem1_{gtl}_{n_nodes}_{px}_{py}_{pz}_{nx}_{ny}_{nz}:
               variables:
-                experiment_setup: ['/usr/tcetmp/bin/enable_mps', '']
-                env_name: 'amg2023-cuda{cuda_version}'
-                xl_version: 'xl/2022.08.19'
-                gtl: 'nogtl'
-                gtlflag: ''
-                processes_per_node: ['8', '4']
-                n_nodes: ['1', '2']
-              #zips:
-              #  gtl_info:
-              #  - gtl
-              #  - gtlflag
+                experiment_setup: '/usr/tcetmp/bin/enable_mps'
+                env_name: amg2023-cuda-10
+                processes_per_node: '8'
+                n_nodes: '1'
               matrices:
                 - size:
-                  - n # TODO: Filter matrix
-                  #- gtl_info
-            amg2023_cuda{cuda_version}_problem1_{gtl}_{n_nodes}_{px}_{py}_{pz}_{nx}_{ny}_{nz}:
+                  - n
+            amg2023_cuda_default_problem1_{gtl}_{n_nodes}_{px}_{py}_{pz}_{nx}_{ny}_{nz}:
               variables:
-                experiment_setup: ['/usr/tcetmp/bin/enable_mps', '']
-                env_name: 'amg2023-cuda{cuda_version}'
-                xl_version: 'xl/2022.08.19'
-                gtl: 'gtl'
-                gtlflag: '-M"-gpu"'
-                processes_per_node: ['8', '4']
-                n_nodes: ['1', '2']
+                env_name: amg2023-cuda-default
+                processes_per_node: ['4', '2']
+                n_nodes: ['2', '4']
+              zips:
+                gtl_info:
+                - gtl
+                - gtlflag
               matrices:
                 - size:
                   - n
-            amg2023_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}:
-              env_vars:
-                set:
-                  OMP_NUM_THREADS: '{omp_num_threads}'
-              variables:
-                env_name: 'amg2023-omp'
-                cuda_version: ''
-                xl_version: 'xl/2022.08.19'
-                gtlflag: ''
-                processes_per_node: ['8', '4']
-                n_nodes: ['1', '2']
-                omp_num_threads: ['1', '2', '4']
-              matrices:
-                - size_threads:
-                  - n # TODO: Filter matrix
-                  - omp_num_threads # TODO: Filter matrix
+                  - gtl_info
   spack:
     concretized: true
     packages:
-      hypre-cuda{cuda_version}:
-        spack_spec: [email protected] +mpi+cuda+mixedint cuda_arch={cuda_arch} ^cuda@{cuda_version}
-        compiler: compiler-cuda{cuda_version}
-      amg2023-cuda{cuda_version}:
-        spack_spec: amg2023@develop +mpi+cuda ^cuda@{cuda_version}
-        compiler: compiler-cuda{cuda_version}
-      hypre-omp:
-        spack_spec: [email protected] +mpi+openmp+mixedint ~cuda
+      hypre-cuda-10:
+        spack_spec: [email protected] +mpi+cuda+mixedint cuda_arch={cuda_arch} ^cuda@{cuda10_version}
+        compiler: compiler-cuda10
+      amg2023-cuda-10:
+        spack_spec: amg2023@develop +mpi+cuda ^cuda@{cuda10_version}
+        compiler: compiler-cuda10
+      hypre-cuda-default:
+        spack_spec: [email protected] +mpi+cuda+mixedint cuda_arch={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
-      amg2023-omp:
-        spack_spec: amg2023@develop +mpi+openmp~cuda
+      amg2023-cuda-default:
+        spack_spec: amg2023@develop +mpi+cuda ^cuda@{default_cuda_version}
         compiler: default-compiler
     environments:
-      amg2023-cuda{cuda_version}:
-        matrix:
-        - cuda_version
+      amg2023-cuda-10:
         packages:
-        - cublas{cuda_version}
+        - cublas-cuda-10
         - lapack
         - default-mpi
-        - hypre-cuda{cuda_version}
-        - amg2023-cuda{cuda_version}
-      amg2023-omp:
+        - hypre-cuda-10
+        - amg2023-cuda-10
+      amg2023-cuda-default:
         packages:
-        - cublas{default_cuda}
+        - cublas-cuda-default
         - lapack
         - default-mpi
-        - hypre-omp
-        - amg2023-omp
+        - hypre-cuda-default
+        - amg2023-cuda-default
diff --git a/experiments/raja-perf/cuda/ramble.yaml b/experiments/raja-perf/cuda/ramble.yaml
@@ -17,18 +17,15 @@ ramble:
               variables:
                 processes_per_node: ['1', '2', '4']
                 n_ranks: '{processes_per_node} * {n_nodes}'
-                batch_time: '02:00'
                 n_nodes: '1'
-                cuda_version: ''
-                compilers: ''
   spack:
     concretized: true
     packages:
       blt:
         spack_spec: '[email protected]:'
         compiler: compiler-gcc
       raja-perf:
-        spack_spec: raja-perf@develop +cuda~openmp+mpi cuda_arch=70 ^cuda@11.8.0
+        spack_spec: raja-perf@develop +cuda~openmp+mpi cuda_arch={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
     environments:
       raja-perf:

diff --git a/experiments/saxpy/cuda/ramble.yaml b/experiments/saxpy/cuda/ramble.yaml
@@ -10,9 +10,8 @@ ramble:
         problem:
           variables:
             n_ranks: '{processes_per_node} * {n_nodes}'
-            batch_time: '120'
           experiments:
-            saxpy_cuda{cuda_version}_{n}_{n_nodes}_{processes_per_node}_{n_ranks}:
+            saxpy_{n}_{n_nodes}_{processes_per_node}_{n_ranks}:
               variables:
                 processes_per_node: '4'
                 n_nodes: '1'
@@ -23,7 +22,7 @@ ramble:
     concretized: true
     packages:
       saxpy:
-        spack_spec: [email protected] +cuda cuda_arch=70 ^cuda@11.8.0
+        spack_spec: [email protected] +cuda cuda_arch={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
     environments:
       saxpy: