CUDA experiment for LLNL-Sierra-IBM-power9-V100-Infiniband

LLNL · Feb 10, 2024 · 0738ba5 · 0738ba5
1 parent 4347564
commit 0738ba5
Show file tree

Hide file tree

Showing 6 changed files with 59 additions and 17 deletions.
diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml
@@ -65,3 +65,21 @@ compilers:
     modules: []
     environment: {}
     extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/clang/clang-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang-gpu
+      cxx: /usr/tce/packages/clang/clang-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang++-gpu
+      f77: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran
+      fc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++17
+      fflags: ''
+    operating_system: rhel7
+    target: ppc64le
+    modules: []
+    environment:
+      set:
+        GXX_PREFIX: "/usr/tce/packages/gcc/gcc-11.2.1"
+    extra_rpaths: []
diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml
@@ -48,26 +48,45 @@ packages:
     - spec: [email protected]
       prefix: /usr/tce/packages/cuda/cuda-11.8.0
     buildable: false
-  blas:
+  cublas:
     externals:
     - spec: [email protected]
       prefix: /usr/tce/packages/cuda/cuda-10.1.243
     - spec: [email protected]
       prefix: /usr/tce/packages/cuda/cuda-11.8.0
     buildable: false
+  cusolver:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0
+    buildable: false
+  blas:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tcetmp/packages/blas/blas-3.6.0-gfortran-4.8.5
+    buildable: false
+  fftw:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tcetmp/packages/fftw/fftw-3.3.10-xl-2023.06.28
+    buildable: false
   lapack:
     externals:
-    - spec: lapack-xl@3.9.0
-      prefix: /usr/tcetmp/packages/lapack/lapack-3.9.0-xl-2020.03.18
+    - spec: lapack-xl@3.6.0
+      prefix: /usr/tcetmp/packages/lapack/lapack-3.6.0-gfortran-4.8.5
     buildable: false
   mpi:
     externals:
-    - spec: [email protected]cuda-11.8.0
+    - spec: [email protected]cuda11.8.0
       prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-11.8.0
       extra_attributes:
         ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm"
-    - spec: [email protected]cuda-10.1.243
+    - spec: [email protected]cuda10.1.243
       prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-10.1.243
       extra_attributes:
         ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm"
+    - spec: [email protected]
+      prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-16.0.6-cuda-11.8.0-gcc-11.2.1
+      extra_attributes:
+        ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm"
     buildable: false
diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml
@@ -6,12 +6,16 @@
 spack:
   packages:
     default-compiler:
-      spack_spec: xl@16.1.1-2022.08.19-cuda{default_cuda_version}
+      spack_spec: clang@16.0.6-cuda{default_cuda_version}
     default-mpi:
-      spack_spec: [email protected]
+      spack_spec: [email protected]-clang16.0.6-cuda{default_cuda_version}
     compiler-gcc:
       spack_spec: [email protected]
+    blas:
+      spack_spec: [email protected]
     cublas-cuda:
-      spack_spec: cublas@{default_cuda_version} 
+      spack_spec: cublas@{default_cuda_version}
     lapack:
-      spack_spec: [email protected]
+      spack_spec: [email protected]
+    fftw:
+      spack_spec: [email protected]
diff --git a/experiments/gromacs/cuda/execute_experiment.tpl b/experiments/gromacs/cuda/execute_experiment.tpl
@@ -11,8 +11,6 @@
 
 cd {experiment_run_dir}
 
-{spack_setup}
-
 {experiment_setup}
 
 {command}
diff --git a/experiments/gromacs/cuda/ramble.yaml b/experiments/gromacs/cuda/ramble.yaml
@@ -22,8 +22,8 @@ ramble:
           variables:
             experiment_setup: ''
             n_ranks: '{processes_per_node} * {n_nodes}'
-            processes_per_node: '4'
-            n_nodes: '2'
+            processes_per_node: '2'
+            n_nodes: '4'
             omp_num_threads: '22'
             target: 'gpu'
           experiments:
@@ -44,15 +44,13 @@ ramble:
   spack:
     concretized: true
     packages:
-      fftw:
-        spack_spec: [email protected] +mpi+openmp
-        compiler: default-compiler
       gromacs:
-        spack_spec: [email protected] +mpi+openmp+cuda~hwloc cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: [email protected] +mpi+openmp+cuda~hwloc~double cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
     environments:
       gromacs:
         packages:
+        - blas
         - lapack
         - default-mpi
         - fftw

diff --git a/repo/gromacs/package.py b/repo/gromacs/package.py
@@ -494,6 +494,9 @@ def cmake_args(self):
             else:
                 options.append("-DGMX_GPLUSPLUS_PATH=%s/g++" % self.spec["gcc"].prefix.bin)
 
+        if target.family == "ppc64le":
+            options.append("-DGMX_GPLUSPLUS_PATH=%s/g++" % '/usr/tce/packages/gcc/gcc-11.2.1/bin')
+
         if "+double" in self.spec:
             options.append("-DGMX_DOUBLE:BOOL=ON")
 
@@ -585,6 +588,8 @@ def cmake_args(self):
         elif target >= "bulldozer":
             # AMD Family 15h
             options.append("-DGMX_SIMD=AVX_128_FMA")
+        elif target.family == "ppc64le":
+            options.append("-DGMX_SIMD=None")
         elif "vsx" in target:
             # IBM Power 7 and beyond
             if self.spec.satisfies("%nvhpc"):