CUDA config for lassen

configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml

@@ -68,8 +68,8 @@ compilers:
 - compiler:
     spec: [email protected]
     paths:
-      cc: /usr/tce/packages/clang/clang-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang-gpu
-      cxx: /usr/tce/packages/clang/clang-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang++-gpu
+      cc: /usr/tce/packages/clang/clang-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang
+      cxx: /usr/tce/packages/clang/clang-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang++
       f77: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran
       fc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran
     flags:

configs/LLNL-Sierra-IBM-power9-V100-Infiniband/variables.yaml

@@ -9,7 +9,7 @@ variables:
   mpi_command: '/usr/tcetmp/bin/lrun -n {n_ranks} -T {processes_per_node} {gtl_flag}'
   batch_submit: 'bsub -q pdebug {execute_experiment}'
   batch_nodes: '#BSUB -nnodes {n_nodes}'
-  batch_ranks: '#BSUB -nnodes {n_nodes}'
+  batch_ranks: ''
   batch_timeout: '#BSUB -W {batch_time}'
   default_cuda_version: '11.8.0'
   cuda_arch: '70'

experiments/gromacs/cuda/ramble.yaml

@@ -24,7 +24,7 @@ ramble:
             n_ranks: '{processes_per_node} * {n_nodes}'
             processes_per_node: '2'
             n_nodes: '4'
-            omp_num_threads: '22'
+            omp_num_threads: '10'
             target: 'gpu'
           experiments:
             gromacs_water_gmx50_adac_size{size}_dlb{dlb}_pin{pin}_target{target}_maxh{maxh}_nsteps{nsteps}_nstlist{nstlist}_npme{npme}:
@@ -33,23 +33,27 @@ ramble:
                 pin: 'off'
                 nb: '{target}'
                 pme: 'auto'
-                bonded: '{target}'
+                bonded: 'cpu'
                 update: '{target}'
                 maxh: '0.05'
                 nsteps: '1000'
                 nstlist: '200'
-                npme: '0'
+                npme: '1'
                 size: '1536'
 
   spack:
     concretized: true
     packages:
       gromacs:
-        spack_spec: [email protected] +mpi+openmp+cuda~hwloc~double cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: [email protected] +mpi+openmp+cuda~hwloc~double cuda_arch=={cuda_arch}
+        compiler: default-compiler
+      cuda:
+        spack_spec: cuda@{default_cuda_version}+allow-unsupported-compilers
         compiler: default-compiler
     environments:
       gromacs:
         packages:
+        - cuda
         - blas
         - lapack
         - default-mpi

repo/gromacs/package.py

@@ -494,10 +494,6 @@ def cmake_args(self):
             else:
                 options.append("-DGMX_GPLUSPLUS_PATH=%s/g++" % self.spec["gcc"].prefix.bin)
 
-        target = self.spec.target
-        if target.family == "ppc64le":
-            options.append("-DGMX_GPLUSPLUS_PATH=%s/g++" % '/usr/tce/packages/gcc/gcc-11.2.1/bin')
-
         if "+double" in self.spec:
             options.append("-DGMX_DOUBLE:BOOL=ON")
 
@@ -543,13 +539,20 @@ def cmake_args(self):
         if "+cuda" in self.spec:
             options.append("-DCUDA_TOOLKIT_ROOT_DIR:STRING=" + self.spec["cuda"].prefix)
 
-        options.append("-DGMX_EXTERNAL_LAPACK:BOOL=ON")
-        if self.spec["lapack"].libs:
-            options.append("-DGMX_LAPACK_USER={0}".format(self.spec["lapack"].libs.joined(";")))
+        target = self.spec.target
+        if "+cuda" in self.spec and target.family == "ppc64le":
+            options.append("-DGMX_EXTERNAL_LAPACK:BOOL=OFF")
+        else:
+            options.append("-DGMX_EXTERNAL_LAPACK:BOOL=ON")
+            if self.spec["lapack"].libs:
+                options.append("-DGMX_LAPACK_USER={0}".format(self.spec["lapack"].libs.joined(";")))
 
-        options.append("-DGMX_EXTERNAL_BLAS:BOOL=ON")
-        if self.spec["blas"].libs:
-            options.append("-DGMX_BLAS_USER={0}".format(self.spec["blas"].libs.joined(";")))
+        if "+cuda" in self.spec and target.family == "ppc64le":
+            options.append("-DGMX_EXTERNAL_BLAS:BOOL=OFF")
+        else:
+            options.append("-DGMX_EXTERNAL_BLAS:BOOL=ON")
+            if self.spec["blas"].libs:
+                options.append("-DGMX_BLAS_USER={0}".format(self.spec["blas"].libs.joined(";")))
 
         if "+cp2k" in self.spec:
             options.append("-DGMX_CP2K:BOOL=ON")