From 3cbfd1ba45d6e0971ad48628b616b6dd34233f85 Mon Sep 17 00:00:00 2001 From: Marty Kandes Date: Wed, 18 Oct 2023 18:17:05 -0700 Subject: [PATCH] Add vasp6@6.4.2 % intel@19.1.3.304 ^intel-mpi@2019.10.317 to exp/0.17.3/cpu/b This deployment is in support of recent user requests for VASP v6.4.x compiled with Wannier90 integration [1]. Please note, however, to satisfy the requests, we have utlized an updated Spack package from @jerrypgreenberg for VASP v6.4.x, which has significant changes to its build process and GPU support --- e.g., moving from CUDA to OpenACC. As such, the Spack package in the sdsc package repo has been update as well, but will not be backwards-compatible with earlier versions of VASP. [1] https://github.com/sdsc/spack/issues/51 --- .../vasp6@6.4.2.o25753366.exp-15-56 | 627 ++++++++++++++++++ .../intel-mpi@2019.10.317/vasp6@6.4.2.sh | 82 +++ .../repos/sdsc/packages/vasp6/package.py | 393 ++++++----- 3 files changed, 923 insertions(+), 179 deletions(-) create mode 100644 etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/intel@19.1.3.304/intel-mpi@2019.10.317/vasp6@6.4.2.o25753366.exp-15-56 create mode 100644 etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/intel@19.1.3.304/intel-mpi@2019.10.317/vasp6@6.4.2.sh diff --git a/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/intel@19.1.3.304/intel-mpi@2019.10.317/vasp6@6.4.2.o25753366.exp-15-56 b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/intel@19.1.3.304/intel-mpi@2019.10.317/vasp6@6.4.2.o25753366.exp-15-56 new file mode 100644 index 00000000000000..44750e4ac9901b --- /dev/null +++ b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/intel@19.1.3.304/intel-mpi@2019.10.317/vasp6@6.4.2.o25753366.exp-15-56 @@ -0,0 +1,627 @@ +1697673944 25753366 33aaa7b16a81d5ba5a9535c03519537e /home/spack_cpu/software/vasp/vasp6@6.4.2.sh + +#!/usr/bin/env bash + +#SBATCH --job-name=vasp6@6.4.2 +#SBATCH --account=use300 +#SBATCH --reservation=root_73 +#SBATCH --partition=ind-shared +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=32G +#SBATCH --time=01:00:00 +#SBATCH --output=%x.o%j.%N + +declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')" +declare -xir UNIX_TIME="$(date +'%s')" + +declare -xr LOCAL_SCRATCH_DIR="/scratch/${USER}/job_${SLURM_JOB_ID}" +declare -xr TMPDIR="${LOCAL_SCRATCH_DIR}" + +declare -xr SYSTEM_NAME='expanse' + +declare -xr SPACK_VERSION='0.17.3' +declare -xr SPACK_INSTANCE_NAME='cpu' +declare -xr SPACK_INSTANCE_VERSION='b' +declare -xr SPACK_INSTANCE_DIR="/cm/shared/apps/spack/${SPACK_VERSION}/${SPACK_INSTANCE_NAME}/${SPACK_INSTANCE_VERSION}" + +declare -xr SLURM_JOB_SCRIPT="$(scontrol show job ${SLURM_JOB_ID} | awk -F= '/Command=/{print $2}')" +declare -xr SLURM_JOB_MD5SUM="$(md5sum ${SLURM_JOB_SCRIPT})" + +declare -xr SCHEDULER_MODULE='slurm' +declare -xr COMPILER_MODULE='intel/19.1.3.304' +declare -xr MPI_MODULE='intel-mpi/2019.10.317' + +echo "${UNIX_TIME} ${SLURM_JOB_ID} ${SLURM_JOB_MD5SUM} ${SLURM_JOB_DEPENDENCY}" +echo "" + +cat "${SLURM_JOB_SCRIPT}" + +module purge +module load "${SCHEDULER_MODULE}" +. "${SPACK_INSTANCE_DIR}/share/spack/setup-env.sh" +module use "${SPACK_ROOT}/share/spack/lmod/linux-rocky8-x86_64/Core" +module load "${COMPILER_MODULE}" +module load "${MPI_MODULE}" +module list +shopt -s expand_aliases + +declare -xr INTEL_LICENSE_FILE='40000@elprado.sdsc.edu:40200@elprado.sdsc.edu' +declare -xr SPACK_PACKAGE='vasp6@6.4.2' +declare -xr SPACK_COMPILER='intel@19.1.3.304' +declare -xr SPACK_VARIANTS='~acc +openmp +scalapack ~vaspsol +wannier90' +declare -xr SPACK_DEPENDENCIES="^wannier90@3.1.0/$(spack find --format '{hash:7}' wannier90@3.1.0 % ${SPACK_COMPILER} ^intel-mkl@2020.4.304 ^intel-mpi@2019.10.317)" +declare -xr SPACK_SPEC="${SPACK_PACKAGE} % ${SPACK_COMPILER} ${SPACK_VARIANTS} ${SPACK_DEPENDENCIES}" + +printenv + +spack config get compilers +spack config get config +spack config get mirrors +spack config get modules +spack config get package +spack config get repos +spack config get upstreams + +time -p spack spec --long --namespaces --types --reuse $(echo "${SPACK_SPEC}") +if [[ "${?}" -ne 0 ]]; then + echo 'ERROR: spack concretization failed.' + exit 1 +fi + +time -p spack install --jobs "${SLURM_CPUS_PER_TASK}" --fail-fast --yes-to-all --reuse $(echo "${SPACK_SPEC}") +#if [[ "${?}" -ne 0 ]]; then +# echo 'ERROR: spack install failed.' +# exit 1 +#fi + +spack module lmod refresh -y +#spack module lmod refresh --delete-tree -y + +#sbatch --dependency="afterok:${SLURM_JOB_ID}" '' + +sleep 600 + +Currently Loaded Modules: + 1) slurm/expanse/21.08.8 3) intel-mpi/2019.10.317/ezrfjne + 2) intel/19.1.3.304/6pv46so + + + +==> Warning: Skipping package at /cm/shared/apps/spack/0.17.3/cpu/b/var/spack/repos/sdsc/packages/amber.configure. "amber.configure" is not a valid Spack module name. +LD_LIBRARY_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/libfabric/lib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/lib/release:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/lib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/compiler/lib/intel64_lin:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/lib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/release:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/tbb/lib/intel64/gcc4.8:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/lib:/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64 +LS_COLORS=rs=0:di=38;5;33:ln=38;5;51:mh=00:pi=40;38;5;11:so=38;5;13:do=38;5;5:bd=48;5;232;38;5;11:cd=48;5;232;38;5;3:or=48;5;232;38;5;9:mi=01;05;37;41:su=48;5;196;38;5;15:sg=48;5;11;38;5;16:ca=48;5;196;38;5;226:tw=48;5;10;38;5;16:ow=48;5;10;38;5;21:st=48;5;21;38;5;15:ex=38;5;40:*.tar=38;5;9:*.tgz=38;5;9:*.arc=38;5;9:*.arj=38;5;9:*.taz=38;5;9:*.lha=38;5;9:*.lz4=38;5;9:*.lzh=38;5;9:*.lzma=38;5;9:*.tlz=38;5;9:*.txz=38;5;9:*.tzo=38;5;9:*.t7z=38;5;9:*.zip=38;5;9:*.z=38;5;9:*.dz=38;5;9:*.gz=38;5;9:*.lrz=38;5;9:*.lz=38;5;9:*.lzo=38;5;9:*.xz=38;5;9:*.zst=38;5;9:*.tzst=38;5;9:*.bz2=38;5;9:*.bz=38;5;9:*.tbz=38;5;9:*.tbz2=38;5;9:*.tz=38;5;9:*.deb=38;5;9:*.rpm=38;5;9:*.jar=38;5;9:*.war=38;5;9:*.ear=38;5;9:*.sar=38;5;9:*.rar=38;5;9:*.alz=38;5;9:*.ace=38;5;9:*.zoo=38;5;9:*.cpio=38;5;9:*.7z=38;5;9:*.rz=38;5;9:*.cab=38;5;9:*.wim=38;5;9:*.swm=38;5;9:*.dwm=38;5;9:*.esd=38;5;9:*.jpg=38;5;13:*.jpeg=38;5;13:*.mjpg=38;5;13:*.mjpeg=38;5;13:*.gif=38;5;13:*.bmp=38;5;13:*.pbm=38;5;13:*.pgm=38;5;13:*.ppm=38;5;13:*.tga=38;5;13:*.xbm=38;5;13:*.xpm=38;5;13:*.tif=38;5;13:*.tiff=38;5;13:*.png=38;5;13:*.svg=38;5;13:*.svgz=38;5;13:*.mng=38;5;13:*.pcx=38;5;13:*.mov=38;5;13:*.mpg=38;5;13:*.mpeg=38;5;13:*.m2v=38;5;13:*.mkv=38;5;13:*.webm=38;5;13:*.ogm=38;5;13:*.mp4=38;5;13:*.m4v=38;5;13:*.mp4v=38;5;13:*.vob=38;5;13:*.qt=38;5;13:*.nuv=38;5;13:*.wmv=38;5;13:*.asf=38;5;13:*.rm=38;5;13:*.rmvb=38;5;13:*.flc=38;5;13:*.avi=38;5;13:*.fli=38;5;13:*.flv=38;5;13:*.gl=38;5;13:*.dl=38;5;13:*.xcf=38;5;13:*.xwd=38;5;13:*.yuv=38;5;13:*.cgm=38;5;13:*.emf=38;5;13:*.ogv=38;5;13:*.ogx=38;5;13:*.aac=38;5;45:*.au=38;5;45:*.flac=38;5;45:*.m4a=38;5;45:*.mid=38;5;45:*.midi=38;5;45:*.mka=38;5;45:*.mp3=38;5;45:*.mpc=38;5;45:*.ogg=38;5;45:*.ra=38;5;45:*.wav=38;5;45:*.oga=38;5;45:*.opus=38;5;45:*.spx=38;5;45:*.xspf=38;5;45: +SRUN_DEBUG=3 +SLURM_STEP_ID=0 +LOCAL_SCRATCH_DIR=/scratch/spack_cpu/job_25753366 +SLURM_NODEID=0 +SLURM_TASK_PID=2069005 +__LMOD_REF_COUNT_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/libfabric/bin:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/bin:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/bin:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/bin/intel64:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/bin:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/bin:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/bin:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin:1;/cm/shared/apps/slurm/current/sbin:2;/cm/shared/apps/slurm/current/bin:2;/cm/shared/apps/spack/0.17.3/cpu/b/bin:1;/home/spack_cpu/.local/bin:3;/home/spack_cpu/bin:3;/cm/shared/apps/sdsc/1.0/bin:1;/cm/shared/apps/sdsc/1.0/sbin:1;/usr/local/bin:2;/usr/bin:2;/usr/local/sbin:2;/usr/sbin:2 +_ModuleTable002_=LHByb3BUPXt9LFsic3RhY2tEZXB0aCJdPTAsWyJzdGF0dXMiXT0iYWN0aXZlIixbInVzZXJOYW1lIl09ImludGVsLW1waS8yMDE5LjEwLjMxNyIsfSxbImludGVsLzE5LjEuMy4zMDQiXT17WyJmbiJdPSIvY20vc2hhcmVkL2FwcHMvc3BhY2svMC4xNy4zL2NwdS9iL3NoYXJlL3NwYWNrL2xtb2QvbGludXgtcm9ja3k4LXg4Nl82NC9Db3JlL2ludGVsLzE5LjEuMy4zMDQvNnB2NDZzby5sdWEiLFsiZnVsbE5hbWUiXT0iaW50ZWwvMTkuMS4zLjMwNC82cHY0NnNvIixbImxvYWRPcmRlciJdPTIscHJvcFQ9e30sWyJzdGFja0RlcHRoIl09MCxbInN0YXR1cyJdPSJhY3RpdmUiLFsidXNlck5hbWUiXT0iaW50ZWwvMTkuMS4zLjMwNCIsfSxzbHVybT17WyJmbiJdPSIvY20vbG9jYWwv +SSH_CONNECTION=198.202.100.13 37052 198.202.100.13 22 +SPACK_PYTHON=/usr/bin/python3 +INTELHOME=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn +SLURM_PRIO_PROCESS=0 +SLURM_CPU_BIND_VERBOSE=quiet +LMOD_FAMILY_MPI_VERSION=ezrfjne +LANG=en_US.UTF-8 +SLURM_SUBMIT_DIR=/home/spack_cpu/software/vasp +HISTCONTROL=ignoredups +COMPILER_MODULE=intel/19.1.3.304 +LMOD_FAMILY_COMPILER_VERSION=6pv46so +HOSTNAME=exp-15-56 +LMOD_SYSTEM_DEFAULT_MODULES=DefaultModules +OLDPWD=/home/spack_cpu +__LMOD_REF_COUNT__LMFILES_=/cm/local/modulefiles/slurm/expanse/21.08.8:1;/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/Core/intel/19.1.3.304/6pv46so.lua:1;/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/intel/19.1.3.304/intel-mpi/2019.10.317/ezrfjne.lua:1 +SLURM_CPUS_PER_TASK=16 +SLURM_STEPID=0 +SLURM_SRUN_COMM_HOST=10.21.0.19 +SPACK_DEPENDENCIES=^wannier90@3.1.0/rhwr6rq +SLURM_DISTRIBUTION=cyclic +ENVIRONMENT=BATCH +SPACK_VERSION=0.17.3 +__LMOD_REF_COUNT_NLSPATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/compiler/lib/intel64/locale/%l_%t/%N:1 +SLURM_PROCID=0 +SPACK_INSTANCE_VERSION=b +SLURM_JOB_GID=11491 +SPACK_VARIANTS=~acc +openmp +scalapack ~vaspsol +wannier90 +SPACK_INSTANCE_DIR=/cm/shared/apps/spack/0.17.3/cpu/b +__LMOD_REF_COUNT_LD_LIBRARY_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/libfabric/lib:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/lib/release:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/lib:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/compiler/lib/intel64_lin:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/lib:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/release:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/tbb/lib/intel64/gcc4.8:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/lib:1;/cm/shared/apps/slurm/current/lib64/slurm:1;/cm/shared/apps/slurm/current/lib64:1 +SLURM_CPU_BIND=quiet,mask_cpu:0x0000000000000000000000000000FFFF +SLURM_JOB_SCRIPT=/home/spack_cpu/software/vasp/vasp6@6.4.2.sh +SLURMD_NODENAME=exp-15-56 +__LMOD_REF_COUNT_FI_PROVIDER_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/libfabric/lib/prov:1;/usr/lib64/libfabric:2;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/lib/prov:1 +FI_PROVIDER_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/libfabric/lib/prov:/usr/lib64/libfabric:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/lib/prov +_ModuleTable004_=c2hhcmVkL2FwcHMvc3BhY2svMC4xNy4zL2NwdS9iL3NoYXJlL3NwYWNrL2xtb2QvbGludXgtcm9ja3k4LXg4Nl82NC9Db3JlIiwiL2NtL2xvY2FsL21vZHVsZWZpbGVzIiwiL2NtL3NoYXJlZC9hcHBzL2FjY2Vzcy9tb2R1bGVmaWxlcyIsIi9ldGMvbW9kdWxlZmlsZXMiLCIvdXNyL3NoYXJlL21vZHVsZWZpbGVzIiwiL3Vzci9zaGFyZS9Nb2R1bGVzL21vZHVsZWZpbGVzIiwiL2NtL3NoYXJlZC9tb2R1bGVmaWxlcyIsfSxbInN5c3RlbUJhc2VNUEFUSCJdPSIvY20vbG9jYWwvbW9kdWxlZmlsZXM6L2NtL3NoYXJlZC9hcHBzL2FjY2Vzcy9tb2R1bGVmaWxlczovY20vc2hhcmVkL21vZHVsZWZpbGVzOi9ldGMvbW9kdWxlZmlsZXM6L3Vzci9zaGFyZS9tb2R1bGVmaWxlczovdXNy +SLURM_TASKS_PER_NODE=1 +S_COLORS=auto +which_declare=declare -f +SPACK_COMPILER=intel@19.1.3.304 +SLURM_JOB_RESERVATION=root_73 +CLASSPATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/lib/mpi.jar:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/mpi.jar +CC=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/icc +MPIF90=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/bin/mpiifort +XDG_SESSION_ID=328868 +USER=spack_cpu +SLURM_NNODES=1 +SLURM_LAUNCH_NODE_IPADDR=10.21.0.19 +SLURM_STEP_TASKS_PER_NODE=1 +MPICXX=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/bin/mpiicpc +SLURM_NTASKS_PER_NODE=1 +__LMOD_REF_COUNT_MODULEPATH=/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/intel-mpi/2019.10.317-ezrfjne/intel/19.1.3.304:1;/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/intel/19.1.3.304:1;/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/Core:1;/cm/local/modulefiles:1;/cm/shared/apps/access/modulefiles:1;/etc/modulefiles:1;/usr/share/modulefiles:1;/usr/share/Modules/modulefiles:1;/cm/shared/modulefiles:1 +__LMOD_REF_COUNT_LOADEDMODULES=slurm/expanse/21.08.8:1;intel/19.1.3.304/6pv46so:1;intel-mpi/2019.10.317/ezrfjne:1 +PWD=/home/spack_cpu/software/vasp +ENABLE_LMOD=1 +SLURM_JOB_NODELIST=exp-15-56 +HOME=/home/spack_cpu +SLURM_CLUSTER_NAME=expanse +INTEL_MPIHOME=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes +CMAKE_PREFIX_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn +LMOD_COLORIZE=yes +LOCAL_TIME=20231018T170544-0700 +LMOD_SYSHOST=expanse +SLURM_NODELIST=exp-15-56 +SSH_CLIENT=198.202.100.13 37052 22 +LMOD_VERSION=8.2.4 +CPATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/pstl/include:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/pstl/stdlib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/tbb/include:/cm/shared/apps/slurm/current/include +SLURM_NTASKS=1 +LMOD_SETTARG_CMD=: +F77=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/ifort +SLURM_JOB_CPUS_PER_NODE=16 +BASH_ENV=/usr/share/lmod/lmod/init/bash +LMOD_MPI_VERSION=2019.10.317-ezrfjne +SLURM_TOPOLOGY_ADDR=exp-15-56 +NLSPATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/compiler/lib/intel64/locale/%l_%t/%N +SLURM_WORKING_CLUSTER=expanse:mgr1:6817:9472:109 +SLURM_JOB_MD5SUM=33aaa7b16a81d5ba5a9535c03519537e /home/spack_cpu/software/vasp/vasp6@6.4.2.sh +SPACK_PACKAGE=vasp6@6.4.2 +__LMOD_REF_COUNT_LIBRARY_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/libfabric/lib:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/lib:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/tbb/lib/intel64/gcc4.8:2;/cm/shared/apps/slurm/current/lib64/slurm:1;/cm/shared/apps/slurm/current/lib64:1 +SLURM_STEP_NODELIST=exp-15-56 +SLURM_JOB_NAME=vasp6@6.4.2 +SLURM_SRUN_COMM_PORT=37059 +TMPDIR=/scratch/spack_cpu/job_25753366 +LIBRARY_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/libfabric/lib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/lib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/tbb/lib/intel64/gcc4.8:/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64 +__LMOD_REF_COUNT_CMAKE_PREFIX_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn:1 +LMOD_sys=Linux +SLURM_JOBID=25753366 +SYSTEM_NAME=expanse +_ModuleTable001_=X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLFsiY19zaG9ydFRpbWUiXT1mYWxzZSxkZXB0aFQ9e30sZmFtaWx5PXtbImNvbXBpbGVyIl09ImludGVsLzE5LjEuMy4zMDQiLFsibXBpIl09ImludGVsLW1waS8yMDE5LjEwLjMxNyIsfSxtVD17WyJpbnRlbC1tcGkvMjAxOS4xMC4zMTciXT17WyJmbiJdPSIvY20vc2hhcmVkL2FwcHMvc3BhY2svMC4xNy4zL2NwdS9iL3NoYXJlL3NwYWNrL2xtb2QvbGludXgtcm9ja3k4LXg4Nl82NC9pbnRlbC8xOS4xLjMuMzA0L2ludGVsLW1waS8yMDE5LjEwLjMxNy9lenJmam5lLmx1YSIsWyJmdWxsTmFtZSJdPSJpbnRlbC1tcGkvMjAxOS4xMC4zMTcvZXpyZmpuZSIsWyJsb2FkT3JkZXIiXT0z +SLURM_CONF=/cm/shared/apps/slurm/var/etc/expanse/slurm.conf +LOADEDMODULES=slurm/expanse/21.08.8:intel/19.1.3.304/6pv46so:intel-mpi/2019.10.317/ezrfjne +FC=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/ifort +__LMOD_REF_COUNT_MANPATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/man:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/man/common:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/man:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/man:1;/cm/shared/apps/slurm/current/man:3;/usr/share/lmod/lmod/share/man:1;/usr/local/share/man:2;/usr/share/man:2;/cm/local/apps/environment-modules/current/share/man:3 +_ModuleTable003_=bW9kdWxlZmlsZXMvc2x1cm0vZXhwYW5zZS8yMS4wOC44IixbImZ1bGxOYW1lIl09InNsdXJtL2V4cGFuc2UvMjEuMDguOCIsWyJsb2FkT3JkZXIiXT0xLHByb3BUPXt9LFsic3RhY2tEZXB0aCJdPTAsWyJzdGF0dXMiXT0iYWN0aXZlIixbInVzZXJOYW1lIl09InNsdXJtIix9LH0sbXBhdGhBPXsiL2NtL3NoYXJlZC9hcHBzL3NwYWNrLzAuMTcuMy9jcHUvYi9zaGFyZS9zcGFjay9sbW9kL2xpbnV4LXJvY2t5OC14ODZfNjQvaW50ZWwtbXBpLzIwMTkuMTAuMzE3LWV6cmZqbmUvaW50ZWwvMTkuMS4zLjMwNCIsIi9jbS9zaGFyZWQvYXBwcy9zcGFjay8wLjE3LjMvY3B1L2Ivc2hhcmUvc3BhY2svbG1vZC9saW51eC1yb2NreTgteDg2XzY0L2ludGVsLzE5LjEuMy4zMDQiLCIvY20v +INTEL_LICENSE_FILE=40000@elprado.sdsc.edu:40200@elprado.sdsc.edu +SLURM_NODE_ALIASES=(null) +MPICC=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/bin/mpiicc +LMOD_ROOT=/usr/share/lmod +SLURM_JOB_QOS=ind-shared-normal +SLURM_TOPOLOGY_ADDR_PATTERN=node +SSH_TTY=/dev/pts/147 +MAIL=/var/spool/mail/spack_cpu +MPI_MODULE=intel-mpi/2019.10.317 +MPIFC=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/bin/mpiifort +SLURM_CPUS_ON_NODE=16 +CXX=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/icpc +LMOD_arch=x86_64 +SLURM_JOB_NUM_NODES=1 +__Init_Default_Modules=1 +CMD_WLM_CLUSTER_NAME=expanse +SLURM_MEM_PER_NODE=32768 +SPACK_ROOT=/cm/shared/apps/spack/0.17.3/cpu/b +TERM=xterm-256color +SHELL=/bin/bash +_ModuleTable_Sz_=5 +SLURM_JOB_UID=527834 +__LMOD_REF_COUNT_CPATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/pstl/include:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/pstl/stdlib:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/tbb/include:2;/cm/shared/apps/slurm/current/include:1 +SLURM_JOB_PARTITION=ind-shared +LMOD_FAMILY_COMPILER=intel/19.1.3.304 +SLURM_PTY_WIN_ROW=24 +SLURM_CPU_BIND_LIST=0x0000000000000000000000000000FFFF +SPACK_INSTANCE_NAME=cpu +SLURM_JOB_USER=spack_cpu +SLURM_PTY_WIN_COL=80 +SLURM_NPROCS=1 +SHLVL=3 +SLURM_SUBMIT_HOST=exp-15-56 +UNIX_TIME=1697673944 +MPIF77=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/bin/mpiifort +SLURM_JOB_ACCOUNT=use300 +MANPATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/man:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/man/common:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/man:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/man:/cm/shared/apps/slurm/current/man:/usr/share/lmod/lmod/share/man:/usr/local/share/man:/usr/share/man:/cm/local/apps/environment-modules/current/share/man +__LMOD_REF_COUNT_CLASSPATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/lib/mpi.jar:1;/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib/mpi.jar:1 +LMOD_PREPEND_BLOCK=normal +SLURM_STEP_LAUNCHER_PORT=37059 +F90=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/ifort +MODULEPATH=/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/intel-mpi/2019.10.317-ezrfjne/intel/19.1.3.304:/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/intel/19.1.3.304:/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/Core:/cm/local/modulefiles:/cm/shared/apps/access/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/usr/share/Modules/modulefiles:/cm/shared/modulefiles +SLURM_PTY_PORT=44191 +SLURM_GTIDS=0 +LOGNAME=spack_cpu +DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/527834/bus +XDG_RUNTIME_DIR=/run/user/527834 +MODULEPATH_ROOT=/usr/share/modulefiles +LMOD_PACKAGE_PATH=/usr/share/lmod/etc/site/lmod +PSTLROOT=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/pstl +LMOD_MPI_NAME=intel-mpi +PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/libfabric/bin:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi/intel64/bin:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/bin:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/bin/intel64:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/bin:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/libfabric/bin:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/mpi/intel64/bin:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin:/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/cm/shared/apps/spack/0.17.3/cpu/b/bin:/home/spack_cpu/.local/bin:/home/spack_cpu/bin:/cm/shared/apps/sdsc/1.0/bin:/cm/shared/apps/sdsc/1.0/sbin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin +SLURM_JOB_ID=25753366 +TBBROOT=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/compilers_and_libraries_2020.4.304/linux/tbb +_LMFILES_=/cm/local/modulefiles/slurm/expanse/21.08.8:/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/Core/intel/19.1.3.304/6pv46so.lua:/cm/shared/apps/spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64/intel/19.1.3.304/intel-mpi/2019.10.317/ezrfjne.lua +SLURM_CPU_BIND_TYPE=mask_cpu: +DEBUGINFOD_URLS=https://debuginfod.centos.org/ +SLURM_STEP_NUM_TASKS=1 +SCHEDULER_MODULE=slurm +MODULESHOME=/usr/share/lmod/lmod +LMOD_SETTARG_FULL_SUPPORT=no +I_MPI_ROOT=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes/compilers_and_libraries_2020.4.317/linux/mpi +HISTSIZE=-1 +LMOD_PKG=/usr/share/lmod/lmod +SLURM_STEP_NUM_NODES=1 +SPACK_SPEC=vasp6@6.4.2 % intel@19.1.3.304 ~acc +openmp +scalapack ~vaspsol +wannier90 ^wannier90@3.1.0/rhwr6rq +_ModuleTable005_=L3NoYXJlL01vZHVsZXMvbW9kdWxlZmlsZXMiLH0= +LMOD_CMD=/usr/share/lmod/lmod/libexec/lmod +SLURM_LOCALID=0 +LESSOPEN=||/usr/bin/lesspipe.sh %s +LMOD_FULL_SETTARG_SUPPORT=no +LMOD_DIR=/usr/share/lmod/lmod/libexec +LMOD_FAMILY_MPI=intel-mpi/2019.10.317 +BASH_FUNC_which%%=() { ( alias; + eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@ +} +BASH_FUNC_module%%=() { eval $($LMOD_CMD bash "$@") && eval $(${LMOD_SETTARG_CMD:-:} -s sh) +} +BASH_FUNC_spack%%=() { : this is a shell function from: /cm/shared/apps/spack/0.17.3/cpu/b/share/spack/setup-env.sh; + : the real spack script is here: /cm/shared/apps/spack/0.17.3/cpu/b/bin/spack; + _spack_shell_wrapper "$@"; + return $? +} +BASH_FUNC__spack_shell_wrapper%%=() { for var in LD_LIBRARY_PATH DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH; + do + eval "if [ -n \"\${${var}-}\" ]; then export SPACK_$var=\${${var}}; fi"; + done; + if [ -n "${ZSH_VERSION:-}" ]; then + emulate -L sh; + fi; + _sp_flags=""; + while [ ! -z ${1+x} ] && [ "${1#-}" != "${1}" ]; do + _sp_flags="$_sp_flags $1"; + shift; + done; + if [ -n "$_sp_flags" ] && [ "${_sp_flags#*h}" != "${_sp_flags}" ] || [ "${_sp_flags#*V}" != "${_sp_flags}" ]; then + command spack $_sp_flags "$@"; + return; + fi; + _sp_subcommand=""; + if [ ! -z ${1+x} ]; then + _sp_subcommand="$1"; + shift; + fi; + case $_sp_subcommand in + "cd") + _sp_arg=""; + if [ -n "$1" ]; then + _sp_arg="$1"; + shift; + fi; + if [ "$_sp_arg" = "-h" ] || [ "$_sp_arg" = "--help" ]; then + command spack cd -h; + else + LOC="$(spack location $_sp_arg "$@")"; + if [ -d "$LOC" ]; then + cd "$LOC"; + else + return 1; + fi; + fi; + return + ;; + "env") + _sp_arg=""; + if [ -n "$1" ]; then + _sp_arg="$1"; + shift; + fi; + if [ "$_sp_arg" = "-h" ] || [ "$_sp_arg" = "--help" ]; then + command spack env -h; + else + case $_sp_arg in + activate) + _a=" $@"; + if [ -z ${1+x} ] || [ "${_a#* --sh}" != "$_a" ] || [ "${_a#* --csh}" != "$_a" ] || [ "${_a#* -h}" != "$_a" ] || [ "${_a#* --help}" != "$_a" ]; then + command spack env activate "$@"; + else + stdout="$(command spack $_sp_flags env activate --sh "$@")" || return; + eval "$stdout"; + fi + ;; + deactivate) + _a=" $@"; + if [ "${_a#* --sh}" != "$_a" ] || [ "${_a#* --csh}" != "$_a" ]; then + command spack env deactivate "$@"; + else + if [ -n "$*" ]; then + command spack env deactivate -h; + else + stdout="$(command spack $_sp_flags env deactivate --sh)" || return; + eval "$stdout"; + fi; + fi + ;; + *) + command spack env $_sp_arg "$@" + ;; + esac; + fi; + return + ;; + "load" | "unload") + _a=" $@"; + if [ "${_a#* --sh}" != "$_a" ] || [ "${_a#* --csh}" != "$_a" ] || [ "${_a#* -h}" != "$_a" ] || [ "${_a#* --list}" != "$_a" ] || [ "${_a#* --help}" != "$_a" ]; then + command spack $_sp_flags $_sp_subcommand "$@"; + else + stdout="$(command spack $_sp_flags $_sp_subcommand --sh "$@")" || return; + eval "$stdout"; + fi + ;; + *) + command spack $_sp_flags $_sp_subcommand "$@" + ;; + esac +} +BASH_FUNC_ml%%=() { eval $($LMOD_DIR/ml_cmd "$@") +} +_=/usr/bin/printenv +compilers: +- compiler: + spec: gcc@8.5.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: + cflags: -O2 -march=native + cxxflags: -O2 -march=native + fflags: -O2 -march=native + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@10.2.0 + paths: + cc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/bin/gcc + cxx: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/bin/g++ + f77: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/bin/gfortran + fc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/bin/gfortran + flags: + cflags: -O2 -march=native + cxxflags: -O2 -march=native + fflags: -O2 -march=native + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: aocc@3.2.0 + paths: + cc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/aocc-3.2.0-io3s466wsnnichqc2o2rikbuloev5bmq/bin/clang + cxx: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/aocc-3.2.0-io3s466wsnnichqc2o2rikbuloev5bmq/bin/clang++ + f77: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/aocc-3.2.0-io3s466wsnnichqc2o2rikbuloev5bmq/bin/flang + fc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/aocc-3.2.0-io3s466wsnnichqc2o2rikbuloev5bmq/bin/flang + flags: + cflags: -O2 -march=native + cxxflags: -O2 -march=native + fflags: -O2 -march=native + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: intel@19.1.3.304 + paths: + cc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/icc + cxx: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/icpc + f77: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/ifort + fc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/ifort + flags: + cflags: -O2 -march=native + cxxflags: -O2 -march=native + fflags: -O2 -march=native + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +config: + verify_ssl: false + install_tree: + root: $spack/opt/spack + projections: + all: ${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH} + template_dirs: + - $spack/share/spack/templates + + # Temporary locations Spack can try to use for builds. + # + # Recommended options are given below. + # + # Builds can be faster in temporary directories on some (e.g., HPC) systems. + # Specifying `$tempdir` will ensure use of the default temporary directory + # (i.e., ``$TMP` or ``$TMPDIR``). + # + # Another option that prevents conflicts and potential permission issues is + # to specify `$user_cache_path/stage`, which ensures each user builds in their + # home directory. + # + # A more traditional path uses the value of `$spack/var/spack/stage`, which + # builds directly inside Spack's instance without staging them in a + # temporary space. Problems with specifying a path inside a Spack instance + # are that it precludes its use as a system package and its ability to be + # pip installable. + # + # In any case, if the username is not already in the path, Spack will append + # the value of `$user` in an attempt to avoid potential conflicts between + # users in shared temporary spaces. + # + # The build stage can be purged with `spack clean --stage` and + # `spack clean -a`, so it is important that the specified directory uniquely + # identifies Spack staging to avoid accidentally wiping out non-Spack work. + build_stage: + - $tempdir/$user/spack-stage + - $user_cache_path/stage + # - $spack/var/spack/stage + + # Directory in which to run tests and store test results. + # Tests will be stored in directories named by date/time and package + # name/hash. + test_stage: $user_cache_path/test + source_cache: $spack/var/spack/cache + misc_cache: $user_cache_path/cache + connect_timeout: 10 + suppress_gpg_warnings: false + install_missing_compilers: false + checksum: true + deprecated: false + dirty: false + build_language: C + locks: true + url_fetch_method: urllib + ccache: false + concretizer: clingo + db_lock_timeout: 3 + package_lock_timeout: null + shared_linking: rpath + allow_sgid: true + terminal_title: false + debug: false + build_jobs: 16 +mirrors: + spack-public: https://mirror.spack.io +modules: + default: + 'enable:': + - lmod + lmod: + core_compilers: + - gcc@8.5.0 + hierarchy: + - mpi + hash_length: 0 + blacklist_implicits: true + whitelist: [py-jupyter-core, hadoop] + naming_scheme: '{name}/{version}/{hash:7}' + projections: + all: '{name}/{version}/{hash:7}' + all: + suffixes: + +openmp: omp + threads=openmp: omp + +ipl64: i64 + environment: + set: + '{name}HOME': '{prefix}' + intel: + environment: + set: + INTEL_LICENSE_FILE: 40000@elprado.sdsc.edu:40200@elprado.sdsc.edu + petsc: + suffixes: + +complex: cmplx + slepc: + suffixes: + ^petsc +complex: cmplx + ^python: + autoload: direct + ^ucx: + autoload: direct + prefix_inspections: + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + bin: + - PATH + man: + - MANPATH + share/man: + - MANPATH + share/aclocal: + - ACLOCAL_PATH + lib/pkgconfig: + - PKG_CONFIG_PATH + lib64/pkgconfig: + - PKG_CONFIG_PATH + share/pkgconfig: + - PKG_CONFIG_PATH + ? '' + : - CMAKE_PREFIX_PATH + + # These are configurations for the module set named "default" +usage: spack config get [-h] [section] +spack config get: error: argument section: invalid choice: 'package' choose from: + bootstrap compilers config mirrors modules packages repos upstreams + +repos: +- $spack/var/spack/repos/sdsc +- $spack/var/spack/repos/builtin +upstreams: {} +==> Warning: Skipping package at /cm/shared/apps/spack/0.17.3/cpu/b/var/spack/repos/sdsc/packages/amber.configure. "amber.configure" is not a valid Spack module name. +Input spec +-------------------------------- +[ ] .vasp6@6.4.2%intel@19.1.3.304~acc+openmp+scalapack~vaspsol+wannier90 +[ ] ^sdsc.wannier90@3.1.0%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" +shared arch=linux-rocky8-zen2 +[bl ] ^builtin.intel-mkl@2020.4.304%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" ~ilp64+shared threads=none arch=linux-rocky8-zen2 +[bl ] ^builtin.intel-mpi@2019.10.317%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2 + +Concretized +-------------------------------- +nfkttop [ ] sdsc.vasp6@6.4.2%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" ~acc+openmp+scalapack~vaspsol+wannier90 arch=linux-rocky8-zen2 +vg6aq26 [bl ] ^builtin.intel-mkl@2020.4.304%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" ~ilp64+shared threads=none arch=linux-rocky8-zen2 +ezrfjne [blr ] ^builtin.intel-mpi@2019.10.317%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2 +gfmobbl [b ] ^builtin.rsync@3.2.3%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2 +ad2tmmw [bl ] ^builtin.lz4@1.9.3%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" libs=shared,static arch=linux-rocky8-zen2 +7jpahoe [bl ] ^builtin.openssl@1.1.1k%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" ~docs certs=system arch=linux-rocky8-zen2 +mr4trum [bl ] ^builtin.popt@1.16%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2 +uqsxtvd [bl ] ^builtin.libiconv@1.16%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" libs=shared,static arch=linux-rocky8-zen2 +kpzf6bj [bl ] ^builtin.xxhash@0.8.0%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2 +ol3dty6 [bl ] ^builtin.zlib@1.2.11%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" +optimize+pic+shared arch=linux-rocky8-zen2 +cmverlm [bl ] ^builtin.zstd@1.5.0%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" ~programs arch=linux-rocky8-zen2 +rhwr6rq [bl ] ^sdsc.wannier90@3.1.0%intel@19.1.3.304 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" +shared arch=linux-rocky8-zen2 + +real 18.09 +user 8.76 +sys 0.24 +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mkl-2020.4.304-vg6aq26t4jbq6aia4xzf3ugsqos5ayuw +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/intel-mpi-2019.10.317-ezrfjnetzkfbr6cme5ogobwzofshdpes +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/lz4-1.9.3-ad2tmmwm6ex3yiigp675l6bbuwfg3cez +[+] /usr (external openssl-1.1.1k-7jpahoelcnehcumz3pyjj3nezazat3dy) +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/libiconv-1.16-uqsxtvdgkn5mwr7fzxkzvq6lt7jdf7qw +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/xxhash-0.8.0-kpzf6bjfrefypkia5kufxquazmzaeadj +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/zlib-1.2.11-ol3dty62y4icftohccbcxpitdm6gf6j3 +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/zstd-1.5.0-cmverlmyc4ot3pxx6ytapyxwcw5hnvhr +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/wannier90-3.1.0-rhwr6rqewhxejgk6nqefihwwjtf6zapq +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/popt-1.16-mr4trumi3p2tfdyfurp226nvsy7uruom +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/rsync-3.2.3-gfmobbl7joy26bp22hbwo7fq3z4w6viy +==> Installing vasp6-6.4.2-nfkttoplgemyx6ocwm5osos76s64h4f6 +==> No binary for vasp6-6.4.2-nfkttoplgemyx6ocwm5osos76s64h4f6 found: installing from source +==> Warning: Skipping package at /cm/shared/apps/spack/0.17.3/cpu/b/var/spack/repos/sdsc/packages/amber.configure. "amber.configure" is not a valid Spack module name. +==> Warning: Expected user 527834 to own /scratch/spack_cpu, but it is owned by 0 +==> Using cached archive: /cm/shared/apps/spack/0.17.3/cpu/b/var/spack/cache/_source-cache/archive/b7/b704637f7384673f91adfbc803edc5cc7fe736d9623453461f7cdc29b123410e.tgz +==> No patches needed for vasp6 +==> Warning: Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for optimizations that are not unique to Intel microprocessors +==> vasp6: Executing phase: 'edit' +==> vasp6: Executing phase: 'build' +==> vasp6: Executing phase: 'install' +==> vasp6: Successfully installed vasp6-6.4.2-nfkttoplgemyx6ocwm5osos76s64h4f6 + Fetch: 0.11s. Build: 35m 39.89s. Total: 35m 40.00s. +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/intel-19.1.3.304/vasp6-6.4.2-nfkttoplgemyx6ocwm5osos76s64h4f6 +real 2153.92 +user 2043.89 +sys 68.76 +==> Warning: Skipping package at /cm/shared/apps/spack/0.17.3/cpu/b/var/spack/repos/sdsc/packages/amber.configure. "amber.configure" is not a valid Spack module name. +==> Regenerating lmod module files +==> OpenFOAM bashrc env: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/aocc-3.2.0/openfoam-2106-jz42us227mirxrhqjvojlaut2giuh74j/etc/bashrc diff --git a/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/intel@19.1.3.304/intel-mpi@2019.10.317/vasp6@6.4.2.sh b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/intel@19.1.3.304/intel-mpi@2019.10.317/vasp6@6.4.2.sh new file mode 100644 index 00000000000000..d48f063b7e5b5b --- /dev/null +++ b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/intel@19.1.3.304/intel-mpi@2019.10.317/vasp6@6.4.2.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +#SBATCH --job-name=vasp6@6.4.2 +#SBATCH --account=use300 +#SBATCH --reservation=root_73 +#SBATCH --partition=ind-shared +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=32G +#SBATCH --time=01:00:00 +#SBATCH --output=%x.o%j.%N + +declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')" +declare -xir UNIX_TIME="$(date +'%s')" + +declare -xr LOCAL_SCRATCH_DIR="/scratch/${USER}/job_${SLURM_JOB_ID}" +declare -xr TMPDIR="${LOCAL_SCRATCH_DIR}" + +declare -xr SYSTEM_NAME='expanse' + +declare -xr SPACK_VERSION='0.17.3' +declare -xr SPACK_INSTANCE_NAME='cpu' +declare -xr SPACK_INSTANCE_VERSION='b' +declare -xr SPACK_INSTANCE_DIR="/cm/shared/apps/spack/${SPACK_VERSION}/${SPACK_INSTANCE_NAME}/${SPACK_INSTANCE_VERSION}" + +declare -xr SLURM_JOB_SCRIPT="$(scontrol show job ${SLURM_JOB_ID} | awk -F= '/Command=/{print $2}')" +declare -xr SLURM_JOB_MD5SUM="$(md5sum ${SLURM_JOB_SCRIPT})" + +declare -xr SCHEDULER_MODULE='slurm' +declare -xr COMPILER_MODULE='intel/19.1.3.304' +declare -xr MPI_MODULE='intel-mpi/2019.10.317' + +echo "${UNIX_TIME} ${SLURM_JOB_ID} ${SLURM_JOB_MD5SUM} ${SLURM_JOB_DEPENDENCY}" +echo "" + +cat "${SLURM_JOB_SCRIPT}" + +module purge +module load "${SCHEDULER_MODULE}" +. "${SPACK_INSTANCE_DIR}/share/spack/setup-env.sh" +module use "${SPACK_ROOT}/share/spack/lmod/linux-rocky8-x86_64/Core" +module load "${COMPILER_MODULE}" +module load "${MPI_MODULE}" +module list +shopt -s expand_aliases + +declare -xr INTEL_LICENSE_FILE='40000@elprado.sdsc.edu:40200@elprado.sdsc.edu' +declare -xr SPACK_PACKAGE='vasp6@6.4.2' +declare -xr SPACK_COMPILER='intel@19.1.3.304' +declare -xr SPACK_VARIANTS='~acc +openmp +scalapack ~vaspsol +wannier90' +declare -xr SPACK_DEPENDENCIES="^wannier90@3.1.0/$(spack find --format '{hash:7}' wannier90@3.1.0 % ${SPACK_COMPILER} ^intel-mkl@2020.4.304 ^intel-mpi@2019.10.317)" +declare -xr SPACK_SPEC="${SPACK_PACKAGE} % ${SPACK_COMPILER} ${SPACK_VARIANTS} ${SPACK_DEPENDENCIES}" + +printenv + +spack config get compilers +spack config get config +spack config get mirrors +spack config get modules +spack config get package +spack config get repos +spack config get upstreams + +time -p spack spec --long --namespaces --types --reuse $(echo "${SPACK_SPEC}") +if [[ "${?}" -ne 0 ]]; then + echo 'ERROR: spack concretization failed.' + exit 1 +fi + +time -p spack install --jobs "${SLURM_CPUS_PER_TASK}" --fail-fast --yes-to-all --reuse $(echo "${SPACK_SPEC}") +#if [[ "${?}" -ne 0 ]]; then +# echo 'ERROR: spack install failed.' +# exit 1 +#fi + +spack module lmod refresh -y +#spack module lmod refresh --delete-tree -y + +#sbatch --dependency="afterok:${SLURM_JOB_ID}" '' + +sleep 600 diff --git a/var/spack/repos/sdsc/packages/vasp6/package.py b/var/spack/repos/sdsc/packages/vasp6/package.py index d47b287004e003..898baf86cc3aae 100644 --- a/var/spack/repos/sdsc/packages/vasp6/package.py +++ b/var/spack/repos/sdsc/packages/vasp6/package.py @@ -1,9 +1,11 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import sys +import glob from spack import * @@ -17,218 +19,251 @@ class Vasp6(MakefilePackage): """ homepage = "https://vasp.at" - url = "file://{0}/vasp.5.4.4.pl2.tgz".format(os.getcwd()) + url = "file://{0}/vasp.5.4.4.pl2.tgz".format(os.getcwd()) manual_download = True - version("6.3.2", sha256="f7595221b0f9236a324ea8afe170637a578cdd5a837cc7679e7f7812f6edf25a") - version("6.2.1", sha256="d25e2f477d83cb20fce6a2a56dcee5dccf86d045dd7f76d3ae19af8343156a13") - version("6.2.0", sha256="49e7ba351bd634bc5f5f67a8ef1e38e64e772857a1c02f602828898a84197e25") - version("6.1.1", sha256="e37a4dfad09d3ad0410833bcd55af6b599179a085299026992c2d8e319bf6927") - version("5.4.4.pl2", sha256="98f75fd75399a23d76d060a6155f4416b340a1704f256a00146f89024035bc8e") - version("5.4.4", sha256="5bd2449462386f01e575f9adf629c08cb03a13142806ffb6a71309ca4431cfb3") - - resource( - name="vaspsol", - git="https://github.com/henniggroup/VASPsol.git", - tag="V1.0", - when="+vaspsol", - ) - - variant("openmp", default=False, description="Enable openmp build") - - variant("scalapack", default=False, description="Enables build with SCALAPACK") - - variant("cuda", default=False, description="Enables running on Nvidia GPUs") - - variant( - "vaspsol", - default=False, - description="Enable VASPsol implicit solvation model\n" - "https://github.com/henniggroup/VASPsol", - ) - - variant("shmem", default=True, description="Enable use_shmem build flag") - - depends_on("rsync", type="build") - depends_on("blas") - depends_on("lapack") - depends_on("fftw-api") - depends_on("mpi", type=("build", "link", "run")) - depends_on("scalapack", when="+scalapack") - depends_on("cuda", when="+cuda") - depends_on("qd", when="%nvhpc") - - conflicts( - "%gcc@:8", msg="GFortran before 9.x does not support all features needed to build VASP" - ) - conflicts("+vaspsol", when="+cuda", msg="+vaspsol only available for CPU") - conflicts("+openmp", when="@:6.1.1", msg="openmp support started from 6.2") + version('6.4.2', sha256='b704637f7384673f91adfbc803edc5cc7fe736d9623453461f7cdc29b123410e') + + resource(name='vaspsol', + git='https://github.com/henniggroup/VASPsol.git', + tag='V1.0', + when='+vaspsol') + + variant('openmp', default=False, + description='Enable openmp build') + + variant('scalapack', default=False, + description='Enables build with SCALAPACK') + + variant('acc', default=False, + description='Enables running on Nvidia GPUs') + + variant('vaspsol', default=False, + description='Enable VASPsol implicit solvation model\n' + 'https://github.com/henniggroup/VASPsol') + variant('wannier90', default=False, + description='Enable wannier90\n') + + + + + depends_on('rsync', type='build') + depends_on('blas') + depends_on('lapack') + depends_on('fftw-api') + depends_on('mpi', type=('build', 'link', 'run')) + depends_on('scalapack', when='+scalapack') + depends_on('qd', when='%nvhpc') + depends_on('wannier90', when='+wannier90') + + conflicts('%gcc@:8', msg='GFortran before 9.x does not support all features needed to build VASP') + conflicts('+vaspsol', when='acc', msg='+vaspsol only available for CPU') + conflicts('+openmp', when='@:6.1.1', msg='openmp support started from 6.2') parallel = False def edit(self, spec, prefix): - if "%gcc" in spec: - if "+openmp" in spec: - make_include = join_path("arch", "makefile.include.linux_gnu_omp") + + if '%gcc' in spec: + if '+openmp' in spec: + if spec['mpi'].name == "openmpi": + if spec['blas'].name == "intel-mkl": + make_include='makefile.include.gnu_ompi_mkl_omp' + else: + sys.exit("makefile does not exist") + else: + make_include = join_path('arch', 'makefile.include.gnu_omp') else: - make_include = join_path("arch", "makefile.include.linux_gnu") - elif "%nvhpc" in spec: - make_include = join_path("arch", "makefile.include.linux_pgi") - filter_file("-pgc++libs", "-c++libs", make_include, string=True) - filter_file("pgcc", spack_cc, make_include) - filter_file("pgc++", spack_cxx, make_include, string=True) - filter_file("pgfortran", spack_fc, make_include) - filter_file( - "/opt/pgi/qd-2.3.17/install/include", spec["qd"].prefix.include, make_include - ) - filter_file("/opt/pgi/qd-2.3.17/install/lib", spec["qd"].prefix.lib, make_include) - elif "%aocc" in spec: - if "+openmp" in spec: + make_include = join_path('arch', 'makefile.include.gnu') + elif '%nvhpc' in spec: + if spec['blas'].name == 'intel-mkl' : + if spec['mpi'].name == "openmpi": + if '+acc' in spec: + make_include = join_path('arch','makefile.include.makefile.include.nvhpc_ompi_mkl_omp_acc') + else: + make_include = join_path('arch','makefile.include.makefile.include.nvhpc_ompi_mkl_omp') + else: + sys.exit('makefile does not exist') + else: + if '+acc' in spec: + if '+openmp' in spec: + make_include = join_path('arch','makefile.include.nvhpc_omp_acc') + else: + make_include = join_path('arch','makefile.include.nvhpc_acc') + elif '%aocc' in spec: + if '+openmp' in spec: copy( - join_path("arch", "makefile.include.linux_gnu_omp"), - join_path("arch", "makefile.include.linux_aocc_omp"), + join_path('arch', 'makefile.include.gnu_omp'), + join_path('arch', 'makefile.include.aocc_omp') ) - make_include = join_path("arch", "makefile.include.linux_aocc_omp") + make_include = join_path('arch', 'makefile.include.aocc_omp') else: copy( - join_path("arch", "makefile.include.linux_gnu"), - join_path("arch", "makefile.include.linux_aocc"), + join_path('arch', 'makefile.include.gnu'), + join_path('arch', 'makefile.include.aocc') ) - make_include = join_path("arch", "makefile.include.linux_aocc") - filter_file("gcc", "{0} {1}".format(spack_cc, "-Mfree"), make_include, string=True) - filter_file("g++", spack_cxx, make_include, string=True) - filter_file("^CFLAGS_LIB[ ]{0,}=.*$", "CFLAGS_LIB = -O3", make_include) - filter_file("^FFLAGS_LIB[ ]{0,}=.*$", "FFLAGS_LIB = -O2", make_include) - filter_file("^OFLAG[ ]{0,}=.*$", "OFLAG = -O3", make_include) - filter_file( - "^FC[ ]{0,}=.*$", "FC = {0}".format(spec["mpi"].mpifc), make_include, string=True - ) + make_include = join_path('arch', 'makefile.include.aocc') filter_file( - "^FCL[ ]{0,}=.*$", "FCL = {0}".format(spec["mpi"].mpifc), make_include, string=True + 'gcc', '{0} {1}'.format(spack_cc, '-Mfree'), + make_include, string=True ) + filter_file('g++', spack_cxx, make_include, string=True) + filter_file('^CFLAGS_LIB[ ]{0,}=.*$', + 'CFLAGS_LIB = -O3', make_include) + filter_file('^FFLAGS_LIB[ ]{0,}=.*$', + 'FFLAGS_LIB = -O2', make_include) + filter_file('^OFLAG[ ]{0,}=.*$', + 'OFLAG = -O3', make_include) + filter_file('^FC[ ]{0,}=.*$', + 'FC = {0}'.format(spec['mpi'].mpifc), + make_include, string=True) + filter_file('^FCL[ ]{0,}=.*$', + 'FCL = {0}'.format(spec['mpi'].mpifc), + make_include, string=True) else: - if "+openmp" in spec: - make_include = join_path( - "arch", "makefile.include.linux_{0}_omp".format(spec.compiler.name) - ) + if '+openmp' in spec: + if spec['mpi'].name == "openmpi": + if spec['blas'].name == "intel-mkl": + make_include = join_path('arch', 'makefile.include.intel_ompi_mkl_omp') + else: + sys.exit('makefile does not exist') + else: + if spec['blas'].name != "intel-mkl": + sys.exit('makefile does not exist') + else: + make_include = join_path('arch', 'makefile.include.intel_omp') else: - make_include = join_path("arch", "makefile.include.linux_" + spec.compiler.name) + make_include = join_path('arch', 'makefile.include.intel') + + + + os.rename(make_include, 'makefile.include') - # Recent versions of vasp have renamed the makefile.include files - # to leave out the linux_ string - if not os.path.exists(make_include): - make_include = make_include.replace("linux_", "") - os.rename(make_include, "makefile.include") + if '+wannier90' in spec: + filter_file('^#WANNIER90_ROOT*','WANNIER90_ROOT = '+spec['wannier90'].prefix,'makefile.include') + filter_file('^#LLIBS.*wannier.*','LLIBS += -L'+spec['wannier90'].prefix+'/lib -lwannier','makefile.include') - # This bunch of 'filter_file()' is to make these options settable - # as environment variables - filter_file("^CPP_OPTIONS[ ]{0,}=[ ]{0,}", "CPP_OPTIONS ?= ", "makefile.include") - filter_file("^FFLAGS[ ]{0,}=[ ]{0,}", "FFLAGS ?= ", "makefile.include") - filter_file("^LIBDIR[ ]{0,}=.*$", "", "makefile.include") - filter_file("^BLAS[ ]{0,}=.*$", "BLAS ?=", "makefile.include") - filter_file("^LAPACK[ ]{0,}=.*$", "LAPACK ?=", "makefile.include") - filter_file("^FFTW[ ]{0,}?=.*$", "FFTW ?=", "makefile.include") - filter_file("^MPI_INC[ ]{0,}=.*$", "MPI_INC ?=", "makefile.include") - filter_file("-DscaLAPACK.*$\n", "", "makefile.include") - filter_file("^SCALAPACK[ ]{0,}=.*$", "SCALAPACK ?=", "makefile.include") - if "+cuda" in spec: - filter_file("^OBJECTS_GPU[ ]{0,}=.*$", "OBJECTS_GPU ?=", "makefile.include") + if '%intel' in spec: + filter_file('qmkl','mkl','makefile.include') + filter_file('^VASP_TARGET_CPU.*','VASP_TARGET_CPU = -march=core-avx2','makefile.include') - filter_file("^CPP_GPU[ ]{0,}=.*$", "CPP_GPU ?=", "makefile.include") + filter_file('OBJECTS_LIB = linpack_double.o','OBJECTS_LIB = linpack_double.o getshmem.o','makefile.include') - filter_file("^CFLAGS[ ]{0,}=.*$", "CFLAGS ?=", "makefile.include") - - if "+vaspsol" in spec: - copy("VASPsol/src/solvation.F", "src/") + # This bunch of 'filter_file()' is to make these options settable + # as environment variables + filter_file('^CPP_OPTIONS[ ]{0,}=[ ]{0,}', + 'CPP_OPTIONS ?= ', + 'makefile.include') + filter_file('^FFLAGS[ ]{0,}=[ ]{0,}', + 'FFLAGS ?= ', + 'makefile.include') + + filter_file('^LIBDIR[ ]{0,}=.*$', '', 'makefile.include') + filter_file('^BLAS[ ]{0,}=.*$', 'BLAS ?=', 'makefile.include') + filter_file('^LAPACK[ ]{0,}=.*$', 'LAPACK ?=', 'makefile.include') + filter_file('^FFTW[ ]{0,}?=.*$', 'FFTW ?=', 'makefile.include') + filter_file('^MPI_INC[ ]{0,}=.*$', 'MPI_INC ?=', 'makefile.include') + filter_file('-DscaLAPACK.*$\n', '', 'makefile.include') + filter_file('^SCALAPACK[ ]{0,}=.*$', 'SCALAPACK ?=', 'makefile.include') + + if '+vaspsol' in spec: + copy('VASPsol/src/solvation.F', 'src/') + + if spec['blas'].name == 'intel-mkl': + filter_file(r'INCS.*=.*FFTW.*/include','INCS = -I'+join_path(spec['intel-mkl'].prefix,'compilers_and_libraries_'+str(spec['intel-mkl'].version),'linux','mkl','include','fftw'),'makefile.include') + filter_file(r'LLIBS.*lfftw3.*','','makefile.include') + + if '+acc' in spec: + filter_file('cuda11.0','cuda11.5','makefile.include') def setup_build_environment(self, spack_env): spec = self.spec - cpp_options = [ - "-DMPI -DMPI_BLOCK=8000", - "-Duse_collective", - "-DCACHE_SIZE=4000", - "-Davoidalloc", - "-Duse_bse_te", - "-Dtbdyn", - ] - - if "+shmem" in spec: - cpp_options.append("-Duse_shmem") - - if "%nvhpc" in self.spec: - cpp_options.extend(['-DHOST=\\"LinuxPGI\\"', "-DPGI16", "-Dqd_emulate"]) - elif "%aocc" in self.spec: - cpp_options.extend(['-DHOST=\\"LinuxGNU\\"', "-Dfock_dblbuf"]) - if "+openmp" in self.spec: - cpp_options.extend(["-D_OPENMP"]) + cpp_options = ['-DMPI -DMPI_BLOCK=8000', + '-Duse_collective', '-DCACHE_SIZE=4000', + '-Davoidalloc', '-Duse_bse_te', + '-Dtbdyn', '-Duse_shmem','-Dfock_dblbuf'] + if '+openmp' in spec: + cpp_options.extend(['-D_OPENMP']) + if '%aocc' in spec: + cpp_options.extend(['-DHOST=\\"LinuxGNU\\"', + '-Dfock_dblbuf']) else: cpp_options.append('-DHOST=\\"LinuxGNU\\"') - if self.spec.satisfies("@6:"): - cpp_options.append("-Dvasp6") + if spec.satisfies('@6:'): + cpp_options.append('-Dvasp6') - cflags = ["-fPIC", "-DADD_"] + cflags = ['-fPIC', '-DADD_'] fflags = [] - if "%gcc" in spec or "%intel" in spec: - fflags.append("-w") - elif "%nvhpc" in spec: - fflags.extend(["-Mnoupcase", "-Mbackslash", "-Mlarge_arrays"]) - elif "%aocc" in spec: - fflags.extend(["-fno-fortran-main", "-Mbackslash", "-ffast-math"]) - - spack_env.set("BLAS", spec["blas"].libs.ld_flags) - spack_env.set("LAPACK", spec["lapack"].libs.ld_flags) - spack_env.set("FFTW", spec["fftw-api"].prefix) - spack_env.set("MPI_INC", spec["mpi"].prefix.include) - - if "%nvhpc" in spec: - spack_env.set("QD", spec["qd"].prefix) - - if "+scalapack" in spec: - cpp_options.append("-DscaLAPACK") - spack_env.set("SCALAPACK", spec["scalapack"].libs.ld_flags) - - if "+cuda" in spec: - cpp_gpu = [ - "-DCUDA_GPU", - "-DRPROMU_CPROJ_OVERLAP", - "-DCUFFT_MIN=28", - "-DUSE_PINNED_MEMORY", - ] - - objects_gpu = [ - "fftmpiw.o", - "fftmpi_map.o", - "fft3dlib.o", - "fftw3d_gpu.o", - "fftmpiw_gpu.o", - ] - - cflags.extend(["-DGPUSHMEM=300", "-DHAVE_CUBLAS"]) - - spack_env.set("CUDA_ROOT", spec["cuda"].prefix) - spack_env.set("CPP_GPU", " ".join(cpp_gpu)) - spack_env.set("OBJECTS_GPU", " ".join(objects_gpu)) - - if "+vaspsol" in spec: - cpp_options.append("-Dsol_compat") - - if spec.satisfies("%gcc@10:"): - fflags.append("-fallow-argument-mismatch") + if '%gcc' in spec or '%intel' in spec: + fflags.append('-w') + elif '%nvhpc' in spec: + fflags.extend(['-Mnoupcase', '-Mbackslash', '-Mlarge_arrays']) + elif '%aocc' in spec: + fflags.extend(['-fno-fortran-main', '-Mbackslash', '-ffast-math']) + + spack_env.set('BLAS', spec['blas'].libs.ld_flags) + spack_env.set('LAPACK', spec['lapack'].libs.ld_flags) + spack_env.set('FFTW', spec['fftw-api'].prefix) + spack_env.set('MPI_INC', self.spec['mpi'].prefix.include) + + if '%nvhpc' in spec: + spack_env.set('QD', spec['qd'].prefix) + + if '+acc' in spec: + cpp_options.extend(['-DMPI_INPLACE','-D_OPENACC','-DUSENCCL','-DUSENCCLP2P']) + + if '%nvhpc' in spec: + cpp_options.append('-Dqd_emulate') + + if '+scalapack' in spec: + cpp_options.append('-DscaLAPACK') + spack_env.set('SCALAPACK', spec['scalapack'].libs.ld_flags) + + if '+vaspsol' in spec: + cpp_options.append('-Dsol_compat') + + if spec.satisfies('%gcc@10:'): + fflags.append('-fallow-argument-mismatch') + + if '+wannier90' in spec: + cpp_options.append('-DVASP2WANNIER90') # Finally - spack_env.set("CPP_OPTIONS", " ".join(cpp_options)) - spack_env.set("CFLAGS", " ".join(cflags)) - spack_env.set("FFLAGS", " ".join(fflags)) + spack_env.set('CPP_OPTIONS', ' '.join(cpp_options)) + spack_env.set('FFLAGS', ' '.join(fflags)) + + if '%nvhpc' in spec: + spack_env.prepend_path('LD_LIBRARY_PATH',join_path(spec['mpi'].prefix,'lib')) + nvc_cuda_path = join_path(ancestor(self.compiler.cc, 3),'cuda') + nvc_cuda_path = glob.glob(join_path(nvc_cuda_path,'*'))[0] + spack_env.set('NVHPC_CUDA_HOME',nvc_cuda_path) + spack_env.prepend_path('PATH',join_path(nvc_cuda_path,'bin')) + spack_env.prepend_path('LD_LIBRARY_PATH',join_path(nvc_cuda_path,'targets','x86_64-linux','lib')) + def build(self, spec, prefix): - if "+cuda" in self.spec: - make("gpu", "gpu_ncl") - else: - make("std", "gam", "ncl") + make('std', 'gam', 'ncl') def install(self, spec, prefix): - install_tree("bin/", prefix.bin) + install_tree('bin/', prefix.bin) + +# @run_after('install') +# def install_test(self): +# mkdirp(join_path(self.prefix,'test')) +# with working_dir(self.prefix.test): +# tar=which('tar') +# tar('xvzf',join_path(os.path.dirname(self.module.__file__),'H2O.tgz')) +# with working_dir('H2O'): +# if self.spec['mpi'].name == 'mvapich2' and 'slurm' in str(self.spec['mpi'].token): +# output=Executable('srun')('-n','4','--mpi=pmi2',join_path(self.prefix.bin,'vasp_std'),output=str,error=str) +# else: +# output=Executable('mpirun')('-np','4',join_path(self.prefix.bin,'vasp_std'),output=str,error=str) +# teststring='F= -.14223' +# if teststring in output: +# print("PASSED") +# else: +# print("FAILED")