diff --git a/environments/.stackhpc/inventory/group_vars/all/hpctests.yml b/environments/.stackhpc/inventory/group_vars/all/hpctests.yml deleted file mode 100644 index 30fde65e2..000000000 --- a/environments/.stackhpc/inventory/group_vars/all/hpctests.yml +++ /dev/null @@ -1,8 +0,0 @@ - -# Configure hpctests pingpong and pingmatrix to use system packages not OpenHPC modules: - -hpctests_extra_paths: - - /usr/lib64/openmpi/bin # for mpitests-openmpi -hpctests_pingmatrix_modules: [] -hpctests_pingpong_modules: [] -hpctests_pingpong_command: mpirun mpitests-IMB-MPI1 pingpong diff --git a/environments/.stackhpc/inventory/group_vars/openhpc/overrides.yml b/environments/.stackhpc/inventory/group_vars/openhpc/overrides.yml index 8c159a1f6..5aac5f8ad 100644 --- a/environments/.stackhpc/inventory/group_vars/openhpc/overrides.yml +++ b/environments/.stackhpc/inventory/group_vars/openhpc/overrides.yml @@ -1,21 +1,3 @@ -openhpc_install_type: generic # i.e. don't want OpenHPC - -openhpc_generic_packages: # system package installs: - # below are default in the role, required to get slurm to work: - - munge - - mariadb-connector-c # only actually needed on slurmdbd - - hwloc-libs # only actuall needed on slurmd - # below added here to get pingpong and pingmatrix hpctests working: - - mpitests-openmpi - -# define paths to slurm on NFS clients: -openhpc_sbin_dir: /nopt/slurm/23.02.5/sbin -openhpc_lib_dir: /nopt/slurm/23.02.5/lib/slurm -openhpc_bin_dir: /nopt/slurm/23.02.5/bin -openhpc_slurm_conf_path: /nopt/slurm/etc/slurm.conf - -# modify slurm.conf: openhpc_config_extra: SlurmctldDebug: debug SlurmdDebug: debug - Epilog: '' # required b/c the default slurm.conf uses an Epilog provided by openhpc pkgs diff --git a/environments/.stackhpc/inventory/local_nfs b/environments/.stackhpc/inventory/local_nfs deleted file mode 100644 index 2f8feaa1c..000000000 --- a/environments/.stackhpc/inventory/local_nfs +++ /dev/null @@ -1,7 +0,0 @@ -nfs: - hosts: - localhost: - ansible_connection: local - ansible_python_interpreter: "{{ansible_playbook_python}}" - ansible_host: '192.168.3.95' # interface on "portal-internal" network - nfs_interface: eth0 diff --git a/environments/.stackhpc/slurm/Dockerfile b/environments/.stackhpc/slurm/Dockerfile deleted file mode 100644 index fa8859b2d..000000000 --- a/environments/.stackhpc/slurm/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -FROM rockylinux:8 as build-stage - -ARG SLURM_VERSION=23.02.5 # From https://www.schedmd.com/downloads.php -ARG SLURM_PREFIX=/slurm # Should match directory Slurm is installed at -ARG SLURM_SYSCONFDIR=/etc/slurm # Should match directory slurm.conf will be in - -RUN set -ex \ - && yum makecache \ - && yum -y update \ - && yum -y install dnf-plugins-core epel-release \ - && yum -y install dnf-plugins-core \ - && yum config-manager --set-enabled powertools \ - && yum -y install \ - wget \ - bzip2 \ - perl \ - gcc \ - gcc-c++\ - git \ - gnupg \ - make \ - munge \ - munge-devel \ - python3-devel \ - python3-pip \ - python3 \ - mariadb-server \ - mariadb-devel \ - psmisc \ - bash-completion \ - vim-enhanced \ - http-parser-devel \ - json-c-devel \ - mpitests-openmpi \ - pmix-devel \ - hwloc \ - hwloc-devel \ - && yum clean all \ - && rm -rf /var/cache/yum - -RUN alternatives --set python /usr/bin/python3 - -RUN pip3 install Cython nose - -RUN set -x \ - && wget https://download.schedmd.com/slurm/slurm-${SLURM_VERSION}.tar.bz2 \ - && tar --bzip -x -f slurm*tar.bz2 - -WORKDIR /slurm-${SLURM_VERSION} - -RUN set -x && ./configure \ - --enable-debug \ - --prefix=${SLURM_PREFIX} \ - --without-rpath \ - --sysconfdir=${SLURM_SYSCONFDIR} \ - --with-mysql_config=/usr/bin - -RUN set -x && make install - -ENTRYPOINT ["/bin/bash"] - - -FROM scratch as export-stage - -ARG SLURM_PREFIX=/slurm # Should match directory Slurm is installed at -# RUN ls ${SLURM_PREFIX} -COPY --from=build-stage ${SLURM_PREFIX}/ . diff --git a/environments/.stackhpc/slurm/README.md b/environments/.stackhpc/slurm/README.md deleted file mode 100644 index 90f524fca..000000000 --- a/environments/.stackhpc/slurm/README.md +++ /dev/null @@ -1,23 +0,0 @@ -This uses a podman container to build Slurm, which is then copied out of the container into `build/23.02.5/`. -The `build/` directory is NFS-exported to the cluster as `/slurm`, so nodes see `/slurm/23.02.5`. - -The following arguments to `./configure` are important: -- `--prefix` must match the path the binaries appear to be at (i.e. from the NFS client side). This is because: - - The `slurm{ctld,d,dbd}` executables hardcode an RPATH, even when passing the `--without-rpath` flag to ./configure. - This means unless the path they are executed at matches the build prefix, they can't find `libslurmfull.so` on startup, - even with entries in `/etc/ld.so.conf.d/`. - - `PluginDir` defaults to being based on the build prefix. Although it can be overriden in `slurm.conf`, the `slurmd`s do not appear to get this parameter when running configless, so they won't start saying the (default) plugin dir doesn't exist -- `--sysconfdir` must match the path the `slurm.conf` file is at on the nodes. Otherwise `s*` commands running on nodes *without* `slurmd` (i.e. the control node only, for a standard Slurm appliance configuration) cannot find the configuration file unless the `SLURM_CONF` environment variable set. - -Assuming `podman` is installed, running the following in this directory will build Slurm and copy it out to the host. As the host's root disk was small, the below includes some TMPDIR/tmpdir options to try to place temporary build artifacts on a larger attached volume. However, ~10GB free was still required on the root disk to complete the build. - - $ export TMPDIR=/mnt/image-storage/tmp # some large persistent storage - $ mkdir -p build/23.02.5/ - $ rm -rf build/23.02.5/* - $ podman --tmpdir=/mnt/image-storage/tmp build \ - --build-arg SLURM_PREFIX=/nopt/slurm/23.02.5/ \ - --build-arg SLURM_SYSCONFDIR=/nopt/slurm/etc/ \ - . -t slurm-23.02.5 \ - --output build/23.02.5/ - -As a convenience, the output of "./configure --help" is included as `configure.help`. diff --git a/environments/.stackhpc/slurm/configure.help b/environments/.stackhpc/slurm/configure.help deleted file mode 100644 index e1398a979..000000000 --- a/environments/.stackhpc/slurm/configure.help +++ /dev/null @@ -1,219 +0,0 @@ -`configure' configures slurm 23.02 to adapt to many kinds of systems. - -Usage: ./configure [OPTION]... [VAR=VALUE]... - -To assign environment variables (e.g., CC, CFLAGS...), specify them as -VAR=VALUE. See below for descriptions of some of the useful variables. - -Defaults for the options are specified in brackets. - -Configuration: - -h, --help display this help and exit - --help=short display options specific to this package - --help=recursive display the short help of all the included packages - -V, --version display version information and exit - -q, --quiet, --silent do not print `checking ...' messages - --cache-file=FILE cache test results in FILE [disabled] - -C, --config-cache alias for `--cache-file=config.cache' - -n, --no-create do not create output files - --srcdir=DIR find the sources in DIR [configure dir or `..'] - -Installation directories: - --prefix=PREFIX install architecture-independent files in PREFIX - [/usr/local] - --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX - [PREFIX] - -By default, `make install' will install all the files in -`/usr/local/bin', `/usr/local/lib' etc. You can specify -an installation prefix other than `/usr/local' using `--prefix', -for instance `--prefix=$HOME'. - -For better control, use the options below. - -Fine tuning of the installation directories: - --bindir=DIR user executables [EPREFIX/bin] - --sbindir=DIR system admin executables [EPREFIX/sbin] - --libexecdir=DIR program executables [EPREFIX/libexec] - --sysconfdir=DIR read-only single-machine data [PREFIX/etc] - --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] - --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] - --libdir=DIR object code libraries [EPREFIX/lib] - --includedir=DIR C header files [PREFIX/include] - --oldincludedir=DIR C header files for non-gcc [/usr/include] - --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] - --datadir=DIR read-only architecture-independent data [DATAROOTDIR] - --infodir=DIR info documentation [DATAROOTDIR/info] - --localedir=DIR locale-dependent data [DATAROOTDIR/locale] - --mandir=DIR man documentation [DATAROOTDIR/man] - --docdir=DIR documentation root [DATAROOTDIR/doc/slurm] - --htmldir=DIR html documentation [DOCDIR] - --dvidir=DIR dvi documentation [DOCDIR] - --pdfdir=DIR pdf documentation [DOCDIR] - --psdir=DIR ps documentation [DOCDIR] - -Program names: - --program-prefix=PREFIX prepend PREFIX to installed program names - --program-suffix=SUFFIX append SUFFIX to installed program names - --program-transform-name=PROGRAM run sed PROGRAM on installed program names - -System types: - --build=BUILD configure for building on BUILD [guessed] - --host=HOST cross-compile to build programs to run on HOST [BUILD] - --target=TARGET configure for building compilers for TARGET [HOST] - -Optional Features: - --disable-option-checking ignore unrecognized --enable/--with options - --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) - --enable-FEATURE[=ARG] include FEATURE [ARG=yes] - --enable-silent-rules less verbose build output (undo: "make V=1") - --disable-silent-rules verbose build output (undo: "make V=0") - --enable-maintainer-mode - enable make rules and dependencies not useful (and - sometimes confusing) to the casual installer - --enable-dependency-tracking - do not reject slow dependency extractors - --disable-dependency-tracking - speeds up one-time build - --enable-shared[=PKGS] build shared libraries [default=yes] - --enable-static[=PKGS] build static libraries [default=yes] - --enable-fast-install[=PKGS] - optimize for fast installation [default=yes] - --disable-libtool-lock avoid locking (might break parallel builds) - --enable-pam enable PAM (Pluggable Authentication Modules) - support - --enable-load-env-no-login - enable --get-user-env option to load user - environment without .login - --disable-x11 disable internal X11 support - --enable-selinux enable internal SELinux support - --disable-glibtest do not try to compile and run a test GLIB program - --disable-gtktest do not try to compile and run a test GTK+ program - --enable-cray-network Run Slurm on a non-Cray system with a Cray network - --enable-really-no-cray Disable cray support for eslogin machines - --disable-optimizations disable optimizations (sets -O0) - --enable-developer enable developer options (asserts, -Werror - also - sets --enable-debug as well) - --disable-debug disable debugging symbols and compile with - optimizations - --enable-memory-leak-debug - enable memory leak debugging code for development - --enable-front-end enable slurmd operation on a front-end - --disable-partial-attach - disable debugger partial task attach support - --enable-salloc-kill-cmd - salloc should kill child processes at job - termination - --disable-slurmrestd disable slurmrestd support - --enable-multiple-slurmd - enable multiple-slurmd support - --enable-deprecated enable deprecated - -Optional Packages: - --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] - --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --without-rpath Do not include rpath in build - --with-mysql_config=PATH - Specify path of directory where mysql_config binary - exists - --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use - both] - --with-aix-soname=aix|svr4|both - shared library versioning (aka "SONAME") variant to - provide on AIX, [default=aix]. - --with-gnu-ld assume the C compiler uses GNU ld [default=no] - --with-sysroot[=DIR] Search for dependent libraries within DIR (or the - compiler's sysroot if not specified). - --with-pam_dir=PATH Specify path to PAM module installation - --without-shared-libslurm - statically link to libslurm.o instead of the shared - libslurm lib - can dramatically increase the - footprint of Slurm. - --with-json=PATH Specify path to json-c installation - --with-jwt=PATH Specify path to jwt installation - --with-http-parser=PATH Specify path to HTTP Parser installation - --with-yaml=PATH Specify path to libyaml installation - --with-dimensions=N set system dimension count for generic computer - system - --with-ofed=PATH Specify path to ofed installation - --with-hdf5=yes/no/PATH location of h5cc or h5pcc for HDF5 configuration - --with-lz4=PATH Specify path to liblz4 installation - --with-hwloc=PATH Specify path to hwloc installation - --with-nvml=PATH Specify path to CUDA installation - --with-rsmi=PATH Specify path to rsmi installation - --with-oneapi=PATH Specify path to oneAPI installation - --with-pmix=PATH Specify path to pmix installation(s). Multiple - version directories can be ':' delimited. - --with-freeipmi=PATH Specify path to freeipmi installation - --with-rrdtool=PATH Specify path to rrdtool-devel installation - --with-ucx=PATH Build with Unified Communication X library support - --with-rdkafka=PATH Specify path to librdkafka installation - --with-bpf=PATH Specify path to bpf header - --with-cray_dir=PATH Specify path to Cray file installation - /opt/cray - by default - --with-datawarp=PATH Specify path to DataWarp installation - --with-hpe-slingshot=PATH - Specify path to HPE Slingshot installation dir - --with-slurmctld-port=N set slurmctld default port [6817] - --with-slurmd-port=N set slurmd default port [6818] - --with-slurmdbd-port=N set slurmdbd default port [6819] - --with-slurmctld-port-count=N - set slurmctld default port count [1] - --with-slurmrestd-port=N - set slurmrestd default port [6820] - --with-netloc=PATH Specify path to netloc installation - --without-readline compile without readline support - --with-systemdsystemunitdir=DIR - Directory for systemd service files - --with-munge=PATH Specify path to munge installation - --with-libcurl=PREFIX look for the curl library in PREFIX/lib and headers - in PREFIX/include - -Some influential environment variables: - CC C compiler command - CFLAGS C compiler flags - LDFLAGS linker flags, e.g. -L if you have libraries in a - nonstandard directory - LIBS libraries to pass to the linker, e.g. -l - CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if - you have headers in a nonstandard directory - CXX C++ compiler command - CXXFLAGS C++ compiler flags - LT_SYS_LIBRARY_PATH - User-defined run-time library search path. - CXXCPP C++ preprocessor - PKG_CONFIG path to pkg-config utility - PKG_CONFIG_PATH - directories to add to pkg-config's search path - PKG_CONFIG_LIBDIR - path overriding pkg-config's built-in search path - CPP C preprocessor - libselinux_CFLAGS - C compiler flags for libselinux, overriding pkg-config - libselinux_LIBS - linker flags for libselinux, overriding pkg-config - dbus_CFLAGS C compiler flags for dbus, overriding pkg-config - dbus_LIBS linker flags for dbus, overriding pkg-config - CHECK_CFLAGS - C compiler flags for CHECK, overriding pkg-config - CHECK_LIBS linker flags for CHECK, overriding pkg-config - GLIB_CFLAGS C compiler flags for GLIB, overriding pkg-config - GLIB_LIBS linker flags for GLIB, overriding pkg-config - GLIB_GENMARSHAL - value of glib_genmarshal for glib-2.0, overriding pkg-config - GOBJECT_QUERY - value of gobject_query for glib-2.0, overriding pkg-config - GLIB_MKENUMS - value of glib_mkenums for glib-2.0, overriding pkg-config - GLIB_COMPILE_RESOURCES - value of glib_compile_resources for gio-2.0, overriding - pkg-config - lua_CFLAGS C compiler flags for lua, overriding pkg-config - lua_LIBS linker flags for lua, overriding pkg-config - -Use these variables to override the choices made by `configure' or to help -it to find libraries and programs with nonstandard names/locations. - -Report bugs to the package provider. -slurm home page: .