diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
new file mode 100644
index 0000000000..7327d876ba
--- /dev/null
+++ b/.github/workflows/end2end.yml
@@ -0,0 +1,61 @@
+name: End-to-end test of EasyBuild in different distros
+on: [push, pull_request]
+jobs:
+ build_publish:
+ name: End-to-end test
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ container:
+ - centos-7.9
+ - centos-8.5
+ - fedora-36
+ - opensuse-15.4
+ - rockylinux-8.8
+ - rockylinux-9.2
+ - ubuntu-20.04
+ - ubuntu-22.04
+ fail-fast: false
+ container:
+ image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+ steps:
+ - name: Check out the repo
+ uses: actions/checkout@v3
+
+ - name: download and unpack easyblocks and easyconfigs repositories
+ run: |
+ cd $HOME
+ for pkg in easyblocks easyconfigs; do
+ curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz
+ tar xfz develop.tar.gz
+ rm -f develop.tar.gz
+ done
+
+ - name: Set up environment
+ shell: bash
+ run: |
+ # collect environment variables to be set in subsequent steps in script that can be sourced
+ echo "export PATH=$PWD:$PATH" > /tmp/eb_env
+ echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env
+
+ - name: Run commands to check test environment
+ shell: bash
+ run: |
+ cmds=(
+ "whoami"
+ "pwd"
+ "env | sort"
+ "eb --version"
+ "eb --show-system-info"
+ "eb --check-eb-deps"
+ "eb --show-config"
+ )
+ for cmd in "${cmds[@]}"; do
+ echo ">>> $cmd"
+ sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd"
+ done
+
+ - name: End-to-end test of installing bzip2 with EasyBuild
+ shell: bash
+ run: |
+ sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 8f5ec0f1e0..5bd07d4306 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -4,6 +4,34 @@ For more detailed information, please see the git log.
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
+v4.8.1 (11 September 2023)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+ - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968)
+ - suggest default title in `--review-pr` (#4287)
+ - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304)
+ - implement support for `cancel_hook` and `fail_hook` (#4315, #4325)
+ - add postiter hook to the list of steps so the corresponding hook can be used (#4316)
+ - add `run_shell_cmd` hook (#4323)
+ - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324)
+ - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329)
+ - add support for using fine grained Github tokens (#4332)
+ - add definitions for ifbf and iofbf toolchain (#4337)
+ - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339)
+- various bug fixes, including:
+ - improve error when checksum dict has no entry for a file (#4150)
+ - avoid error being logged when `checksums.json` is not found (#4261)
+ - don't fail in `mkdir` if path gets created while processing it (#4300, #4328)
+ - ignore request for external module (meta)data when no modules tool is active (#4308)
+ - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309)
+ - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312)
+ - enhance `Toolchain.get_flag` to handle lists (#4319)
+ - only add extensions in module file if there are extensions (#4331)
+
+
v4.8.0 (7 July 2023)
--------------------
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
index c85104d687..3ac26eb476 100644
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -395,13 +395,13 @@ def get_checksums_from_json(self, always_read=False):
:param always_read: always read the checksums.json file, even if it has been read before
"""
if always_read or self.json_checksums is None:
- try:
- path = self.obtain_file("checksums.json", no_download=True)
+ path = self.obtain_file("checksums.json", no_download=True, warning_only=True)
+ if path is not None:
self.log.info("Loading checksums from file %s", path)
json_txt = read_file(path)
self.json_checksums = json.loads(json_txt)
- # if the file can't be found, return an empty dict
- except EasyBuildError:
+ else:
+ # if the file can't be found, return an empty dict
self.json_checksums = {}
return self.json_checksums
@@ -736,7 +736,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
return exts_sources
def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
- git_config=None, no_download=False, download_instructions=None, alt_location=None):
+ git_config=None, no_download=False, download_instructions=None, alt_location=None,
+ warning_only=False):
"""
Locate the file with the given name
- searches in different subdirectories of source path
@@ -789,7 +790,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
return fullpath
except IOError as err:
- raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
+ if not warning_only:
+ raise EasyBuildError("Downloading file %s "
+ "from url %s to %s failed: %s", filename, url, fullpath, err)
+ else:
+ self.log.warning("Downloading file %s "
+ "from url %s to %s failed: %s", filename, url, fullpath, err)
+ return None
else:
# try and find file in various locations
@@ -866,8 +873,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
self.dry_run_msg(" * %s (MISSING)", filename)
return filename
else:
- raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+ if not warning_only:
+ raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+ "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+ else:
+ self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... "
"Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+ return None
elif git_config:
return get_source_tarball_from_git(filename, targetdir, git_config)
else:
@@ -959,7 +971,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
error_msg += "and downloading it didn't work either... "
error_msg += "Paths attempted (in order): %s " % failedpaths_msg
- raise EasyBuildError(error_msg, filename)
+ if not warning_only:
+ raise EasyBuildError(error_msg, filename)
+ else:
+ self.log.warning(error_msg, filename)
+ return None
#
# GETTER/SETTER UTILITY FUNCTIONS
@@ -4174,6 +4190,10 @@ def build_and_install_one(ecdict, init_env):
dry_run_msg('', silent=silent)
print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
+ if ecdict['ec']['build_info_msg']:
+ msg = "This easyconfig provides the following build information:\n\n%s\n"
+ print_msg(msg % ecdict['ec']['build_info_msg'], log=_log, silent=silent)
+
if dry_run:
# print note on interpreting dry run output (argument is reference to location of dry run messages)
print_dry_run_note('below', silent=silent)
diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py
index b37c3660f0..dd91229d1e 100644
--- a/easybuild/framework/easyconfig/default.py
+++ b/easybuild/framework/easyconfig/default.py
@@ -228,6 +228,8 @@
'buildstats': [None, "A list of dicts with build statistics", OTHER],
'deprecated': [False, "String specifying reason why this easyconfig file is deprecated "
"and will be archived in the next major release of EasyBuild", OTHER],
+ 'build_info_msg': [None, "String with information to be printed to stdout and logged during the building "
+ "of the easyconfig", OTHER],
}
diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
index f4c4be464e..55fe3c8578 100644
--- a/easybuild/framework/easyconfig/easyconfig.py
+++ b/easybuild/framework/easyconfig/easyconfig.py
@@ -73,7 +73,7 @@
from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
-from easybuild.tools.modules import modules_tool
+from easybuild.tools.modules import modules_tool, NoModulesTool
from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
@@ -1306,6 +1306,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None):
:param existing_metadata: already available metadata for this external module (if any)
"""
res = {}
+ if isinstance(self.modules_tool, NoModulesTool):
+ self.log.debug('Ignoring request for external module data for %s as no modules tool is active', mod_name)
+ return res
if existing_metadata is None:
existing_metadata = {}
diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
index 62968f9de0..abe8f998ea 100644
--- a/easybuild/framework/easyconfig/tools.py
+++ b/easybuild/framework/easyconfig/tools.py
@@ -60,7 +60,7 @@
from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
from easybuild.tools.filetools import read_file, resolve_path, which, write_file
from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
-from easybuild.tools.github import det_pr_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data
+from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data
from easybuild.tools.github import fetch_files_from_pr
from easybuild.tools.multidiff import multidiff
from easybuild.tools.py2vs3 import OrderedDict
@@ -561,6 +561,11 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False
lines.extend(['', "This PR is associated with a generic '.x' milestone, "
"it should be associated to the next release milestone once merged"])
+ default_new_title = det_pr_title([ec['ec'] for ec in ecs])
+ if default_new_title != pr_data['title']:
+ lines.extend(['', "If this PR contains only new easyconfigs and has not been edited from the default, "
+ "then the title should be: %s" % default_new_title])
+
return '\n'.join(lines)
diff --git a/easybuild/main.py b/easybuild/main.py
index 319c80b5ae..1a9aa736a7 100644
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -69,7 +69,8 @@
from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr
from easybuild.tools.github import new_pr_from_branch
from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr
-from easybuild.tools.hooks import START, END, load_hooks, run_hook
+from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, FAIL
+from easybuild.tools.hooks import load_hooks, run_hook
from easybuild.tools.modules import modules_tool
from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color
from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm
@@ -545,8 +546,10 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
exit_on_failure = not (options.dump_test_report or options.upload_test_report)
with rich_live_cm():
+ run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs])
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
exit_on_failure=exit_on_failure)
+ run_hook(POST_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ecs_with_res])
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
@@ -577,30 +580,20 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
return overall_success
-def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
+def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
+ :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None)
"""
+ if prepared_cfg_data is None or any([args, logfile, testing]):
+ init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing)
+ else:
+ init_session_state, eb_go, cfg_settings = prepared_cfg_data
- register_lock_cleanup_signal_handlers()
-
- # if $CDPATH is set, unset it, it'll only cause trouble...
- # see https://github.com/easybuilders/easybuild-framework/issues/2944
- if 'CDPATH' in os.environ:
- del os.environ['CDPATH']
-
- # When EB is run via `exec` the special bash variable $_ is not set
- # So emulate this here to allow (module) scripts depending on that to work
- if '_' not in os.environ:
- os.environ['_'] = sys.executable
-
- # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
- init_session_state = session_state()
- eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
options, orig_paths = eb_go.options, eb_go.args
if 'python2' not in build_option('silence_deprecation_warnings'):
@@ -729,10 +722,41 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
cleanup(logfile, eb_tmpdir, testing, silent=False)
+def prepare_main(args=None, logfile=None, testing=None):
+ """
+ Prepare for calling main function by setting up the EasyBuild configuration
+ :param args: command line arguments to take into account when parsing the EasyBuild configuration settings
+ :param logfile: log file to use
+ :param testing: enable testing mode
+ :return: 3-tuple with initial session state data, EasyBuildOptions instance, and tuple with configuration settings
+ """
+ register_lock_cleanup_signal_handlers()
+
+ # if $CDPATH is set, unset it, it'll only cause trouble...
+ # see https://github.com/easybuilders/easybuild-framework/issues/2944
+ if 'CDPATH' in os.environ:
+ del os.environ['CDPATH']
+
+ # When EB is run via `exec` the special bash variable $_ is not set
+ # So emulate this here to allow (module) scripts depending on that to work
+ if '_' not in os.environ:
+ os.environ['_'] = sys.executable
+
+ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
+ init_session_state = session_state()
+ eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
+
+ return init_session_state, eb_go, cfg_settings
+
+
if __name__ == "__main__":
+ init_session_state, eb_go, cfg_settings = prepare_main()
+ hooks = load_hooks(eb_go.options.hooks)
try:
- main()
+ main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
except EasyBuildError as err:
+ run_hook(FAIL, hooks, args=[err])
print_error(err.msg)
except KeyboardInterrupt as err:
+ run_hook(CANCEL, hooks, args=[err])
print_error("Cancelled by user: %s" % err)
diff --git a/easybuild/toolchains/ifbf.py b/easybuild/toolchains/ifbf.py
new file mode 100644
index 0000000000..3507e1eab9
--- /dev/null
+++ b/easybuild/toolchains/ifbf.py
@@ -0,0 +1,44 @@
+##
+# Copyright 2012-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+EasyBuild support for ifbf toolchain (includes Intel compilers, FlexiBLAS, and FFTW).
+
+Authors:
+
+* Sebastian Achilles (Juelich Supercomputing Centre)
+"""
+
+from easybuild.toolchains.intel_compilers import IntelCompilersToolchain
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+
+
+class Ifbf(IntelCompilersToolchain, FlexiBLAS, Fftw):
+ """
+ Compiler toolchain with Intel compilers, FlexiBLAS, and FFTW
+ """
+ NAME = 'ifbf'
+ SUBTOOLCHAIN = IntelCompilersToolchain.NAME
+ OPTIONAL = True
diff --git a/easybuild/toolchains/iofbf.py b/easybuild/toolchains/iofbf.py
new file mode 100644
index 0000000000..3410ffaf9f
--- /dev/null
+++ b/easybuild/toolchains/iofbf.py
@@ -0,0 +1,47 @@
+##
+# Copyright 2012-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+EasyBuild support for iofbf toolchain (includes Intel compilers, OpenMPI,
+FlexiBLAS, LAPACK, ScaLAPACK and FFTW).
+
+Authors:
+
+* Sebastian Achilles (Juelich Supercomputing Centre)
+"""
+
+from easybuild.toolchains.iompi import Iompi
+from easybuild.toolchains.ifbf import Ifbf
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+from easybuild.toolchains.linalg.scalapack import ScaLAPACK
+
+
+class Iofbf(Iompi, FlexiBLAS, ScaLAPACK, Fftw):
+ """
+ Compiler toolchain with Intel compilers (icc/ifort), OpenMPI,
+ FlexiBLAS, LAPACK, ScaLAPACK and FFTW.
+ """
+ NAME = 'iofbf'
+ SUBTOOLCHAIN = [Iompi.NAME, Ifbf.NAME]
diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py
index 3f8acec4f5..6c0a173fe4 100644
--- a/easybuild/tools/config.py
+++ b/easybuild/tools/config.py
@@ -301,6 +301,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'sequential',
'set_default_module',
'set_gid_bit',
+ 'silence_hook_trigger',
'skip_extensions',
'skip_test_cases',
'skip_test_step',
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
index 19fbc280a4..6e4bd2d83e 100644
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -65,7 +65,7 @@
from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
from easybuild.tools.config import build_option, install_path
from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
-from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type
+from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type
from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg
try:
@@ -1264,14 +1264,11 @@ def verify_checksum(path, checksums):
for checksum in checksums:
if isinstance(checksum, dict):
- if filename in checksum:
+ try:
# Set this to a string-type checksum
checksum = checksum[filename]
- elif build_option('enforce_checksums'):
- raise EasyBuildError("Missing checksum for %s", filename)
- else:
- # Set to None and allow to fail elsewhere
- checksum = None
+ except KeyError:
+ raise EasyBuildError("Missing checksum for %s in %s", filename, checksum)
if isinstance(checksum, string_type):
# if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters)
@@ -1301,7 +1298,8 @@ def verify_checksum(path, checksums):
# no matching checksums
return False
else:
- raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value).",
+ raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), "
+ "2-tuple (type, value), or tuple of alternative checksum specs.",
checksum)
actual_checksum = compute_checksum(path, typ)
@@ -1918,7 +1916,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None):
# climb up until we hit an existing path or the empty string (for relative paths)
while existing_parent_path and not os.path.exists(existing_parent_path):
existing_parent_path = os.path.dirname(existing_parent_path)
- os.makedirs(path)
+ makedirs(path, exist_ok=True)
else:
os.mkdir(path)
except OSError as err:
@@ -2619,6 +2617,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
recursive = git_config.pop('recursive', False)
clone_into = git_config.pop('clone_into', False)
keep_git_dir = git_config.pop('keep_git_dir', False)
+ extra_config_params = git_config.pop('extra_config_params', None)
+ recurse_submodules = git_config.pop('recurse_submodules', None)
# input validation of git_config dict
if git_config:
@@ -2644,7 +2644,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
targetpath = os.path.join(targetdir, filename)
# compose 'git clone' command, and run it
- clone_cmd = ['git', 'clone']
+ if extra_config_params:
+ git_cmd = 'git ' + ' '.join(['-c %s' % param for param in extra_config_params])
+ else:
+ git_cmd = 'git'
+ clone_cmd = [git_cmd, 'clone']
if not keep_git_dir and not commit:
# Speed up cloning by only fetching the most recent commit, not the whole history
@@ -2655,6 +2659,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
clone_cmd.extend(['--branch', tag])
if recursive:
clone_cmd.append('--recursive')
+ if recurse_submodules:
+ clone_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
else:
# checkout is done separately below for specific commits
clone_cmd.append('--no-checkout')
@@ -2674,16 +2680,21 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
# if a specific commit is asked for, check it out
if commit:
- checkout_cmd = ['git', 'checkout', commit]
- if recursive:
- checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive'])
+ checkout_cmd = [git_cmd, 'checkout', commit]
+
+ if recursive or recurse_submodules:
+ checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init'])
+ if recursive:
+ checkout_cmd.append('--recursive')
+ if recurse_submodules:
+ checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name)
elif not build_option('extended_dry_run'):
# If we wanted to get a tag make sure we actually got a tag and not a branch with the same name
# This doesn't make sense in dry-run mode as we don't have anything to check
- cmd = 'git describe --exact-match --tags HEAD'
+ cmd = '%s describe --exact-match --tags HEAD' % git_cmd
# Note: Disable logging to also disable the error handling in run_cmd
(out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name)
if ec != 0 or tag not in out.splitlines():
@@ -2696,13 +2707,16 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
# make the repo unshallow first;
# this is equivalent with 'git fetch -unshallow' in Git 1.8.3+
# (first fetch seems to do nothing, unclear why)
- cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647')
+ cmds.append('%s fetch --depth=2147483647 && git fetch --depth=2147483647' % git_cmd)
- cmds.append('git checkout refs/tags/' + tag)
+ cmds.append('%s checkout refs/tags/' % git_cmd + tag)
# Clean all untracked files, e.g. from left-over submodules
- cmds.append('git clean --force -d -x')
+ cmds.append('%s clean --force -d -x' % git_cmd)
if recursive:
- cmds.append('git submodule update --init --recursive')
+ cmds.append('%s submodule update --init --recursive' % git_cmd)
+ elif recurse_submodules:
+ cmds.append('%s submodule update --init ' % git_cmd)
+ cmds[-1] += ' '.join(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
for cmd in cmds:
run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name)
diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py
index 78846ac539..983dc977c4 100644
--- a/easybuild/tools/github.py
+++ b/easybuild/tools/github.py
@@ -2250,15 +2250,18 @@ def install_github_token(github_user, silent=False):
def validate_github_token(token, github_user):
"""
Check GitHub token:
- * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40)
- * see if it can be used for authenticated access
+ * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters
+ and length of 40 starting with 'ghp_', or fine-grained GitHub token with only
+ alphanumeric ([a-zA-Z0-9]) characters + '_' and length of 93 starting with 'github_pat_'),
+ * see if it can be used for authenticated access.
"""
# cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$')
token_regex_old_format = re.compile('^[0-9a-f]{40}$')
+ # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
+ token_regex_fine_grained = re.compile('github_pat_[a-zA-Z0-9_]{82}')
- # token should be 40 characters long, and only contain characters in [0-9a-f]
- sanity_check = bool(token_regex.match(token))
+ sanity_check = bool(token_regex.match(token)) or bool(token_regex_fine_grained.match(token))
if sanity_check:
_log.info("Sanity check on token passed")
else:
diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py
index 3c21d4e104..f454974edb 100644
--- a/easybuild/tools/hooks.py
+++ b/easybuild/tools/hooks.py
@@ -61,23 +61,30 @@
START = 'start'
PARSE = 'parse'
+BUILD_AND_INSTALL_LOOP = 'build_and_install_loop'
SINGLE_EXTENSION = 'single_extension'
MODULE_WRITE = 'module_write'
END = 'end'
+CANCEL = 'cancel'
+FAIL = 'fail'
+
+RUN_SHELL_CMD = 'run_shell_cmd'
+
PRE_PREF = 'pre_'
POST_PREF = 'post_'
HOOK_SUFF = '_hook'
# list of names for steps in installation procedure (in order of execution)
STEP_NAMES = [FETCH_STEP, READY_STEP, SOURCE_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP,
- INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
+ INSTALL_STEP, EXTENSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
# hook names (in order of being triggered)
HOOK_NAMES = [
START,
PARSE,
+ PRE_PREF + BUILD_AND_INSTALL_LOOP,
] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)]
for p in [PRE_PREF, POST_PREF]] + [
# pre-extensions hook is triggered before starting installation of extensions,
@@ -97,7 +104,12 @@
POST_PREF + MODULE_STEP,
] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:]
for p in [PRE_PREF, POST_PREF]] + [
+ POST_PREF + BUILD_AND_INSTALL_LOOP,
END,
+ CANCEL,
+ FAIL,
+ PRE_PREF + RUN_SHELL_CMD,
+ POST_PREF + RUN_SHELL_CMD,
]
KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]
@@ -195,7 +207,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
return res
-def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None):
+def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, kwargs=None, msg=None):
"""
Run hook with specified label and return result of calling the hook or None.
@@ -211,6 +223,8 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
if hook:
if args is None:
args = []
+ if kwargs is None:
+ kwargs = {}
if pre_step_hook:
label = 'pre-' + label
@@ -219,9 +233,9 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
if msg is None:
msg = "Running %s hook..." % label
- if build_option('debug'):
+ if build_option('debug') and not build_option('silence_hook_trigger'):
print_msg(msg)
- _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
- res = hook(*args)
+ _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs)
+ res = hook(*args, **kwargs)
return res
diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py
index aaf97d3194..72afdd0896 100644
--- a/easybuild/tools/module_generator.py
+++ b/easybuild/tools/module_generator.py
@@ -622,7 +622,8 @@ def _generate_extensions_list(self):
"""
Generate a list of all extensions in name/version format
"""
- return self.app.make_extension_string(name_version_sep='/', ext_sep=',').split(',')
+ exts_str = self.app.make_extension_string(name_version_sep='/', ext_sep=',')
+ return exts_str.split(',') if exts_str else []
def _generate_help_text(self):
"""
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
index 761cdc262f..b60e57513d 100644
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -501,6 +501,8 @@ def override_options(self):
'silence-deprecation-warnings': (
"Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations),
'strlist', 'extend', []),
+ 'silence-hook-trigger': ("Supress printing of debug message every time a hook is triggered",
+ None, 'store_true', False),
'skip-extensions': ("Skip installation of extensions", None, 'store_true', False),
'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False),
@@ -1894,7 +1896,7 @@ def set_tmpdir(tmpdir=None, raise_error=False):
os.close(fd)
os.chmod(tmptest_file, 0o700)
if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False,
- stream_output=False):
+ stream_output=False, with_hooks=False):
msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir()
msg += "This can cause problems in the build process, consider using --tmpdir."
if raise_error:
diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py
index 99f8cb0f4a..1fec4650b1 100644
--- a/easybuild/tools/py2vs3/py2.py
+++ b/easybuild/tools/py2vs3/py2.py
@@ -35,6 +35,7 @@
import ConfigParser as configparser # noqa
import imp
import json
+import os
import subprocess
import time
import urllib2 as std_urllib # noqa
@@ -115,3 +116,11 @@ def sort_looseversions(looseversions):
# with Python 2, we can safely use 'sorted' on LooseVersion instances
# (but we can't in Python 3, see https://bugs.python.org/issue14894)
return sorted(looseversions)
+
+
+def makedirs(name, mode=0o777, exist_ok=False):
+ try:
+ os.makedirs(name, mode)
+ except OSError:
+ if not exist_ok or not os.path.isdir(name):
+ raise
diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py
index 77f786cbec..8d9145179d 100644
--- a/easybuild/tools/py2vs3/py3.py
+++ b/easybuild/tools/py2vs3/py3.py
@@ -44,6 +44,7 @@
from html.parser import HTMLParser # noqa
from itertools import zip_longest
from io import StringIO # noqa
+from os import makedirs # noqa
from string import ascii_letters, ascii_lowercase # noqa
from urllib.request import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa
from urllib.parse import urlencode # noqa
diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py
index 1b8c385563..8916d80795 100644
--- a/easybuild/tools/run.py
+++ b/easybuild/tools/run.py
@@ -50,6 +50,7 @@
from easybuild.base import fancylogger
from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since
from easybuild.tools.config import ERROR, IGNORE, WARN, build_option
+from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook
from easybuild.tools.py2vs3 import string_type
from easybuild.tools.utilities import trace_msg
@@ -131,7 +132,8 @@ def get_output_from_process(proc, read_size=None, asynchronous=False):
@run_cmd_cache
def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
- force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False):
+ force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False,
+ with_hooks=True):
"""
Run specified command (in a subshell)
:param cmd: command to run
@@ -148,6 +150,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
:param trace: print command being executed as part of trace output
:param stream_output: enable streaming command output to stdout
:param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
+ :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
"""
cwd = os.getcwd()
@@ -233,6 +236,13 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
else:
raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
+ if with_hooks:
+ hooks = load_hooks(build_option('hooks'))
+ hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()})
+ if isinstance(hook_res, string_type):
+ cmd, old_cmd = hook_res, cmd
+ _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd)
+
_log.info('running cmd: %s ' % cmd)
try:
proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
@@ -248,7 +258,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
return (proc, cmd, cwd, start_time, cmd_log)
else:
return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple,
- regexp=regexp, stream_output=stream_output, trace=trace)
+ regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks)
def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''):
@@ -293,7 +303,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out
def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False,
- regexp=True, stream_output=None, trace=True, output=''):
+ regexp=True, stream_output=None, trace=True, output='', with_hook=True):
"""
Complete running of command represented by passed subprocess.Popen instance.
@@ -308,6 +318,7 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False
:param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error)
:param stream_output: enable streaming command output to stdout
:param trace: print command being executed as part of trace output
+ :param with_hook: trigger post run_shell_cmd hooks (if defined)
"""
# use small read size when streaming output, to make it stream more fluently
# read size should not be too small though, to avoid too much overhead
@@ -343,6 +354,15 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False
sys.stdout.write(output)
stdouterr += output
+ if with_hook:
+ hooks = load_hooks(build_option('hooks'))
+ run_hook_kwargs = {
+ 'exit_code': ec,
+ 'output': stdouterr,
+ 'work_dir': os.getcwd(),
+ }
+ run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+
if trace:
trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
@@ -485,6 +505,17 @@ def check_answers_list(answers):
# Part 2: Run the command and answer questions
# - this needs asynchronous stdout
+ hooks = load_hooks(build_option('hooks'))
+ run_hook_kwargs = {
+ 'interactive': True,
+ 'work_dir': os.getcwd(),
+ }
+ hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+ if isinstance(hook_res, string_type):
+ cmd, old_cmd = hook_res, cmd
+ _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')",
+ RUN_SHELL_CMD, cmd, old_cmd)
+
# # Log command output
if cmd_log:
cmd_log.write("# output for interactive command: %s\n\n" % cmd)
@@ -599,6 +630,13 @@ def get_proc():
except IOError as err:
_log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err)
+ run_hook_kwargs.update({
+ 'interactive': True,
+ 'exit_code': ec,
+ 'output': stdout_err,
+ })
+ run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+
if trace:
trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time)))
diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
index cab4b00055..b81e018bba 100644
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -274,7 +274,7 @@ def get_avail_core_count():
core_cnt = int(sum(sched_getaffinity()))
else:
# BSD-type systems
- out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False)
+ out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
try:
if int(out) > 0:
core_cnt = int(out)
@@ -311,7 +311,7 @@ def get_total_memory():
elif os_type == DARWIN:
cmd = "sysctl -n hw.memsize"
_log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd)
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
+ out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
if ec == 0:
memtotal = int(out.strip()) // (1024**2)
@@ -393,14 +393,15 @@ def get_cpu_vendor():
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.vendor"
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False)
+ out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, with_hooks=False)
out = out.strip()
if ec == 0 and out in VENDOR_IDS:
vendor = VENDOR_IDS[out]
_log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
else:
cmd = "sysctl -n machdep.cpu.brand_string"
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False)
+ out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False,
+ with_hooks=False)
out = out.strip().split(' ')[0]
if ec == 0 and out in CPU_VENDORS:
vendor = out
@@ -503,7 +504,7 @@ def get_cpu_model():
elif os_type == DARWIN:
cmd = "sysctl -n machdep.cpu.brand_string"
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
+ out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
if ec == 0:
model = out.strip()
_log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model))
@@ -548,7 +549,7 @@ def get_cpu_speed():
elif os_type == DARWIN:
cmd = "sysctl -n hw.cpufrequency_max"
_log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd)
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
+ out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
out = out.strip()
cpu_freq = None
if ec == 0 and out:
@@ -596,7 +597,8 @@ def get_cpu_features():
for feature_set in ['extfeatures', 'features', 'leaf7_features']:
cmd = "sysctl -n machdep.cpu.%s" % feature_set
_log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd)
- out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False)
+ out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False,
+ with_hooks=False)
if ec == 0:
cpu_feat.extend(out.strip().lower().split())
@@ -624,7 +626,7 @@ def get_gpu_info():
cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader"
_log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd)
out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False,
- force_in_dry_run=True, trace=False, stream_output=False)
+ force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
if ec == 0:
for line in out.strip().split('\n'):
nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {})
@@ -643,14 +645,14 @@ def get_gpu_info():
cmd = "rocm-smi --showdriverversion --csv"
_log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd)
out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False,
- force_in_dry_run=True, trace=False, stream_output=False)
+ force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
if ec == 0:
amd_driver = out.strip().split('\n')[1].split(',')[1]
cmd = "rocm-smi --showproductname --csv"
_log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd)
out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False,
- force_in_dry_run=True, trace=False, stream_output=False)
+ force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
if ec == 0:
for line in out.strip().split('\n')[1:]:
amd_card_series = line.split(',')[1]
@@ -898,7 +900,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False):
Output is returned as a single-line string (newlines are replaced by '; ').
"""
out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True,
- trace=False, stream_output=False)
+ trace=False, stream_output=False, with_hooks=False)
if not ignore_ec and ec:
_log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out))
return UNKNOWN
diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py
index 308678a555..a16fe35dc0 100644
--- a/easybuild/tools/toolchain/toolchain.py
+++ b/easybuild/tools/toolchain/toolchain.py
@@ -1129,8 +1129,11 @@ def _setenv_variables(self, donotset=None, verbose=True):
setvar("EBVAR%s" % key, val, verbose=False)
def get_flag(self, name):
- """Get compiler flag for a certain option."""
- return "-%s" % self.options.option(name)
+ """Get compiler flag(s) for a certain option."""
+ if isinstance(self.options.option(name), list):
+ return " ".join("-%s" % x for x in list(self.options.option(name)))
+ else:
+ return "-%s" % self.options.option(name)
def toolchain_family(self):
"""Return toolchain family for this toolchain."""
diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
index 0b569741bb..e0016710ae 100644
--- a/easybuild/tools/version.py
+++ b/easybuild/tools/version.py
@@ -45,7 +45,7 @@
# recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
# UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
# This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.8.0')
+VERSION = LooseVersion('4.8.1')
UNKNOWN = 'UNKNOWN'
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
index 4f27e86fb2..6afc9db94f 100644
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -39,6 +39,7 @@
from unittest import TextTestRunner
import easybuild.tools.systemtools as st
+from easybuild.base import fancylogger
from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.easyconfig.easyconfig import EasyConfig
@@ -2424,6 +2425,30 @@ def test_checksum_step(self):
eb.fetch_sources()
eb.checksum_step()
+ with self.mocked_stdout_stderr() as (stdout, stderr):
+
+ # using checksum-less test easyconfig in location that does not provide checksums.json
+ test_ec = os.path.join(self.test_prefix, 'test-no-checksums.eb')
+ copy_file(toy_ec, test_ec)
+ write_file(test_ec, 'checksums = []', append=True)
+ ec = process_easyconfig(test_ec)[0]
+
+ # enable logging to screen, so we can check whether error is logged when checksums.json is not found
+ fancylogger.logToScreen(enable=True, stdout=True)
+
+ eb = get_easyblock_instance(ec)
+ eb.fetch_sources()
+ eb.checksum_step()
+
+ fancylogger.logToScreen(enable=False, stdout=True)
+ stdout = self.get_stdout()
+
+ # make sure there's no error logged for not finding checksums.json,
+ # see also https://github.com/easybuilders/easybuild-framework/issues/4301
+ regex = re.compile("ERROR .*Couldn't find file checksums.json anywhere", re.M)
+ regex.search(stdout)
+ self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in log" % regex.pattern)
+
# fiddle with checksum to check whether faulty checksum is catched
copy_file(toy_ec, self.test_prefix)
toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec))
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
index 32d72c7b83..400a05c1b3 100644
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -45,6 +45,7 @@
from unittest import TextTestRunner
from easybuild.tools import run
import easybuild.tools.filetools as ft
+import easybuild.tools.py2vs3 as py2vs3
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option
from easybuild.tools.multidiff import multidiff
@@ -348,6 +349,14 @@ def test_checksums(self):
alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', broken_checksums['sha256'])
self.assertFalse(ft.verify_checksum(fp, alt_checksums))
+ # Check dictionary
+ alt_checksums = (known_checksums['sha256'],)
+ self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): known_checksums['sha256']}))
+ faulty_dict = {'wrong-name': known_checksums['sha256']}
+ self.assertErrorRegex(EasyBuildError,
+ "Missing checksum for " + os.path.basename(fp) + " in .*wrong-name.*",
+ ft.verify_checksum, fp, faulty_dict)
+
# check whether missing checksums are enforced
build_options = {
'enforce_checksums': True,
@@ -362,6 +371,8 @@ def test_checksums(self):
for checksum in [known_checksums[x] for x in ('md5', 'sha256')]:
dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'}
self.assertTrue(ft.verify_checksum(fp, dict_checksum))
+ del dict_checksum[os.path.basename(fp)]
+ self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum)
def test_common_path_prefix(self):
"""Test get common path prefix for a list of paths."""
@@ -2808,6 +2819,32 @@ def run_check():
]) % git_repo
run_check()
+ git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite']
+ expected = '\n'.join([
+ ' running command "git clone --depth 1 --branch tag_for_tests --recursive'
+ + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"',
+ r" \(in .*/tmp.*\)",
+ r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ r" \(in .*/tmp.*\)",
+ ]) % git_repo
+ run_check()
+
+ git_config['extra_config_params'] = [
+ 'submodule."fastahack".active=false',
+ 'submodule."sha1".active=false',
+ ]
+ expected = '\n'.join([
+ ' running command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false'
+ + ' clone --depth 1 --branch tag_for_tests --recursive'
+ + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"',
+ r" \(in .*/tmp.*\)",
+ r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ r" \(in .*/tmp.*\)",
+ ]) % git_repo
+ run_check()
+ del git_config['recurse_submodules']
+ del git_config['extra_config_params']
+
git_config['keep_git_dir'] = True
expected = '\n'.join([
r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"',
@@ -2830,7 +2867,20 @@ def run_check():
]) % git_repo
run_check()
+ git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite']
+ expected = '\n'.join([
+ r' running command "git clone --no-checkout %(git_repo)s"',
+ r" \(in .*/tmp.*\)",
+ ' running command "git checkout 8456f86 && git submodule update --init --recursive'
+ + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"',
+ r" \(in testrepository\)",
+ r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+ r" \(in .*/tmp.*\)",
+ ]) % git_repo
+ run_check()
+
del git_config['recursive']
+ del git_config['recurse_submodules']
expected = '\n'.join([
r' running command "git clone --no-checkout %(git_repo)s"',
r" \(in .*/tmp.*\)",
@@ -3389,6 +3439,17 @@ def test_set_gid_sticky_bits(self):
self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
+ def test_compat_makedirs(self):
+ """Test compatibility layer for Python3 os.makedirs"""
+ name = os.path.join(self.test_prefix, 'folder')
+ self.assertNotExists(name)
+ py2vs3.makedirs(name)
+ self.assertExists(name)
+ # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3)
+ self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name)
+ py2vs3.makedirs(name, exist_ok=True) # No error
+ self.assertExists(name)
+
def test_create_unused_dir(self):
"""Test create_unused_dir function."""
path = ft.create_unused_dir(self.test_prefix, 'folder')
diff --git a/test/framework/github.py b/test/framework/github.py
index c6962a4623..c9bfbaedd3 100644
--- a/test/framework/github.py
+++ b/test/framework/github.py
@@ -648,6 +648,11 @@ def test_validate_github_token(self):
if token_old_format:
self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT))
+ # if a fine-grained token is available, test with that too
+ finegrained_token = os.getenv('TEST_GITHUB_TOKEN_FINEGRAINED')
+ if finegrained_token:
+ self.assertTrue(gh.validate_github_token(finegrained_token, GITHUB_TEST_ACCOUNT))
+
def test_github_find_easybuild_easyconfig(self):
"""Test for find_easybuild_easyconfig function"""
if self.skip_github_tests:
diff --git a/test/framework/hooks.py b/test/framework/hooks.py
index fad251b040..152d2352a4 100644
--- a/test/framework/hooks.py
+++ b/test/framework/hooks.py
@@ -32,8 +32,9 @@
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
-import easybuild.tools.hooks
+import easybuild.tools.hooks # so we can reset cached hooks
from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import update_build_option
from easybuild.tools.filetools import remove_file, write_file
from easybuild.tools.hooks import find_hook, load_hooks, run_hook, verify_hooks
@@ -52,6 +53,9 @@ def setUp(self):
'def parse_hook(ec):',
' print("Parse hook with argument %s" % ec)',
'',
+ 'def pre_build_and_install_loop_hook(ecs):',
+ ' print("About to start looping for %d easyconfigs!" % len(ecs))',
+ '',
'def foo():',
' print("running foo helper method")',
'',
@@ -64,9 +68,28 @@ def setUp(self):
'',
'def pre_single_extension_hook(ext):',
' print("this is run before installing an extension")',
+ '',
+ 'def pre_run_shell_cmd_hook(cmd, interactive=False):',
+ ' if interactive:',
+ ' print("this is run before running interactive command \'%s\'" % cmd)',
+ ' else:',
+ ' print("this is run before running command \'%s\'" % cmd)',
+ ' if cmd == "make install":',
+ ' return "sudo " + cmd',
+ '',
+ 'def fail_hook(err):',
+ ' print("EasyBuild FAIL: %s" % err)',
])
write_file(self.test_hooks_pymod, test_hooks_pymod_txt)
+ def tearDown(self):
+ """Cleanup."""
+
+ # reset cached hooks
+ easybuild.tools.hooks._cached_hooks.clear()
+
+ super(HooksTest, self).tearDown()
+
def test_load_hooks(self):
"""Test for load_hooks function."""
@@ -74,11 +97,14 @@ def test_load_hooks(self):
hooks = load_hooks(self.test_hooks_pymod)
- self.assertEqual(len(hooks), 5)
+ self.assertEqual(len(hooks), 8)
expected = [
+ 'fail_hook',
'parse_hook',
'post_configure_hook',
+ 'pre_build_and_install_loop_hook',
'pre_install_hook',
+ 'pre_run_shell_cmd_hook',
'pre_single_extension_hook',
'start_hook',
]
@@ -113,6 +139,9 @@ def test_find_hook(self):
pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0]
pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0]
start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0]
+ pre_run_shell_cmd_hook = [hooks[k] for k in hooks if k == 'pre_run_shell_cmd_hook'][0]
+ fail_hook = [hooks[k] for k in hooks if k == 'fail_hook'][0]
+ pre_build_and_install_loop_hook = [hooks[k] for k in hooks if k == 'pre_build_and_install_loop_hook'][0]
self.assertEqual(find_hook('configure', hooks), None)
self.assertEqual(find_hook('configure', hooks, pre_step_hook=True), None)
@@ -138,6 +167,19 @@ def test_find_hook(self):
self.assertEqual(find_hook('start', hooks, pre_step_hook=True), None)
self.assertEqual(find_hook('start', hooks, post_step_hook=True), None)
+ self.assertEqual(find_hook('run_shell_cmd', hooks), None)
+ self.assertEqual(find_hook('run_shell_cmd', hooks, pre_step_hook=True), pre_run_shell_cmd_hook)
+ self.assertEqual(find_hook('run_shell_cmd', hooks, post_step_hook=True), None)
+
+ self.assertEqual(find_hook('fail', hooks), fail_hook)
+ self.assertEqual(find_hook('fail', hooks, pre_step_hook=True), None)
+ self.assertEqual(find_hook('fail', hooks, post_step_hook=True), None)
+
+ hook_name = 'build_and_install_loop'
+ self.assertEqual(find_hook(hook_name, hooks), None)
+ self.assertEqual(find_hook(hook_name, hooks, pre_step_hook=True), pre_build_and_install_loop_hook)
+ self.assertEqual(find_hook(hook_name, hooks, post_step_hook=True), None)
+
def test_run_hook(self):
"""Test for run_hook function."""
@@ -145,43 +187,74 @@ def test_run_hook(self):
init_config(build_options={'debug': True})
- self.mock_stdout(True)
- self.mock_stderr(True)
- run_hook('start', hooks)
- run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...")
- run_hook('configure', hooks, pre_step_hook=True, args=[None])
- run_hook('configure', hooks, post_step_hook=True, args=[None])
- run_hook('build', hooks, pre_step_hook=True, args=[None])
- run_hook('build', hooks, post_step_hook=True, args=[None])
- run_hook('install', hooks, pre_step_hook=True, args=[None])
- run_hook('install', hooks, post_step_hook=True, args=[None])
- run_hook('extensions', hooks, pre_step_hook=True, args=[None])
- for _ in range(3):
- run_hook('single_extension', hooks, pre_step_hook=True, args=[None])
- run_hook('single_extension', hooks, post_step_hook=True, args=[None])
- run_hook('extensions', hooks, post_step_hook=True, args=[None])
- stdout = self.get_stdout()
- stderr = self.get_stderr()
- self.mock_stdout(False)
- self.mock_stderr(False)
-
- expected_stdout = '\n'.join([
+ def run_hooks():
+ self.mock_stdout(True)
+ self.mock_stderr(True)
+ run_hook('start', hooks)
+ run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...")
+ run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True)
+ run_hook('configure', hooks, pre_step_hook=True, args=[None])
+ run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True})
+ run_hook('configure', hooks, post_step_hook=True, args=[None])
+ run_hook('build', hooks, pre_step_hook=True, args=[None])
+ run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"])
+ run_hook('build', hooks, post_step_hook=True, args=[None])
+ run_hook('install', hooks, pre_step_hook=True, args=[None])
+ res = run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={})
+ self.assertEqual(res, "sudo make install")
+ run_hook('install', hooks, post_step_hook=True, args=[None])
+ run_hook('extensions', hooks, pre_step_hook=True, args=[None])
+ for _ in range(3):
+ run_hook('single_extension', hooks, pre_step_hook=True, args=[None])
+ run_hook('single_extension', hooks, post_step_hook=True, args=[None])
+ run_hook('extensions', hooks, post_step_hook=True, args=[None])
+ run_hook('fail', hooks, args=[EasyBuildError('oops')])
+ stdout = self.get_stdout()
+ stderr = self.get_stderr()
+ self.mock_stdout(False)
+ self.mock_stderr(False)
+
+ return stdout, stderr
+
+ stdout, stderr = run_hooks()
+
+ expected_stdout_lines = [
"== Running start hook...",
"this is triggered at the very beginning",
"== Running parse hook for example.eb...",
"Parse hook with argument ",
+ "== Running pre-build_and_install_loop hook...",
+ "About to start looping for 2 easyconfigs!",
+ "== Running pre-run_shell_cmd hook...",
+ "this is run before running interactive command 'configure.sh'",
"== Running post-configure hook...",
"this is run after configure step",
"running foo helper method",
+ "== Running pre-run_shell_cmd hook...",
+ "this is run before running command 'make -j 3'",
"== Running pre-install hook...",
"this is run before install step",
+ "== Running pre-run_shell_cmd hook...",
+ "this is run before running command 'make install'",
"== Running pre-single_extension hook...",
"this is run before installing an extension",
"== Running pre-single_extension hook...",
"this is run before installing an extension",
"== Running pre-single_extension hook...",
"this is run before installing an extension",
- ])
+ "== Running fail hook...",
+ "EasyBuild FAIL: 'oops'",
+ ]
+ expected_stdout = '\n'.join(expected_stdout_lines)
+
+ self.assertEqual(stdout.strip(), expected_stdout)
+ self.assertEqual(stderr, '')
+
+ # test silencing of hook trigger
+ update_build_option('silence_hook_trigger', True)
+ stdout, stderr = run_hooks()
+
+ expected_stdout = '\n'.join(x for x in expected_stdout_lines if not x.startswith('== Running'))
self.assertEqual(stdout.strip(), expected_stdout)
self.assertEqual(stderr, '')
diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py
index b6bec17093..a0c6131140 100644
--- a/test/framework/module_generator.py
+++ b/test/framework/module_generator.py
@@ -756,6 +756,16 @@ def test_module_extensions(self):
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(desc), "Pattern '%s' found in: %s" % (regex.pattern, desc))
+ # check if the extensions is missing if there are no extensions
+ test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb')
+
+ ec = EasyConfig(test_ec)
+ eb = EasyBlock(ec)
+ modgen = self.MODULE_GENERATOR_CLASS(eb)
+ desc = modgen.get_description()
+
+ self.assertFalse(re.search(r"\s*extensions\(", desc), "No extensions found in: %s" % desc)
+
def test_prepend_paths(self):
"""Test generating prepend-paths statements."""
# test prepend_paths
diff --git a/test/framework/modules.py b/test/framework/modules.py
index 4f2b3fc5d6..7bf87130bd 100644
--- a/test/framework/modules.py
+++ b/test/framework/modules.py
@@ -1273,24 +1273,33 @@ def get_resolved_module_path():
test_dir1_relative = os.path.join(test_dir1, '..', os.path.basename(test_dir1))
test_dir2_dot = os.path.join(os.path.dirname(test_dir2), '.', os.path.basename(test_dir2))
self.modtool.add_module_path(test_dir1_relative)
- self.assertEqual(get_resolved_module_path(), test_dir1)
+ self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1))
# Adding the same path, but in a different form may be possible, but may also be ignored, e.g. in EnvModules
self.modtool.add_module_path(test_dir1)
if get_resolved_module_path() != test_dir1:
- self.assertEqual(get_resolved_module_path(), os.pathsep.join([test_dir1, test_dir1]))
+ modpath = get_resolved_module_path().split(os.pathsep)
+ self.assertEqual(len(modpath), 2)
+ self.assertTrue(os.path.samefile(modpath[0], test_dir1))
+ self.assertTrue(os.path.samefile(modpath[1], test_dir1))
self.modtool.remove_module_path(test_dir1)
- self.assertEqual(get_resolved_module_path(), test_dir1)
+ self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1))
+
self.modtool.add_module_path(test_dir2_dot)
- self.assertEqual(get_resolved_module_path(), test_dir_2_and_1)
+ modpath = get_resolved_module_path().split(os.pathsep)
+ self.assertEqual(len(modpath), 2)
+ self.assertTrue(os.path.samefile(modpath[0], test_dir2))
+ self.assertTrue(os.path.samefile(modpath[1], test_dir1))
+
self.modtool.remove_module_path(test_dir2_dot)
- self.assertEqual(get_resolved_module_path(), test_dir1)
+ self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1))
+
# Force adding such a dot path which can be removed with either variant
os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative])
self.modtool.remove_module_path(test_dir2_dot)
- self.assertEqual(get_resolved_module_path(), test_dir1)
+ self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1))
os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative])
self.modtool.remove_module_path(test_dir2)
- self.assertEqual(get_resolved_module_path(), test_dir1)
+ self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1))
os.environ['MODULEPATH'] = old_module_path # Restore
diff --git a/test/framework/options.py b/test/framework/options.py
index 106f951374..219aa4a39a 100644
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -679,6 +679,7 @@ def test_avail_hooks(self):
"List of supported hooks (in order of execution):",
" start_hook",
" parse_hook",
+ " pre_build_and_install_loop_hook",
" pre_fetch_hook",
" post_fetch_hook",
" pre_ready_hook",
@@ -701,6 +702,8 @@ def test_avail_hooks(self):
" pre_single_extension_hook",
" post_single_extension_hook",
" post_extensions_hook",
+ " pre_postiter_hook",
+ " post_postiter_hook",
" pre_postproc_hook",
" post_postproc_hook",
" pre_sanitycheck_hook",
@@ -716,7 +719,12 @@ def test_avail_hooks(self):
" post_package_hook",
" pre_testcases_hook",
" post_testcases_hook",
+ " post_build_and_install_loop_hook",
" end_hook",
+ " cancel_hook",
+ " fail_hook",
+ " pre_run_shell_cmd_hook",
+ " post_run_shell_cmd_hook",
'',
])
self.assertEqual(stdout, expected)
@@ -2571,7 +2579,8 @@ def test_cleanup_builddir(self):
'--force',
'--try-amend=prebuildopts=nosuchcommand &&',
]
- self.eb_main(args, do_build=True)
+ with self.mocked_stdout_stderr():
+ self.eb_main(args, do_build=True)
self.assertExists(toy_buildpath, "Build dir %s is retained after failed build" % toy_buildpath)
def test_filter_deps(self):
@@ -5073,19 +5082,22 @@ def test_fetch(self):
lock_path = os.path.join(self.test_installpath, 'software', '.locks', lock_fn)
mkdir(lock_path, parents=True)
- args = ['toy-0.0.eb', '--fetch']
- stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False)
+ # Run for a "regular" EC and one with an external module dependency
+ # which might trip up the dependency resolution (see #4298)
+ for ec in ('toy-0.0.eb', 'toy-0.0-deps.eb'):
+ args = [ec, '--fetch']
+ stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False)
- patterns = [
- r"^== fetching files\.\.\.$",
- r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$",
- ]
- for pattern in patterns:
- regex = re.compile(pattern, re.M)
- self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout))
+ patterns = [
+ r"^== fetching files\.\.\.$",
+ r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$",
+ ]
+ for pattern in patterns:
+ regex = re.compile(pattern, re.M)
+ self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout))
- regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$")
- self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
+ regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$")
+ self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
def test_parse_external_modules_metadata(self):
"""Test parse_external_modules_metadata function."""
diff --git a/test/framework/run.py b/test/framework/run.py
index 908a9e9d93..1a93ff1a75 100644
--- a/test/framework/run.py
+++ b/test/framework/run.py
@@ -39,6 +39,7 @@
import subprocess
import sys
import tempfile
+import textwrap
import time
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
@@ -47,6 +48,7 @@
import easybuild.tools.asyncprocess as asyncprocess
import easybuild.tools.utilities
from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging
+from easybuild.tools.config import update_build_option
from easybuild.tools.filetools import adjust_permissions, read_file, write_file
from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process
from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa
@@ -543,7 +545,7 @@ def test_run_cmd_script(self):
"""Testing use of run_cmd with shell=False to call external scripts"""
py_test_script = os.path.join(self.test_prefix, 'test.py')
write_file(py_test_script, '\n'.join([
- '#!/usr/bin/env python',
+ '#!%s' % sys.executable,
'print("hello")',
]))
adjust_permissions(py_test_script, stat.S_IXUSR)
@@ -736,6 +738,60 @@ def test_check_log_for_errors(self):
expected_msg = "Found 1 potential error(s) in command output (output: the process crashed with 0)"
self.assertIn(expected_msg, read_file(logfile))
+ def test_run_cmd_with_hooks(self):
+ """
+ Test running command with run_cmd with pre/post run_shell_cmd hooks in place.
+ """
+ cwd = os.getcwd()
+
+ hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
+ hooks_file_txt = textwrap.dedent("""
+ def pre_run_shell_cmd_hook(cmd, *args, **kwargs):
+ work_dir = kwargs['work_dir']
+ if kwargs.get('interactive'):
+ print("pre-run hook interactive '%s' in %s" % (cmd, work_dir))
+ else:
+ print("pre-run hook '%s' in %s" % (cmd, work_dir))
+ if not cmd.startswith('echo'):
+ cmds = cmd.split(';')
+ return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()])
+
+ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
+ exit_code = kwargs.get('exit_code')
+ output = kwargs.get('output')
+ work_dir = kwargs['work_dir']
+ if kwargs.get('interactive'):
+ msg = "post-run hook interactive '%s'" % cmd
+ else:
+ msg = "post-run hook '%s'" % cmd
+ msg += " (exit code: %s, output: '%s')" % (exit_code, output)
+ print(msg)
+ """)
+ write_file(hooks_file, hooks_file_txt)
+ update_build_option('hooks', hooks_file)
+
+ with self.mocked_stdout_stderr():
+ run_cmd("make")
+ stdout = self.get_stdout()
+
+ expected_stdout = '\n'.join([
+ "pre-run hook 'make' in %s" % cwd,
+ "post-run hook 'echo make' (exit code: 0, output: 'make\n')",
+ '',
+ ])
+ self.assertEqual(stdout, expected_stdout)
+
+ with self.mocked_stdout_stderr():
+ run_cmd_qa("sleep 2; make", qa={})
+ stdout = self.get_stdout()
+
+ expected_stdout = '\n'.join([
+ "pre-run hook interactive 'sleep 2; make' in %s" % cwd,
+ "post-run hook interactive 'sleep 2; echo make' (exit code: 0, output: 'make\n')",
+ '',
+ ])
+ self.assertEqual(stdout, expected_stdout)
+
def suite():
""" returns all the testcases in this module """
diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py
index 88ff864454..c4614e3333 100644
--- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py
+++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py
@@ -34,7 +34,7 @@
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, print_warning
from easybuild.tools.environment import setvar
from easybuild.tools.filetools import mkdir, write_file
from easybuild.tools.modules import get_software_root, get_software_version
@@ -122,7 +122,11 @@ def build_step(self, name=None, cfg=None):
name = self.name
cmd = compose_toy_build_cmd(self.cfg, name, cfg['prebuildopts'], cfg['buildopts'])
- run_cmd(cmd)
+ # purposely run build command without checking exit code;
+ # we rely on this in test_toy_build_hooks
+ (out, ec) = run_cmd(cmd, log_ok=False, log_all=False)
+ if ec:
+ print_warning("Command '%s' failed, but we'll ignore it..." % cmd)
def install_step(self, name=None):
"""Install toy."""
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
index 1c36431c45..d80d41c788 100644
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -2950,6 +2950,21 @@ def test_env_vars_external_module(self):
expected = {}
self.assertEqual(res, expected)
+ def test_get_flag(self):
+ """Test get_flag function"""
+ tc = self.get_toolchain('gompi', version='2018a')
+
+ checks = {
+ '-a': 'a',
+ '-openmp': 'openmp',
+ '-foo': ['foo'],
+ '-foo -bar': ['foo', 'bar'],
+ }
+
+ for flagstring, flags in checks.items():
+ tc.options.options_map['openmp'] = flags
+ self.assertEqual(tc.get_flag('openmp'), flagstring)
+
def suite():
""" return all the tests"""
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
index 52aa18aae9..fb8ec907dc 100644
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -58,6 +58,7 @@
from easybuild.tools.modules import Lmod
from easybuild.tools.py2vs3 import reload, string_type
from easybuild.tools.run import run_cmd
+from easybuild.tools.utilities import nub
from easybuild.tools.systemtools import get_shared_lib_ext
from easybuild.tools.version import VERSION as EASYBUILD_VERSION
@@ -100,6 +101,9 @@ def tearDown(self):
del sys.modules['easybuild.easyblocks.toytoy']
del sys.modules['easybuild.easyblocks.generic.toy_extension']
+ # reset cached hooks
+ easybuild.tools.hooks._cached_hooks.clear()
+
super(ToyBuildTest, self).tearDown()
# remove logs
@@ -154,7 +158,7 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi
def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True,
raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True,
- raise_systemexit=False, force=True, test_report_regexs=None):
+ raise_systemexit=False, force=True, test_report_regexs=None, debug=True):
"""Perform a toy build."""
if extra_args is None:
extra_args = []
@@ -168,10 +172,11 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True
'--sourcepath=%s' % self.test_sourcepath,
'--buildpath=%s' % self.test_buildpath,
'--installpath=%s' % self.test_installpath,
- '--debug',
'--unittest-file=%s' % self.logfile,
'--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]),
]
+ if debug:
+ args.append('--debug')
if force:
args.append('--force')
if tmpdir is not None:
@@ -2885,6 +2890,9 @@ def test_toy_build_hooks(self):
hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
hooks_file_txt = textwrap.dedent("""
import os
+ from easybuild.tools.filetools import change_dir, copy_file
+
+ TOY_COMP_CMD = "gcc toy.c -o toy"
def start_hook():
print('start hook triggered')
@@ -2908,6 +2916,9 @@ def post_install_hook(self):
print('in post-install hook for %s v%s' % (self.name, self.version))
print(', '.join(sorted(os.listdir(self.installdir))))
+ copy_of_toy = os.path.join(self.start_dir, 'copy_of_toy')
+ copy_file(copy_of_toy, os.path.join(self.installdir, 'bin'))
+
def module_write_hook(self, module_path, module_txt):
print('in module-write hook hook for %s' % os.path.basename(module_path))
return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore')
@@ -2920,12 +2931,30 @@ def pre_sanitycheck_hook(self):
def end_hook():
print('end hook triggered, all done!')
+
+ def pre_run_shell_cmd_hook(cmd, *args, **kwargs):
+ if cmd.strip() == TOY_COMP_CMD:
+ print("pre_run_shell_cmd_hook triggered for '%s'" % cmd)
+ # 'copy_toy_file' command doesn't exist, but don't worry,
+ # this problem will be fixed in post_run_shell_cmd_hook
+ cmd += " && copy_toy_file toy copy_of_toy"
+ return cmd
+
+ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
+ exit_code = kwargs['exit_code']
+ output = kwargs['output']
+ work_dir = kwargs['work_dir']
+ if cmd.strip().startswith(TOY_COMP_CMD) and exit_code:
+ cwd = change_dir(work_dir)
+ copy_file('toy', 'copy_of_toy')
+ change_dir(cwd)
+ print("'%s' command failed (exit code %s), but I fixed it!" % (cmd, exit_code))
""")
write_file(hooks_file, hooks_file_txt)
self.mock_stderr(True)
self.mock_stdout(True)
- self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True)
+ self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True, debug=False)
stderr = self.get_stderr()
stdout = self.get_stdout()
self.mock_stderr(False)
@@ -2936,7 +2965,9 @@ def end_hook():
if get_module_syntax() == 'Lua':
toy_mod_file += '.lua'
- self.assertEqual(stderr, '')
+ warnings = nub([x for x in stderr.strip().splitlines() if x])
+ self.assertEqual(warnings, ["WARNING: Command ' gcc toy.c -o toy ' failed, but we'll ignore it..."])
+
# parse hook is triggered 3 times: once for main install, and then again for each extension;
# module write hook is triggered 5 times:
# - before installing extensions
@@ -2944,44 +2975,32 @@ def end_hook():
# - for final module file
# - for devel module file
expected_output = textwrap.dedent("""
- == Running start hook...
start hook triggered
- == Running parse hook for test.eb...
toy 0.0
['%(name)s-%(version)s.tar.gz']
echo toy
- == Running pre-configure hook...
pre-configure: toy.source: True
- == Running post-configure hook...
post-configure: toy.source: False
- == Running post-install hook...
+ pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy '
+ ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it!
in post-install hook for toy v0.0
bin, lib
- == Running module_write hook...
in module-write hook hook for {mod_name}
- == Running parse hook...
toy 0.0
['%(name)s-%(version)s.tar.gz']
echo toy
- == Running parse hook...
toy 0.0
['%(name)s-%(version)s.tar.gz']
echo toy
- == Running post-single_extension hook...
installing of extension bar is done!
- == Running post-single_extension hook...
+ pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy '
+ ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it!
installing of extension toy is done!
- == Running pre-sanitycheck hook...
pre_sanity_check_hook
- == Running module_write hook...
in module-write hook hook for {mod_name}
- == Running module_write hook...
in module-write hook hook for {mod_name}
- == Running module_write hook...
in module-write hook hook for {mod_name}
- == Running module_write hook...
in module-write hook hook for {mod_name}
- == Running end hook...
end hook triggered, all done!
""").strip().format(mod_name=os.path.basename(toy_mod_file))
self.assertEqual(stdout.strip(), expected_output)
@@ -2989,6 +3008,10 @@ def end_hook():
toy_mod = read_file(toy_mod_file)
self.assertIn('Not a toy anymore', toy_mod)
+ toy_bin_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin')
+ toy_bins = sorted(os.listdir(toy_bin_dir))
+ self.assertEqual(toy_bins, ['bar', 'copy_of_toy', 'toy'])
+
def test_toy_multi_deps(self):
"""Test installation of toy easyconfig that uses multi_deps."""
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
@@ -3900,6 +3923,30 @@ def test_toy_post_install_messages(self):
regex = re.compile(pattern, re.M)
self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+ def test_toy_build_info_msg(self):
+ """
+ Test use of build info message
+ """
+ test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+ toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+ test_ec_txt = read_file(toy_ec)
+ test_ec_txt += '\nbuild_info_msg = "Are you sure you want to install this toy software?"'
+ test_ec = os.path.join(self.test_prefix, 'test.eb')
+ write_file(test_ec, test_ec_txt)
+
+ with self.mocked_stdout_stderr():
+ self.test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True)
+ stdout = self.get_stdout()
+
+ pattern = '\n'.join([
+ r"== This easyconfig provides the following build information:",
+ r'',
+ r"Are you sure you want to install this toy software\?",
+ ])
+ regex = re.compile(pattern, re.M)
+ self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
def suite():
""" return all the tests in this file """