From 5dc338ed43e2182cdfbe65b50a0b6e91275d32f7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 19 Feb 2022 18:18:33 +0100 Subject: [PATCH 001/111] add end-to-end test for running EasyBuild in different Linux distros using containers --- .github/workflows/end2end.yml | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/end2end.yml diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml new file mode 100644 index 0000000000..7afd352776 --- /dev/null +++ b/.github/workflows/end2end.yml @@ -0,0 +1,62 @@ +name: End-to-end test of EasyBuild in different distros +on: [push, pull_request] +jobs: + build_publish: + name: End-to-end test + runs-on: ubuntu-latest + strategy: + matrix: + container: + - centos-7.9 + - centos-8.4 + - fedora-35 + - opensuse-15.4 + - rockylinux-8.5 + - ubuntu-20.04 + fail-fast: false + container: + image: ghcr.io/easybuilders/${{ matrix.container }} + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: download and unpack easyblocks and easyconfigs repositories + run: | + cd $HOME + for pkg in easyblocks easyconfigs; do + curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz + tar xfz develop.tar.gz + rm -f develop.tar.gz + done + + - name: End-to-end test of EasyBuild + shell: bash + run: | + export PATH=$PWD:$PATH + export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop + + # initialize environment (for Lmod) + if [ -f /etc/profile.d/modules.sh ]; then + # for Rocky, CentOS 8, Fedora + echo ">>> sourcing /etc/profile.d/modules.sh" + . /etc/profile.d/modules.sh + else + echo ">>> sourcing /etc/profile.d/*lmod*.sh" + . /etc/profile.d/*lmod*.sh + fi + + # tests are run with root privileges, so we need to tell EasyBuild that's OK... + export EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1 + + cmds=( + "env | sort" + "eb --version" + "eb --show-system-info" + "eb --check-eb-deps" + "eb --show-config" + "eb bzip2-1.0.8.eb --trace --robot" + ) + for cmd in "${cmds[@]}"; do + echo ">>> $cmd" + eval "$cmd" + done From c1d5ca8b2e611604b6efbd4d2f38a8dcb065be3d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Mar 2022 18:03:43 +0100 Subject: [PATCH 002/111] stick to openSUSE 15.3 (stable) rather than using 15.4 (alpha) for now for testing EasyBuild --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7afd352776..7594b79e59 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -10,7 +10,7 @@ jobs: - centos-7.9 - centos-8.4 - fedora-35 - - opensuse-15.4 + - opensuse-15.3 - rockylinux-8.5 - ubuntu-20.04 fail-fast: false From aaf65645f7b2da9dba2ccf1474a7e6d11713dd0b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 21 Dec 2022 16:57:57 +0100 Subject: [PATCH 003/111] Improve error when checksum dict has no entry for a file When the dict didn't contain the filename EB will crash with > Invalid checksum spec 'None', should be a string (MD5) or 2-tuple (type, value). This will now raise a more descriptive error --- easybuild/tools/filetools.py | 9 +++------ test/framework/filetools.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index daa143b46c..f3bc626b2b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1265,14 +1265,11 @@ def verify_checksum(path, checksums): for checksum in checksums: if isinstance(checksum, dict): - if filename in checksum: + try: # Set this to a string-type checksum checksum = checksum[filename] - elif build_option('enforce_checksums'): - raise EasyBuildError("Missing checksum for %s", filename) - else: - # Set to None and allow to fail elsewhere - checksum = None + except KeyError: + raise EasyBuildError("Missing checksum for %s in %s", filename, checksum) if isinstance(checksum, string_type): # if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 32d72c7b83..aa6374e965 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -348,6 +348,14 @@ def test_checksums(self): alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', broken_checksums['sha256']) self.assertFalse(ft.verify_checksum(fp, alt_checksums)) + # Check dictionary + alt_checksums = (known_checksums['sha256'],) + self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): known_checksums['sha256']})) + faulty_dict = {'wrong-name': known_checksums['sha256']} + self.assertErrorRegex(EasyBuildError, + "Missing checksum for " + os.path.basename(fp) + " in .*wrong-name.*", + ft.verify_checksum, fp, faulty_dict) + # check whether missing checksums are enforced build_options = { 'enforce_checksums': True, @@ -362,6 +370,8 @@ def test_checksums(self): for checksum in [known_checksums[x] for x in ('md5', 'sha256')]: dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'} self.assertTrue(ft.verify_checksum(fp, dict_checksum)) + del dict_checksum[os.path.basename(fp)] + self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum) def test_common_path_prefix(self): """Test get common path prefix for a list of paths.""" From f140a328ba6a2398931e92ce5858d36ac0668e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Fri, 19 May 2023 15:22:45 +0200 Subject: [PATCH 004/111] fix error message when checksums set to [None] --- easybuild/framework/easyblock.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index df73e28db4..33d96166e0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -395,13 +395,13 @@ def get_checksums_from_json(self, always_read=False): :param always_read: always read the checksums.json file, even if it has been read before """ if always_read or self.json_checksums is None: - try: - path = self.obtain_file("checksums.json", no_download=True) + path = self.obtain_file("checksums.json", no_download=True, warning_only=True) + if path is not None: self.log.info("Loading checksums from file %s", path) json_txt = read_file(path) self.json_checksums = json.loads(json_txt) - # if the file can't be found, return an empty dict - except EasyBuildError: + else: + # if the file can't be found, return an empty dict self.json_checksums = {} return self.json_checksums @@ -736,7 +736,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None, no_download=False, download_instructions=None, alt_location=None): + git_config=None, no_download=False, download_instructions=None, alt_location=None, + warning_only=False): """ Locate the file with the given name - searches in different subdirectories of source path @@ -789,7 +790,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No return fullpath except IOError as err: - raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err) + if not warning_only: + raise EasyBuildError("Downloading file %s " + "from url %s to %s failed: %s", filename, url, fullpath, err) + else: + self.log.warning("Downloading file %s " + "from url %s to %s failed: %s", filename, url, fullpath, err) + return None else: # try and find file in various locations @@ -866,8 +873,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + if not warning_only: + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + else: + self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + return None elif git_config: return get_source_tarball_from_git(filename, targetdir, git_config) else: @@ -959,7 +971,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No error_msg += "and downloading it didn't work either... " error_msg += "Paths attempted (in order): %s " % failedpaths_msg - raise EasyBuildError(error_msg, filename) + if not warning_only: + raise EasyBuildError(error_msg, filename) + else: + self.log.warning(error_msg, filename) + return None # # GETTER/SETTER UTILITY FUNCTIONS From ea926f1e1a998534532058242e5f55b2b427ba85 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 21 Dec 2022 16:57:57 +0100 Subject: [PATCH 005/111] Improve error when checksum dict has no entry for a file When the dict didn't contain the filename EB will crash with > Invalid checksum spec 'None', should be a string (MD5) or 2-tuple (type, value). This will now raise a more descriptive error --- easybuild/tools/filetools.py | 9 +++------ test/framework/filetools.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index daa143b46c..f3bc626b2b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1265,14 +1265,11 @@ def verify_checksum(path, checksums): for checksum in checksums: if isinstance(checksum, dict): - if filename in checksum: + try: # Set this to a string-type checksum checksum = checksum[filename] - elif build_option('enforce_checksums'): - raise EasyBuildError("Missing checksum for %s", filename) - else: - # Set to None and allow to fail elsewhere - checksum = None + except KeyError: + raise EasyBuildError("Missing checksum for %s in %s", filename, checksum) if isinstance(checksum, string_type): # if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 32d72c7b83..aa6374e965 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -348,6 +348,14 @@ def test_checksums(self): alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', broken_checksums['sha256']) self.assertFalse(ft.verify_checksum(fp, alt_checksums)) + # Check dictionary + alt_checksums = (known_checksums['sha256'],) + self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): known_checksums['sha256']})) + faulty_dict = {'wrong-name': known_checksums['sha256']} + self.assertErrorRegex(EasyBuildError, + "Missing checksum for " + os.path.basename(fp) + " in .*wrong-name.*", + ft.verify_checksum, fp, faulty_dict) + # check whether missing checksums are enforced build_options = { 'enforce_checksums': True, @@ -362,6 +370,8 @@ def test_checksums(self): for checksum in [known_checksums[x] for x in ('md5', 'sha256')]: dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'} self.assertTrue(ft.verify_checksum(fp, dict_checksum)) + del dict_checksum[os.path.basename(fp)] + self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum) def test_common_path_prefix(self): """Test get common path prefix for a list of paths.""" From 0f9ca9236e349d8fccc49bebc9cb494418c13bcc Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Fri, 23 Jun 2023 15:06:25 +0100 Subject: [PATCH 006/111] suggest default title in --- easybuild/framework/easyconfig/tools.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 62968f9de0..abe8f998ea 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -60,7 +60,7 @@ from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files from easybuild.tools.filetools import read_file, resolve_path, which, write_file from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO -from easybuild.tools.github import det_pr_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data +from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.github import fetch_files_from_pr from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict @@ -561,6 +561,11 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False lines.extend(['', "This PR is associated with a generic '.x' milestone, " "it should be associated to the next release milestone once merged"]) + default_new_title = det_pr_title([ec['ec'] for ec in ecs]) + if default_new_title != pr_data['title']: + lines.extend(['', "If this PR contains only new easyconfigs and has not been edited from the default, " + "then the title should be: %s" % default_new_title]) + return '\n'.join(lines) From 6adee629033385af2b764c09a20cf573d7eba59f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 7 Jul 2023 22:38:55 +0200 Subject: [PATCH 007/111] bump version to 4.8.1dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 0b569741bb..61f8473a0e 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.0') +VERSION = LooseVersion('4.8.1.dev0') UNKNOWN = 'UNKNOWN' From d43ae94854b7bf03fa4d12d5edab1fc1029f0676 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 24 Jul 2023 09:44:25 +0200 Subject: [PATCH 008/111] Don't fail `mkdir` if path gets created while processing it This may happen if a parallel build creates e.g. the sources-directory --- easybuild/tools/filetools.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 19fbc280a4..4a77346f98 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1921,6 +1921,12 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): os.makedirs(path) else: os.mkdir(path) + except FileExistsError as err: + if os.path.exists(path): + # This may happen if a parallel build creates the directory after we checked for its existance + _log.debug("Directory creation aborted as it seems it was already created: %s", err) + else: + raise EasyBuildError("Failed to create directory %s: %s", path, err) except OSError as err: raise EasyBuildError("Failed to create directory %s: %s", path, err) From 7e0d42dbe36527433541be40afc44d70623adcb7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 24 Jul 2023 10:03:51 +0200 Subject: [PATCH 009/111] Fix spelling Co-authored-by: ocaisa --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4a77346f98..d16d4f4778 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1923,7 +1923,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): os.mkdir(path) except FileExistsError as err: if os.path.exists(path): - # This may happen if a parallel build creates the directory after we checked for its existance + # This may happen if a parallel build creates the directory after we checked for its existence _log.debug("Directory creation aborted as it seems it was already created: %s", err) else: raise EasyBuildError("Failed to create directory %s: %s", path, err) From ccc6ef7644969b7117224aa5b6f58e47ad34ca66 Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Tue, 25 Jul 2023 11:04:49 +0100 Subject: [PATCH 010/111] Add cancel and fail hooks --- easybuild/main.py | 5 ++++- easybuild/tools/hooks.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 319c80b5ae..32a3e922ef 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -734,5 +734,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): main() except EasyBuildError as err: print_error(err.msg) - except KeyboardInterrupt as err: + except KeyboardInterrupt as err: + eb_go, cfg_settings = set_up_configuration(args=None, logfile=None, testing=None) + options, orig_paths = eb_go.options, eb_go.args + run_hook('cancel', load_hooks(options.hooks)) print_error("Cancelled by user: %s" % err) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 3c21d4e104..f072a2bcd4 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -65,6 +65,9 @@ MODULE_WRITE = 'module_write' END = 'end' +FAIL = 'fail' +CANCEL = 'cancel' + PRE_PREF = 'pre_' POST_PREF = 'post_' HOOK_SUFF = '_hook' @@ -98,6 +101,8 @@ ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] for p in [PRE_PREF, POST_PREF]] + [ END, + FAIL, + CANCEL, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] From 3eee39508317195cae141fefb076ac32f5a8fecf Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 25 Jul 2023 13:30:00 +0100 Subject: [PATCH 011/111] Run hooks in main with arguments --- easybuild/main.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 32a3e922ef..a3186b68cd 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import START, END, load_hooks, run_hook +from easybuild.tools.hooks import START, END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -730,12 +730,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if __name__ == "__main__": + eb_go, _ = set_up_configuration(args=None, logfile=None, testing=None) + hooks = load_hooks(eb_go.options.hooks) try: main() except EasyBuildError as err: + run_hook(FAIL, hooks, args=[err]) print_error(err.msg) - except KeyboardInterrupt as err: - eb_go, cfg_settings = set_up_configuration(args=None, logfile=None, testing=None) - options, orig_paths = eb_go.options, eb_go.args - run_hook('cancel', load_hooks(options.hooks)) + except KeyboardInterrupt as err: + run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) From a80ac7f57f8b6728e104ff8d3326f63bcd94668a Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 25 Jul 2023 14:17:10 +0100 Subject: [PATCH 012/111] Init eb_go with parse_options() --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index a3186b68cd..e93af02890 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -71,7 +71,7 @@ from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool -from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color +from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs @@ -730,7 +730,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if __name__ == "__main__": - eb_go, _ = set_up_configuration(args=None, logfile=None, testing=None) + eb_go = parse_options() hooks = load_hooks(eb_go.options.hooks) try: main() From 3a083e696e0bc1a1e1273181b59ba2be7d9d47e9 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 25 Jul 2023 14:58:40 +0100 Subject: [PATCH 013/111] Add poised hook to provide list of easyconfigs to external tools --- easybuild/main.py | 3 ++- easybuild/tools/hooks.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 319c80b5ae..6acf78aa5e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import START, END, load_hooks, run_hook +from easybuild.tools.hooks import READY_STEP, START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -543,6 +543,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) + run_hook(READY_STEP, options.hooks, args=[ordered_ecs]) with rich_live_cm(): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 3c21d4e104..3b7299aaa9 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -60,6 +60,7 @@ TESTCASES_STEP = 'testcases' START = 'start' +POISED = 'poised' PARSE = 'parse' SINGLE_EXTENSION = 'single_extension' MODULE_WRITE = 'module_write' @@ -77,6 +78,7 @@ # hook names (in order of being triggered) HOOK_NAMES = [ START, + POISED, PARSE, ] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)] for p in [PRE_PREF, POST_PREF]] + [ From d4eebd9488a988ad50613765c77dcdceb3813c7a Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 25 Jul 2023 15:10:20 +0100 Subject: [PATCH 014/111] Pass correct arguments to poised_hook --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 6acf78aa5e..716197388f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import READY_STEP, START, END, load_hooks, run_hook +from easybuild.tools.hooks import POISED, START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -543,7 +543,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - run_hook(READY_STEP, options.hooks, args=[ordered_ecs]) + run_hook(POISED, hooks, args=[ordered_ecs]) with rich_live_cm(): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, From 2a82b0dafc8ddb337126f28e3b2c8a7d55b2e59c Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 25 Jul 2023 16:16:53 +0100 Subject: [PATCH 015/111] Swap order of parse and poised hooks --- easybuild/tools/hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 3b7299aaa9..a227c3c09f 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -60,8 +60,8 @@ TESTCASES_STEP = 'testcases' START = 'start' -POISED = 'poised' PARSE = 'parse' +POISED = 'poised' SINGLE_EXTENSION = 'single_extension' MODULE_WRITE = 'module_write' END = 'end' @@ -78,8 +78,8 @@ # hook names (in order of being triggered) HOOK_NAMES = [ START, - POISED, PARSE, + POISED, ] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)] for p in [PRE_PREF, POST_PREF]] + [ # pre-extensions hook is triggered before starting installation of extensions, From 65111fb413ec48a142ec433bcac21b953bf48720 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 3 Aug 2023 13:46:16 +0200 Subject: [PATCH 016/111] Ignore request for external module (meta)data when no modules tool is active E.g. for `--fetch` the modules tool is explicitly set to `None`/`NoModulesTool` to allow running on e.g. laptops to fetch sources. This however fails for external module dependencies in which it tries to resolve them using the modules tool which fails with `NotImplementedError`. Simply ignore the request in that case as-if the module metadata wasn't found. Fixes #4298 --- easybuild/framework/easyconfig/easyconfig.py | 5 +++- test/framework/options.py | 25 +++++++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f4c4be464e..55fe3c8578 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -73,7 +73,7 @@ from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name -from easybuild.tools.modules import modules_tool +from easybuild.tools.modules import modules_tool, NoModulesTool from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type from easybuild.tools.systemtools import check_os_dependency, pick_dep_version from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain @@ -1306,6 +1306,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None): :param existing_metadata: already available metadata for this external module (if any) """ res = {} + if isinstance(self.modules_tool, NoModulesTool): + self.log.debug('Ignoring request for external module data for %s as no modules tool is active', mod_name) + return res if existing_metadata is None: existing_metadata = {} diff --git a/test/framework/options.py b/test/framework/options.py index 106f951374..265e154e55 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5073,19 +5073,22 @@ def test_fetch(self): lock_path = os.path.join(self.test_installpath, 'software', '.locks', lock_fn) mkdir(lock_path, parents=True) - args = ['toy-0.0.eb', '--fetch'] - stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) + # Run for a "regular" EC and one with an external module dependency + # which might trip up the dependency resolution (see #4298) + for ec in ('toy-0.0.eb', 'toy-0.0-deps.eb'): + args = [ec, '--fetch'] + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) - patterns = [ - r"^== fetching files\.\.\.$", - r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$", - ] - for pattern in patterns: - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout)) + patterns = [ + r"^== fetching files\.\.\.$", + r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout)) - regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$") - self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$") + self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) def test_parse_external_modules_metadata(self): """Test parse_external_modules_metadata function.""" From 78559a3a24a593f2f6a7e46fa468ec0666813a15 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Aug 2023 16:16:52 +0200 Subject: [PATCH 017/111] use sys.executable to obtain path to python command, rather than assuming that 'python' command is available in $PATH --- test/framework/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/run.py b/test/framework/run.py index 908a9e9d93..f97d3b240f 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -543,7 +543,7 @@ def test_run_cmd_script(self): """Testing use of run_cmd with shell=False to call external scripts""" py_test_script = os.path.join(self.test_prefix, 'test.py') write_file(py_test_script, '\n'.join([ - '#!/usr/bin/env python', + '#!%s' % sys.executable, 'print("hello")', ])) adjust_permissions(py_test_script, stat.S_IXUSR) From e1a9cde4b3dc5a7b38aba001e47029b0f71fcea8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Aug 2023 21:17:12 +0200 Subject: [PATCH 018/111] fix test_add_and_remove_module_path by replacing string comparison on paths by checking whether they point to the same path (since symlinks may cause trouble) --- test/framework/modules.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 4f2b3fc5d6..7bf87130bd 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1273,24 +1273,33 @@ def get_resolved_module_path(): test_dir1_relative = os.path.join(test_dir1, '..', os.path.basename(test_dir1)) test_dir2_dot = os.path.join(os.path.dirname(test_dir2), '.', os.path.basename(test_dir2)) self.modtool.add_module_path(test_dir1_relative) - self.assertEqual(get_resolved_module_path(), test_dir1) + self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1)) # Adding the same path, but in a different form may be possible, but may also be ignored, e.g. in EnvModules self.modtool.add_module_path(test_dir1) if get_resolved_module_path() != test_dir1: - self.assertEqual(get_resolved_module_path(), os.pathsep.join([test_dir1, test_dir1])) + modpath = get_resolved_module_path().split(os.pathsep) + self.assertEqual(len(modpath), 2) + self.assertTrue(os.path.samefile(modpath[0], test_dir1)) + self.assertTrue(os.path.samefile(modpath[1], test_dir1)) self.modtool.remove_module_path(test_dir1) - self.assertEqual(get_resolved_module_path(), test_dir1) + self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1)) + self.modtool.add_module_path(test_dir2_dot) - self.assertEqual(get_resolved_module_path(), test_dir_2_and_1) + modpath = get_resolved_module_path().split(os.pathsep) + self.assertEqual(len(modpath), 2) + self.assertTrue(os.path.samefile(modpath[0], test_dir2)) + self.assertTrue(os.path.samefile(modpath[1], test_dir1)) + self.modtool.remove_module_path(test_dir2_dot) - self.assertEqual(get_resolved_module_path(), test_dir1) + self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1)) + # Force adding such a dot path which can be removed with either variant os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative]) self.modtool.remove_module_path(test_dir2_dot) - self.assertEqual(get_resolved_module_path(), test_dir1) + self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1)) os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative]) self.modtool.remove_module_path(test_dir2) - self.assertEqual(get_resolved_module_path(), test_dir1) + self.assertTrue(os.path.samefile(get_resolved_module_path(), test_dir1)) os.environ['MODULEPATH'] = old_module_path # Restore From 2ef4cf6239eab39c97db787f4409423ae9db695f Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Mon, 7 Aug 2023 11:27:49 +0100 Subject: [PATCH 019/111] Merge with main --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 eb | 0 test/framework/suite.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100755 => 100644 easybuild/scripts/bootstrap_eb.py mode change 100755 => 100644 easybuild/scripts/clean_gists.py mode change 100755 => 100644 easybuild/scripts/createSubmoduleDeps.sh mode change 100755 => 100644 easybuild/scripts/findPythonDeps.py mode change 100755 => 100644 easybuild/scripts/findUpdatedEcs.sh mode change 100755 => 100644 easybuild/scripts/fix_docs.py mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-develop.sh mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100755 => 100644 easybuild/scripts/install_eb_dep.sh mode change 100755 => 100644 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100755 => 100644 easybuild/scripts/rpath_args.py mode change 100755 => 100644 eb mode change 100755 => 100644 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100755 new mode 100644 diff --git a/eb b/eb old mode 100755 new mode 100644 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100755 new mode 100644 From 7c3a948937b5f0c276e17d6aa7310abdf74fc9de Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Mon, 7 Aug 2023 11:30:04 +0100 Subject: [PATCH 020/111] Merge develop into 4303_poised_hooK --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 eb | 0 test/framework/suite.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100755 => 100644 easybuild/scripts/bootstrap_eb.py mode change 100755 => 100644 easybuild/scripts/clean_gists.py mode change 100755 => 100644 easybuild/scripts/createSubmoduleDeps.sh mode change 100755 => 100644 easybuild/scripts/findPythonDeps.py mode change 100755 => 100644 easybuild/scripts/findUpdatedEcs.sh mode change 100755 => 100644 easybuild/scripts/fix_docs.py mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-develop.sh mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100755 => 100644 easybuild/scripts/install_eb_dep.sh mode change 100755 => 100644 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100755 => 100644 easybuild/scripts/rpath_args.py mode change 100755 => 100644 eb mode change 100755 => 100644 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100755 new mode 100644 diff --git a/eb b/eb old mode 100755 new mode 100644 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100755 new mode 100644 From 863d6e66a06080d380ee47018650b1420dcbab58 Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Mon, 7 Aug 2023 12:03:37 +0100 Subject: [PATCH 021/111] Metadata change --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 eb | 0 test/framework/suite.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100755 => 100644 easybuild/scripts/bootstrap_eb.py mode change 100755 => 100644 easybuild/scripts/clean_gists.py mode change 100755 => 100644 easybuild/scripts/createSubmoduleDeps.sh mode change 100755 => 100644 easybuild/scripts/findPythonDeps.py mode change 100755 => 100644 easybuild/scripts/findUpdatedEcs.sh mode change 100755 => 100644 easybuild/scripts/fix_docs.py mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-develop.sh mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100755 => 100644 easybuild/scripts/install_eb_dep.sh mode change 100755 => 100644 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100755 => 100644 easybuild/scripts/rpath_args.py mode change 100755 => 100644 eb mode change 100755 => 100644 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100755 new mode 100644 diff --git a/eb b/eb old mode 100755 new mode 100644 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100755 new mode 100644 From 90ac497fbc60958f06ba64a4fc01a2302f67ba3f Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Mon, 7 Aug 2023 12:50:14 +0100 Subject: [PATCH 022/111] Fix metadata --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 eb | 0 test/framework/suite.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100644 => 100755 easybuild/scripts/bootstrap_eb.py mode change 100644 => 100755 easybuild/scripts/clean_gists.py mode change 100644 => 100755 easybuild/scripts/createSubmoduleDeps.sh mode change 100644 => 100755 easybuild/scripts/findPythonDeps.py mode change 100644 => 100755 easybuild/scripts/findUpdatedEcs.sh mode change 100644 => 100755 easybuild/scripts/fix_docs.py mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-develop.sh mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100644 => 100755 easybuild/scripts/install_eb_dep.sh mode change 100644 => 100755 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100644 => 100755 easybuild/scripts/rpath_args.py mode change 100644 => 100755 eb mode change 100644 => 100755 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100644 new mode 100755 diff --git a/eb b/eb old mode 100644 new mode 100755 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100644 new mode 100755 From b376941702ed1dfe5eb0545755dee2da23f56bed Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Mon, 7 Aug 2023 13:45:13 +0100 Subject: [PATCH 023/111] Add pre and post_loop hooks --- easybuild/main.py | 4 ++-- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 easybuild/tools/hooks.py | 5 +++-- test/framework/suite.py | 0 15 files changed, 5 insertions(+), 4 deletions(-) mode change 100755 => 100644 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100755 => 100644 easybuild/scripts/bootstrap_eb.py mode change 100755 => 100644 easybuild/scripts/clean_gists.py mode change 100755 => 100644 easybuild/scripts/createSubmoduleDeps.sh mode change 100755 => 100644 easybuild/scripts/findPythonDeps.py mode change 100755 => 100644 easybuild/scripts/findUpdatedEcs.sh mode change 100755 => 100644 easybuild/scripts/fix_docs.py mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-develop.sh mode change 100755 => 100644 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100755 => 100644 easybuild/scripts/install_eb_dep.sh mode change 100755 => 100644 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100755 => 100644 easybuild/scripts/rpath_args.py mode change 100755 => 100644 test/framework/suite.py diff --git a/easybuild/main.py b/easybuild/main.py index 716197388f..cb0a55723f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import POISED, START, END, load_hooks, run_hook +from easybuild.tools.hooks import LOOP, PRE_PREF, POST_PREF, START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -543,7 +543,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - run_hook(POISED, hooks, args=[ordered_ecs]) + run_hook(PRE_PREF + LOOP, hooks, args=[ordered_ecs]) with rich_live_cm(): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100755 new mode 100644 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100755 new mode 100644 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100755 new mode 100644 diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index a227c3c09f..8676f9171d 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -61,7 +61,7 @@ START = 'start' PARSE = 'parse' -POISED = 'poised' +LOOP = 'loop' SINGLE_EXTENSION = 'single_extension' MODULE_WRITE = 'module_write' END = 'end' @@ -79,7 +79,8 @@ HOOK_NAMES = [ START, PARSE, - POISED, + PRE_PREF + LOOP, + POST_PREF + LOOP, ] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)] for p in [PRE_PREF, POST_PREF]] + [ # pre-extensions hook is triggered before starting installation of extensions, diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100755 new mode 100644 From 3ad8787dac20bdf331dcd519c5482652a1912639 Mon Sep 17 00:00:00 2001 From: "Xavier Elwell (Advanced Research Computing)" Date: Mon, 7 Aug 2023 13:45:50 +0100 Subject: [PATCH 024/111] Fix permissions --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 test/framework/suite.py | 0 13 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100644 => 100755 easybuild/scripts/bootstrap_eb.py mode change 100644 => 100755 easybuild/scripts/clean_gists.py mode change 100644 => 100755 easybuild/scripts/createSubmoduleDeps.sh mode change 100644 => 100755 easybuild/scripts/findPythonDeps.py mode change 100644 => 100755 easybuild/scripts/findUpdatedEcs.sh mode change 100644 => 100755 easybuild/scripts/fix_docs.py mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-develop.sh mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100644 => 100755 easybuild/scripts/install_eb_dep.sh mode change 100644 => 100755 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100644 => 100755 easybuild/scripts/rpath_args.py mode change 100644 => 100755 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100644 new mode 100755 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100644 new mode 100755 From bdb3363b4c472e7ca27c201db7bea34a1b2111b6 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Mon, 7 Aug 2023 13:53:52 +0100 Subject: [PATCH 025/111] Revert permission --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 eb | 0 test/framework/suite.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100644 => 100755 easybuild/scripts/bootstrap_eb.py mode change 100644 => 100755 easybuild/scripts/clean_gists.py mode change 100644 => 100755 easybuild/scripts/createSubmoduleDeps.sh mode change 100644 => 100755 easybuild/scripts/findPythonDeps.py mode change 100644 => 100755 easybuild/scripts/findUpdatedEcs.sh mode change 100644 => 100755 easybuild/scripts/fix_docs.py mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-develop.sh mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100644 => 100755 easybuild/scripts/install_eb_dep.sh mode change 100644 => 100755 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100644 => 100755 easybuild/scripts/rpath_args.py mode change 100644 => 100755 eb mode change 100644 => 100755 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100644 new mode 100755 diff --git a/eb b/eb old mode 100644 new mode 100755 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100644 new mode 100755 From aa864a9bea3a25b0126fdff6e82f7932de618dfb Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Mon, 7 Aug 2023 15:54:34 +0100 Subject: [PATCH 026/111] Implement post_loop hook --- easybuild/main.py | 3 ++- easybuild/tools/hooks.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index cb0a55723f..8322cce8cc 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -543,11 +543,12 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) - run_hook(PRE_PREF + LOOP, hooks, args=[ordered_ecs]) with rich_live_cm(): + run_hook(PRE_PREF + LOOP, hooks, args=[ordered_ecs]) ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) + run_hook(POST_PREF + LOOP, hooks, args=[ecs_with_res]) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 8676f9171d..936f7e5518 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -80,7 +80,6 @@ START, PARSE, PRE_PREF + LOOP, - POST_PREF + LOOP, ] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)] for p in [PRE_PREF, POST_PREF]] + [ # pre-extensions hook is triggered before starting installation of extensions, @@ -100,6 +99,7 @@ POST_PREF + MODULE_STEP, ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] for p in [PRE_PREF, POST_PREF]] + [ + POST_PREF + LOOP, END, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] From 014fd42092bb5309b5117adf9d8be1fb9f351843 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 10:41:29 +0100 Subject: [PATCH 027/111] Add crash hook --- easybuild/main.py | 5 ++++- easybuild/tools/hooks.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index e93af02890..b5c2f9a01e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import START, END, CANCEL, FAIL, load_hooks, run_hook +from easybuild.tools.hooks import CRASH, START, END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -740,3 +740,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): except KeyboardInterrupt as err: run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) + except Exception as err: + run_hook(CRASH, hooks, args=[err]) + print_error("Encountered an unrecoverable error: %s" % err) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index f072a2bcd4..ee1af63b6c 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -67,6 +67,7 @@ FAIL = 'fail' CANCEL = 'cancel' +CRASH = 'crash' PRE_PREF = 'pre_' POST_PREF = 'post_' @@ -103,6 +104,7 @@ END, FAIL, CANCEL, + CRASH, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] From 1a589dce3c7a9801a1717cee3e2d73faeb1c10e1 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 10:47:40 +0100 Subject: [PATCH 028/111] Remove trailing whitespace --- easybuild/tools/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index ee1af63b6c..de07866c16 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -102,7 +102,7 @@ ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] for p in [PRE_PREF, POST_PREF]] + [ END, - FAIL, + FAIL, CANCEL, CRASH, ] From 565c11311524ab1ad172df01cf2fdb0d7999b6bd Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 10:51:03 +0100 Subject: [PATCH 029/111] Revert file permissions --- easybuild/scripts/EasyBuild-reinstall-dev.sh | 0 easybuild/scripts/bootstrap_eb.py | 0 easybuild/scripts/clean_gists.py | 0 easybuild/scripts/createSubmoduleDeps.sh | 0 easybuild/scripts/findPythonDeps.py | 0 easybuild/scripts/findUpdatedEcs.sh | 0 easybuild/scripts/fix_docs.py | 0 easybuild/scripts/install-EasyBuild-develop.sh | 0 easybuild/scripts/install-EasyBuild-sprint.sh | 0 easybuild/scripts/install_eb_dep.sh | 0 easybuild/scripts/mk_tmpl_easyblock_for.py | 0 easybuild/scripts/rpath_args.py | 0 eb | 0 test/framework/suite.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 easybuild/scripts/EasyBuild-reinstall-dev.sh mode change 100644 => 100755 easybuild/scripts/bootstrap_eb.py mode change 100644 => 100755 easybuild/scripts/clean_gists.py mode change 100644 => 100755 easybuild/scripts/createSubmoduleDeps.sh mode change 100644 => 100755 easybuild/scripts/findPythonDeps.py mode change 100644 => 100755 easybuild/scripts/findUpdatedEcs.sh mode change 100644 => 100755 easybuild/scripts/fix_docs.py mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-develop.sh mode change 100644 => 100755 easybuild/scripts/install-EasyBuild-sprint.sh mode change 100644 => 100755 easybuild/scripts/install_eb_dep.sh mode change 100644 => 100755 easybuild/scripts/mk_tmpl_easyblock_for.py mode change 100644 => 100755 easybuild/scripts/rpath_args.py mode change 100644 => 100755 eb mode change 100644 => 100755 test/framework/suite.py diff --git a/easybuild/scripts/EasyBuild-reinstall-dev.sh b/easybuild/scripts/EasyBuild-reinstall-dev.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/clean_gists.py b/easybuild/scripts/clean_gists.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/createSubmoduleDeps.sh b/easybuild/scripts/createSubmoduleDeps.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/findUpdatedEcs.sh b/easybuild/scripts/findUpdatedEcs.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/fix_docs.py b/easybuild/scripts/fix_docs.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/install_eb_dep.sh b/easybuild/scripts/install_eb_dep.sh old mode 100644 new mode 100755 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py old mode 100644 new mode 100755 diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py old mode 100644 new mode 100755 diff --git a/eb b/eb old mode 100644 new mode 100755 diff --git a/test/framework/suite.py b/test/framework/suite.py old mode 100644 new mode 100755 From 9e1c72e63aa97d5b034fd78b08453632a9a0dc18 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 11:00:25 +0100 Subject: [PATCH 030/111] Rename LOOP to BUILD_AND_INSTALL_LOOP --- easybuild/main.py | 6 +++--- easybuild/tools/hooks.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 8322cce8cc..bbeff64f33 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import LOOP, PRE_PREF, POST_PREF, START, END, load_hooks, run_hook +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -545,10 +545,10 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session exit_on_failure = not (options.dump_test_report or options.upload_test_report) with rich_live_cm(): - run_hook(PRE_PREF + LOOP, hooks, args=[ordered_ecs]) + run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs]) ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) - run_hook(POST_PREF + LOOP, hooks, args=[ecs_with_res]) + run_hook(POST_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ecs_with_res]) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 936f7e5518..dc7fdea8df 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -61,7 +61,7 @@ START = 'start' PARSE = 'parse' -LOOP = 'loop' +BUILD_AND_INSTALL_LOOP = 'build_and_install_loop' SINGLE_EXTENSION = 'single_extension' MODULE_WRITE = 'module_write' END = 'end' @@ -79,7 +79,7 @@ HOOK_NAMES = [ START, PARSE, - PRE_PREF + LOOP, + PRE_PREF + BUILD_AND_INSTALL_LOOP, ] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)] for p in [PRE_PREF, POST_PREF]] + [ # pre-extensions hook is triggered before starting installation of extensions, @@ -99,7 +99,7 @@ POST_PREF + MODULE_STEP, ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] for p in [PRE_PREF, POST_PREF]] + [ - POST_PREF + LOOP, + POST_PREF + BUILD_AND_INSTALL_LOOP, END, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] From d11e7d238a1d340b8b07b42d3dd5edd692879d40 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 12:03:27 +0100 Subject: [PATCH 031/111] Add new hooks to tests expected output --- test/framework/options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 106f951374..354ba9d4c0 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -679,6 +679,7 @@ def test_avail_hooks(self): "List of supported hooks (in order of execution):", " start_hook", " parse_hook", + " pre_build_and_install_loop_hook" " pre_fetch_hook", " post_fetch_hook", " pre_ready_hook", @@ -716,7 +717,9 @@ def test_avail_hooks(self): " post_package_hook", " pre_testcases_hook", " post_testcases_hook", + " post_build_and_install_loop_hook" " end_hook", + " " '', ]) self.assertEqual(stdout, expected) From 45c8eb5a6a8f2ab0b34656184b245170859e7266 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 12:03:57 +0100 Subject: [PATCH 032/111] Remove accidental whitespace --- test/framework/options.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 354ba9d4c0..02d829ce4f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -719,7 +719,6 @@ def test_avail_hooks(self): " post_testcases_hook", " post_build_and_install_loop_hook" " end_hook", - " " '', ]) self.assertEqual(stdout, expected) From 852e4c902d91bb8c442abc4cf145fc91912efbc9 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 12:15:00 +0100 Subject: [PATCH 033/111] Updated unit tests expected output with new hooks --- test/framework/options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 106f951374..8d67dda656 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -717,6 +717,9 @@ def test_avail_hooks(self): " pre_testcases_hook", " post_testcases_hook", " end_hook", + " fail_hook", + " cancel_hook" + " crash_hook" '', ]) self.assertEqual(stdout, expected) From 1f5e372cb0be30832a23d96cdd5b4fcf86f4e6fb Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 12:21:55 +0100 Subject: [PATCH 034/111] Insert missing commas --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 02d829ce4f..ad1754d05b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -679,7 +679,7 @@ def test_avail_hooks(self): "List of supported hooks (in order of execution):", " start_hook", " parse_hook", - " pre_build_and_install_loop_hook" + " pre_build_and_install_loop_hook", " pre_fetch_hook", " post_fetch_hook", " pre_ready_hook", @@ -717,7 +717,7 @@ def test_avail_hooks(self): " post_package_hook", " pre_testcases_hook", " post_testcases_hook", - " post_build_and_install_loop_hook" + " post_build_and_install_loop_hook", " end_hook", '', ]) From 3d61e4aadfe18faa815cd83be7f86be78cdbe48d Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 12:22:43 +0100 Subject: [PATCH 035/111] Insert missing commas --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 8d67dda656..0aebead19d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -718,8 +718,8 @@ def test_avail_hooks(self): " post_testcases_hook", " end_hook", " fail_hook", - " cancel_hook" - " crash_hook" + " cancel_hook", + " crash_hook", '', ]) self.assertEqual(stdout, expected) From 74afd7deea6e8050964be76a3282fd667c4f15c7 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 8 Aug 2023 12:32:45 +0100 Subject: [PATCH 036/111] tabs instead of spaces --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index ad1754d05b..65bdbb290b 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -679,7 +679,7 @@ def test_avail_hooks(self): "List of supported hooks (in order of execution):", " start_hook", " parse_hook", - " pre_build_and_install_loop_hook", + " pre_build_and_install_loop_hook", " pre_fetch_hook", " post_fetch_hook", " pre_ready_hook", @@ -717,7 +717,7 @@ def test_avail_hooks(self): " post_package_hook", " pre_testcases_hook", " post_testcases_hook", - " post_build_and_install_loop_hook", + " post_build_and_install_loop_hook", " end_hook", '', ]) From b8808564b649b56270b1187a8cd09f3f54945a0a Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Tue, 8 Aug 2023 13:35:09 +0100 Subject: [PATCH 037/111] Replace spaces with tabs to match expected options output --- test/framework/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 0aebead19d..1d7d2c3471 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -717,9 +717,9 @@ def test_avail_hooks(self): " pre_testcases_hook", " post_testcases_hook", " end_hook", - " fail_hook", - " cancel_hook", - " crash_hook", + " fail_hook", + " cancel_hook", + " crash_hook", '', ]) self.assertEqual(stdout, expected) From b5ec611bd4a4daae9bc098eb861cb4b0d42fe2a9 Mon Sep 17 00:00:00 2001 From: XavierCS-dev <64903011+XavierCS-dev@users.noreply.github.com> Date: Tue, 8 Aug 2023 16:08:33 +0100 Subject: [PATCH 038/111] Shorten long line of imports --- easybuild/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 37c2c31318..f0932e9916 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,8 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, END, CANCEL, FAIL, load_hooks, run_hook +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, \ + END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm From 95c788f3e7f239e5c18a7d24cc4a3a4f450c0c06 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Aug 2023 09:28:15 +0200 Subject: [PATCH 039/111] Update checksum type error message --- easybuild/tools/filetools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index f3bc626b2b..e7034eca5c 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1299,7 +1299,8 @@ def verify_checksum(path, checksums): # no matching checksums return False else: - raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value).", + raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5 or SHA256) " + "2-tuple (type, value) or tuple of alternative checksum specs.", checksum) actual_checksum = compute_checksum(path, typ) From 6917ef93bba76e90953aed858a33af644af37786 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 11:35:38 +0100 Subject: [PATCH 040/111] add postiter hook to the list of hooks --- easybuild/tools/hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index dc7fdea8df..45d0477889 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -85,10 +85,12 @@ # pre-extensions hook is triggered before starting installation of extensions, # pre/post extension (singular) hook is triggered when installing individual extensions, # post-extensions hook is triggered after installation of extensions + # postiter hook is triggered to restore options that were iterated over PRE_PREF + EXTENSIONS_STEP, PRE_PREF + SINGLE_EXTENSION, POST_PREF + SINGLE_EXTENSION, POST_PREF + EXTENSIONS_STEP, + POSTITER_STEP, ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(EXTENSIONS_STEP)+1:STEP_NAMES.index(MODULE_STEP)] for p in [PRE_PREF, POST_PREF]] + [ # pre-module hook hook is triggered before starting module step which creates module file, From 4dcf1c189d4ac9828e55b4a63df86b91d896cf9b Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 11:37:58 +0100 Subject: [PATCH 041/111] and the test --- test/framework/options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/options.py b/test/framework/options.py index 65bdbb290b..c55e18e1f5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -702,6 +702,7 @@ def test_avail_hooks(self): " pre_single_extension_hook", " post_single_extension_hook", " post_extensions_hook", + " postiter_hook", " pre_postproc_hook", " post_postproc_hook", " pre_sanitycheck_hook", From f296dc5d937fd9280fc7570a0aa1679669cea532 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 11:44:58 +0100 Subject: [PATCH 042/111] formatting of imports --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index f0932e9916..aaf7f71274 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,8 +69,8 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, \ - END, CANCEL, FAIL, load_hooks, run_hook +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, END, CANCEL, FAIL +from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm From b7d4291084cea335342233c6b7877636973cec79 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:07:57 +0100 Subject: [PATCH 043/111] there is a pre_ and post_ version of the hook --- easybuild/tools/hooks.py | 4 +--- test/framework/options.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 45d0477889..6099e04b01 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -72,7 +72,7 @@ # list of names for steps in installation procedure (in order of execution) STEP_NAMES = [FETCH_STEP, READY_STEP, SOURCE_STEP, PATCH_STEP, PREPARE_STEP, CONFIGURE_STEP, BUILD_STEP, TEST_STEP, - INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, + INSTALL_STEP, EXTENSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP, PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP] # hook names (in order of being triggered) @@ -85,12 +85,10 @@ # pre-extensions hook is triggered before starting installation of extensions, # pre/post extension (singular) hook is triggered when installing individual extensions, # post-extensions hook is triggered after installation of extensions - # postiter hook is triggered to restore options that were iterated over PRE_PREF + EXTENSIONS_STEP, PRE_PREF + SINGLE_EXTENSION, POST_PREF + SINGLE_EXTENSION, POST_PREF + EXTENSIONS_STEP, - POSTITER_STEP, ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(EXTENSIONS_STEP)+1:STEP_NAMES.index(MODULE_STEP)] for p in [PRE_PREF, POST_PREF]] + [ # pre-module hook hook is triggered before starting module step which creates module file, diff --git a/test/framework/options.py b/test/framework/options.py index c55e18e1f5..80015baeab 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -702,7 +702,8 @@ def test_avail_hooks(self): " pre_single_extension_hook", " post_single_extension_hook", " post_extensions_hook", - " postiter_hook", + " pre_postiter_hook", + " post_postiter_hook", " pre_postproc_hook", " post_postproc_hook", " pre_sanitycheck_hook", From 0143b5cd0aa44d2b7ee9a48b0d39c1cba96dfeef Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:39:03 +0100 Subject: [PATCH 044/111] enhance `get_flag` to handle lists --- easybuild/tools/toolchain/toolchain.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 308678a555..38d2f139d3 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1129,8 +1129,14 @@ def _setenv_variables(self, donotset=None, verbose=True): setvar("EBVAR%s" % key, val, verbose=False) def get_flag(self, name): - """Get compiler flag for a certain option.""" - return "-%s" % self.options.option(name) + """Get compiler flag(s) for a certain option.""" + if isinstance(self.options.option(name), str): + return "-%s" % self.options.option(name) + elif isinstance(self.options.option(name), list): + return " ".join("-%s" % x for x in self.options.option(name)) + else: + msg = "Do not know how to convert toolchain flag %s, of type %s, for use" + raise EasyBuildError(msg % (name, type(self.options.option(name)))) def toolchain_family(self): """Return toolchain family for this toolchain.""" From f2ec534da2a3caf5b98fe5a2dbd2f00cb50e8e7e Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:59:18 +0100 Subject: [PATCH 045/111] maintain existing behaviour unless it is a list --- easybuild/tools/toolchain/toolchain.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 38d2f139d3..4ada51995b 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1130,13 +1130,10 @@ def _setenv_variables(self, donotset=None, verbose=True): def get_flag(self, name): """Get compiler flag(s) for a certain option.""" - if isinstance(self.options.option(name), str): - return "-%s" % self.options.option(name) - elif isinstance(self.options.option(name), list): + if isinstance(self.options.option(name), list): return " ".join("-%s" % x for x in self.options.option(name)) else: - msg = "Do not know how to convert toolchain flag %s, of type %s, for use" - raise EasyBuildError(msg % (name, type(self.options.option(name)))) + return "-%s" % self.options.option(name) def toolchain_family(self): """Return toolchain family for this toolchain.""" From 3052696ca49b3ed062d68e3e900758b8b3bbe454 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Wed, 9 Aug 2023 17:31:20 +0100 Subject: [PATCH 046/111] also accept tuple --- easybuild/tools/toolchain/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 4ada51995b..b9d489852e 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1130,7 +1130,7 @@ def _setenv_variables(self, donotset=None, verbose=True): def get_flag(self, name): """Get compiler flag(s) for a certain option.""" - if isinstance(self.options.option(name), list): + if isinstance(self.options.option(name), list) or isinstance(self.options.option(name), tuple): return " ".join("-%s" % x for x in self.options.option(name)) else: return "-%s" % self.options.option(name) From 2646f7584a1d0253a9afea4cdd0f766f6f53e26d Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Wed, 9 Aug 2023 18:24:58 +0100 Subject: [PATCH 047/111] add tests --- easybuild/tools/toolchain/toolchain.py | 4 ++-- test/framework/toolchain.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index b9d489852e..a16fe35dc0 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1130,8 +1130,8 @@ def _setenv_variables(self, donotset=None, verbose=True): def get_flag(self, name): """Get compiler flag(s) for a certain option.""" - if isinstance(self.options.option(name), list) or isinstance(self.options.option(name), tuple): - return " ".join("-%s" % x for x in self.options.option(name)) + if isinstance(self.options.option(name), list): + return " ".join("-%s" % x for x in list(self.options.option(name))) else: return "-%s" % self.options.option(name) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 1c36431c45..cf0369764c 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2950,6 +2950,20 @@ def test_env_vars_external_module(self): expected = {} self.assertEqual(res, expected) + def test_get_flag(self): + """Test get_flag function""" + tc = self.get_toolchain('gompi', version='2018a') + + checks = { + '-a': 'a', + '-openmp': 'openmp', + '-foo': ['foo'], + '-foo -bar': ['foo', 'bar'], + } + + for flagstring, flags in checks.items(): + tc.options.options_map['openmp'] = flags + self.assertEqual(tc.get_flag('openmp'), flagstring) def suite(): """ return all the tests""" From 587f64d893529c1b12b65de8e45a2741b5358f8c Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Wed, 9 Aug 2023 18:27:03 +0100 Subject: [PATCH 048/111] f8 --- test/framework/toolchain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index cf0369764c..d80d41c788 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2965,6 +2965,7 @@ def test_get_flag(self): tc.options.options_map['openmp'] = flags self.assertEqual(tc.get_flag('openmp'), flagstring) + def suite(): """ return all the tests""" return TestLoaderFiltered().loadTestsFromTestCase(ToolchainTest, sys.argv[1:]) From 0e751c9319be06b1c1e60f284510e37961159059 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 10:33:12 +0100 Subject: [PATCH 049/111] Reorder failure hooks to be in alphabetical order --- easybuild/main.py | 3 ++- easybuild/tools/hooks.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index b5c2f9a01e..621871cbb0 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,8 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import CRASH, START, END, CANCEL, FAIL, load_hooks, run_hook +from easybuild.tools.hooks import CRASH, START, \ + END, CANCEL, FAIL, load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index de07866c16..8e8eb2302c 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -65,9 +65,9 @@ MODULE_WRITE = 'module_write' END = 'end' -FAIL = 'fail' CANCEL = 'cancel' CRASH = 'crash' +FAIL = 'fail' PRE_PREF = 'pre_' POST_PREF = 'post_' @@ -102,9 +102,9 @@ ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:] for p in [PRE_PREF, POST_PREF]] + [ END, - FAIL, CANCEL, CRASH, + FAIL, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] From 8a474bb8ad69d756c7c606fcd3dc80c76a382c16 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 10:55:32 +0100 Subject: [PATCH 050/111] Fix output tests for hook order --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index d889fc6b6c..5b1ef19825 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -719,9 +719,9 @@ def test_avail_hooks(self): " post_testcases_hook", " post_build_and_install_loop_hook", " end_hook", - " fail_hook", " cancel_hook", " crash_hook", + " fail_hook", '', ]) self.assertEqual(stdout, expected) From ec87c8608cc4748aaa6367b23bfed8403f545cd2 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 14:35:42 +0100 Subject: [PATCH 051/111] Alter refactor to define variables --- easybuild/main.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index aaf7f71274..a05fae4604 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -580,7 +580,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): +def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use @@ -600,6 +600,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # So emulate this here to allow (module) scripts depending on that to work if '_' not in os.environ: os.environ['_'] = sys.executable + + if any([args, logfile, testing]): + eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() @@ -732,11 +735,16 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): cleanup(logfile, eb_tmpdir, testing, silent=False) +def prepare_main(args=None, logfile=None, testing=None): + eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) + return eb_go, cfg_settings + + if __name__ == "__main__": - eb_go = parse_options() + eb_go, cfg_settings = prepare_main() hooks = load_hooks(eb_go.options.hooks) try: - main() + main(eb_go, cfg_settings) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg) From 2e8b80f8986397281fa7a788b5b564fae47fe8ac Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 14:37:22 +0100 Subject: [PATCH 052/111] Comment out line overriding variables defined elsewhere --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index a05fae4604..ebcc2bd917 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -606,7 +606,7 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() - eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) + # eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args if 'python2' not in build_option('silence_deprecation_warnings'): From 81e534420a365177a7cbb161b8746dfc1548a987 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 14:47:52 +0100 Subject: [PATCH 053/111] Fix style issues --- easybuild/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index ebcc2bd917..f376f0d43f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -72,7 +72,7 @@ from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, END, CANCEL, FAIL from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool -from easybuild.tools.options import opts_dict_to_eb_opts, parse_options, set_up_configuration, use_color +from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm from easybuild.tools.output import start_progress_bar, stop_progress_bar, update_progress_bar from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs @@ -600,7 +600,7 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, # So emulate this here to allow (module) scripts depending on that to work if '_' not in os.environ: os.environ['_'] = sys.executable - + if any([args, logfile, testing]): eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) From 0ac9daf0c979ce5d15bee18f53fde3ad9560f1b3 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 15:42:50 +0100 Subject: [PATCH 054/111] Update main docstring and add prepare_main docstring --- easybuild/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index f376f0d43f..123a0e965a 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -583,6 +583,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. + :param: eb_go: easybuild general options object + :param: cfg_settings: configuration settings of easybuild :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build @@ -736,6 +738,13 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, def prepare_main(args=None, logfile=None, testing=None): + """ + Prepare Main: Set up eb_go and consequently, cfg_settings in order to obtain the hooks, + and prevent the configuration being set up twice. + :param args: command line arguments to take into account when parsing the EasyBuild configuration settings + :param logfile: log file to use + :param testing: enable testing mode + """ eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) return eb_go, cfg_settings From 22b44297912d32824c395b41c7ab247d781c11d1 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:08:57 +0100 Subject: [PATCH 055/111] Remove uncessary comment --- easybuild/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 123a0e965a..86360006d1 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -608,7 +608,6 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() - # eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args if 'python2' not in build_option('silence_deprecation_warnings'): From 1751fe309e81e5e7e1d1d4b9504424d97575c15d Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:24:17 +0100 Subject: [PATCH 056/111] Only set eb_go and cfg_settings if not already --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 86360006d1..78f4eb7343 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -603,7 +603,7 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, if '_' not in os.environ: os.environ['_'] = sys.executable - if any([args, logfile, testing]): + if not all([eb_go, cfg_settings]): eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) # purposely session state very early, to avoid modules loaded by EasyBuild meddling in From c4be4ca3d9e54622502ccccc05c0fb83810f8e95 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:25:29 +0100 Subject: [PATCH 057/111] Move setup config to occure before os.environ is checked --- easybuild/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 78f4eb7343..848c7f1baf 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -593,6 +593,9 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, register_lock_cleanup_signal_handlers() + if not all([eb_go, cfg_settings]): + eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) + # if $CDPATH is set, unset it, it'll only cause trouble... # see https://github.com/easybuilders/easybuild-framework/issues/2944 if 'CDPATH' in os.environ: @@ -603,9 +606,6 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, if '_' not in os.environ: os.environ['_'] = sys.executable - if not all([eb_go, cfg_settings]): - eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() options, orig_paths = eb_go.options, eb_go.args From 58623de7eebe1a4215d588978e617b9974e674be Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:26:55 +0100 Subject: [PATCH 058/111] Change pprepare_main description --- easybuild/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 848c7f1baf..ef69ca19d3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -738,8 +738,7 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, def prepare_main(args=None, logfile=None, testing=None): """ - Prepare Main: Set up eb_go and consequently, cfg_settings in order to obtain the hooks, - and prevent the configuration being set up twice. + Prepare for calling main function by setting up the EasyBuild configuration. :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode From 8148fb5f0a4af7d253ce30a7ac3fdf03161e0d40 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:28:58 +0100 Subject: [PATCH 059/111] Add return to prepare_main docstring --- easybuild/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index ef69ca19d3..1a22daf318 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -738,10 +738,11 @@ def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, def prepare_main(args=None, logfile=None, testing=None): """ - Prepare for calling main function by setting up the EasyBuild configuration. + Prepare for calling main function by setting up the EasyBuild configuration :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode + return: easybuild options and configuration settings """ eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) return eb_go, cfg_settings From f2f6b45230f64dc18eef80a695e69fb0ee0f468b Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:31:07 +0100 Subject: [PATCH 060/111] Move eb_go and cfg_settings to end of parameters to prevent breaking function calls --- easybuild/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 1a22daf318..6413b42c6c 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -580,7 +580,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(eb_go=None, cfg_settings=None, args=None, logfile=None, do_build=None, testing=False, modtool=None): +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb_go=None, cfg_settings=None): """ Main function: parse command line options, and act accordingly. :param: eb_go: easybuild general options object @@ -742,7 +742,7 @@ def prepare_main(args=None, logfile=None, testing=None): :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode - return: easybuild options and configuration settings + :return: easybuild options and configuration settings """ eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) return eb_go, cfg_settings @@ -752,7 +752,7 @@ def prepare_main(args=None, logfile=None, testing=None): eb_go, cfg_settings = prepare_main() hooks = load_hooks(eb_go.options.hooks) try: - main(eb_go, cfg_settings) + main(eb_go=eb_go, cfg_settings=cfg_settings) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg) From 56def6bf5f740fe754f742aa712e6adcd0d7f79e Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:46:12 +0100 Subject: [PATCH 061/111] Add init_session_state and hooks to prepare_main returns --- easybuild/main.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 6413b42c6c..500ffabfe9 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -580,7 +580,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb_go=None, cfg_settings=None): +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb_go=None, cfg_settings=None, + init_session_state=None, hooks=None): """ Main function: parse command line options, and act accordingly. :param: eb_go: easybuild general options object @@ -591,23 +592,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb :param testing: enable testing mode """ - register_lock_cleanup_signal_handlers() - - if not all([eb_go, cfg_settings]): - eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - - # if $CDPATH is set, unset it, it'll only cause trouble... - # see https://github.com/easybuilders/easybuild-framework/issues/2944 - if 'CDPATH' in os.environ: - del os.environ['CDPATH'] + if not all([eb_go, cfg_settings, init_session_state, hooks]): + eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) - # When EB is run via `exec` the special bash variable $_ is not set - # So emulate this here to allow (module) scripts depending on that to work - if '_' not in os.environ: - os.environ['_'] = sys.executable - - # purposely session state very early, to avoid modules loaded by EasyBuild meddling in - init_session_state = session_state() options, orig_paths = eb_go.options, eb_go.args if 'python2' not in build_option('silence_deprecation_warnings'): @@ -742,17 +729,31 @@ def prepare_main(args=None, logfile=None, testing=None): :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode - :return: easybuild options and configuration settings + :return: easybuild options, configuration settings and session state """ + register_lock_cleanup_signal_handlers() + + # if $CDPATH is set, unset it, it'll only cause trouble... + # see https://github.com/easybuilders/easybuild-framework/issues/2944 + if 'CDPATH' in os.environ: + del os.environ['CDPATH'] + + # When EB is run via `exec` the special bash variable $_ is not set + # So emulate this here to allow (module) scripts depending on that to work + if '_' not in os.environ: + os.environ['_'] = sys.executable + + # purposely session state very early, to avoid modules loaded by EasyBuild meddling in + init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - return eb_go, cfg_settings + hooks = load_hooks(eb_go.options.hooks) + return eb_go, cfg_settings, init_session_state, hooks if __name__ == "__main__": - eb_go, cfg_settings = prepare_main() - hooks = load_hooks(eb_go.options.hooks) + eb_go, cfg_settings, init_session_state, hooks = prepare_main() try: - main(eb_go=eb_go, cfg_settings=cfg_settings) + main(eb_go=eb_go, cfg_settings=cfg_settings, init_session_state=init_session_state, hooks=hooks) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg) From d8a34e4a898c6a3e72f0760a6e26a3c0e6a751d6 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:47:57 +0100 Subject: [PATCH 062/111] Update docstrings to match new parameters and return values --- easybuild/main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 500ffabfe9..1f71c60ba3 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -584,12 +584,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb init_session_state=None, hooks=None): """ Main function: parse command line options, and act accordingly. - :param: eb_go: easybuild general options object - :param: cfg_settings: configuration settings of easybuild :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode + :param eb_go: easybuild general options object + :param cfg_settings: configuration settings of easybuild + :param init_session_state: session state + :param hooks: available hooks """ if not all([eb_go, cfg_settings, init_session_state, hooks]): @@ -729,7 +731,7 @@ def prepare_main(args=None, logfile=None, testing=None): :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode - :return: easybuild options, configuration settings and session state + :return: easybuild options, configuration settings, session state and hooks """ register_lock_cleanup_signal_handlers() From fcc6fee8a246dd4c4c2d004c65d7372a718580d8 Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Thu, 10 Aug 2023 16:57:36 +0100 Subject: [PATCH 063/111] Add or condition to ensure test passes --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 1f71c60ba3..82c5fbac3e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -594,7 +594,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb :param hooks: available hooks """ - if not all([eb_go, cfg_settings, init_session_state, hooks]): + if not all([eb_go, cfg_settings, init_session_state, hooks]) or any([args, logfile, testing]): eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args From 4146b46ef0ff78920169255a8d93d5bc1754d08b Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Fri, 11 Aug 2023 09:50:00 +0100 Subject: [PATCH 064/111] Separate check on hooks to be None --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 82c5fbac3e..2a3dbe5781 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -594,7 +594,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb :param hooks: available hooks """ - if not all([eb_go, cfg_settings, init_session_state, hooks]) or any([args, logfile, testing]): + if not all([eb_go, cfg_settings, init_session_state, hooks]) or hooks is None or any([args, logfile, testing]): eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args From d9b15f04dcc5abc5fa93baa654a9a5d6965ccead Mon Sep 17 00:00:00 2001 From: XavierCS-dev Date: Fri, 11 Aug 2023 10:01:36 +0100 Subject: [PATCH 065/111] Improve clarity when checking for None values in main --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 2a3dbe5781..8b4929d035 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -594,7 +594,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb :param hooks: available hooks """ - if not all([eb_go, cfg_settings, init_session_state, hooks]) or hooks is None or any([args, logfile, testing]): + if any(i is None for i in [eb_go, cfg_settings, init_session_state, hooks]) or any([args, logfile, testing]): eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args From 7d31bb8e2b209a77ecddbf966c7f3a7bdf1dfeb6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 12:06:47 +0200 Subject: [PATCH 066/111] clean up passing of data produced by prepare_main into main --- easybuild/main.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 8b4929d035..f20f5b4152 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -580,22 +580,22 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, eb_go=None, cfg_settings=None, - init_session_state=None, hooks=None): +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode + :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None) :param eb_go: easybuild general options object :param cfg_settings: configuration settings of easybuild :param init_session_state: session state - :param hooks: available hooks """ - - if any(i is None for i in [eb_go, cfg_settings, init_session_state, hooks]) or any([args, logfile, testing]): - eb_go, cfg_settings, init_session_state, hooks = prepare_main(args=args, logfile=logfile, testing=testing) + if prepared_cfg_data is None or any([args, logfile, testing]): + init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing) + else: + init_session_state, eb_go, cfg_settings = prepared_cfg_data options, orig_paths = eb_go.options, eb_go.args @@ -731,7 +731,7 @@ def prepare_main(args=None, logfile=None, testing=None): :param args: command line arguments to take into account when parsing the EasyBuild configuration settings :param logfile: log file to use :param testing: enable testing mode - :return: easybuild options, configuration settings, session state and hooks + :return: 3-tuple with initial session state data, EasyBuildOptions instance, and tuple with configuration settings """ register_lock_cleanup_signal_handlers() @@ -748,14 +748,15 @@ def prepare_main(args=None, logfile=None, testing=None): # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) - hooks = load_hooks(eb_go.options.hooks) - return eb_go, cfg_settings, init_session_state, hooks + + return init_session_state, eb_go, cfg_settings if __name__ == "__main__": - eb_go, cfg_settings, init_session_state, hooks = prepare_main() + init_session_state, eb_go, cfg_settings = prepare_main() + hooks = load_hooks(eb_go.options.hooks) try: - main(eb_go=eb_go, cfg_settings=cfg_settings, init_session_state=init_session_state, hooks=hooks) + main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: run_hook(FAIL, hooks, args=[err]) print_error(err.msg) From 20c99cdaf00878bb4583396c930a049bd6ee8a20 Mon Sep 17 00:00:00 2001 From: XavierCS-dev <64903011+XavierCS-dev@users.noreply.github.com> Date: Fri, 11 Aug 2023 11:15:21 +0100 Subject: [PATCH 067/111] Remove unnecessary parameter descriptions Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index f20f5b4152..24f994da14 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -588,9 +588,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr :param do_build: whether or not to actually perform the build :param testing: enable testing mode :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None) - :param eb_go: easybuild general options object - :param cfg_settings: configuration settings of easybuild - :param init_session_state: session state """ if prepared_cfg_data is None or any([args, logfile, testing]): init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing) From ddf0c947cba3fe05e1172c7faee98d56b581ee70 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 19:02:53 +0200 Subject: [PATCH 068/111] trigger hook before/after running a shell command (disabled in tools.systemtools for functions that are called before configuration is fully in place so build options are defined) --- easybuild/tools/hooks.py | 4 ++++ easybuild/tools/options.py | 2 +- easybuild/tools/run.py | 12 +++++++++++- easybuild/tools/systemtools.py | 24 +++++++++++++----------- test/framework/options.py | 2 ++ 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 7a104d6277..a3586cfb4b 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -70,6 +70,8 @@ CRASH = 'crash' FAIL = 'fail' +RUN_SHELL_CMD = 'run_shell_cmd' + PRE_PREF = 'pre_' POST_PREF = 'post_' HOOK_SUFF = '_hook' @@ -108,6 +110,8 @@ CANCEL, CRASH, FAIL, + PRE_PREF + RUN_SHELL_CMD, + POST_PREF + RUN_SHELL_CMD, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 761cdc262f..5acf1780a2 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1894,7 +1894,7 @@ def set_tmpdir(tmpdir=None, raise_error=False): os.close(fd) os.chmod(tmptest_file, 0o700) if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False): + stream_output=False, with_hooks=False): msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() msg += "This can cause problems in the build process, consider using --tmpdir." if raise_error: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 1b8c385563..8f2ba68c5d 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -50,6 +50,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import ERROR, IGNORE, WARN, build_option +from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.py2vs3 import string_type from easybuild.tools.utilities import trace_msg @@ -131,7 +132,8 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): @run_cmd_cache def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, - force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False): + force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False, + with_hooks=True): """ Run specified command (in a subshell) :param cmd: command to run @@ -148,6 +150,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) + :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) """ cwd = os.getcwd() @@ -233,6 +236,10 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True else: raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) + if with_hooks: + hooks = load_hooks(build_option('hooks')) + run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd]) + _log.info('running cmd: %s ' % cmd) try: proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -240,6 +247,9 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) + if with_hooks: + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd]) + if inp: proc.stdin.write(inp.encode()) proc.stdin.close() diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index cab4b00055..b81e018bba 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -274,7 +274,7 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False) + out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) try: if int(out) > 0: core_cnt = int(out) @@ -311,7 +311,7 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: memtotal = int(out.strip()) // (1024**2) @@ -393,14 +393,15 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, with_hooks=False) out = out.strip() if ec == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) else: cmd = "sysctl -n machdep.cpu.brand_string" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, + with_hooks=False) out = out.strip().split(' ')[0] if ec == 0 and out in CPU_VENDORS: vendor = out @@ -503,7 +504,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: model = out.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -548,7 +549,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) out = out.strip() cpu_freq = None if ec == 0 and out: @@ -596,7 +597,8 @@ def get_cpu_features(): for feature_set in ['extfeatures', 'features', 'leaf7_features']: cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) - out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False) + out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, + with_hooks=False) if ec == 0: cpu_feat.extend(out.strip().lower().split()) @@ -624,7 +626,7 @@ def get_gpu_info(): cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False) + force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: for line in out.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -643,14 +645,14 @@ def get_gpu_info(): cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False) + force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: amd_driver = out.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, - force_in_dry_run=True, trace=False, stream_output=False) + force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False) if ec == 0: for line in out.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -898,7 +900,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Output is returned as a single-line string (newlines are replaced by '; '). """ out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True, - trace=False, stream_output=False) + trace=False, stream_output=False, with_hooks=False) if not ignore_ec and ec: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out)) return UNKNOWN diff --git a/test/framework/options.py b/test/framework/options.py index 2ef6488d16..78c3a495e6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -724,6 +724,8 @@ def test_avail_hooks(self): " cancel_hook", " crash_hook", " fail_hook", + " pre_run_shell_cmd_hook", + " post_run_shell_cmd_hook", '', ]) self.assertEqual(stdout, expected) From 1b798e1dacc78925f8eda8725a198071a142926a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 19:26:29 +0200 Subject: [PATCH 069/111] extend hook tests to also check use of run_shell_cmd, fail, and build_and_install_loop hooks --- test/framework/hooks.py | 42 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index fad251b040..0743cc16b9 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -52,6 +52,9 @@ def setUp(self): 'def parse_hook(ec):', ' print("Parse hook with argument %s" % ec)', '', + 'def pre_build_and_install_loop_hook(ecs):', + ' print("About to start looping for %d easyconfigs!" % len(ecs))', + '', 'def foo():', ' print("running foo helper method")', '', @@ -64,6 +67,12 @@ def setUp(self): '', 'def pre_single_extension_hook(ext):', ' print("this is run before installing an extension")', + '', + 'def pre_run_shell_cmd_hook(cmd):', + ' print("this is run before running command \'%s\'" % cmd)', + '', + 'def fail_hook(err):', + ' print("EasyBuild FAIL: %s" % err)', ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) @@ -74,11 +83,14 @@ def test_load_hooks(self): hooks = load_hooks(self.test_hooks_pymod) - self.assertEqual(len(hooks), 5) + self.assertEqual(len(hooks), 8) expected = [ + 'fail_hook', 'parse_hook', 'post_configure_hook', + 'pre_build_and_install_loop_hook', 'pre_install_hook', + 'pre_run_shell_cmd_hook', 'pre_single_extension_hook', 'start_hook', ] @@ -113,6 +125,9 @@ def test_find_hook(self): pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0] pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0] start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0] + pre_run_shell_cmd_hook = [hooks[k] for k in hooks if k == 'pre_run_shell_cmd_hook'][0] + fail_hook = [hooks[k] for k in hooks if k == 'fail_hook'][0] + pre_build_and_install_loop_hook = [hooks[k] for k in hooks if k == 'pre_build_and_install_loop_hook'][0] self.assertEqual(find_hook('configure', hooks), None) self.assertEqual(find_hook('configure', hooks, pre_step_hook=True), None) @@ -138,6 +153,19 @@ def test_find_hook(self): self.assertEqual(find_hook('start', hooks, pre_step_hook=True), None) self.assertEqual(find_hook('start', hooks, post_step_hook=True), None) + self.assertEqual(find_hook('run_shell_cmd', hooks), None) + self.assertEqual(find_hook('run_shell_cmd', hooks, pre_step_hook=True), pre_run_shell_cmd_hook) + self.assertEqual(find_hook('run_shell_cmd', hooks, post_step_hook=True), None) + + self.assertEqual(find_hook('fail', hooks), fail_hook) + self.assertEqual(find_hook('fail', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('fail', hooks, post_step_hook=True), None) + + hook_name = 'build_and_install_loop' + self.assertEqual(find_hook(hook_name, hooks), None) + self.assertEqual(find_hook(hook_name, hooks, pre_step_hook=True), pre_build_and_install_loop_hook) + self.assertEqual(find_hook(hook_name, hooks, post_step_hook=True), None) + def test_run_hook(self): """Test for run_hook function.""" @@ -149,17 +177,21 @@ def test_run_hook(self): self.mock_stderr(True) run_hook('start', hooks) run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") + run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) run_hook('configure', hooks, pre_step_hook=True, args=[None]) run_hook('configure', hooks, post_step_hook=True, args=[None]) run_hook('build', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"]) run_hook('install', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, pre_step_hook=True, args=[None]) for _ in range(3): run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) run_hook('single_extension', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, post_step_hook=True, args=[None]) + run_hook('fail', hooks, args=[EasyBuildError('oops')]) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -170,17 +202,25 @@ def test_run_hook(self): "this is triggered at the very beginning", "== Running parse hook for example.eb...", "Parse hook with argument ", + "== Running pre-build_and_install_loop hook...", + "About to start looping for 2 easyconfigs!", "== Running post-configure hook...", "this is run after configure step", "running foo helper method", + "== Running pre-run_shell_cmd hook...", + "this is run before running command 'make -j 3'", "== Running pre-install hook...", "this is run before install step", + "== Running pre-run_shell_cmd hook...", + "this is run before running command 'make install'", "== Running pre-single_extension hook...", "this is run before installing an extension", "== Running pre-single_extension hook...", "this is run before installing an extension", "== Running pre-single_extension hook...", "this is run before installing an extension", + "== Running fail hook...", + "EasyBuild FAIL: 'oops'", ]) self.assertEqual(stdout.strip(), expected_stdout) From 7ffa35682314865f11ca63c886d9942ce6971526 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 20:04:50 +0200 Subject: [PATCH 070/111] also run pre/post run_shell_cmd hooks when running interactive commands via run_cmd_qa --- easybuild/tools/hooks.py | 8 +++++--- easybuild/tools/run.py | 5 +++++ test/framework/hooks.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index a3586cfb4b..88fa3a85d3 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -209,7 +209,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): return res -def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None): +def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, kwargs=None, msg=None): """ Run hook with specified label and return result of calling the hook or None. @@ -225,6 +225,8 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if hook: if args is None: args = [] + if kwargs is None: + kwargs = {} if pre_step_hook: label = 'pre-' + label @@ -236,6 +238,6 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if build_option('debug'): print_msg(msg) - _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) - res = hook(*args) + _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) + res = hook(*args, **kwargs) return res diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 8f2ba68c5d..c49790a497 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -519,6 +519,9 @@ def get_proc(): if cmd_log: cmd_log.close() + hooks = load_hooks(build_option('hooks')) + run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'interactive': True}) + with get_proc() as proc: ec = proc.poll() stdout_err = '' @@ -609,6 +612,8 @@ def get_proc(): except IOError as err: _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs={'interactive': True}) + if trace: trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 0743cc16b9..4737f1a991 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -68,8 +68,11 @@ def setUp(self): 'def pre_single_extension_hook(ext):', ' print("this is run before installing an extension")', '', - 'def pre_run_shell_cmd_hook(cmd):', - ' print("this is run before running command \'%s\'" % cmd)', + 'def pre_run_shell_cmd_hook(cmd, interactive=False):', + ' if interactive:', + ' print("this is run before running interactive command \'%s\'" % cmd)', + ' else:', + ' print("this is run before running command \'%s\'" % cmd)', '', 'def fail_hook(err):', ' print("EasyBuild FAIL: %s" % err)', @@ -179,12 +182,13 @@ def test_run_hook(self): run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) run_hook('configure', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True}) run_hook('configure', hooks, post_step_hook=True, args=[None]) run_hook('build', hooks, pre_step_hook=True, args=[None]) run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) - run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) run_hook('install', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, pre_step_hook=True, args=[None]) for _ in range(3): @@ -204,6 +208,8 @@ def test_run_hook(self): "Parse hook with argument ", "== Running pre-build_and_install_loop hook...", "About to start looping for 2 easyconfigs!", + "== Running pre-run_shell_cmd hook...", + "this is run before running interactive command 'configure.sh'", "== Running post-configure hook...", "this is run after configure step", "running foo helper method", From 63271d430c483b48ee6540621b598bed90f6d489 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 20:11:07 +0200 Subject: [PATCH 071/111] allow tweaking command to run via pre-run_shell_cmd hook --- easybuild/tools/run.py | 15 +++++++++++---- test/framework/hooks.py | 5 ++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c49790a497..7d6e66ccf5 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -238,7 +238,10 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True if with_hooks: hooks = load_hooks(build_option('hooks')) - run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd]) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd]) + if isinstance(hook_res, string_type): + cmd, old_cmd = hook_res, cmd + self.log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) _log.info('running cmd: %s ' % cmd) try: @@ -495,6 +498,13 @@ def check_answers_list(answers): # Part 2: Run the command and answer questions # - this needs asynchronous stdout + hooks = load_hooks(build_option('hooks')) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'interactive': True}) + if isinstance(hook_res, string_type): + cmd, old_cmd = hook_res, cmd + self.log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", + RUN_SHELL_CMD, cmd, old_cmd) + # # Log command output if cmd_log: cmd_log.write("# output for interactive command: %s\n\n" % cmd) @@ -519,9 +529,6 @@ def get_proc(): if cmd_log: cmd_log.close() - hooks = load_hooks(build_option('hooks')) - run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'interactive': True}) - with get_proc() as proc: ec = proc.poll() stdout_err = '' diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 4737f1a991..1be55b8ac7 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -73,6 +73,8 @@ def setUp(self): ' print("this is run before running interactive command \'%s\'" % cmd)', ' else:', ' print("this is run before running command \'%s\'" % cmd)', + ' if cmd == "make install":', + ' return "sudo " + cmd', '', 'def fail_hook(err):', ' print("EasyBuild FAIL: %s" % err)', @@ -188,7 +190,8 @@ def test_run_hook(self): run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) run_hook('build', hooks, post_step_hook=True, args=[None]) run_hook('install', hooks, pre_step_hook=True, args=[None]) - run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) + res = run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) + self.assertEqual(res, "sudo make install") run_hook('install', hooks, post_step_hook=True, args=[None]) run_hook('extensions', hooks, pre_step_hook=True, args=[None]) for _ in range(3): From 2d6f1a951110243aa94ff25ada5313bbd9160b47 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 20:22:33 +0200 Subject: [PATCH 072/111] pass working directory to pre run_shell_cmd hook + pass output, exit code, and working directory of command to post run_shell_cmd hook --- easybuild/tools/run.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7d6e66ccf5..d8d99f28ce 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -238,10 +238,10 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True if with_hooks: hooks = load_hooks(build_option('hooks')) - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd]) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()}) if isinstance(hook_res, string_type): cmd, old_cmd = hook_res, cmd - self.log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) + _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) _log.info('running cmd: %s ' % cmd) try: @@ -250,9 +250,6 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True except OSError as err: raise EasyBuildError("run_cmd init cmd %s failed:%s", cmd, err) - if with_hooks: - run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd]) - if inp: proc.stdin.write(inp.encode()) proc.stdin.close() @@ -261,7 +258,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True return (proc, cmd, cwd, start_time, cmd_log) else: return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, - regexp=regexp, stream_output=stream_output, trace=trace) + regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks) def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''): @@ -306,7 +303,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, - regexp=True, stream_output=None, trace=True, output=''): + regexp=True, stream_output=None, trace=True, output='', with_hook=True): """ Complete running of command represented by passed subprocess.Popen instance. @@ -321,6 +318,7 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param stream_output: enable streaming command output to stdout :param trace: print command being executed as part of trace output + :param with_hook: trigger post run_shell_cmd hooks (if defined) """ # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead @@ -356,6 +354,15 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False sys.stdout.write(output) stdouterr += output + if with_hook: + hooks = load_hooks(build_option('hooks')) + run_hook_kwargs = { + 'exit_code': ec, + 'output': stdouterr, + 'work_dir': os.getcwd(), + } + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) + if trace: trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) @@ -499,10 +506,14 @@ def check_answers_list(answers): # - this needs asynchronous stdout hooks = load_hooks(build_option('hooks')) - hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'interactive': True}) + run_hook_kwargs = { + 'interactive': True, + 'work_dir': os.getcwd(), + } + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) if isinstance(hook_res, string_type): cmd, old_cmd = hook_res, cmd - self.log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", + _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) # # Log command output @@ -619,7 +630,12 @@ def get_proc(): except IOError as err: _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) - run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs={'interactive': True}) + run_hook_kwargs.update({ + 'interactive': True, + 'exit_code': ec, + 'output': stdout_err, + }) + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) if trace: trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) From 1f92ccc6930611d7db8dbb18b77dbc6d1cb2610a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 21:26:55 +0200 Subject: [PATCH 073/111] add test for running shell commands with pre/post run_cmd_shell hooks in place --- test/framework/run.py | 56 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/test/framework/run.py b/test/framework/run.py index f97d3b240f..1a93ff1a75 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -39,6 +39,7 @@ import subprocess import sys import tempfile +import textwrap import time from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -47,6 +48,7 @@ import easybuild.tools.asyncprocess as asyncprocess import easybuild.tools.utilities from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging +from easybuild.tools.config import update_build_option from easybuild.tools.filetools import adjust_permissions, read_file, write_file from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process from easybuild.tools.run import parse_log_for_error, run_cmd, run_cmd_qa @@ -736,6 +738,60 @@ def test_check_log_for_errors(self): expected_msg = "Found 1 potential error(s) in command output (output: the process crashed with 0)" self.assertIn(expected_msg, read_file(logfile)) + def test_run_cmd_with_hooks(self): + """ + Test running command with run_cmd with pre/post run_shell_cmd hooks in place. + """ + cwd = os.getcwd() + + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') + hooks_file_txt = textwrap.dedent(""" + def pre_run_shell_cmd_hook(cmd, *args, **kwargs): + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + print("pre-run hook interactive '%s' in %s" % (cmd, work_dir)) + else: + print("pre-run hook '%s' in %s" % (cmd, work_dir)) + if not cmd.startswith('echo'): + cmds = cmd.split(';') + return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()]) + + def post_run_shell_cmd_hook(cmd, *args, **kwargs): + exit_code = kwargs.get('exit_code') + output = kwargs.get('output') + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + msg = "post-run hook interactive '%s'" % cmd + else: + msg = "post-run hook '%s'" % cmd + msg += " (exit code: %s, output: '%s')" % (exit_code, output) + print(msg) + """) + write_file(hooks_file, hooks_file_txt) + update_build_option('hooks', hooks_file) + + with self.mocked_stdout_stderr(): + run_cmd("make") + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook 'make' in %s" % cwd, + "post-run hook 'echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + + with self.mocked_stdout_stderr(): + run_cmd_qa("sleep 2; make", qa={}) + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook interactive 'sleep 2; make' in %s" % cwd, + "post-run hook interactive 'sleep 2; echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + def suite(): """ returns all the testcases in this module """ From f0ef6272a37dad11c2fc4d05168e155c5161893b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 21:27:40 +0200 Subject: [PATCH 074/111] extend test_toy_build_hooks with run_shell_cmd hooks --- .../sandbox/easybuild/easyblocks/t/toy.py | 8 ++- test/framework/toy_build.py | 69 ++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 88ff864454..c4614e3333 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -34,7 +34,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root, get_software_version @@ -122,7 +122,11 @@ def build_step(self, name=None, cfg=None): name = self.name cmd = compose_toy_build_cmd(self.cfg, name, cfg['prebuildopts'], cfg['buildopts']) - run_cmd(cmd) + # purposely run build command without checking exit code; + # we rely on this in test_toy_build_hooks + (out, ec) = run_cmd(cmd, log_ok=False, log_all=False) + if ec: + print_warning("Command '%s' failed, but we'll ignore it..." % cmd) def install_step(self, name=None): """Install toy.""" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 52aa18aae9..934438a549 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -58,6 +58,7 @@ from easybuild.tools.modules import Lmod from easybuild.tools.py2vs3 import reload, string_type from easybuild.tools.run import run_cmd +from easybuild.tools.utilities import nub from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -2885,6 +2886,9 @@ def test_toy_build_hooks(self): hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') hooks_file_txt = textwrap.dedent(""" import os + from easybuild.tools.filetools import change_dir, copy_file + + TOY_COMP_CMD = "gcc toy.c -o toy" def start_hook(): print('start hook triggered') @@ -2908,6 +2912,9 @@ def post_install_hook(self): print('in post-install hook for %s v%s' % (self.name, self.version)) print(', '.join(sorted(os.listdir(self.installdir)))) + copy_of_toy = os.path.join(self.start_dir, 'copy_of_toy') + copy_file(copy_of_toy, os.path.join(self.installdir, 'bin')) + def module_write_hook(self, module_path, module_txt): print('in module-write hook hook for %s' % os.path.basename(module_path)) return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore') @@ -2920,6 +2927,24 @@ def pre_sanitycheck_hook(self): def end_hook(): print('end hook triggered, all done!') + + def pre_run_shell_cmd_hook(cmd, *args, **kwargs): + if cmd.strip() == TOY_COMP_CMD: + print("pre_run_shell_cmd_hook triggered for '%s'" % cmd) + # 'copy_toy_file' command doesn't exist, but don't worry, + # this problem will be fixed in post_run_shell_cmd_hook + cmd += " && copy_toy_file toy copy_of_toy" + return cmd + + def post_run_shell_cmd_hook(cmd, *args, **kwargs): + exit_code = kwargs['exit_code'] + output = kwargs['output'] + work_dir = kwargs['work_dir'] + if cmd.strip().startswith(TOY_COMP_CMD) and exit_code: + cwd = change_dir(work_dir) + copy_file('toy', 'copy_of_toy') + change_dir(cwd) + print("'%s' command failed (exit code %s), but I fixed it!" % (cmd, exit_code)) """) write_file(hooks_file, hooks_file_txt) @@ -2936,7 +2961,9 @@ def end_hook(): if get_module_syntax() == 'Lua': toy_mod_file += '.lua' - self.assertEqual(stderr, '') + warnings = nub([x for x in stderr.strip().splitlines() if x]) + self.assertEqual(warnings, ["WARNING: Command ' gcc toy.c -o toy ' failed, but we'll ignore it..."]) + # parse hook is triggered 3 times: once for main install, and then again for each extension; # module write hook is triggered 5 times: # - before installing extensions @@ -2950,10 +2977,22 @@ def end_hook(): toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running pre-configure hook... pre-configure: toy.source: True + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running post-configure hook... post-configure: toy.source: False + == Running pre-run_shell_cmd hook... + pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' + == Running post-run_shell_cmd hook... + ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! == Running post-install hook... in post-install hook for toy v0.0 bin, lib @@ -2967,16 +3006,40 @@ def end_hook(): toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running post-single_extension hook... installing of extension bar is done! + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' + == Running post-run_shell_cmd hook... + ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! == Running post-single_extension hook... installing of extension toy is done! + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running pre-sanitycheck hook... pre_sanity_check_hook == Running module_write hook... in module-write hook hook for {mod_name} + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running module_write hook... in module-write hook hook for {mod_name} + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... + == Running pre-run_shell_cmd hook... + == Running post-run_shell_cmd hook... == Running module_write hook... in module-write hook hook for {mod_name} == Running module_write hook... @@ -2989,6 +3052,10 @@ def end_hook(): toy_mod = read_file(toy_mod_file) self.assertIn('Not a toy anymore', toy_mod) + toy_bin_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') + toy_bins = sorted(os.listdir(toy_bin_dir)) + self.assertEqual(toy_bins, ['bar', 'copy_of_toy', 'toy']) + def test_toy_multi_deps(self): """Test installation of toy easyconfig that uses multi_deps.""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 6a1e5b8ef452ca559cd90d30a9bf697430e03273 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 11 Aug 2023 22:04:03 +0200 Subject: [PATCH 075/111] trivial code style fix --- easybuild/tools/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index d8d99f28ce..8916d80795 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -514,7 +514,7 @@ def check_answers_list(answers): if isinstance(hook_res, string_type): cmd, old_cmd = hook_res, cmd _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", - RUN_SHELL_CMD, cmd, old_cmd) + RUN_SHELL_CMD, cmd, old_cmd) # # Log command output if cmd_log: From 29dbb7175bf439bd0115145481f37890ae0f9771 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Aug 2023 09:59:22 +0200 Subject: [PATCH 076/111] omit '== Running ... hook' messages in test_toy_build_hooks by not using --debug --- test/framework/toy_build.py | 55 +++---------------------------------- 1 file changed, 4 insertions(+), 51 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 934438a549..a7e32971a6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -155,7 +155,7 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True, - raise_systemexit=False, force=True, test_report_regexs=None): + raise_systemexit=False, force=True, test_report_regexs=None, debug=True): """Perform a toy build.""" if extra_args is None: extra_args = [] @@ -169,10 +169,11 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, - '--debug', '--unittest-file=%s' % self.logfile, '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), ] + if debug: + args.append('--debug') if force: args.append('--force') if tmpdir is not None: @@ -2950,7 +2951,7 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): self.mock_stderr(True) self.mock_stdout(True) - self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True) + self.test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True, debug=False) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) @@ -2971,80 +2972,32 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): # - for final module file # - for devel module file expected_output = textwrap.dedent(""" - == Running start hook... start hook triggered - == Running parse hook for test.eb... toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-configure hook... pre-configure: toy.source: True - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running post-configure hook... post-configure: toy.source: False - == Running pre-run_shell_cmd hook... pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' - == Running post-run_shell_cmd hook... ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! - == Running post-install hook... in post-install hook for toy v0.0 bin, lib - == Running module_write hook... in module-write hook hook for {mod_name} - == Running parse hook... toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy - == Running parse hook... toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running post-single_extension hook... installing of extension bar is done! - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' - == Running post-run_shell_cmd hook... ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! - == Running post-single_extension hook... installing of extension toy is done! - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-sanitycheck hook... pre_sanity_check_hook - == Running module_write hook... in module-write hook hook for {mod_name} - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running module_write hook... in module-write hook hook for {mod_name} - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running pre-run_shell_cmd hook... - == Running post-run_shell_cmd hook... - == Running module_write hook... in module-write hook hook for {mod_name} - == Running module_write hook... in module-write hook hook for {mod_name} - == Running end hook... end hook triggered, all done! """).strip().format(mod_name=os.path.basename(toy_mod_file)) self.assertEqual(stdout.strip(), expected_output) From 6c5b8bff57ee7c09544889194c545bb1482e32dd Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 12 Aug 2023 10:32:59 +0100 Subject: [PATCH 077/111] add build_info_msg easyconfig parameter --- easybuild/framework/easyblock.py | 4 ++++ easybuild/framework/easyconfig/default.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c85104d687..55d73d3ba5 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4174,6 +4174,10 @@ def build_and_install_one(ecdict, init_env): dry_run_msg('', silent=silent) print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) + if ecdict['ec']['build_info_msg']: + msg = "This easyconfig provides the following build information:\n\n%s\n" + print_msg(msg % ecdict['ec']['build_info_msg'], log=_log, silent=silent) + if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) print_dry_run_note('below', silent=silent) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index b37c3660f0..dd91229d1e 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -228,6 +228,8 @@ 'buildstats': [None, "A list of dicts with build statistics", OTHER], 'deprecated': [False, "String specifying reason why this easyconfig file is deprecated " "and will be archived in the next major release of EasyBuild", OTHER], + 'build_info_msg': [None, "String with information to be printed to stdout and logged during the building " + "of the easyconfig", OTHER], } From a3cde1a8b0634c759b77236b926c2bb6e5f25049 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Aug 2023 12:21:51 +0200 Subject: [PATCH 078/111] reset cached hooks in tests to avoid test hooks leaking into other tests --- test/framework/hooks.py | 10 +++++++++- test/framework/toy_build.py | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 1be55b8ac7..2d150ae869 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -32,7 +32,7 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner -import easybuild.tools.hooks +import easybuild.tools.hooks # so we can reset cached hooks from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import remove_file, write_file from easybuild.tools.hooks import find_hook, load_hooks, run_hook, verify_hooks @@ -81,6 +81,14 @@ def setUp(self): ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) + def tearDown(self): + """Cleanup.""" + + # reset cached hooks + easybuild.tools.hooks._cached_hooks.clear() + + super(HooksTest, self).tearDown() + def test_load_hooks(self): """Test for load_hooks function.""" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index a7e32971a6..f61a4be33c 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -101,6 +101,9 @@ def tearDown(self): del sys.modules['easybuild.easyblocks.toytoy'] del sys.modules['easybuild.easyblocks.generic.toy_extension'] + # reset cached hooks + easybuild.tools.hooks._cached_hooks.clear() + super(ToyBuildTest, self).tearDown() # remove logs From ecbaf34414d6cf0b1b8110880a62fe15bc97a159 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Aug 2023 18:54:05 +0200 Subject: [PATCH 079/111] remove crash hook (since it doesn't work) --- easybuild/main.py | 5 +---- easybuild/tools/hooks.py | 2 -- test/framework/options.py | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 24f994da14..1a9aa736a7 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -69,7 +69,7 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, CRASH, START, END, CANCEL, FAIL +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, FAIL from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color @@ -760,6 +760,3 @@ def prepare_main(args=None, logfile=None, testing=None): except KeyboardInterrupt as err: run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) - except Exception as err: - run_hook(CRASH, hooks, args=[err]) - print_error("Encountered an unrecoverable error: %s" % err) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 7a104d6277..785e255171 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -67,7 +67,6 @@ END = 'end' CANCEL = 'cancel' -CRASH = 'crash' FAIL = 'fail' PRE_PREF = 'pre_' @@ -106,7 +105,6 @@ POST_PREF + BUILD_AND_INSTALL_LOOP, END, CANCEL, - CRASH, FAIL, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] diff --git a/test/framework/options.py b/test/framework/options.py index 2ef6488d16..545d18cdda 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -722,7 +722,6 @@ def test_avail_hooks(self): " post_build_and_install_loop_hook", " end_hook", " cancel_hook", - " crash_hook", " fail_hook", '', ]) From c48615e5a05662e018216266d91f04c4b0df074c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 12 Aug 2023 19:26:45 +0200 Subject: [PATCH 080/111] add test to check whether build_info_msg works as expected --- test/framework/toy_build.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 52aa18aae9..40e967fcf0 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -3900,6 +3900,30 @@ def test_toy_post_install_messages(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_toy_build_info_msg(self): + """ + Test use of build info message + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nbuild_info_msg = "Are you sure you want to install this toy software?"' + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self.test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True) + stdout = self.get_stdout() + + pattern = '\n'.join([ + r"== This easyconfig provides the following build information:", + r'', + r"Are you sure you want to install this toy software\?", + ]) + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def suite(): """ return all the tests in this file """ From 5b45c240c00feec269ac4af7e093dee9545a524c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 13 Aug 2023 18:06:16 +0200 Subject: [PATCH 081/111] ignore stdout produced by toy easyblock in test_cleanup_builddir --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 78c3a495e6..74877765e5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2580,7 +2580,8 @@ def test_cleanup_builddir(self): '--force', '--try-amend=prebuildopts=nosuchcommand &&', ] - self.eb_main(args, do_build=True) + with self.mocked_stdout_stderr(): + self.eb_main(args, do_build=True) self.assertExists(toy_buildpath, "Build dir %s is retained after failed build" % toy_buildpath) def test_filter_deps(self): From 886d7bf2bb7bc76902edae69ed67b78af695c01d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 15 Aug 2023 16:26:30 +0200 Subject: [PATCH 082/111] Add `makedirs(..., exits_ok)` for Python2 --- easybuild/tools/py2vs3/py2.py | 10 ++++++++++ easybuild/tools/py2vs3/py3.py | 1 + test/framework/filetools.py | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 99f8cb0f4a..08a3b52b72 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -33,8 +33,10 @@ """ # these are not used here, but imported from here in other places import ConfigParser as configparser # noqa +import errno import imp import json +import os import subprocess import time import urllib2 as std_urllib # noqa @@ -115,3 +117,11 @@ def sort_looseversions(looseversions): # with Python 2, we can safely use 'sorted' on LooseVersion instances # (but we can't in Python 3, see https://bugs.python.org/issue14894) return sorted(looseversions) + + +def makedirs(name, mode=0o777, exist_ok=False): + try: + os.makedirs(name, mode) + except OSError as e: + if not exist_ok or e.errno != errno.EEXIST: + raise diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index 77f786cbec..8d9145179d 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -44,6 +44,7 @@ from html.parser import HTMLParser # noqa from itertools import zip_longest from io import StringIO # noqa +from os import makedirs # noqa from string import ascii_letters, ascii_lowercase # noqa from urllib.request import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa from urllib.parse import urlencode # noqa diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 32d72c7b83..14011e2cbc 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,6 +45,7 @@ from unittest import TextTestRunner from easybuild.tools import run import easybuild.tools.filetools as ft +import easybuild.tools.py2vs3 as py2vs3 from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff @@ -3389,6 +3390,16 @@ def test_set_gid_sticky_bits(self): self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + def test_compat_makedirs(self): + """Test compatibility layer for Python3 os.makedirs""" + name = os.path.join(self.test_prefix, 'folder') + self.assertNotExists(name) + py2vs3.makedirs(name) + self.assertExists(name) + self.assertErrorRegex(Exception, os.path.basename(name), py2vs3.makedirs, name) + py2vs3.makedirs(name, exist_ok=True) # No error + self.assertExists(name) + def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') From 22def640ae0bfb96280389e96347a015837aecc3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 15 Aug 2023 16:32:51 +0200 Subject: [PATCH 083/111] Don't rely on errno Same as the Python3 implementation which has a comment: > Cannot rely on checking for EEXIST, since the operating system could give priority to other errors like EACCES or EROFS --- easybuild/tools/py2vs3/py2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/py2vs3/py2.py b/easybuild/tools/py2vs3/py2.py index 08a3b52b72..1fec4650b1 100644 --- a/easybuild/tools/py2vs3/py2.py +++ b/easybuild/tools/py2vs3/py2.py @@ -33,7 +33,6 @@ """ # these are not used here, but imported from here in other places import ConfigParser as configparser # noqa -import errno import imp import json import os @@ -122,6 +121,6 @@ def sort_looseversions(looseversions): def makedirs(name, mode=0o777, exist_ok=False): try: os.makedirs(name, mode) - except OSError as e: - if not exist_ok or e.errno != errno.EEXIST: + except OSError: + if not exist_ok or not os.path.isdir(name): raise From 204814347b05ce0b5977783aae59e83ee2e219ef Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 15 Aug 2023 16:36:02 +0200 Subject: [PATCH 084/111] Remove use of Python3 `FileExistsError` Use the new py2vpy3.makedirs compat layer. --- easybuild/tools/filetools.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d16d4f4778..a5e5f63ec6 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -65,7 +65,7 @@ from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar -from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type +from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -1918,15 +1918,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # climb up until we hit an existing path or the empty string (for relative paths) while existing_parent_path and not os.path.exists(existing_parent_path): existing_parent_path = os.path.dirname(existing_parent_path) - os.makedirs(path) + makedirs(path, exist_ok=True) else: os.mkdir(path) - except FileExistsError as err: - if os.path.exists(path): - # This may happen if a parallel build creates the directory after we checked for its existence - _log.debug("Directory creation aborted as it seems it was already created: %s", err) - else: - raise EasyBuildError("Failed to create directory %s: %s", path, err) except OSError as err: raise EasyBuildError("Failed to create directory %s: %s", path, err) From 40f9fe761c1810e45ec407c9343647a3283982e8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 15 Aug 2023 18:37:21 +0200 Subject: [PATCH 085/111] use liberal pattern to check whether makedirs wrapper raises an exception if directory already exists, so the check passes with both Python 2 and 3 --- test/framework/filetools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 14011e2cbc..b87b9359b0 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -3396,7 +3396,8 @@ def test_compat_makedirs(self): self.assertNotExists(name) py2vs3.makedirs(name) self.assertExists(name) - self.assertErrorRegex(Exception, os.path.basename(name), py2vs3.makedirs, name) + # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3) + self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name) py2vs3.makedirs(name, exist_ok=True) # No error self.assertExists(name) From f5ec5e57ea1fd64cc4ce7da2f1b6563388c28485 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 07:53:47 +0200 Subject: [PATCH 086/111] use amd64 version of containers to run end2end test --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7594b79e59..63d5013f15 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -15,7 +15,7 @@ jobs: - ubuntu-20.04 fail-fast: false container: - image: ghcr.io/easybuilders/${{ matrix.container }} + image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 steps: - name: Check out the repo uses: actions/checkout@v2 From e46d5415775e816e0895b650bbc38dbfa320e94e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 08:23:15 +0200 Subject: [PATCH 087/111] add --silence-hook-trigger configuration option to supress printing of debug message every time a hook is triggered --- easybuild/tools/config.py | 1 + easybuild/tools/hooks.py | 2 +- easybuild/tools/options.py | 2 ++ test/framework/hooks.py | 72 +++++++++++++++++++++++--------------- 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 3f8acec4f5..6c0a173fe4 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -301,6 +301,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'sequential', 'set_default_module', 'set_gid_bit', + 'silence_hook_trigger', 'skip_extensions', 'skip_test_cases', 'skip_test_step', diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 9c61820090..f454974edb 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -233,7 +233,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if msg is None: msg = "Running %s hook..." % label - if build_option('debug'): + if build_option('debug') and not build_option('silence_hook_trigger'): print_msg(msg) _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5acf1780a2..b60e57513d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -501,6 +501,8 @@ def override_options(self): 'silence-deprecation-warnings': ( "Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations), 'strlist', 'extend', []), + 'silence-hook-trigger': ("Supress printing of debug message every time a hook is triggered", + None, 'store_true', False), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), diff --git a/test/framework/hooks.py b/test/framework/hooks.py index 2d150ae869..152d2352a4 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -34,6 +34,7 @@ import easybuild.tools.hooks # so we can reset cached hooks from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import update_build_option from easybuild.tools.filetools import remove_file, write_file from easybuild.tools.hooks import find_hook, load_hooks, run_hook, verify_hooks @@ -186,33 +187,38 @@ def test_run_hook(self): init_config(build_options={'debug': True}) - self.mock_stdout(True) - self.mock_stderr(True) - run_hook('start', hooks) - run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") - run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) - run_hook('configure', hooks, pre_step_hook=True, args=[None]) - run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True}) - run_hook('configure', hooks, post_step_hook=True, args=[None]) - run_hook('build', hooks, pre_step_hook=True, args=[None]) - run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) - run_hook('build', hooks, post_step_hook=True, args=[None]) - run_hook('install', hooks, pre_step_hook=True, args=[None]) - res = run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) - self.assertEqual(res, "sudo make install") - run_hook('install', hooks, post_step_hook=True, args=[None]) - run_hook('extensions', hooks, pre_step_hook=True, args=[None]) - for _ in range(3): - run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) - run_hook('single_extension', hooks, post_step_hook=True, args=[None]) - run_hook('extensions', hooks, post_step_hook=True, args=[None]) - run_hook('fail', hooks, args=[EasyBuildError('oops')]) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) - - expected_stdout = '\n'.join([ + def run_hooks(): + self.mock_stdout(True) + self.mock_stderr(True) + run_hook('start', hooks) + run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") + run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) + run_hook('configure', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True}) + run_hook('configure', hooks, post_step_hook=True, args=[None]) + run_hook('build', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) + run_hook('build', hooks, post_step_hook=True, args=[None]) + run_hook('install', hooks, pre_step_hook=True, args=[None]) + res = run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) + self.assertEqual(res, "sudo make install") + run_hook('install', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, pre_step_hook=True, args=[None]) + for _ in range(3): + run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) + run_hook('single_extension', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, post_step_hook=True, args=[None]) + run_hook('fail', hooks, args=[EasyBuildError('oops')]) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + return stdout, stderr + + stdout, stderr = run_hooks() + + expected_stdout_lines = [ "== Running start hook...", "this is triggered at the very beginning", "== Running parse hook for example.eb...", @@ -238,7 +244,17 @@ def test_run_hook(self): "this is run before installing an extension", "== Running fail hook...", "EasyBuild FAIL: 'oops'", - ]) + ] + expected_stdout = '\n'.join(expected_stdout_lines) + + self.assertEqual(stdout.strip(), expected_stdout) + self.assertEqual(stderr, '') + + # test silencing of hook trigger + update_build_option('silence_hook_trigger', True) + stdout, stderr = run_hooks() + + expected_stdout = '\n'.join(x for x in expected_stdout_lines if not x.startswith('== Running')) self.assertEqual(stdout.strip(), expected_stdout) self.assertEqual(stderr, '') From c9d45e4dbad23bfeaf898b1ebd344e254a95c2eb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 08:28:45 +0200 Subject: [PATCH 088/111] update list of containers to use in end2end tests --- .github/workflows/end2end.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 63d5013f15..06a192661f 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -8,11 +8,12 @@ jobs: matrix: container: - centos-7.9 - - centos-8.4 - - fedora-35 - - opensuse-15.3 - - rockylinux-8.5 + - centos-8.5 + - fedora-36 + - opensuse-15.4 + - rockylinux-8.7 - ubuntu-20.04 + - ubuntu-22.04 fail-fast: false container: image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 From 5deae99370a27ade745df72e08f7e7ab71cb5b51 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 09:18:01 +0200 Subject: [PATCH 089/111] split end2end test in separate steps --- .github/workflows/end2end.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 06a192661f..9ab4605bd0 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -30,7 +30,7 @@ jobs: rm -f develop.tar.gz done - - name: End-to-end test of EasyBuild + - name: Set up environment shell: bash run: | export PATH=$PWD:$PATH @@ -49,15 +49,28 @@ jobs: # tests are run with root privileges, so we need to tell EasyBuild that's OK... export EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1 + # save full environment to use in subsequent steps, + # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + while read line; do echo "$line" >> $GITHUB_ENV ; done < <(env) + + - name: Run commands to check test environment + shell: bash + run: | cmds=( + "whoami" + "pwd" "env | sort" "eb --version" "eb --show-system-info" "eb --check-eb-deps" "eb --show-config" - "eb bzip2-1.0.8.eb --trace --robot" ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" eval "$cmd" done + + - name: End-to-end test of installing bzip2 with EasyBuild + shell: bash + run: | + eb bzip2-1.0.8.eb --trace --robot From e21dfc04225914a9836941a890747cfb26329a76 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 11:29:21 +0200 Subject: [PATCH 090/111] run commands in end2end test workflow via login shell by using 'bash -l -c' --- .github/workflows/end2end.yml | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 9ab4605bd0..1c70d20e8e 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -33,25 +33,13 @@ jobs: - name: Set up environment shell: bash run: | - export PATH=$PWD:$PATH - export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop - - # initialize environment (for Lmod) - if [ -f /etc/profile.d/modules.sh ]; then - # for Rocky, CentOS 8, Fedora - echo ">>> sourcing /etc/profile.d/modules.sh" - . /etc/profile.d/modules.sh - else - echo ">>> sourcing /etc/profile.d/*lmod*.sh" - . /etc/profile.d/*lmod*.sh - fi + # set environment variables to be used in subsequent steps, + # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable + echo "PATH=$PWD:$PATH" >> $GITHUB_ENV + echo "PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> $GITHUB_ENV # tests are run with root privileges, so we need to tell EasyBuild that's OK... - export EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1 - - # save full environment to use in subsequent steps, - # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - while read line; do echo "$line" >> $GITHUB_ENV ; done < <(env) + echo "EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1" >> $GITHUB_ENV - name: Run commands to check test environment shell: bash @@ -67,10 +55,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - eval "$cmd" + bash -l -c "$cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - eb bzip2-1.0.8.eb --trace --robot + bash -l -c "eb bzip2-1.0.8.eb --trace --robot" From 34846525b186856891a49b77446fe26f1b286e3f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 11:46:45 +0200 Subject: [PATCH 091/111] update to end2end test workflow to actions/checkout@v3 --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 1c70d20e8e..5a0f2d768d 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -19,7 +19,7 @@ jobs: image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: download and unpack easyblocks and easyconfigs repositories run: | From a47d43b1e939de49e45f83af73b137b9f3943cec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 11:49:37 +0200 Subject: [PATCH 092/111] set $PATH directory in command being run in login shell --- .github/workflows/end2end.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 5a0f2d768d..7b38978796 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -34,8 +34,8 @@ jobs: shell: bash run: | # set environment variables to be used in subsequent steps, - # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable - echo "PATH=$PWD:$PATH" >> $GITHUB_ENV + # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable; + # not: we can't extend $PATH this way, because it will not always be picked up in login shell used in later steps echo "PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> $GITHUB_ENV # tests are run with root privileges, so we need to tell EasyBuild that's OK... @@ -55,10 +55,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - bash -l -c "$cmd" + bash -l -c "export PATH=$PWD:$PATH; $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - bash -l -c "eb bzip2-1.0.8.eb --trace --robot" + bash -l -c "export PATH=$PWD:$PATH; eb bzip2-1.0.8.eb --trace --robot" From 7ea111ec18f78783c72cca3d88f2d54e6b782e25 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 13:47:28 +0200 Subject: [PATCH 093/111] run commands in end2end test worfklow via 'sudo -u easybuild' --- .github/workflows/end2end.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 7b38978796..1207c3888d 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -35,11 +35,8 @@ jobs: run: | # set environment variables to be used in subsequent steps, # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable; - # not: we can't extend $PATH this way, because it will not always be picked up in login shell used in later steps - echo "PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> $GITHUB_ENV - - # tests are run with root privileges, so we need to tell EasyBuild that's OK... - echo "EASYBUILD_ALLOW_USE_AS_ROOT_AND_ACCEPT_CONSEQUENCES=1" >> $GITHUB_ENV + ENV_CMDS="export PATH=$PWD:$PATH; export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" + echo "ENV_CMDS='${ENV_CMDS}'" >> $GITHUB_ENV - name: Run commands to check test environment shell: bash @@ -55,10 +52,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - bash -l -c "export PATH=$PWD:$PATH; $cmd" + sudo -u easybuild bash -l -c "${ENV_CMDS}; $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - bash -l -c "export PATH=$PWD:$PATH; eb bzip2-1.0.8.eb --trace --robot" + sudo -u easybuild bash -l -c "${ENV_CMDS}; eb bzip2-1.0.8.eb --trace --robot" From fbbc048c965a02dcd64bb711960c3de3e1570509 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 14:25:58 +0200 Subject: [PATCH 094/111] fix updating $PATH and $PYTHONPATH before running command in end2end test workflow --- .github/workflows/end2end.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 1207c3888d..14aa31b9a1 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -35,8 +35,8 @@ jobs: run: | # set environment variables to be used in subsequent steps, # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable; - ENV_CMDS="export PATH=$PWD:$PATH; export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" - echo "ENV_CMDS='${ENV_CMDS}'" >> $GITHUB_ENV + EB_ENV="PATH=$PWD:$PATH PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" + echo "EB_ENV='${EB_ENV}'" >> $GITHUB_ENV - name: Run commands to check test environment shell: bash @@ -52,10 +52,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - sudo -u easybuild bash -l -c "${ENV_CMDS}; $cmd" + sudo -u easybuild bash -l -c "${EB_ENV} $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - sudo -u easybuild bash -l -c "${ENV_CMDS}; eb bzip2-1.0.8.eb --trace --robot" + sudo -u easybuild bash -l -c "${EB_ENV} eb bzip2-1.0.8.eb --trace --robot" From 0d49930a9ab187a1fcdf8674a0357523ee8615fd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 16 Aug 2023 14:54:46 +0200 Subject: [PATCH 095/111] use source script rather than modifying $GITHUB_ENV --- .github/workflows/end2end.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 14aa31b9a1..ba6b02ea8e 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -33,10 +33,9 @@ jobs: - name: Set up environment shell: bash run: | - # set environment variables to be used in subsequent steps, - # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-environment-variable; - EB_ENV="PATH=$PWD:$PATH PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" - echo "EB_ENV='${EB_ENV}'" >> $GITHUB_ENV + # collect environment variables to be set in subsequent steps in script that can be sourced + echo "export PATH=$PWD:$PATH" > /tmp/eb_env + echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env - name: Run commands to check test environment shell: bash @@ -52,10 +51,10 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - sudo -u easybuild bash -l -c "${EB_ENV} $cmd" + sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild shell: bash run: | - sudo -u easybuild bash -l -c "${EB_ENV} eb bzip2-1.0.8.eb --trace --robot" + sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot" From 773f9b97bea7615e29d9350e83aab9c1d1a61c6b Mon Sep 17 00:00:00 2001 From: Rovanion Luckey Date: Wed, 16 Aug 2023 18:54:50 +0200 Subject: [PATCH 096/111] Add support for fine grained Github tokens --- easybuild/tools/github.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index bc3a6b3e27..3bb21a6b95 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2243,15 +2243,16 @@ def install_github_token(github_user, silent=False): def validate_github_token(token, github_user): """ Check GitHub token: - * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40) - * see if it can be used for authenticated access + * see if it conforms expectations (character classes depending on type, length of 40-93), + * see if it can be used for authenticated access. """ # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$') token_regex_old_format = re.compile('^[0-9a-f]{40}$') + # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats + token_regex_fine_grained = re.compile('github_pat_[a-zA-Z0-9_]{82}') - # token should be 40 characters long, and only contain characters in [0-9a-f] - sanity_check = bool(token_regex.match(token)) + sanity_check = bool(token_regex.match(token)) or bool(token_regex_fine_grained.match(token)) if sanity_check: _log.info("Sanity check on token passed") else: From 65efc4abdf00e00f281909a50b14811aa122ac2b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 24 Aug 2023 21:16:48 +0200 Subject: [PATCH 097/111] also test with Rocky Linux 8.8 and 9.2 --- .github/workflows/end2end.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index ba6b02ea8e..7327d876ba 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -11,7 +11,8 @@ jobs: - centos-8.5 - fedora-36 - opensuse-15.4 - - rockylinux-8.7 + - rockylinux-8.8 + - rockylinux-9.2 - ubuntu-20.04 - ubuntu-22.04 fail-fast: false From 481bf95d2e65aed546813d4b35e2d95de5acd934 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Tue, 29 Aug 2023 21:04:06 +0200 Subject: [PATCH 098/111] add definitions for ifbf and iofbf toolchain --- easybuild/toolchains/ifbf.py | 44 ++++++++++++++++++++++++++++++++ easybuild/toolchains/iofbf.py | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 easybuild/toolchains/ifbf.py create mode 100644 easybuild/toolchains/iofbf.py diff --git a/easybuild/toolchains/ifbf.py b/easybuild/toolchains/ifbf.py new file mode 100644 index 0000000000..3507e1eab9 --- /dev/null +++ b/easybuild/toolchains/ifbf.py @@ -0,0 +1,44 @@ +## +# Copyright 2012-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for ifbf toolchain (includes Intel compilers, FlexiBLAS, and FFTW). + +Authors: + +* Sebastian Achilles (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.intel_compilers import IntelCompilersToolchain +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS + + +class Ifbf(IntelCompilersToolchain, FlexiBLAS, Fftw): + """ + Compiler toolchain with Intel compilers, FlexiBLAS, and FFTW + """ + NAME = 'ifbf' + SUBTOOLCHAIN = IntelCompilersToolchain.NAME + OPTIONAL = True diff --git a/easybuild/toolchains/iofbf.py b/easybuild/toolchains/iofbf.py new file mode 100644 index 0000000000..3410ffaf9f --- /dev/null +++ b/easybuild/toolchains/iofbf.py @@ -0,0 +1,47 @@ +## +# Copyright 2012-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iofbf toolchain (includes Intel compilers, OpenMPI, +FlexiBLAS, LAPACK, ScaLAPACK and FFTW). + +Authors: + +* Sebastian Achilles (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.iompi import Iompi +from easybuild.toolchains.ifbf import Ifbf +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Iofbf(Iompi, FlexiBLAS, ScaLAPACK, Fftw): + """ + Compiler toolchain with Intel compilers (icc/ifort), OpenMPI, + FlexiBLAS, LAPACK, ScaLAPACK and FFTW. + """ + NAME = 'iofbf' + SUBTOOLCHAIN = [Iompi.NAME, Ifbf.NAME] From 98402430e2407160c44eb0b44d064c7214c1c831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 11:55:25 +0200 Subject: [PATCH 099/111] Add submodule filtering and extra configuration to git cloning commands --- easybuild/tools/filetools.py | 40 +++++++++++++++++++++++++++--------- test/framework/filetools.py | 38 +++++++++++++++++++++++----------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a5e5f63ec6..74e2c13369 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -65,7 +65,7 @@ from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar -from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type +from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -1918,9 +1918,15 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # climb up until we hit an existing path or the empty string (for relative paths) while existing_parent_path and not os.path.exists(existing_parent_path): existing_parent_path = os.path.dirname(existing_parent_path) - makedirs(path, exist_ok=True) + os.makedirs(path) else: os.mkdir(path) + except FileExistsError as err: + if os.path.exists(path): + # This may happen if a parallel build creates the directory after we checked for its existence + _log.debug("Directory creation aborted as it seems it was already created: %s", err) + else: + raise EasyBuildError("Failed to create directory %s: %s", path, err) except OSError as err: raise EasyBuildError("Failed to create directory %s: %s", path, err) @@ -2619,6 +2625,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config): recursive = git_config.pop('recursive', False) clone_into = git_config.pop('clone_into', False) keep_git_dir = git_config.pop('keep_git_dir', False) + extra_config_params = git_config.pop('extra_config_params', None) + recurse_submodules = git_config.pop('recurse_submodules', None) # input validation of git_config dict if git_config: @@ -2644,7 +2652,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config): targetpath = os.path.join(targetdir, filename) # compose 'git clone' command, and run it - clone_cmd = ['git', 'clone'] + if extra_config_params: + git_cmd = 'git ' + ' '.join([f'-c {param}' for param in extra_config_params]) + else: + git_cmd = 'git' + clone_cmd = [git_cmd, 'clone'] if not keep_git_dir and not commit: # Speed up cloning by only fetching the most recent commit, not the whole history @@ -2655,6 +2667,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.extend(['--branch', tag]) if recursive: clone_cmd.append('--recursive') + if recurse_submodules: + clone_cmd.extend([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) else: # checkout is done separately below for specific commits clone_cmd.append('--no-checkout') @@ -2674,16 +2688,19 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # if a specific commit is asked for, check it out if commit: - checkout_cmd = ['git', 'checkout', commit] + checkout_cmd = [git_cmd, 'checkout', commit] if recursive: - checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) + checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init', '--recursive']) + elif recurse_submodules: + checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) + checkout_cmd.extend([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check - cmd = 'git describe --exact-match --tags HEAD' + cmd = f'{git_cmd} describe --exact-match --tags HEAD' # Note: Disable logging to also disable the error handling in run_cmd (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) if ec != 0 or tag not in out.splitlines(): @@ -2696,13 +2713,16 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # make the repo unshallow first; # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ # (first fetch seems to do nothing, unclear why) - cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append(f'{git_cmd} fetch --depth=2147483647 && git fetch --depth=2147483647') - cmds.append('git checkout refs/tags/' + tag) + cmds.append(f'{git_cmd} checkout refs/tags/' + tag) # Clean all untracked files, e.g. from left-over submodules - cmds.append('git clean --force -d -x') + cmds.append(f'{git_cmd} clean --force -d -x') if recursive: - cmds.append('git submodule update --init --recursive') + cmds.append(f'{git_cmd} submodule update --init --recursive') + elif recurse_submodules: + cmds.append(f'{git_cmd} submodule update --init ') + cmds[-1] += ' '.join([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) for cmd in cmds: run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index b87b9359b0..d8215ce979 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,7 +45,6 @@ from unittest import TextTestRunner from easybuild.tools import run import easybuild.tools.filetools as ft -import easybuild.tools.py2vs3 as py2vs3 from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff @@ -2809,6 +2808,32 @@ def run_check(): ]) % git_repo run_check() + git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] + expected = '\n'.join([ + ' running command "git clone --depth 1 --branch tag_for_tests --recursive' + + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', + r" \(in .*/tmp.*\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", + ]) % git_repo + run_check() + + git_config['extra_config_params'] = [ + 'submodule."fastahack".active=false', + 'submodule."sha1".active=false', + ] + expected = '\n'.join([ + ' running command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + + ' clone --depth 1 --branch tag_for_tests --recursive' + + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', + r" \(in .*/tmp.*\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", + ]) % git_repo + run_check() + del git_config['recurse_submodules'] + del git_config['extra_config_params'] + git_config['keep_git_dir'] = True expected = '\n'.join([ r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', @@ -3390,17 +3415,6 @@ def test_set_gid_sticky_bits(self): self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) - def test_compat_makedirs(self): - """Test compatibility layer for Python3 os.makedirs""" - name = os.path.join(self.test_prefix, 'folder') - self.assertNotExists(name) - py2vs3.makedirs(name) - self.assertExists(name) - # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3) - self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name) - py2vs3.makedirs(name, exist_ok=True) # No error - self.assertExists(name) - def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') From 012fa749c9e7b4401040715aa3de64c7b1d41d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 12:00:28 +0200 Subject: [PATCH 100/111] fix conflicts --- easybuild/tools/filetools.py | 10 ++-------- test/framework/filetools.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 74e2c13369..d6b1c5cf90 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -65,7 +65,7 @@ from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN from easybuild.tools.config import build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar -from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type +from easybuild.tools.py2vs3 import HTMLParser, load_source, makedirs, std_urllib, string_type from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -1918,15 +1918,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # climb up until we hit an existing path or the empty string (for relative paths) while existing_parent_path and not os.path.exists(existing_parent_path): existing_parent_path = os.path.dirname(existing_parent_path) - os.makedirs(path) + makedirs(path, exist_ok=True) else: os.mkdir(path) - except FileExistsError as err: - if os.path.exists(path): - # This may happen if a parallel build creates the directory after we checked for its existence - _log.debug("Directory creation aborted as it seems it was already created: %s", err) - else: - raise EasyBuildError("Failed to create directory %s: %s", path, err) except OSError as err: raise EasyBuildError("Failed to create directory %s: %s", path, err) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index d8215ce979..3921d422e7 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -45,6 +45,7 @@ from unittest import TextTestRunner from easybuild.tools import run import easybuild.tools.filetools as ft +import easybuild.tools.py2vs3 as py2vs3 from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff @@ -3415,6 +3416,17 @@ def test_set_gid_sticky_bits(self): self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + def test_compat_makedirs(self): + """Test compatibility layer for Python3 os.makedirs""" + name = os.path.join(self.test_prefix, 'folder') + self.assertNotExists(name) + py2vs3.makedirs(name) + self.assertExists(name) + # exception is raised because file exists (OSError in Python 2, FileExistsError in Python 3) + self.assertErrorRegex(Exception, '.*', py2vs3.makedirs, name) + py2vs3.makedirs(name, exist_ok=True) # No error + self.assertExists(name) + def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') From b6f3e7278ef79ebd0bdb7d19a10224af5a10988a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 12:52:59 +0200 Subject: [PATCH 101/111] fix string formating syntax --- easybuild/tools/filetools.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d6b1c5cf90..812b52a420 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2647,7 +2647,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # compose 'git clone' command, and run it if extra_config_params: - git_cmd = 'git ' + ' '.join([f'-c {param}' for param in extra_config_params]) + git_cmd = 'git ' + ' '.join(['-c %s' % param for param in extra_config_params]) else: git_cmd = 'git' clone_cmd = [git_cmd, 'clone'] @@ -2662,7 +2662,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): if recursive: clone_cmd.append('--recursive') if recurse_submodules: - clone_cmd.extend([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) + clone_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) else: # checkout is done separately below for specific commits clone_cmd.append('--no-checkout') @@ -2687,14 +2687,14 @@ def get_source_tarball_from_git(filename, targetdir, git_config): checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init', '--recursive']) elif recurse_submodules: checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) - checkout_cmd.extend([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) + checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check - cmd = f'{git_cmd} describe --exact-match --tags HEAD' + cmd = '%s describe --exact-match --tags HEAD' % git_cmd # Note: Disable logging to also disable the error handling in run_cmd (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name) if ec != 0 or tag not in out.splitlines(): @@ -2707,16 +2707,16 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # make the repo unshallow first; # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ # (first fetch seems to do nothing, unclear why) - cmds.append(f'{git_cmd} fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append('%s fetch --depth=2147483647 && git fetch --depth=2147483647' % git_cmd) - cmds.append(f'{git_cmd} checkout refs/tags/' + tag) + cmds.append('%s checkout refs/tags/' % git_cmd + tag) # Clean all untracked files, e.g. from left-over submodules - cmds.append(f'{git_cmd} clean --force -d -x') + cmds.append('%s clean --force -d -x' % git_cmd) if recursive: - cmds.append(f'{git_cmd} submodule update --init --recursive') + cmds.append('%s submodule update --init --recursive' % git_cmd) elif recurse_submodules: - cmds.append(f'{git_cmd} submodule update --init ') - cmds[-1] += ' '.join([f"--recurse-submodules='{pat}'" for pat in recurse_submodules]) + cmds.append('%s submodule update --init ' % git_cmd) + cmds[-1] += ' '.join(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) for cmd in cmds: run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name) From ad3d411e80adb91dfa2bd754aca1dc5eb1f2b80b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 30 Aug 2023 14:40:28 +0200 Subject: [PATCH 102/111] improve docstring for validate_github_token to cover both classic and fine-grained GitHub tokens --- easybuild/tools/github.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 3bb21a6b95..6feb86284d 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2243,7 +2243,8 @@ def install_github_token(github_user, silent=False): def validate_github_token(token, github_user): """ Check GitHub token: - * see if it conforms expectations (character classes depending on type, length of 40-93), + * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters and length of 40 starting with 'ghp_'), + or fine-grained GitHub token with only alphanumeric ([a-zA-Z0-9]) + '_' and length of 93 starting with 'github_pat_'), * see if it can be used for authenticated access. """ # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ From 4ab0b76e656450ef2ed7ee5001e9aae867dfb9af Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 30 Aug 2023 14:42:52 +0200 Subject: [PATCH 103/111] fix excessively long lines in docstring for validate_github_token --- easybuild/tools/github.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 6feb86284d..d6093a9df3 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2243,8 +2243,9 @@ def install_github_token(github_user, silent=False): def validate_github_token(token, github_user): """ Check GitHub token: - * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters and length of 40 starting with 'ghp_'), - or fine-grained GitHub token with only alphanumeric ([a-zA-Z0-9]) + '_' and length of 93 starting with 'github_pat_'), + * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters + and length of 40 starting with 'ghp_', or fine-grained GitHub token with only + alphanumeric ([a-zA-Z0-9]) characters + '_' and length of 93 starting with 'github_pat_'), * see if it can be used for authenticated access. """ # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ From 3e94aafd744851847c976cd145bfbde4a4d24e0a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 30 Aug 2023 14:59:30 +0200 Subject: [PATCH 104/111] enhance test for validate_github_token so testing with fine-grained GitHub token can also be done --- test/framework/github.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/github.py b/test/framework/github.py index 9df53736e4..fb0d59cc49 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -583,6 +583,11 @@ def test_validate_github_token(self): if token_old_format: self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT)) + # if a fine-grained token is available, test with that too + finegrained_token = os.getenv('TEST_GITHUB_TOKEN_FINEGRAINED') + if finegrained_token: + self.assertTrue(gh.validate_github_token(finegrained_token, GITHUB_TEST_ACCOUNT)) + def test_github_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" if self.skip_github_tests: From 4c36bb21e016b5d4f437407f3d056fb093411252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 16:48:46 +0200 Subject: [PATCH 105/111] Fix typo in submodule filtering of --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 812b52a420..26ebb0782e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2685,7 +2685,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): checkout_cmd = [git_cmd, 'checkout', commit] if recursive: checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init', '--recursive']) - elif recurse_submodules: + if recurse_submodules: checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) From 2e94335c55e17514098fda26a52bb8badcfaf661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kr=C3=A1l?= Date: Wed, 30 Aug 2023 17:19:40 +0200 Subject: [PATCH 106/111] fix submodule filtering bug, add test --- easybuild/tools/filetools.py | 10 ++++++---- test/framework/filetools.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 26ebb0782e..3a99978db9 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2683,11 +2683,13 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # if a specific commit is asked for, check it out if commit: checkout_cmd = [git_cmd, 'checkout', commit] - if recursive: - checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init', '--recursive']) - if recurse_submodules: + + if recursive or recurse_submodules: checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) - checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) + if recursive: + checkout_cmd.append('--recursive') + if recurse_submodules: + checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3921d422e7..80a23d83cc 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2857,7 +2857,20 @@ def run_check(): ]) % git_repo run_check() + git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] + expected = '\n'.join([ + r' running command "git clone --no-checkout %(git_repo)s"', + r" \(in .*/tmp.*\)", + ' running command "git checkout 8456f86 && git submodule update --init --recursive' + + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"', + r" \(in testrepository\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", + ]) % git_repo + run_check() + del git_config['recursive'] + del git_config['recurse_submodules'] expected = '\n'.join([ r' running command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", From f725ac265c45fe015b19b894a561a4613994d793 Mon Sep 17 00:00:00 2001 From: Ward Poelmans Date: Wed, 16 Aug 2023 17:32:58 +0200 Subject: [PATCH 107/111] Only add extensions in module file if there are extensions This fixes #4330. --- easybuild/tools/module_generator.py | 3 ++- test/framework/module_generator.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index aaf97d3194..72afdd0896 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -622,7 +622,8 @@ def _generate_extensions_list(self): """ Generate a list of all extensions in name/version format """ - return self.app.make_extension_string(name_version_sep='/', ext_sep=',').split(',') + exts_str = self.app.make_extension_string(name_version_sep='/', ext_sep=',') + return exts_str.split(',') if exts_str else [] def _generate_help_text(self): """ diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index b6bec17093..a0c6131140 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -756,6 +756,16 @@ def test_module_extensions(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(desc), "Pattern '%s' found in: %s" % (regex.pattern, desc)) + # check if the extensions is missing if there are no extensions + test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + + ec = EasyConfig(test_ec) + eb = EasyBlock(ec) + modgen = self.MODULE_GENERATOR_CLASS(eb) + desc = modgen.get_description() + + self.assertFalse(re.search(r"\s*extensions\(", desc), "No extensions found in: %s" % desc) + def test_prepend_paths(self): """Test generating prepend-paths statements.""" # test prepend_paths From 0981458eb701b24f06211dbea5bd60df29923853 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 9 Sep 2023 18:16:10 +0200 Subject: [PATCH 108/111] enhance test_checksum_step to check lack of error message in log when checksums.json is not found --- test/framework/easyblock.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 4f27e86fb2..6afc9db94f 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -39,6 +39,7 @@ from unittest import TextTestRunner import easybuild.tools.systemtools as st +from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import EasyConfig @@ -2424,6 +2425,30 @@ def test_checksum_step(self): eb.fetch_sources() eb.checksum_step() + with self.mocked_stdout_stderr() as (stdout, stderr): + + # using checksum-less test easyconfig in location that does not provide checksums.json + test_ec = os.path.join(self.test_prefix, 'test-no-checksums.eb') + copy_file(toy_ec, test_ec) + write_file(test_ec, 'checksums = []', append=True) + ec = process_easyconfig(test_ec)[0] + + # enable logging to screen, so we can check whether error is logged when checksums.json is not found + fancylogger.logToScreen(enable=True, stdout=True) + + eb = get_easyblock_instance(ec) + eb.fetch_sources() + eb.checksum_step() + + fancylogger.logToScreen(enable=False, stdout=True) + stdout = self.get_stdout() + + # make sure there's no error logged for not finding checksums.json, + # see also https://github.com/easybuilders/easybuild-framework/issues/4301 + regex = re.compile("ERROR .*Couldn't find file checksums.json anywhere", re.M) + regex.search(stdout) + self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in log" % regex.pattern) + # fiddle with checksum to check whether faulty checksum is catched copy_file(toy_ec, self.test_prefix) toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) From f60d2c6bd01e45facf01305d53444333194512f9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 9 Sep 2023 19:07:41 +0200 Subject: [PATCH 109/111] improve error message produced by verify_checksum --- easybuild/tools/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index e66a532c10..6e4bd2d83e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1298,8 +1298,8 @@ def verify_checksum(path, checksums): # no matching checksums return False else: - raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5 or SHA256) " - "2-tuple (type, value) or tuple of alternative checksum specs.", + raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), " + "2-tuple (type, value), or tuple of alternative checksum specs.", checksum) actual_checksum = compute_checksum(path, typ) From 22273a2df0503fb466184bbfb95bad4523bc9d69 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Sat, 9 Sep 2023 22:53:43 +0200 Subject: [PATCH 110/111] prepare release notes for EasyBuild v4.8.1 + bump version to 4.8.1 --- RELEASE_NOTES | 30 ++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8f5ec0f1e0..5cd325dd66 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,36 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.8.1 (11 September 2023) +-------------------------- + +update/bugfix release + +- various enhancements, including: + - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968) + - suggest default title in `--review-pr` (#4287) + - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304) + - Implement hooks for failure scenarios: `crash_hook`, `cancel_hook`, `fail_hook` (#4315) + - add postiter hook to the list of steps so the corresponding hook can be used (#4316) + - add `run_shell_cmd` hook (#4323) + - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324) + - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329) + - add support for using fine grained Github tokens (#4332) + - add definitions for ifbf and iofbf toolchain (#4337) + - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338) +- various bug fixes, including: + - Improve error when checksum dict has no entry for a file (#4150) + - don't fail in `mkdir` if path gets created while processing it (#4300) + - Ignore request for external module (meta)data when no modules tool is active (#4308) + - use sys.executable to obtain path to `python` command, rather than assuming that `python` command is available in `$PATH` (#4309) + - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312) + - enhance `Toolchain.get_flag` to handle lists (#4319) + - remove crash hook (since it doesn't work) (#4325) + - don't use `FileExistsError` in `mkdir` function (doesn't exist in Python 2.7) (#4328) + - only add extensions in module file if there are extensions (#4331) + - Fix submodule filtering bug in `git_config` (#4339) + + v4.8.0 (7 July 2023) -------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 61f8473a0e..e0016710ae 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.1.dev0') +VERSION = LooseVersion('4.8.1') UNKNOWN = 'UNKNOWN' From 6e05649c242e5bf0750885e7b924d4debdd2d74c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 10 Sep 2023 10:57:26 +0200 Subject: [PATCH 111/111] minor tweaks to 4.8.1 release notes --- RELEASE_NOTES | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 5cd325dd66..5bd07d4306 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -13,25 +13,23 @@ update/bugfix release - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968) - suggest default title in `--review-pr` (#4287) - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304) - - Implement hooks for failure scenarios: `crash_hook`, `cancel_hook`, `fail_hook` (#4315) + - implement support for `cancel_hook` and `fail_hook` (#4315, #4325) - add postiter hook to the list of steps so the corresponding hook can be used (#4316) - add `run_shell_cmd` hook (#4323) - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324) - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329) - add support for using fine grained Github tokens (#4332) - add definitions for ifbf and iofbf toolchain (#4337) - - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338) + - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339) - various bug fixes, including: - - Improve error when checksum dict has no entry for a file (#4150) - - don't fail in `mkdir` if path gets created while processing it (#4300) - - Ignore request for external module (meta)data when no modules tool is active (#4308) - - use sys.executable to obtain path to `python` command, rather than assuming that `python` command is available in `$PATH` (#4309) + - improve error when checksum dict has no entry for a file (#4150) + - avoid error being logged when `checksums.json` is not found (#4261) + - don't fail in `mkdir` if path gets created while processing it (#4300, #4328) + - ignore request for external module (meta)data when no modules tool is active (#4308) + - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309) - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312) - enhance `Toolchain.get_flag` to handle lists (#4319) - - remove crash hook (since it doesn't work) (#4325) - - don't use `FileExistsError` in `mkdir` function (doesn't exist in Python 2.7) (#4328) - only add extensions in module file if there are extensions (#4331) - - Fix submodule filtering bug in `git_config` (#4339) v4.8.0 (7 July 2023)