Merge pull request #4342 from easybuilders/4.8.x

release EasyBuild v4.8.1

.github/workflows/end2end.yml

@@ -0,0 +1,61 @@
+name: End-to-end test of EasyBuild in different distros
+on: [push, pull_request]
+jobs:
+  build_publish:
+    name: End-to-end test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        container:
+          - centos-7.9
+          - centos-8.5
+          - fedora-36
+          - opensuse-15.4
+          - rockylinux-8.8
+          - rockylinux-9.2
+          - ubuntu-20.04
+          - ubuntu-22.04
+      fail-fast: false
+    container:
+      image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: download and unpack easyblocks and easyconfigs repositories
+        run: |
+          cd $HOME
+          for pkg in easyblocks easyconfigs; do
+              curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz
+              tar xfz develop.tar.gz
+              rm -f develop.tar.gz
+          done
+
+      - name: Set up environment
+        shell: bash
+        run: |
+          # collect environment variables to be set in subsequent steps in script that can be sourced
+          echo "export PATH=$PWD:$PATH" > /tmp/eb_env
+          echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env
+
+      - name: Run commands to check test environment
+        shell: bash
+        run: |
+          cmds=(
+            "whoami"
+            "pwd"
+            "env | sort"
+            "eb --version"
+            "eb --show-system-info"
+            "eb --check-eb-deps"
+            "eb --show-config"
+          )
+          for cmd in "${cmds[@]}"; do
+              echo ">>> $cmd"
+              sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd"
+          done
+
+      - name: End-to-end test of installing bzip2 with EasyBuild
+        shell: bash
+        run: |
+          sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"

RELEASE_NOTES

@@ -4,6 +4,34 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.8.1 (11 September 2023)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968)
+  - suggest default title in `--review-pr` (#4287)
+  - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304)
+  - implement support for `cancel_hook` and `fail_hook` (#4315, #4325)
+  - add postiter hook to the list of steps so the corresponding hook can be used (#4316)
+  - add `run_shell_cmd` hook (#4323)
+  - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324)
+  - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329)
+  - add support for using fine grained Github tokens (#4332)
+  - add definitions for ifbf and iofbf toolchain (#4337)
+  - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339)
+- various bug fixes, including:
+  - improve error when checksum dict has no entry for a file (#4150)
+  - avoid error being logged when `checksums.json` is not found (#4261)
+  - don't fail in `mkdir` if path gets created while processing it (#4300, #4328)
+  - ignore request for external module (meta)data when no modules tool is active (#4308)
+  - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309)
+  - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312)
+  - enhance `Toolchain.get_flag` to handle lists (#4319)
+  - only add extensions in module file if there are extensions (#4331)
+
+
 v4.8.0 (7 July 2023)
 --------------------
 

easybuild/framework/easyblock.py

@@ -395,13 +395,13 @@ def get_checksums_from_json(self, always_read=False):
         :param always_read: always read the checksums.json file, even if it has been read before
         """
         if always_read or self.json_checksums is None:
-            try:
-                path = self.obtain_file("checksums.json", no_download=True)
+            path = self.obtain_file("checksums.json", no_download=True, warning_only=True)
+            if path is not None:
                 self.log.info("Loading checksums from file %s", path)
                 json_txt = read_file(path)
                 self.json_checksums = json.loads(json_txt)
-            # if the file can't be found, return an empty dict
-            except EasyBuildError:
+            else:
+                # if the file can't be found, return an empty dict
                 self.json_checksums = {}
 
         return self.json_checksums
@@ -736,7 +736,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None, no_download=False, download_instructions=None, alt_location=None):
+                    git_config=None, no_download=False, download_instructions=None, alt_location=None,
+                    warning_only=False):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -789,7 +790,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     return fullpath
 
             except IOError as err:
-                raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
+                if not warning_only:
+                    raise EasyBuildError("Downloading file %s "
+                                         "from url %s to %s failed: %s", filename, url, fullpath, err)
+                else:
+                    self.log.warning("Downloading file %s "
+                                     "from url %s to %s failed: %s", filename, url, fullpath, err)
+                    return None
 
         else:
             # try and find file in various locations
@@ -866,8 +873,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     self.dry_run_msg("  * %s (MISSING)", filename)
                     return filename
                 else:
-                    raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                    if not warning_only:
+                        raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                                             "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                    else:
+                        self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... "
                                          "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                        return None
             elif git_config:
                 return get_source_tarball_from_git(filename, targetdir, git_config)
             else:
@@ -959,7 +971,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         error_msg += "and downloading it didn't work either... "
                         error_msg += "Paths attempted (in order): %s " % failedpaths_msg
 
-                    raise EasyBuildError(error_msg, filename)
+                    if not warning_only:
+                        raise EasyBuildError(error_msg, filename)
+                    else:
+                        self.log.warning(error_msg, filename)
+                        return None
 
     #
     # GETTER/SETTER UTILITY FUNCTIONS
@@ -4174,6 +4190,10 @@ def build_and_install_one(ecdict, init_env):
         dry_run_msg('', silent=silent)
     print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
 
+    if ecdict['ec']['build_info_msg']:
+        msg = "This easyconfig provides the following build information:\n\n%s\n"
+        print_msg(msg % ecdict['ec']['build_info_msg'], log=_log, silent=silent)
+
     if dry_run:
         # print note on interpreting dry run output (argument is reference to location of dry run messages)
         print_dry_run_note('below', silent=silent)

easybuild/framework/easyconfig/default.py

@@ -228,6 +228,8 @@
     'buildstats': [None, "A list of dicts with build statistics", OTHER],
     'deprecated': [False, "String specifying reason why this easyconfig file is deprecated "
                           "and will be archived in the next major release of EasyBuild", OTHER],
+    'build_info_msg': [None, "String with information to be printed to stdout and logged during the building "
+                             "of the easyconfig", OTHER],
 }
 
 

easybuild/framework/easyconfig/easyconfig.py

@@ -73,7 +73,7 @@
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
-from easybuild.tools.modules import modules_tool
+from easybuild.tools.modules import modules_tool, NoModulesTool
 from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
 from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
@@ -1306,6 +1306,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None):
         :param existing_metadata: already available metadata for this external module (if any)
         """
         res = {}
+        if isinstance(self.modules_tool, NoModulesTool):
+            self.log.debug('Ignoring request for external module data for %s as no modules tool is active', mod_name)
+            return res
 
         if existing_metadata is None:
             existing_metadata = {}

easybuild/framework/easyconfig/tools.py

@@ -60,7 +60,7 @@
 from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
 from easybuild.tools.filetools import read_file, resolve_path, which, write_file
 from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
-from easybuild.tools.github import det_pr_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data
+from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data
 from easybuild.tools.github import fetch_files_from_pr
 from easybuild.tools.multidiff import multidiff
 from easybuild.tools.py2vs3 import OrderedDict
@@ -561,6 +561,11 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False
             lines.extend(['', "This PR is associated with a generic '.x' milestone, "
                               "it should be associated to the next release milestone once merged"])
 
+        default_new_title = det_pr_title([ec['ec'] for ec in ecs])
+        if default_new_title != pr_data['title']:
+            lines.extend(['', "If this PR contains only new easyconfigs and has not been edited from the default, "
+                              "then the title should be: %s" % default_new_title])
+
     return '\n'.join(lines)
 
 

easybuild/main.py

@@ -69,7 +69,8 @@
 from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr
 from easybuild.tools.github import new_pr_from_branch
 from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr
-from easybuild.tools.hooks import START, END, load_hooks, run_hook
+from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, FAIL
+from easybuild.tools.hooks import load_hooks, run_hook
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color
 from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm
@@ -545,8 +546,10 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
 
         with rich_live_cm():
+            run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs])
             ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
                                                       exit_on_failure=exit_on_failure)
+            run_hook(POST_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ecs_with_res])
     else:
         ecs_with_res = [(ec, {}) for ec in ordered_ecs]
 
@@ -577,30 +580,20 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
     return overall_success
 
 
-def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
+def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None):
     """
     Main function: parse command line options, and act accordingly.
     :param args: command line arguments to use
     :param logfile: log file to use
     :param do_build: whether or not to actually perform the build
     :param testing: enable testing mode
+    :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None)
     """
+    if prepared_cfg_data is None or any([args, logfile, testing]):
+        init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing)
+    else:
+        init_session_state, eb_go, cfg_settings = prepared_cfg_data
 
-    register_lock_cleanup_signal_handlers()
-
-    # if $CDPATH is set, unset it, it'll only cause trouble...
-    # see https://github.com/easybuilders/easybuild-framework/issues/2944
-    if 'CDPATH' in os.environ:
-        del os.environ['CDPATH']
-
-    # When EB is run via `exec` the special bash variable $_ is not set
-    # So emulate this here to allow (module) scripts depending on that to work
-    if '_' not in os.environ:
-        os.environ['_'] = sys.executable
-
-    # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
-    init_session_state = session_state()
-    eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
     options, orig_paths = eb_go.options, eb_go.args
 
     if 'python2' not in build_option('silence_deprecation_warnings'):
@@ -729,10 +722,41 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         cleanup(logfile, eb_tmpdir, testing, silent=False)
 
 
+def prepare_main(args=None, logfile=None, testing=None):
+    """
+    Prepare for calling main function by setting up the EasyBuild configuration
+    :param args: command line arguments to take into account when parsing the EasyBuild configuration settings
+    :param logfile: log file to use
+    :param testing: enable testing mode
+    :return: 3-tuple with initial session state data, EasyBuildOptions instance, and tuple with configuration settings
+    """
+    register_lock_cleanup_signal_handlers()
+
+    # if $CDPATH is set, unset it, it'll only cause trouble...
+    # see https://github.com/easybuilders/easybuild-framework/issues/2944
+    if 'CDPATH' in os.environ:
+        del os.environ['CDPATH']
+
+    # When EB is run via `exec` the special bash variable $_ is not set
+    # So emulate this here to allow (module) scripts depending on that to work
+    if '_' not in os.environ:
+        os.environ['_'] = sys.executable
+
+    # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
+    init_session_state = session_state()
+    eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
+
+    return init_session_state, eb_go, cfg_settings
+
+
 if __name__ == "__main__":
+    init_session_state, eb_go, cfg_settings = prepare_main()
+    hooks = load_hooks(eb_go.options.hooks)
     try:
-        main()
+        main(prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
     except EasyBuildError as err:
+        run_hook(FAIL, hooks, args=[err])
         print_error(err.msg)
     except KeyboardInterrupt as err:
+        run_hook(CANCEL, hooks, args=[err])
         print_error("Cancelled by user: %s" % err)

easybuild/toolchains/ifbf.py

@@ -0,0 +1,44 @@
+##
+# Copyright 2012-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for ifbf toolchain (includes Intel compilers, FlexiBLAS, and FFTW).
+
+Authors:
+
+* Sebastian Achilles (Juelich Supercomputing Centre)
+"""
+
+from easybuild.toolchains.intel_compilers import IntelCompilersToolchain
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+
+
+class Ifbf(IntelCompilersToolchain, FlexiBLAS, Fftw):
+    """
+    Compiler toolchain with Intel compilers, FlexiBLAS, and FFTW
+    """
+    NAME = 'ifbf'
+    SUBTOOLCHAIN = IntelCompilersToolchain.NAME
+    OPTIONAL = True