diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 06ca1dddb5..973f83e1f5 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -42,11 +42,13 @@ """ import datetime import difflib +import filecmp import glob import hashlib import inspect import itertools import os +import platform import re import shutil import signal @@ -59,7 +61,7 @@ from functools import partial from easybuild.base import fancylogger -from easybuild.tools import run +from easybuild.tools import LooseVersion, run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN @@ -2435,8 +2437,42 @@ def copy_file(path, target_path, force_in_dry_run=False): else: mkdir(os.path.dirname(target_path), parents=True) if path_exists: - shutil.copy2(path, target_path) - _log.info("%s copied to %s", path, target_path) + try: + # on filesystems that support extended file attributes, copying read-only files with + # shutil.copy2() will give a PermissionError, when using Python < 3.7 + # see https://bugs.python.org/issue24538 + shutil.copy2(path, target_path) + _log.info("%s copied to %s", path, target_path) + # catch the more general OSError instead of PermissionError, + # since Python 2.7 doesn't support PermissionError + except OSError as err: + # if file is writable (not read-only), then we give up since it's not a simple permission error + if os.path.exists(target_path) and os.stat(target_path).st_mode & stat.S_IWUSR: + raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) + + pyver = LooseVersion(platform.python_version()) + if pyver >= LooseVersion('3.7'): + raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) + elif LooseVersion('3.7') > pyver >= LooseVersion('3'): + if not isinstance(err, PermissionError): + raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) + + # double-check whether the copy actually succeeded + if not os.path.exists(target_path) or not filecmp.cmp(path, target_path, shallow=False): + raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) + + try: + # re-enable user write permissions in target, copy xattrs, then remove write perms again + adjust_permissions(target_path, stat.S_IWUSR) + shutil._copyxattr(path, target_path) + adjust_permissions(target_path, stat.S_IWUSR, add=False) + except OSError as err: + raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) + + msg = ("Failed to copy extended attributes from file %s to %s, due to a bug in shutil (see " + "https://bugs.python.org/issue24538). Copy successful with workaround.") + _log.info(msg, path, target_path) + elif os.path.islink(path): if os.path.isdir(target_path): target_path = os.path.join(target_path, os.path.basename(path)) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 1a0294a02f..f6dd6fd7ac 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -32,6 +32,7 @@ @author: Maxime Boissonneault (Compute Canada, Universite Laval) """ import datetime +import filecmp import glob import logging import os @@ -51,6 +52,8 @@ from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import StringIO, std_urllib +from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import LINUX, get_os_type class FileToolsTest(EnhancedTestCase): @@ -1912,6 +1915,47 @@ def test_copy_file(self): # However, if we add 'force_in_dry_run=True' it should throw an exception self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target, force_in_dry_run=True) + def test_copy_file_xattr(self): + """Test copying a file with extended attributes using copy_file.""" + # test copying a read-only files with extended attributes set + # first, create a special file with extended attributes + special_file = os.path.join(self.test_prefix, 'special.txt') + ft.write_file(special_file, 'special') + # make read-only, and set extended attributes + attr = ft.which('attr') + xattr = ft.which('xattr') + # try to attr (Linux) or xattr (macOS) to set extended attributes foo=bar + cmd = None + if attr: + cmd = "attr -s foo -V bar %s" % special_file + elif xattr: + cmd = "xattr -w foo bar %s" % special_file + + if cmd: + (_, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False) + + # need to make file read-only after setting extended attribute + ft.adjust_permissions(special_file, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False) + + # only proceed if setting extended attribute worked + if ec == 0: + target = os.path.join(self.test_prefix, 'copy.txt') + ft.copy_file(special_file, target) + self.assertTrue(os.path.exists(target)) + self.assertTrue(filecmp.cmp(special_file, target, shallow=False)) + + # only verify wheter extended attributes were also copied on Linux, + # since shutil.copy2 doesn't copy them on macOS; + # see warning at https://docs.python.org/3/library/shutil.html + if get_os_type() == LINUX: + if attr: + cmd = "attr -g foo %s" % target + else: + cmd = "xattr -l %s" % target + (out, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False) + self.assertEqual(ec, 0) + self.assertTrue(out.endswith('\nbar\n')) + def test_copy_files(self): """Test copy_files function.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')