Copying special files before tar

PILOTVERSION

@@ -1 +1 @@
-3.6.10.30
+3.6.10.31

pilot/control/data.py

@@ -70,7 +70,8 @@
     remove,
     write_file,
     copy,
-    get_directory_size
+    get_directory_size,
+    find_files_with_pattern
 )
 from pilot.util.processes import threads_aborted
 from pilot.util.queuehandling import (
@@ -901,19 +902,20 @@ def copy_special_files(tardir):
     :param tardir: path to tar directory (str).
     """
 
+    # general pattern, typically xrdlog.txt. The pilot might produce multiple files, xrdlog.txt-LFN1..N
     xrd_logfile = os.environ.get('XRD_LOGFILE', None)
     if xrd_logfile:
         # xrootd is then expected to have produced a corresponding log file
         pilot_home = os.environ.get('PILOT_HOME', None)
         if pilot_home:
-            path = os.path.join(pilot_home, xrd_logfile)
-            if os.path.exists(path):
+            # find all log files
+            matching_files = find_files_with_pattern(pilot_home, f'{xrd_logfile}*')
+            for logfile in matching_files:
+                path = os.path.join(pilot_home, logfile)
                 try:
                     copy(path, tardir)
                 except (NoSuchFile, FileHandlingFailure) as exc:
-                    logger.warning(f'caught exception when copying {xrd_logfile}: {exc}')
-            else:
-                logger.warning(f'could not find the expected {xrd_logfile} in {pilot_home}')
+                    logger.warning(f'caught exception when copying {logfile}: {exc}')
         else:
             logger.warning(f'cannot look for {xrd_logfile} since PILOT_HOME was not set')
 

pilot/copytool/common.py

@@ -239,3 +239,27 @@ def resolve_common_transfer_errors(output: str, is_stagein: bool = True) -> dict
 
     # reg exp the output to get real error message
     return output_line_scan(ret, output)
+
+
+def rename_xrdlog(lfn):
+    """
+    In case an xroot client logfile was created, rename it.
+
+    :param lfn: local file name (str).
+    """
+
+    xrd_logfile = os.environ.get('XRD_LOGFILE', None)
+    if xrd_logfile:
+        # xrootd is then expected to have produced a corresponding log file
+        pilot_home = os.environ.get('PILOT_HOME', None)
+        if pilot_home:
+            path = os.path.join(pilot_home, xrd_logfile)
+            if os.path.exists(path):
+                try:
+                    os.rename(path, f'{path}-{lfn}')
+                except Exception as exc:
+                    logger.warning(f'exception caught while renaming file: {exc}')
+            else:
+                logger.warning(f'did not find the expected {xrd_logfile} in {pilot_home}')
+        else:
+            logger.warning(f'cannot look for {xrd_logfile} since PILOT_HOME was not set')

pilot/copytool/rucio.py

@@ -44,6 +44,7 @@
     resolve_common_transfer_errors,
     verify_catalog_checksum,
     get_timeout,
+    rename_xrdlog
 )
 from pilot.util.config import config
 
@@ -119,6 +120,7 @@ def copy_in(files, **kwargs):
         else:
             protocol = get_protocol(trace_report_out)
             trace_report.update(protocol=protocol)
+            rename_xrdlog(fspec.lfn)
 
         # make sure there was no missed failure (only way to deal with this until rucio API has been fixed)
         # (using the timeout decorator prevents the trace_report_out from being updated - rucio API should return
@@ -401,6 +403,7 @@ def copy_out(files, **kwargs):  # noqa: C901
         else:
             protocol = get_protocol(trace_report_out)
             trace_report.update(protocol=protocol)
+            rename_xrdlog(fspec.lfn)
 
         # make sure there was no missed failure (only way to deal with this until rucio API has been fixed)
         # (using the timeout decorator prevents the trace_report_out from being updated - rucio API should return

pilot/util/constants.py

@@ -26,7 +26,7 @@
 RELEASE = '3'   # released number should be fixed at 3 for Pilot 3
 VERSION = '6'   # version number is '1' for first release, '0' until then, increased for bigger updates
 REVISION = '10'  # revision number should be reset to '0' for every new version release, increased for small updates
-BUILD = '30'     # build number should be reset to '1' for every new development cycle
+BUILD = '31'     # build number should be reset to '1' for every new development cycle
 
 SUCCESS = 0
 FAILURE = 1

pilot/util/filehandling.py

@@ -19,6 +19,7 @@
 # Authors:
 # - Paul Nilsson, [email protected], 2017-23
 
+import fnmatch
 import hashlib
 import io
 import logging
@@ -1339,3 +1340,23 @@ def append_to_file(from_file: str, to_file: str) -> bool:
         logger.warning(f"an error occurred while processing the file: {exc}")
 
     return status
+
+
+def find_files_with_pattern(directory, pattern):
+    """
+    Find files in a directory that match a specified pattern.
+
+    :param directory: The directory to search for files (str)
+    :param pattern: The pattern to match filenames (str)
+    :return: a list of matching filenames found in the directory (list).
+    """
+
+    try:
+        if not os.path.exists(directory):
+            raise FileNotFoundError(f"directory '{directory}' does not exist")
+
+        # return all matching files
+        return [f for f in os.listdir(directory) if fnmatch.fnmatch(f, pattern)]
+    except (FileNotFoundError, PermissionError) as exc:
+        logger.warning(f"exception caught while finding files: {exc}")
+        return []