diff --git a/pilot/copytool/common.py b/pilot/copytool/common.py index 3646a5f6..9233a7b8 100644 --- a/pilot/copytool/common.py +++ b/pilot/copytool/common.py @@ -21,6 +21,8 @@ # - Paul Nilsson, paul.nilsson@cern.ch, 2017-2023 # - Mario Lassnig, mario.lassnig@cern.ch, 2020 +"""Commnon functions for copytools.""" + import logging import os import re @@ -40,11 +42,10 @@ def get_timeout(filesize: int, add: int = 0) -> int: """ Get a proper time-out limit based on the file size. - :param filesize: file size (int). + :param filesize: file size (int) :param add: optional additional time to be added [s] (int) :return: time-out in seconds (int). """ - timeout_max = 3 * 3600 # 3 hours timeout_min = 300 # self.timeout @@ -56,13 +57,13 @@ def get_timeout(filesize: int, add: int = 0) -> int: def verify_catalog_checksum(fspec: Any, path: str) -> (str, str): """ Verify that the local and remote (fspec) checksum values are the same. + The function will update the fspec object. - :param fspec: FileSpec object for a given file. - :param path: path to local file (string). - :return: state (string), diagnostics (string). + :param fspec: FileSpec object for a given file (Any) + :param path: path to local file (str) + :return: state (str), diagnostics (str). """ - diagnostics = "" state = "" @@ -104,10 +105,10 @@ def verify_catalog_checksum(fspec: Any, path: str) -> (str, str): def merge_destinations(files: list) -> dict: """ - Converts the file-with-destination dict to a destination-with-files dict + Convert the file-with-destination dict to a destination-with-files dictionary. :param files: files to merge (list) - :return: destination-with-files dictionary. + :return: destination-with-files dictionary (dict). """ destinations = {} # ensure type(files) == list @@ -132,9 +133,9 @@ def get_copysetup(copytools: list, copytool_name: str) -> str: """ Return the copysetup for the given copytool. - :param copytools: copytools list from infosys. - :param copytool_name: name of copytool (string). - :return: copysetup (string). + :param copytools: copytools list from infosys (list) + :param copytool_name: name of copytool (str) + :return: copysetup (str). """ copysetup = "" @@ -152,27 +153,27 @@ def get_copysetup(copytools: list, copytool_name: str) -> str: def get_error_info(rcode: int, state: str, error_msg: str) -> dict: """ Return an error info dictionary specific to transfer errors. + Helper function to resolve_common_transfer_errors(). - :param rcode: return code (int). - :param state: state string used in Rucio traces. - :param error_msg: transfer command stdout (string). - :return: dictionary with format {'rcode': rcode, 'state': state, 'error': error_msg}. + :param rcode: return code (int) + :param state: state string used in Rucio traces (str) + :param error_msg: transfer command stdout (str) + :return: dictionary with format {'rcode': rcode, 'state': state, 'error': error_msg} (dict). """ - return {'rcode': rcode, 'state': state, 'error': error_msg} def output_line_scan(ret: dict, output: str) -> dict: """ Do some reg exp on the transfer command output to search for special errors. + Helper function to resolve_common_transfer_errors(). - :param ret: pre-filled error info dictionary with format {'rcode': rcode, 'state': state, 'error': error_msg} - :param output: transfer command stdout (string). - :return: updated error info dictionary. + :param ret: pre-filled error info dictionary with format {'rcode': rcode, 'state': state, 'error': error_msg} (dict) + :param output: transfer command stdout (str) + :return: updated error info dictionary (dict). """ - for line in output.split('\n'): match = re.search(r"[Dd]etails\s*:\s*(?P.*)", line) # Python 3 (added r) if match: @@ -188,11 +189,10 @@ def resolve_common_transfer_errors(output: str, is_stagein: bool = True) -> dict """ Resolve any common transfer related errors. - :param output: stdout from transfer command (string). - :param is_stagein: optional (boolean). - :return: dict {'rcode': rcode, 'state': state, 'error': error_msg}. + :param output: stdout from transfer command (str) + :param is_stagein: optional (bool) + :return: dict {'rcode': rcode, 'state': state, 'error': error_msg} (dict). """ - # default to make sure dictionary exists and all fields are populated (some of which might be overwritten below) ret = get_error_info(ErrorCodes.STAGEINFAILED if is_stagein else ErrorCodes.STAGEOUTFAILED, 'COPY_ERROR', output) if not output: