Protocols: Deprecate GridFTP; fix rucio#7347

etc/docker/test/extra/rucio_autotests_common.cfg

@@ -41,7 +41,7 @@ carbon_port = 8125
 user_scope = travis
 
 [conveyor]
-scheme = srm,root,davs,gsiftp,http,https,mock,file,magnet
+scheme = srm,root,davs,http,https,mock,file,magnet
 transfertool = fts3
 ftshosts = https://fts:8446
 cacert = /opt/rucio/etc/rucio_ca.pem

etc/docker/test/extra/rucio_default.cfg

@@ -48,7 +48,7 @@ carbon_port = 8125
 user_scope = docker
 
 [conveyor]
-scheme = https,davs,gsiftp,root,srm,mock,file,magnet
+scheme = https,davs,root,srm,mock,file,magnet
 #scheme = https
 #user_transfers = cms
 #user_activities = ['dummy_user_activity']

etc/rucio.cfg.template

@@ -114,7 +114,7 @@ enable_metrics = False
 metrics_port = 8080
 
 [conveyor]
-scheme = srm,gsiftp,root,http,https
+scheme = srm,root,http,https
 transfertool = fts3
 ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
 cacert = /opt/rucio/etc/web/ca.crt

etc/rucio_multi_vo.cfg.template

@@ -95,7 +95,7 @@ enable_metrics = False
 metrics_port = 8080
 
 [conveyor]
-scheme = srm,gsiftp,root,http,https
+scheme = srm,root,http,https
 transfertool = fts3
 ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
 cacert = /opt/rucio/etc/web/ca.crt

lib/rucio/client/commands/bin_legacy/rucio.py

@@ -1757,7 +1757,7 @@ def list_rse_usage(args, client, logger, console, spinner):
         spinner.start()
 
     all_usages = client.get_rse_usage(rse=args.rse, filters={'per_account': args.show_accounts})
-    select_usages = [u for u in all_usages if u['source'] not in ('srm', 'gsiftp', 'webdav')]
+    select_usages = [u for u in all_usages if u['source'] not in ('srm', 'webdav')]
 
     if cli_config == 'rich':
         output = []

lib/rucio/client/commands/bin_legacy/rucio_admin.py

@@ -1972,7 +1972,7 @@ def get_parser():
                                                               '"""""""""""""\n'
                                                               '::\n'
                                                               '\n'
-                                                              '    $ rucio-admin rse add-protocol --hostname jdoes.test.org --scheme gsiftp --prefix \'/atlasdatadisk/rucio/\' --port 8443 JDOE_DATADISK\n'
+                                                              '    $ rucio-admin rse add-protocol --hostname jdoes.test.org --scheme davs --prefix \'/atlasdatadisk/rucio/\' --port 8443 JDOE_DATADISK\n'
                                                               '\n'
                                                               'Note: no printed stdout.\n'
                                                               'Note: examples of optional parameters::\n'
@@ -2006,7 +2006,7 @@ def get_parser():
                                                               '"""""""""""""\n'
                                                               '::\n'
                                                               '\n'
-                                                              '   $ rucio-admin rse delete-protocol  --scheme gsiftp JDOE_DATADISK\n'
+                                                              '   $ rucio-admin rse delete-protocol  --scheme davs JDOE_DATADISK\n'
                                                               '\n'
                                                               'Note: no printed stdout.\n'
                                                               '\n')

lib/rucio/client/commands/rse.py

@@ -204,8 +204,8 @@ def namespace(self, parser: "ArgumentParser") -> None:
 
     def usage_example(self) -> list[str]:
         return [
-            "$ rucio rse protocol --hostname jdoes.test.org --scheme gsiftp --prefix '/atlasdatadisk/rucio/' --port 8443 --rse JDOE_DATADISK  # Add a new protocol on jdoe.test.org that uses gsiftp",
-            "$ rucio rse protocol remove --scheme gsiftp --rse JDOE_DATADISK # Remove the existing gsiftp protocol",
+            "$ rucio rse protocol --hostname jdoes.test.org --scheme davs --prefix '/atlasdatadisk/rucio/' --port 8443 --rse JDOE_DATADISK  # Add a new protocol on jdoe.test.org that uses davs",
+            "$ rucio rse protocol remove --scheme davs --rse JDOE_DATADISK # Remove the existing davs protocol",
         ]
 
     def add(self):

lib/rucio/client/rseclient.py

@@ -299,7 +299,7 @@ def lfns2pfns(
         :param protocol_domain: The scope of the protocol. Supported are 'LAN', 'WAN', and 'ALL' (as default).
         :param operation: The name of the requested operation (read, write, or delete).
                           If None, all operations are queried.
-        :param scheme: The identifier of the requested protocol (gsiftp, https, davs, etc).
+        :param scheme: The identifier of the requested protocol (https, davs, etc).
 
         :returns: A dictionary of LFN / PFN pairs.
         :raises RSENotFound: if the RSE doesn't exist.

lib/rucio/common/constants.py

@@ -32,8 +32,7 @@
 KEY_TYPES = ['ALL', 'COLLECTION', 'FILE', 'DERIVED']
 # all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection)
 
-SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
-              'gsiftp': ['srm', 'gsiftp'],
+SCHEME_MAP = {'srm': ['srm'],
               'https': ['https', 'davs', 'srm+https', 'cs3s'],
               'davs': ['https', 'davs', 'srm+https', 'cs3s'],
               'srm+https': ['https', 'davs', 'srm+https', 'cs3s'],
@@ -51,7 +50,7 @@
 SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'custom_table', 'random']
 SORTING_ALGORITHMS = list(get_args(SORTING_ALGORITHMS_LITERAL))
 
-SUPPORTED_PROTOCOLS_LITERAL = Literal['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone', 'magnet']
+SUPPORTED_PROTOCOLS_LITERAL = Literal['srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone', 'magnet']
 SUPPORTED_PROTOCOLS: list[str] = list(get_args(SUPPORTED_PROTOCOLS_LITERAL))
 
 RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL = Literal['ALL', 'LAN', 'WAN']

lib/rucio/daemons/auditor/srmdumps.py

@@ -174,10 +174,6 @@ def http_links(base_url: str) -> list[str]:
         'links': gfal_links,
         'download': gfal_download_to_file,
     },
-    'gsiftp': {
-        'links': gfal_links,
-        'download': gfal_download_to_file,
-    },
     'root': {
         'links': gfal_links,
         'download': gfal_download_to_file,

lib/rucio/rse/protocols/gfal.py

@@ -183,7 +183,7 @@ def connect(self):
             gfal2.set_verbose(gfal2.verbose_level.warning)
 
         self.__ctx = gfal2.creat_context()  # pylint: disable=no-member
-        self.__ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ["gsiftp", "rfio", "gsidcap", "dcap", "kdcap"])
+        self.__ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ["rfio", "gsidcap", "dcap", "kdcap"])
         self.__ctx.set_opt_string("XROOTD PLUGIN", "XRD.WANTPROT", "gsi,unix")
         self.__ctx.set_opt_boolean("XROOTD PLUGIN", "NORMALIZE_PATH", False)
         auth_configured = False

lib/rucio/rse/rsemanager.py

@@ -751,7 +751,7 @@ def _retry_protocol_stat(protocol, pfn):
 
 def __check_compatible_scheme(dest_scheme, src_scheme):
     """
-    Check if two schemes are compatible, such as srm and gsiftp
+    Check if two schemes are compatible, such as srm and davs
 
     :param dest_scheme:    Destination scheme
     :param src_scheme:     Source scheme

lib/rucio/web/rest/flaskapi/v1/redirect.py

@@ -124,7 +124,7 @@ def get(self, scope_name):
         }
 
         dids = [{'scope': scope, 'name': name}]
-        schemes = request.args.getlist('schemes') or ['http', 'https', 'root', 'gsiftp', 'srm', 'davs']
+        schemes = request.args.getlist('schemes') or ['http', 'https', 'root', 'srm', 'davs']
         sortby = request.args.get('select', default=None)
         sortby = request.args.get('sort', default=sortby)
 

lib/rucio/web/rest/flaskapi/v1/rses.py

@@ -863,7 +863,7 @@ def get(self, rse):
           required: True
         - name: scheme
           in: query
-          description: Optional argument to help with the protocol selection (e.g., http / gsiftp / srm)
+          description: Optional argument to help with the protocol selection (e.g., http / davs / srm)
           schema:
             type: string
         - name: domain

tests/test_auditor_srmdumps.py

@@ -88,7 +88,6 @@ def test_returns_a_list_of_links():
 def test_identifies_known_protocols():
     """ test_protocol_identifies_known_protocols """
     assert srmdumps.protocol('davs://some/example') == 'davs'
-    assert srmdumps.protocol('gsiftp://some/example') == 'gsiftp'
     assert srmdumps.protocol('http://some/example') == 'http'
     assert srmdumps.protocol('https://some/example') == 'https'
     assert srmdumps.protocol('root://some/example') == 'root'

tests/test_replica_sorting.py

@@ -55,7 +55,7 @@
     {'site': 'APERTURE', 'address': 'aperture.com', 'ip': LOCATION_TO_IP['Austria']},
     {'site': 'BLACKMESA', 'address': 'blackmesa.com', 'ip': LOCATION_TO_IP['Japan']},
 ]
-schemes = ['root', 'gsiftp', 'davs']
+schemes = ['root', 'davs']
 
 
 @pytest.fixture
@@ -138,17 +138,17 @@ def protocols_setup(vo, root_account, mock_scope):
                                      'domains': {
                                          'lan': {'read': 2, 'write': 2, 'delete': 2},
                                          'wan': {'read': 2, 'write': 2, 'delete': 2}}})
-    add_protocol(rse_info[0]['id'], {'scheme': 'gsiftp',
-                                     'hostname': ('gsiftp.%s' % base_rse_info[0]['address']),
+    add_protocol(rse_info[0]['id'], {'scheme': 'davs',
+                                     'hostname': ('davs.2.%s' % base_rse_info[0]['address']),
                                      'port': 8446,
                                      'prefix': '/test/chamber/',
                                      'impl': 'rucio.rse.protocols.gfal.Default',
                                      'domains': {
                                          'lan': {'read': None, 'write': None, 'delete': None},
                                          'wan': {'read': 3, 'write': 3, 'delete': 3}}})
 
-    add_protocol(rse_info[1]['id'], {'scheme': 'gsiftp',
-                                     'hostname': ('gsiftp.%s' % base_rse_info[1]['address']),
+    add_protocol(rse_info[1]['id'], {'scheme': 'davs',
+                                     'hostname': ('davs.2.%s' % base_rse_info[1]['address']),
                                      'port': 8446,
                                      'prefix': '/lambda/complex/',
                                      'impl': 'rucio.rse.protocols.gfal.Default',
@@ -201,8 +201,8 @@ def test_sort_geoip_wan_client_location(vo, rest_client, auth_token, protocols_s
         test_with_cache = True
         data['client_location'] = {'site': CLIENT_SITE}
 
-    first_aut_then_jpn = ['root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com', 'gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com']
-    first_jpn_then_aut = ['gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com', 'root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com']
+    first_aut_then_jpn = ['root.aperture.com', 'davs.aperture.com', 'davs.blackmesa.com', 'root.blackmesa.com']
+    first_jpn_then_aut = ['davs.blackmesa.com', 'root.blackmesa.com', 'root.aperture.com', 'davs.aperture.com']
     for client_location, expected_order in (
             ('Switzerland', first_aut_then_jpn),
             ('Romania', first_aut_then_jpn),
@@ -289,7 +289,7 @@ def fake_get_distance(se1, se2, *args, **kwargs):
         assert len(replicas) == 1
         sources_list = replicas[0]['sources']
         print(sources_list)
-        assert len(sources_list) == 6
+        assert len(sources_list) == 4
 
         sorted_replica_hosts = list(sorted(sources_list, key=lambda source: source['priority']))
         sorted_replica_hosts = list(map(lambda source: urlparse(source['pfn']).hostname, sorted_replica_hosts))
@@ -300,7 +300,7 @@ def fake_get_distance(se1, se2, *args, **kwargs):
         print(replicas)
         assert len(replicas) == 1
         sources_dict = replicas[0]['pfns']
-        assert len(sources_dict) == 6
+        assert len(sources_dict) == 4
 
         sorted_replica_hosts = list(sorted(sources_dict, key=lambda pfn: sources_dict[pfn]['priority']))
         sorted_replica_hosts = list(map(lambda source: urlparse(source).hostname, sorted_replica_hosts))
@@ -364,13 +364,13 @@ def fake_get_distance(se1, se2, *args, **kwargs):
         assert len(replicas) == 1
         sources_list = replicas[0]['sources']
         print(sources_list)
-        # 3 for wan and 2 for lan, since one is blocked for lan for each site
-        assert len(sources_list) == 5
+        # 3 for wan and 1 for lan, since one is blocked for lan for each site
+        assert len(sources_list) == 4
 
         sorted_replica_hosts = list(sorted(sources_list, key=lambda source: source['priority']))
         print(sorted_replica_hosts)
         lan_pfns = list(filter(lambda source: source['domain'] == 'lan', sorted_replica_hosts))
-        assert len(lan_pfns) == 2
+        assert len(lan_pfns) == 1
         for lanpfn in lan_pfns:
             assert protocols_setup['rse_info'][info_id]['name'] == lanpfn['rse']
 
@@ -383,8 +383,8 @@ def fake_get_distance(se1, se2, *args, **kwargs):
         print(replicas)
         assert len(replicas) == 1
         sources_dict = replicas[0]['pfns']
-        # 3 for wan and 2 for lan, since one is blocked for lan for each site
-        assert len(sources_dict) == 5
+        # 1 for wan and 2 for lan, since one is blocked for lan for each site
+        assert len(sources_dict) == 3
 
         sorted_replica_hosts = list(sorted(sources_dict, key=lambda pfn: sources_dict[pfn]['priority']))
         lan_pfns = list(filter(lambda pfn: sources_dict[pfn]['domain'] == 'lan', sorted_replica_hosts))
@@ -434,16 +434,16 @@ def fake_sort_replicas(dictreplica, *args, **kwargs):
         assert len(replicas) == 1
         sources_list = replicas[0]['sources']
         print(sources_list)
-        # 4 for lan, since one is blocked for lan for each site
-        assert len(sources_list) == 4
+        # 3 for lan, since one is blocked for lan for each site
+        assert len(sources_list) == 3
 
     elif content_type == Mime.JSON_STREAM:
         replicas = list(map(json.loads, filter(bool, map(str.strip, replicas_response.splitlines(keepends=False)))))
         print(replicas)
         assert len(replicas) == 1
         sources_dict = replicas[0]['pfns']
-        # 4 for lan, since one is blocked for lan for each site
-        assert len(sources_dict) == 4
+        # 3 for lan, since one is blocked for lan for each site
+        assert len(sources_dict) == 3
 
 
 @pytest.mark.noparallel(reason='fails when run in parallel')
@@ -482,7 +482,7 @@ def fake_get_geoip_db(*args, **kwargs):
 
     # now set config to not ignore errors
     core_config.set("core", "geoip_ignore_error", False)
-    
+
     # invalidate cache for __get_distance so that __get_geoip_db is called
     replica_sorter.REGION.invalidate()
 
@@ -499,7 +499,7 @@ def fake_get_geoip_db(*args, **kwargs):
 
     # reset to ignore errors for other tests
     core_config.set("core", "geoip_ignore_error", True)
-    
+
 
 @pytest.mark.noparallel(reason='fails when run in parallel, replicas should not be changed')
 def test_get_sorted_list_replicas_no_metalink(vo, rest_client, auth_token, protocols_setup, mock_scope):