Repository: rucio/rucio

Issue:
Drop references to the AGIS-API endpoint to fix the auditor
Motivation
----------
There are several references to the atlas-agis-api.cern.ch endpoint in the codebase.
For example, the auditor daemon relies on it to fetch information on the RSEs and and to download the site storage reports.
With the shutdown of AGIS this does not work anymore and so the auditor, the way we currently deploy it, has become ineffective. 

Modification
------------
We could either replace the AGIS endpoint with [CRIC](https://atlas-cric.cern.ch/api/atlas/ddmendpoint/query/?json) or, better, not rely on external services at all and expolit the rucio.client.rseclient classes to expose the same information to the auditor. The latter has been tested and it showed to work.


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/dumper/__init__.py b/lib/rucio/common/dumper/__init__.py
--- a/lib/rucio/common/dumper/__init__.py
+++ b/lib/rucio/common/dumper/__init__.py
@@ -28,7 +28,6 @@
 import contextlib
 import datetime
 import gzip
-import json
 import logging
 import os
 import re
@@ -41,6 +40,7 @@
 import bz2
 
 from rucio.common import config
+from rucio.core.rse import get_rse_id, get_rse_protocols
 
 
 class HTTPDownloadFailed(Exception):
@@ -242,46 +242,14 @@ def to_datetime(str_or_datetime):
     return date
 
 
-def agis_endpoints_data(cache=True):
-    '''
-    Returns the result of querying the ddmendpoints list from AGIS
-    and parsing the results with json.loads().
-
-    :param cache: If True only makes one query to AGIS and returns
-    the same result for subsequent invocations. Consider the effects
-    of the cache when testing code which uses this function.
-    :returns: a list of dicts with the information returned by AGIS.
-    '''
-    if cache:
-        agis_data = getattr(agis_endpoints_data, '__agis_data', None)
-    else:
-        agis_data = None
-
-    if agis_data is None:
-        ddmendpoints_data_url = ('http://atlas-agis-api.cern.ch/request/ddmendpoint'
-                                 '/query/list/?json')
-
-        r = requests.get(ddmendpoints_data_url)
-        assert r.status_code == 200
-        agis_data = json.loads(r.text)
-        agis_endpoints_data.__agis_data = agis_data
-    return agis_data
+def ddmendpoint_preferred_protocol(ddmendpoint):
+    return next(p for p in get_rse_protocols(get_rse_id(ddmendpoint))['protocols'] if p['domains']['wan']['read'] == 1)
 
 
 def ddmendpoint_url(ddmendpoint):
-    agisdata = agis_endpoints_data()
-    endpointdata = next(entry for entry in agisdata if entry['name'] == ddmendpoint)
-    protocoldata = endpointdata['arprotocols']['read_wan'][0]
-    return str(''.join([protocoldata['endpoint'],
-                       re.sub(r'rucio/$', '', protocoldata['path'])]))
-
-# Using AGIS is better to avoid the use of ATLAS credentials, but using
-# rucio.client for the same task is also possible:
-# def ddmendpoint_url(ddmendpoint):
-#     protocols = rucio.client.RSEClient().get_rse(ddmendpoint)['protocols']
-#     pdata = next(proto for proto in protocols if proto['scheme'] == 'srm')
-#     pdata['prefix'] = '/'.join(pdata['prefix'].split('/')[:-1])
-#     return 'srm://{hostname}:{port}/{prefix}'.format(pdata)
+    preferred_protocol = ddmendpoint_preferred_protocol(ddmendpoint)
+    prefix = re.sub(r'rucio/$', '', preferred_protocol['prefix'])
+    return '{scheme}://{hostname}:{port}'.format(**preferred_protocol) + prefix
 
 
 def http_download_to_file(url, file_, session=None):
diff --git a/lib/rucio/common/dumper/consistency.py b/lib/rucio/common/dumper/consistency.py
--- a/lib/rucio/common/dumper/consistency.py
+++ b/lib/rucio/common/dumper/consistency.py
@@ -1,4 +1,5 @@
-# Copyright 2015-2019 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2015-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +14,12 @@
 # limitations under the License.
 #
 # Authors:
-# - Fernando Lopez <fernando.e.lopez@gmail.com>, 2015
-# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
+# - Fernando López <felopez@cern.ch>, 2015
+# - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2019-2020
 # - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2020
-#
-# PY3K COMPATIBLE
+# - Fabio Luchetti <fabio.luchetti@cern.ch>, 2022
 
 import datetime
 import logging
@@ -56,11 +56,7 @@ def dump(cls, subcommand, ddm_endpoint, storage_dump, prev_date_fname=None, next
         else:
             assert subcommand == 'consistency-manual'
 
-        prefix = path_parsing.prefix(
-            dumper.agis_endpoints_data(),
-            ddm_endpoint,
-        )
-        prefix_components = path_parsing.components(prefix)
+        prefix_components = path_parsing.components(dumper.ddmendpoint_url(ddm_endpoint))
 
         def parser(line):
             '''


Test patch:
diff --git a/lib/rucio/tests/test_dumper.py b/lib/rucio/tests/test_dumper.py
--- a/lib/rucio/tests/test_dumper.py
+++ b/lib/rucio/tests/test_dumper.py
@@ -53,17 +53,17 @@
 DATE_SECONDS = "2015-03-10 14:00:35"
 DATE_TENTHS = "2015-03-10T14:00:35.5"
 DATE_MILLISECONDS = "2015-03-10T14:00:35.5"
-AGISDATA = [{
-    'arprotocols': {
-        'read_wan': [
-            {
-                'endpoint': 'srm://example.com',
-                'path': '/atlasdatadisk/rucio/'
-            }
-        ]
+RSEPROTOCOL = {
+    "hostname": "example.com",
+    "scheme": "root",
+    "port": 1094,
+    "prefix": "//atlasdatadisk/rucio/",
+    "domains": {
+        "wan": {
+            "read": 1,
+        }
     },
-    'name': 'SOMEENDPOINT',
-}]
+}
 
 
 class MockResponse:
@@ -163,21 +163,15 @@ def test_to_date_format():
     assert isinstance(dumper.to_datetime(DATE_MILLISECONDS), datetime)
 
 
-@mock.patch('requests.get')
-def test_agis_endpoints_data_parses_proper_json(mock_get):
-    mock_get.return_value = MockResponse(AGISDATA, 200)
-    assert dumper.agis_endpoints_data(cache=False) == AGISDATA
-
-
-@mock.patch('rucio.common.dumper.agis_endpoints_data')
-def test_ddmendpoint_url_builds_url_from_agis_records(mock_get):
-    mock_get.return_value = AGISDATA
-    assert dumper.ddmendpoint_url('SOMEENDPOINT') == 'srm://example.com/atlasdatadisk/'
+@mock.patch('rucio.common.dumper.ddmendpoint_preferred_protocol')
+def test_ddmendpoint_url_builds_url_from_ddmendpoint_preferred_protocol(mock_get):
+    mock_get.return_value = RSEPROTOCOL
+    assert dumper.ddmendpoint_url('SOMEENDPOINT') == 'root://example.com:1094//atlasdatadisk/'
 
 
-@mock.patch('rucio.common.dumper.agis_endpoints_data')
+@mock.patch('rucio.common.dumper.ddmendpoint_preferred_protocol')
 def test_ddmendpoint_url_fails_on_unexistent_entry(mock_get):
-    mock_get.return_value = []
+    mock_get.side_effect = StopIteration()
     with pytest.raises(StopIteration):
         dumper.ddmendpoint_url('SOMEENDPOINT')
 
diff --git a/lib/rucio/tests/test_dumper_consistency.py b/lib/rucio/tests/test_dumper_consistency.py
--- a/lib/rucio/tests/test_dumper_consistency.py
+++ b/lib/rucio/tests/test_dumper_consistency.py
@@ -1,4 +1,5 @@
-# Copyright 2015-2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2015-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +14,11 @@
 # limitations under the License.
 #
 # Authors:
-# - Fernando Lopez <fernando.e.lopez@gmail.com>, 2015
+# - Fernando López <felopez@cern.ch>, 2015
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2017
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Fabio Luchetti <fabio.luchetti@cern.ch>, 2022
 
 import json
 import os
@@ -42,6 +44,19 @@
     import mock
 
 
+RSEPROTOCOL = {
+    "hostname": "example.com",
+    "scheme": "root",
+    "port": 1094,
+    "prefix": "//atlasdatadisk/rucio/",
+    "domains": {
+        "wan": {
+            "read": 1,
+        }
+    },
+}
+
+
 def mocked_requests(*args, **kwargs):
     class MockResponse:
         def __init__(self, json_data, status_code):
@@ -96,16 +111,11 @@ class TestConsistency(unittest.TestCase):
 
     def setUp(self):  # pylint: disable=invalid-name
         self.tmp_dir = tempfile.mkdtemp()
-        self.fake_agis_data = [{
-            'name': 'MOCK_SCRATCHDISK',
-            'se': 'srm://example.com:8446/',
-            'endpoint': '/pnfs/example.com/atlas/atlasdatadisk/'
-        }]
 
     def tearDown(self):  # pylint: disable=invalid-name
         shutil.rmtree(self.tmp_dir)
 
-    @mock.patch('rucio.common.dumper.agis_endpoints_data')
+    @mock.patch('rucio.common.dumper.ddmendpoint_preferred_protocol')
     def test_consistency_manual_correct_file_default_args(self, mock_get):
         ''' DUMPER '''
         rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
@@ -115,7 +125,7 @@ def test_consistency_manual_correct_file_default_args(self, mock_get):
         rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
         sdf = make_temp_file(self.tmp_dir, storage_dump)
 
-        mock_get.return_value = self.fake_agis_data
+        mock_get.return_value = RSEPROTOCOL
         consistency = Consistency.dump(
             'consistency-manual',
             'MOCK_SCRATCHDISK',
@@ -126,7 +136,7 @@ def test_consistency_manual_correct_file_default_args(self, mock_get):
         )
         assert len(list(consistency)) == 0
 
-    @mock.patch('rucio.common.dumper.agis_endpoints_data')
+    @mock.patch('rucio.common.dumper.ddmendpoint_preferred_protocol')
     def test_consistency_manual_lost_file(self, mock_get):
         ''' DUMPER '''
         rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
@@ -137,7 +147,7 @@ def test_consistency_manual_lost_file(self, mock_get):
         rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
         sdf = make_temp_file(self.tmp_dir, storage_dump)
 
-        mock_get.return_value = self.fake_agis_data
+        mock_get.return_value = RSEPROTOCOL
 
         consistency = Consistency.dump(
             'consistency-manual',
@@ -152,7 +162,7 @@ def test_consistency_manual_lost_file(self, mock_get):
         assert consistency[0].apparent_status == 'LOST'
         assert consistency[0].path == 'user/someuser/aa/bb/user.someuser.filename2'
 
-    @mock.patch('rucio.common.dumper.agis_endpoints_data')
+    @mock.patch('rucio.common.dumper.ddmendpoint_preferred_protocol')
     def test_consistency_manual_transient_file_is_not_lost(self, mock_get):
         ''' DUMPER '''
         rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
@@ -164,7 +174,7 @@ def test_consistency_manual_transient_file_is_not_lost(self, mock_get):
         rrdf2 = make_temp_file(self.tmp_dir, rucio_dump_2)
         sdf = make_temp_file(self.tmp_dir, storage_dump)
 
-        mock_get.return_value = self.fake_agis_data
+        mock_get.return_value = RSEPROTOCOL
 
         consistency = Consistency.dump(
             'consistency-manual',
@@ -176,7 +186,7 @@ def test_consistency_manual_transient_file_is_not_lost(self, mock_get):
         )
         assert len(list(consistency)) == 0
 
-    @mock.patch('rucio.common.dumper.agis_endpoints_data')
+    @mock.patch('rucio.common.dumper.ddmendpoint_preferred_protocol')
     def test_consistency_manual_dark_file(self, mock_get):
         ''' DUMPER '''
         rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
@@ -187,7 +197,7 @@ def test_consistency_manual_dark_file(self, mock_get):
         rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
         sdf = make_temp_file(self.tmp_dir, storage_dump)
 
-        mock_get.return_value = self.fake_agis_data
+        mock_get.return_value = RSEPROTOCOL
 
         consistency = Consistency.dump(
             'consistency-manual',
@@ -203,17 +213,17 @@ def test_consistency_manual_dark_file(self, mock_get):
         assert consistency[0].apparent_status == 'DARK'
         assert consistency[0].path == 'user/someuser/aa/bb/user.someuser.filename2'
 
-    @mock.patch('rucio.common.dumper.agis_endpoints_data')
+    @mock.patch('rucio.common.dumper.ddmendpoint_preferred_protocol')
     def test_consistency_manual_multiple_slashes_in_storage_dump_do_not_generate_false_positive(self, mock_get):
         ''' DUMPER '''
         rucio_dump = 'MOCK_SCRATCHDISK\tuser.someuser\tuser.someuser.filename\t19028d77\t189468\t2015-09-20 21:22:04\tuser/someuser/aa/bb/user.someuser.filename\t2015-09-20 21:22:17\tA\n'
-        storage_dump = '/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n'
+        storage_dump = '/example.com:1094////atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n'
 
         rrdf1 = make_temp_file(self.tmp_dir, rucio_dump)
         rrdf2 = make_temp_file(self.tmp_dir, rucio_dump)
         sdf = make_temp_file(self.tmp_dir, storage_dump)
 
-        mock_get.return_value = self.fake_agis_data
+        mock_get.return_value = RSEPROTOCOL
 
         consistency = Consistency.dump(
             'consistency-manual',
@@ -229,21 +239,15 @@ def test_consistency_manual_multiple_slashes_in_storage_dump_do_not_generate_fal
 
     @mock.patch('requests.Session.head', side_effect=mocked_requests)
     @mock.patch('requests.Session.get', side_effect=mocked_requests)
-    @mock.patch('rucio.common.dumper.agis_endpoints_data')
+    @mock.patch('rucio.common.dumper.ddmendpoint_preferred_protocol', return_value=RSEPROTOCOL)
     def test_consistency(self, mock_dumper_get, mock_request_get, mock_request_head):
         ''' DUMPER '''
         storage_dump = (
-            '/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.filename\n'
-            '/pnfs/example.com/atlas///atlasdatadisk/rucio//user/someuser/aa/bb/user.someuser.dark\n'
+            '//atlasdatadisk/rucio/user/someuser/aa/bb/user.someuser.filename\n'
+            '//atlasdatadisk/rucio/user/someuser/aa/bb/user.someuser.dark\n'
         )
         sd = make_temp_file(self.tmp_dir, storage_dump)
 
-        agisdata = [{
-            'name': 'MOCK_SCRATCHDISK',
-            'se': 'srm://example.com/',
-            'endpoint': 'pnfs/example.com/atlas/atlasdatadisk/',
-        }]
-        mock_dumper_get.return_value = agisdata
         consistency = Consistency.dump('consistency',
                                        'MOCK_SCRATCHDISK',
                                        storage_dump=sd,