Repository: rucio/rucio

Issue:
rse_settings dictionary datatype
Motivation
----------
at the moment, the `rsemanager` takes most of its settings from a dictionary, often called `rse_settings`, eg in rsemanager.py::create_protocol. this dictionary is unnecessarily complex and hence obscure. it is also not easy to type-hint


Modification
------------
it would be minimally invasive to type-hint the dictionary, which is possible starting from python 3.8 -- but we currently still want to support py3.6

i would suggest replacing it with a NamedTuple (added in py2.6 or py3.0, respectively) (with the benefit of being immutable, easy to type-hint, and more transparent), or possibly even a custom dataclass.




Discussion:

Resolving patch:
diff --git a/lib/rucio/common/types.py b/lib/rucio/common/types.py
--- a/lib/rucio/common/types.py
+++ b/lib/rucio/common/types.py
@@ -13,6 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Any, Optional, TypedDict, Union
+
+
 class InternalType(object):
     '''
     Base for Internal representations of string types
@@ -99,3 +102,57 @@ class InternalScope(InternalType):
     '''
     def __init__(self, scope, vo='def', fromExternal=True):
         super(InternalScope, self).__init__(value=scope, vo=vo, fromExternal=fromExternal)
+
+
+class RSEDomainLANDict(TypedDict):
+    read: Optional[int]
+    write: Optional[int]
+    delete: Optional[int]
+
+
+class RSEDomainWANDict(TypedDict):
+    read: Optional[int]
+    write: Optional[int]
+    delete: Optional[int]
+    third_party_copy_read: Optional[int]
+    third_party_copy_write: Optional[int]
+
+
+class RSEDomainsDict(TypedDict):
+    lan: RSEDomainLANDict
+    wan: RSEDomainWANDict
+
+
+class RSEProtocolDict(TypedDict):
+    auth_token: Optional[str]  # FIXME: typing.NotRequired
+    hostname: str
+    scheme: str
+    port: int
+    prefix: str
+    impl: str
+    domains: RSEDomainsDict
+    extended_attributes: Optional[Union[str, dict[str, Any]]]
+
+
+class RSESettingsDict(TypedDict):
+    availability_delete: bool
+    availability_read: bool
+    availability_write: bool
+    credentials: Optional[dict[str, Any]]
+    lfn2pfn_algorithm: str
+    qos_class: Optional[str]
+    staging_area: bool
+    rse_type: str
+    sign_url: Optional[str]
+    read_protocol: int
+    write_protocol: int
+    delete_protocol: int
+    third_party_copy_read_protocol: int
+    third_party_copy_write_protocol: int
+    id: str
+    rse: str
+    volatile: bool
+    verify_checksum: bool
+    deterministic: bool
+    domain: list[str]
+    protocols: list[RSEProtocolDict]
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -30,6 +30,7 @@
 from rucio.common import exception, utils
 from rucio.common.cache import make_region_memcached
 from rucio.common.config import get_lfn2pfn_algorithm_default, config_get_bool
+from rucio.common import types
 from rucio.common.utils import CHECKSUM_KEY, GLOBALLY_SUPPORTED_CHECKSUMS, Availability
 from rucio.core.rse_counter import add_counter, get_counter
 from rucio.db.sqla import models
@@ -1340,7 +1341,7 @@ def add_protocol(
 
 
 @read_session
-def get_rse_protocols(rse_id, schemes=None, *, session: "Session"):
+def get_rse_protocols(rse_id, schemes=None, *, session: "Session") -> types.RSESettingsDict:
     """
     Returns protocol information. Parameter combinations are: (operation OR default) XOR scheme.
 
@@ -1379,7 +1380,7 @@ def _format_get_rse_protocols(
         rse_attributes: Optional[dict[str, Any]] = None,
         *,
         session: "Session"
-) -> dict[str, Any]:
+) -> types.RSESettingsDict:
     _rse = rse
     if rse_attributes:
         lfn2pfn_algorithm = rse_attributes.get('lfn2pfn_algorithm')
@@ -1451,7 +1452,7 @@ def _format_get_rse_protocols(
 
 
 @read_session
-def get_rse_info(rse_id, *, session: "Session"):
+def get_rse_info(rse_id, *, session: "Session") -> types.RSESettingsDict:
     """
     For historical reasons, related to usage of rsemanager, "rse_info" is equivalent to
     a cached call to get_rse_protocols without any schemes set.
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -20,13 +20,14 @@
 from urllib.parse import urlparse
 
 from rucio.common import exception, utils, constants
+from rucio.common import types
 from rucio.common.config import config_get_int
 from rucio.common.constraints import STRING_TYPES
 from rucio.common.logging import formatted_logger
 from rucio.common.utils import make_valid_did, GLOBALLY_SUPPORTED_CHECKSUMS
 
 
-def get_rse_info(rse=None, vo='def', rse_id=None, session=None):
+def get_rse_info(rse=None, vo='def', rse_id=None, session=None) -> types.RSESettingsDict:
     """
         Returns all protocol related RSE attributes.
         Call with either rse and vo, or (in server mode) rse_id
@@ -40,20 +41,20 @@ def get_rse_info(rse=None, vo='def', rse_id=None, session=None):
                     id                ...     an internal identifier
                     rse               ...     the name of the RSE as string
                     type              ...     the storage type odf the RSE e.g. DISK
-                    volatile          ...     boolean indictaing if the RSE is volatile
+                    volatile          ...     boolean indicating if the RSE is volatile
                     verify_checksum   ...     boolean indicating whether RSE supports requests for checksums
-                    deteministic      ...     boolean indicating of the nameing of the files follows the defined determinism
-                    domain            ...     indictaing the domain that should be assumed for transfers. Values are 'ALL', 'LAN', or 'WAN'
-                    protocols         ...     all supported protocol in form of a list of dict objects with the followig structure
+                    deterministic      ...     boolean indicating of the naming of the files follows the defined determinism
+                    domain            ...     indicating the domain that should be assumed for transfers. Values are 'ALL', 'LAN', or 'WAN'
+                    protocols         ...     all supported protocol in form of a list of dict objects with the following structure
                     - scheme              ...     protocol scheme e.g. http, srm, ...
                     - hostname            ...     hostname of the site
                     - prefix              ...     path to the folder where the files are stored
                     - port                ...     port used for this protocol
                     - impl                ...     naming the python class of the protocol implementation
                     - extended_attributes ...     additional information for the protocol
-                    - domains             ...     a dict naming each domain and the priority of the protocol for each operation (lower is better, zero is not upported)
+                    - domains             ...     a dict naming each domain and the priority of the protocol for each operation (lower is better, zero is not supported)
 
-        :raises RSENotFound: if the provided RSE coud not be found in the database.
+        :raises RSENotFound: if the provided RSE could not be found in the database.
     """
     # __request_rse_info will be assigned when the module is loaded as it depends on the rucio environment (server or client)
     # __request_rse_info, rse_region are defined in /rucio/rse/__init__.py
@@ -66,7 +67,7 @@ def get_rse_info(rse=None, vo='def', rse_id=None, session=None):
     return rse_info
 
 
-def _get_possible_protocols(rse_settings, operation, scheme=None, domain=None, impl=None):
+def _get_possible_protocols(rse_settings: types.RSESettingsDict, operation, scheme=None, domain=None, impl=None):
     """
     Filter the list of available protocols or provided by the supported ones.
 
@@ -116,7 +117,7 @@ def _get_possible_protocols(rse_settings, operation, scheme=None, domain=None, i
     return [c for c in candidates if c not in tbr]
 
 
-def get_protocols_ordered(rse_settings, operation, scheme=None, domain='wan', impl=None):
+def get_protocols_ordered(rse_settings: types.RSESettingsDict, operation, scheme=None, domain='wan', impl=None):
     if operation not in utils.rse_supported_protocol_operations():
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
 
@@ -128,7 +129,7 @@ def get_protocols_ordered(rse_settings, operation, scheme=None, domain='wan', im
     return candidates
 
 
-def select_protocol(rse_settings, operation, scheme=None, domain='wan'):
+def select_protocol(rse_settings: types.RSESettingsDict, operation, scheme=None, domain='wan'):
     if operation not in utils.rse_supported_protocol_operations():
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
 
@@ -141,9 +142,9 @@ def select_protocol(rse_settings, operation, scheme=None, domain='wan'):
     return min(candidates, key=lambda k: k['domains'][domain][operation])
 
 
-def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log, impl=None):
+def create_protocol(rse_settings: types.RSESettingsDict, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log, impl=None):
     """
-    Instanciates the protocol defined for the given operation.
+    Instantiates the protocol defined for the given operation.
 
     :param rse_settings:  RSE attributes
     :param operation:     Intended operation for this protocol
@@ -191,11 +192,11 @@ def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_tok
     return protocol
 
 
-def lfns2pfns(rse_settings, lfns, operation='write', scheme=None, domain='wan', auth_token=None, logger=logging.log, impl=None):
+def lfns2pfns(rse_settings: types.RSESettingsDict, lfns, operation='write', scheme=None, domain='wan', auth_token=None, logger=logging.log, impl=None):
     """
         Convert the lfn to a pfn
 
-        :rse_settings:      RSE attributes
+        :param rse_settings:      RSE attributes
         :param lfns:        logical file names as a dict containing 'scope' and 'name' as keys. For bulk a list of dicts can be provided
         :param operation:   Intended operation for this protocol
         :param scheme:      Optional filter if no specific protocol is defined in rse_setting for the provided operation
@@ -209,11 +210,11 @@ def lfns2pfns(rse_settings, lfns, operation='write', scheme=None, domain='wan',
     return create_protocol(rse_settings, operation, scheme, domain, auth_token=auth_token, logger=logger, impl=impl).lfns2pfns(lfns)
 
 
-def parse_pfns(rse_settings, pfns, operation='read', domain='wan', auth_token=None):
+def parse_pfns(rse_settings: types.RSESettingsDict, pfns, operation='read', domain='wan', auth_token=None):
     """
         Checks if a PFN is feasible for a given RSE. If so it splits the pfn in its various components.
 
-        :rse_settings:   RSE attributes
+        :param rse_settings:   RSE attributes
         :param pfns:        list of PFNs
         :param operation: Intended operation for this protocol
         :param domain:    Optional specification of the domain
@@ -231,12 +232,12 @@ def parse_pfns(rse_settings, pfns, operation='read', domain='wan', auth_token=No
     return create_protocol(rse_settings, operation, urlparse(pfns[0]).scheme, domain, auth_token=auth_token).parse_pfns(pfns)
 
 
-def exists(rse_settings, files, domain='wan', scheme=None, impl=None, auth_token=None, vo='def', logger=logging.log):
+def exists(rse_settings: types.RSESettingsDict, files, domain='wan', scheme=None, impl=None, auth_token=None, vo='def', logger=logging.log):
     """
         Checks if a file is present at the connected storage.
         Providing a list indicates the bulk mode.
 
-        :rse_settings:      RSE attributes
+        :param rse_settings:      RSE attributes
         :param files:       a single dict or a list with dicts containing 'scope' and 'name'
                             if LFNs are used and only 'name' if PFNs are used.
                             E.g. {'name': '2_rse_remote_get.raw', 'scope': 'user.jdoe'}, {'name': 'user/jdoe/5a/98/3_rse_remote_get.raw'}
@@ -251,7 +252,7 @@ def exists(rse_settings, files, domain='wan', scheme=None, impl=None, auth_token
     """
 
     ret = {}
-    gs = True  # gs represents the global status which inidcates if every operation workd in bulk mode
+    gs = True  # gs represents the global status which indicates if every operation worked in bulk mode
 
     protocol = create_protocol(rse_settings, 'read', scheme=scheme, impl=impl, domain=domain, auth_token=auth_token, logger=logger)
     protocol.connect()
@@ -292,12 +293,12 @@ def exists(rse_settings, files, domain='wan', scheme=None, impl=None, auth_token
     return [gs, ret]
 
 
-def upload(rse_settings, lfns, domain='wan', source_dir=None, force_pfn=None, force_scheme=None, transfer_timeout=None, delete_existing=False, sign_service=None, auth_token=None, vo='def', logger=logging.log, impl=None):
+def upload(rse_settings: types.RSESettingsDict, lfns, domain='wan', source_dir=None, force_pfn=None, force_scheme=None, transfer_timeout=None, delete_existing=False, sign_service=None, auth_token=None, vo='def', logger=logging.log, impl=None):
     """
         Uploads a file to the connected storage.
         Providing a list indicates the bulk mode.
 
-        :rse_settings:            RSE attributes
+        :param rse_settings:            RSE attributes
         :param lfns:              a single dict or a list with dicts containing 'scope' and 'name'.
                                   Examples:
                                   [
@@ -491,12 +492,12 @@ def upload(rse_settings, lfns, domain='wan', source_dir=None, force_pfn=None, fo
     return {0: gs, 1: ret, 'success': gs, 'pfn': pfn}
 
 
-def delete(rse_settings, lfns, domain='wan', auth_token=None, logger=logging.log, impl=None):
+def delete(rse_settings: types.RSESettingsDict, lfns, domain='wan', auth_token=None, logger=logging.log, impl=None):
     """
         Delete a file from the connected storage.
         Providing a list indicates the bulk mode.
 
-        :rse_settings:     RSE attributes
+        :param rse_settings:     RSE attributes
         :param lfns:       a single dict or a list with dicts containing 'scope' and 'name'. E.g. [{'name': '1_rse_remote_delete.raw', 'scope': 'user.jdoe'}, {'name': '2_rse_remote_delete.raw', 'scope': 'user.jdoe'}]
         :param domain:     The network domain, either 'wan' (default) or 'lan'
         :param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
@@ -509,7 +510,7 @@ def delete(rse_settings, lfns, domain='wan', auth_token=None, logger=logging.log
 
     """
     ret = {}
-    gs = True  # gs represents the global status which inidcates if every operation workd in bulk mode
+    gs = True  # gs represents the global status which indicates if every operation worked in bulk mode
 
     protocol = create_protocol(rse_settings, 'delete', domain=domain, auth_token=auth_token, logger=logger, impl=impl)
     protocol.connect()
@@ -534,12 +535,12 @@ def delete(rse_settings, lfns, domain='wan', auth_token=None, logger=logging.log
     return [gs, ret]
 
 
-def rename(rse_settings, files, domain='wan', auth_token=None, logger=logging.log, impl=None):
+def rename(rse_settings: types.RSESettingsDict, files, domain='wan', auth_token=None, logger=logging.log, impl=None):
     """
         Rename files stored on the connected storage.
         Providing a list indicates the bulk mode.
 
-        :rse_settings:     RSE attributes
+        :param rse_settings:     RSE attributes
         :param files:      a single dict or a list with dicts containing 'scope', 'name', 'new_scope' and 'new_name'
                            if LFNs are used or only 'name' and 'new_name' if PFNs are used.
                            If 'new_scope' or 'new_name' are not provided, the current one is used.
@@ -560,7 +561,7 @@ def rename(rse_settings, files, domain='wan', auth_token=None, logger=logging.lo
         :raises ServiceUnavailable: for any other reason
     """
     ret = {}
-    gs = True  # gs represents the global status which inidcates if every operation workd in bulk mode
+    gs = True  # gs represents the global status which indicates if every operation worked in bulk mode
 
     protocol = create_protocol(rse_settings, 'write', domain=domain, auth_token=auth_token, logger=logger, impl=impl)
     protocol.connect()
@@ -610,11 +611,11 @@ def rename(rse_settings, files, domain='wan', auth_token=None, logger=logging.lo
     return [gs, ret]
 
 
-def get_space_usage(rse_settings, scheme=None, domain='wan', auth_token=None, logger=logging.log, impl=None):
+def get_space_usage(rse_settings: types.RSESettingsDict, scheme=None, domain='wan', auth_token=None, logger=logging.log, impl=None):
     """
         Get RSE space usage information.
 
-        :rse_settings:     RSE attributes
+        :param rse_settings:     RSE attributes
         :param scheme:     optional filter to select which protocol to be used.
         :param domain:     The network domain, either 'wan' (default) or 'lan'
         :param auth_token: Optionally passing JSON Web Token (OIDC) string for authentication
@@ -622,7 +623,7 @@ def get_space_usage(rse_settings, scheme=None, domain='wan', auth_token=None, lo
 
         :returns:          a list with dict containing 'totalsize' and 'unusedsize'
 
-        :raises ServiceUnavailable: if some generic error occured in the library.
+        :raises ServiceUnavailable: if some generic error occurred in the library.
     """
     gs = True
     ret = {}


Test patch: