Repository: rucio/rucio

Issue:
Move rule permission check is not working
Motivation
----------
For rucio move_rule the permission check is not working, thus the default permission is used.



Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -11,6 +11,7 @@
 # - Martin Barisits, <martin.barisits@cern.ch>, 2017
 # - Mario Lassnig, <mario.lassnig@cern.ch>, 2017
 # - Frank Berghaus, <frank.berghaus@cern.ch>, 2017
+# - Martin Barisits, <martin.barisits@cern.ch>, 2017-2018
 
 import base64
 import datetime
@@ -265,7 +266,7 @@ def execute(cmd):
 
 def rse_supported_protocol_operations():
     """ Returns a list with operations supported by all RSE protocols."""
-    return ['read', 'write', 'delete']
+    return ['read', 'write', 'delete', 'third_party_copy']
 
 
 def rse_supported_protocol_domains():
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -9,7 +9,7 @@
 # - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2016
 # - Mario Lassnig, <mario.lassnig@cern.ch>, 2012-2013, 2017
 # - Ralph Vigne, <ralph.vigne@cern.ch>, 2013-2015
-# - Martin Barisits, <martin.barisits@cern.ch>, 2013-2016
+# - Martin Barisits, <martin.barisits@cern.ch>, 2013-2018
 # - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2016
 # - Thomas Beermann, <thomas.beermann@cern.ch>, 2014, 2017
 # - Wen Guan, <wen.guan@cern.ch>, 2015-2016
@@ -949,14 +949,15 @@ def del_protocols(rse, scheme, hostname=None, port=None, session=None):
     for domain in utils.rse_supported_protocol_domains():
         for op in utils.rse_supported_protocol_operations():
             op_name = ''.join([op, '_', domain])
-            prots = session.query(models.RSEProtocols).\
-                filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
-                                       getattr(models.RSEProtocols, op_name) > 0)).\
-                order_by(getattr(models.RSEProtocols, op_name).asc())
-            i = 1
-            for p in prots:
-                p.update({op_name: i})
-                i += 1
+            if getattr(models.RSEProtocols, op_name, None):
+                prots = session.query(models.RSEProtocols).\
+                    filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
+                                           getattr(models.RSEProtocols, op_name) > 0)).\
+                    order_by(getattr(models.RSEProtocols, op_name).asc())
+                i = 1
+                for p in prots:
+                    p.update({op_name: i})
+                    i += 1
 
 
 @transactional_session
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -12,7 +12,7 @@
  - Vincent Garonne, <vincent.garonne@cern.ch>, 2013-2017
  - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2014, 2017
  - Wen Guan, <wen.guan@cern.ch>, 2014-2015
- - Martin Barisits, <martin.barisits@cern.ch>, 2017
+ - Martin Barisits, <martin.barisits@cern.ch>, 2017-2018
  - Tobias Wegner, <tobias.wegner@cern.ch>, 2017
 '''
 
@@ -65,25 +65,19 @@ def get_rse_info(rse, session=None):
     return rse_info
 
 
-def _get_possible_protocols(rse_settings, operation, scheme=None):
-    """ Filter the list of available protocols or provided by the supported ones.
-
-        :param rse_settings: The rse settings.
-        :param operation: The operation (write, read).
-        :param scheme: optional filter if no specific protocol is defined in
-                       rse_setting for the provided operation.
-
-        :returns: The list of possible protocols.
+def _get_possible_protocols(rse_settings, operation, scheme=None, domain=None):
+    """
+    Filter the list of available protocols or provided by the supported ones.
+
+    :param rse_settings: The rse settings.
+    :param operation:    The operation (write, read).
+    :param scheme:       Optional filter if no specific protocol is defined in
+                         rse_setting for the provided operation.
+    :param domain:       Optional domain (lan/wan), if not specified, both will be returned
+    :returns:            The list of possible protocols.
     """
     operation = operation.lower()
     candidates = rse_settings['protocols']
-    if type(rse_settings['domain']) is not list:
-        raise exception.RSEProtocolDomainNotSupported('Domain setting must be list.')
-
-    for d in rse_settings['domain']:
-        if d not in utils.rse_supported_protocol_domains():
-            raise exception.RSEProtocolDomainNotSupported('Domain %s is not supported'
-                                                          ' by Rucio.' % d)
 
     # convert scheme to list, if given as string
     if scheme and not isinstance(scheme, list):
@@ -97,9 +91,15 @@ def _get_possible_protocols(rse_settings, operation, scheme=None):
             continue
 
         filtered = True
-        for d in rse_settings['domain']:
-            if protocol['domains'][d][operation] != 0:
+
+        if not domain:
+            for d in protocol['domains'].keys():
+                if protocol['domains'][d][operation] != 0:
+                    filtered = False
+        else:
+            if protocol['domains'].get(domain, {operation: 0}).get(operation) != 0:
                 filtered = False
+
         if filtered:
             tbr.append(protocol)
 
@@ -110,45 +110,49 @@ def _get_possible_protocols(rse_settings, operation, scheme=None):
     return [c for c in candidates if c not in tbr]
 
 
-def get_protocols_ordered(rse_settings, operation, scheme=None):
-    candidates = _get_possible_protocols(rse_settings, operation, scheme)
-    domain = rse_settings['domain'][0]
+def get_protocols_ordered(rse_settings, operation, scheme=None, domain='wan'):
+    if operation not in utils.rse_supported_protocol_operations():
+        raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
+
+    if domain and domain not in utils.rse_supported_protocol_domains():
+        raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
+
+    candidates = _get_possible_protocols(rse_settings, operation, scheme, domain)
     candidates.sort(key=lambda k: k['domains'][domain][operation])
     return candidates
 
 
-def select_protocol(rse_settings, operation, scheme=None):
-    candidates = _get_possible_protocols(rse_settings, operation, scheme)
-    domain = rse_settings['domain'][0]
-    return min(candidates, key=lambda k: k['domains'][domain][operation])
+def select_protocol(rse_settings, operation, scheme=None, domain='wan'):
+    if operation not in utils.rse_supported_protocol_operations():
+        raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
+
+    if domain and domain not in utils.rse_supported_protocol_domains():
+        raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
 
+    candidates = _get_possible_protocols(rse_settings, operation, scheme, domain)
+    return min(candidates, key=lambda k: k['domains'][domain][operation])
 
-def create_protocol(rse_settings, operation, scheme=None):
-    """ Instanciates the protocol defined for the given operation.
 
-        :param rse_attr: RSE attributes
-        :param operation: the intended operation for this protocol
-        :param scheme: optional filter if no specific protocol is defined in rse_setting for the provided operation
+def create_protocol(rse_settings, operation, scheme=None, domain='wan'):
+    """
+    Instanciates the protocol defined for the given operation.
 
-        :returns: an instance of the requested protocol
+    :param rse_attr:  RSE attributes
+    :param operation: Intended operation for this protocol
+    :param scheme:    Optional filter if no specific protocol is defined in rse_setting for the provided operation
+    :parma domain:    Optional specification of the domain
+    :returns:         An instance of the requested protocol
     """
 
     # Verify feasibility of Protocol
     operation = operation.lower()
     if operation not in utils.rse_supported_protocol_operations():
         raise exception.RSEOperationNotSupported('Operation %s is not supported' % operation)
-    rse_settings['domain'] = [rse_settings['domain']] if type(rse_settings['domain']) is not list else rse_settings['domain']
-    for domain in rse_settings['domain']:
-        if domain.lower() not in utils.rse_supported_protocol_domains():
-            raise exception.RSEOperationNotSupported('Domain %s not supported' % rse_settings['domain'])
-
-    if rse_settings['%s_protocol' % operation] == DEFAULT_PROTOCOL:
-        protocol_attr = select_protocol(rse_settings, operation, scheme)
-    else:
-        protocol_attr = rse_settings['%s_protocol' % operation]
-        for d in rse_settings['domain']:
-            if protocol_attr['domains'][d][operation] == 0:
-                raise exception.RSEOperationNotSupported('Operation %s for domain %s not supported by %s' % (operation, rse_settings['domain'], protocol_attr['scheme']))
+
+    if domain and domain not in utils.rse_supported_protocol_domains():
+        raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
+
+    protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
 
     # Instanciate protocol
     comp = protocol_attr['impl'].split('.')
@@ -163,7 +167,7 @@ def create_protocol(rse_settings, operation, scheme=None):
     return protocol
 
 
-def lfns2pfns(rse_settings, lfns, operation='write', scheme=None):
+def lfns2pfns(rse_settings, lfns, operation='write', scheme=None, domain='wan'):
     """
         Convert the lfn to a pfn
 
@@ -173,10 +177,10 @@ def lfns2pfns(rse_settings, lfns, operation='write', scheme=None):
         :returns: a dict with scope:name as key and the PFN as value
 
     """
-    return create_protocol(rse_settings, operation, scheme).lfns2pfns(lfns)
+    return create_protocol(rse_settings, operation, scheme, domain).lfns2pfns(lfns)
 
 
-def parse_pfns(rse_settings, pfns, operation='read'):
+def parse_pfns(rse_settings, pfns, operation='read', domain='wan'):
     """
         Checks if a PFN is feasible for a given RSE. If so it splits the pfn in its various components.
 
@@ -192,10 +196,10 @@ def parse_pfns(rse_settings, pfns, operation='read'):
     """
     if len(set([urlparse(pfn).scheme for pfn in pfns])) != 1:
         raise ValueError('All PFNs must provide the same protocol scheme')
-    return create_protocol(rse_settings, operation, urlparse(pfns[0]).scheme).parse_pfns(pfns)
+    return create_protocol(rse_settings, operation, urlparse(pfns[0]).scheme, domain).parse_pfns(pfns)
 
 
-def download(rse_settings, files, dest_dir=None, force_scheme=None, ignore_checksum=False, printstatements=False):
+def download(rse_settings, files, dest_dir=None, force_scheme=None, ignore_checksum=False, printstatements=False, domain='wan'):
     """
         Copy a file from the connected storage to the local file system.
         Providing a list indicates the bulk mode.
@@ -221,7 +225,7 @@ def download(rse_settings, files, dest_dir=None, force_scheme=None, ignore_check
     ret = {}
     gs = True  # gs represents the global status which inidcates if every operation workd in bulk mode
 
-    protocol = create_protocol(rse_settings, 'read', scheme=force_scheme)
+    protocol = create_protocol(rse_settings, 'read', scheme=force_scheme, domain=domain)
     protocol.connect()
 
     files = [files] if not type(files) is list else files
@@ -581,7 +585,7 @@ def get_space_usage(rse_settings, scheme=None):
     return [gs, ret]
 
 
-def find_matching_scheme(rse_settings_dest, rse_settings_src, operation_src, operation_dest, domain=None, scheme=None):
+def find_matching_scheme(rse_settings_dest, rse_settings_src, operation_src, operation_dest, domain='wan', scheme=None):
     """
     Find the best matching scheme between two RSEs
 
@@ -599,11 +603,6 @@ def find_matching_scheme(rse_settings_dest, rse_settings_src, operation_src, ope
     src_candidates = copy.copy(rse_settings_src['protocols'])
     dest_candidates = copy.copy(rse_settings_dest['protocols'])
 
-    if not domain:
-        if type(rse_settings_dest['domain']) is not list:
-            raise exception.RSEProtocolDomainNotSupported('Domain setting must be list.')
-        domain = rse_settings_dest['domain'][0]
-
     # Clean up src_candidates
     tbr = list()
     for protocol in src_candidates:


Test patch:
diff --git a/lib/rucio/tests/test_rse.py b/lib/rucio/tests/test_rse.py
--- a/lib/rucio/tests/test_rse.py
+++ b/lib/rucio/tests/test_rse.py
@@ -11,7 +11,7 @@
  - Thomas Beermann, <thomas.beermann@cern.ch>, 2012
  - Angelos Molfetas, <angelos.molfetas@cern.ch>, 2012
  - Mario Lassnig, <mario.lassnig@cern.ch>, 2012
- - Martin Barisits, <martin.barisits@cern.ch>, 2013-2017
+ - Martin Barisits, <martin.barisits@cern.ch>, 2013-2018
  - Cedric Serfon, <cedric.serfon@cern.ch>, 2014-2015
  - Ralph Vigne, <ralph.vigne@cern.ch>, 2013-2015
  - Wen Guan, <wen.guan@cern.ch>, 2015
@@ -24,7 +24,8 @@
 from rucio.client.rseclient import RSEClient
 from rucio.client.replicaclient import ReplicaClient
 from rucio.common.exception import (Duplicate, RSENotFound, RSEProtocolNotSupported,
-                                    InvalidObject, RSEProtocolDomainNotSupported, RSEProtocolPriorityError, ResourceTemporaryUnavailable)
+                                    InvalidObject, RSEProtocolDomainNotSupported, RSEProtocolPriorityError,
+                                    ResourceTemporaryUnavailable)
 from rucio.common.utils import generate_uuid
 from rucio.core.rse import (add_rse, get_rse_id, del_rse, list_rses, rse_exists, add_rse_attribute, list_rse_attributes,
                             set_rse_transfer_limits, get_rse_transfer_limits, delete_rse_transfer_limits)
@@ -693,10 +694,9 @@ def test_get_protocols_operations(self):
 
         ops = {'read': 1, 'write': 2, 'delete': 3}
         rse_attr = mgr.get_rse_info(protocol_rse)
-        rse_attr['domain'] = ['lan']
         for op in ops:
             # resp = self.client.get_protocols(protocol_rse, operation=op, protocol_domain='lan')
-            p = mgr.select_protocol(rse_attr, op)
+            p = mgr.select_protocol(rse_attr, op, domain='lan')
             if op not in p['scheme'].lower():
                 for p in protocols:
                     self.client.delete_protocols(protocol_rse, p['scheme'])
@@ -743,10 +743,9 @@ def test_get_protocols_defaults(self):
             self.client.add_protocol(protocol_rse, p)
 
         rse_attr = mgr.get_rse_info(protocol_rse)
-        rse_attr['domain'] = ['lan']
         for op in ['delete', 'read', 'write']:
             # resp = self.client.get_protocols(protocol_rse, operation=op, default=True, protocol_domain='lan')
-            p = mgr.select_protocol(rse_attr, op)
+            p = mgr.select_protocol(rse_attr, op, domain='lan')
             print p['scheme']
             print op
             if op not in p['scheme'].lower():
@@ -754,10 +753,9 @@ def test_get_protocols_defaults(self):
                     self.client.delete_protocols(protocol_rse, p['scheme'])
                 self.client.delete_rse(protocol_rse)
                 raise Exception('Unexpected protocols returned for %s: %s' % (op, p))
-        rse_attr['domain'] = ['wan']
         for op in ['delete', 'read', 'write']:
             # resp = self.client.get_protocols(protocol_rse, operation=op, default=True, protocol_domain='wan')
-            p = mgr.select_protocol(rse_attr, op)
+            p = mgr.select_protocol(rse_attr, op, domain='wan')
             if ((op == 'delete') and (p['port'] != 17)) or ((op == 'read') and (p['port'] != 42)) or ((op == 'write') and (p['port'] != 19)):
                 for p in protocols:
                     self.client.delete_protocols(protocol_rse, p['scheme'])
@@ -860,8 +858,7 @@ def test_get_protocols_domain_not_exist(self):
 
         try:
             rse_attr = mgr.get_rse_info(protocol_rse)
-            rse_attr['domain'] = ['FRIENDS']
-            mgr.select_protocol(rse_attr, 'write')
+            mgr.select_protocol(rse_attr, 'write', domain='FRIENDS')
         except Exception, e:
             self.client.delete_protocols(protocol_rse, 'MOCK')
             self.client.delete_rse(protocol_rse)
@@ -964,7 +961,7 @@ def test_update_protocols_port_exist(self):
                       'prefix': '/the/one/with/all/the/files',
                       'impl': 'rucio.rse.protocols.SomeProtocol.SomeImplementation',
                       'domains': {
-                          'lan': {'read': 1,
+                          'wan': {'read': 1,
                                   'write': 1,
                                   'delete': 0}},
                       'extended_attributes': 'TheOneWithAllTheRest'},
@@ -974,7 +971,7 @@ def test_update_protocols_port_exist(self):
                       'prefix': '/the/one/with/all/the/files',
                       'impl': 'rucio.rse.protocols.SomeProtocol.SomeImplementation',
                       'domains': {
-                          'lan': {'read': 1,
+                          'wan': {'read': 1,
                                   'write': 1,
                                   'delete': 0}},
                       'extended_attributes': 'TheOneWithAllTheRest'}]
@@ -1009,8 +1006,7 @@ def test_update_protocols_various_attributes(self):
 
         self.client.update_protocols(protocol_rse, scheme='MOCK', hostname='localhost', port=17, data={'prefix': 'where/the/files/are', 'extended_attributes': 'Something else', 'port': '12'})
         rse_attr = mgr.get_rse_info(protocol_rse)
-        rse_attr['domain'] = ['lan']
-        p = mgr.select_protocol(rse_attr, 'read', scheme='MOCK')
+        p = mgr.select_protocol(rse_attr, 'read', scheme='MOCK', domain='lan')
         if p['prefix'] != 'where/the/files/are' and p['extended_attributes'] != 'Something else':
             raise Exception('Update gave unexpected results: %s' % p)
         self.client.delete_protocols(protocol_rse, 'MOCK')