Repository: rucio/rucio

Issue:
Handling of istape=False in the client
Motivation
----------
Currently, the clients only allow downloads from RSEs which are flagged with `istape=False`. There are two issues with this. For some reason, in ATLAS, the `istape` attribute has been stored with the value `"False"` instead of the value `0`, which means the rse_expression only works with the direct string check on `"False"`. Secondly, as this is hardcoded in the client, for communities which do not have their RSEs flagged with `"istape=False"` the download will not work.
We need a better workflow for this.

Some options are:
- Extending the RSE Expression parameter with a system-default `*` parameter, which lists all RSEs. Currently, in ATLAS, the `ALL` flag is used for this, but again, this needs to be created by a script. A system internal parameter would be good. With this a new attribute could be used to flag tape rses: `tape=1`. The client can now do the download with the EXPRESSION `*\tape=1`. Even for communities who do not have the tape flag, this query will work.



Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -1151,7 +1151,7 @@ def _get_sources(self, merged_items, resolve_archives=True):
             # extend RSE expression to exclude tape RSEs for non-admin accounts
             rse_expression = item.get('rse')
             if self.is_tape_excluded:
-                rse_expression = 'istape=False' if not rse_expression else '(%s)&istape=False' % rse_expression
+                rse_expression = '*\istape=true' if not rse_expression else '(%s)\istape=true' % rse_expression
             logger.debug('rse_expression: %s' % rse_expression)
 
             # get PFNs of files and datasets
diff --git a/lib/rucio/core/rse_expression_parser.py b/lib/rucio/core/rse_expression_parser.py
--- a/lib/rucio/core/rse_expression_parser.py
+++ b/lib/rucio/core/rse_expression_parser.py
@@ -10,7 +10,7 @@
   - Vincent Garonne, <vincent.garonne@cern.ch>, 2013
   - Mario Lassnig, <mario.lassnig@cern.ch>, 2013
   - Cedric Serfon, <cedric.serfon@cern.ch>, 2014
-  - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
+  - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018-2019
 
   PY3K COMPATIBLE
 '''
@@ -31,8 +31,7 @@
 
 DEFAULT_RSE_ATTRIBUTE = schema.DEFAULT_RSE_ATTRIBUTE['pattern']
 RSE_ATTRIBUTE = schema.RSE_ATTRIBUTE['pattern']
-PRIMITIVE = r'(\(*(%s|%s)\)*)' % (RSE_ATTRIBUTE, DEFAULT_RSE_ATTRIBUTE)
-
+PRIMITIVE = r'(\(*(%s|%s|%s)\)*)' % (RSE_ATTRIBUTE, DEFAULT_RSE_ATTRIBUTE, '\*')
 UNION = r'(\|%s)' % (PRIMITIVE)
 INTERSECTION = r'(\&%s)' % (PRIMITIVE)
 COMPLEMENT = r'(\\%s)' % (PRIMITIVE)
@@ -176,6 +175,8 @@ def __resolve_primitive_expression(expression):
     elif ('>' in primitiveexpression):
         keyvalue = primitiveexpression.split(">")
         return (RSEAttributeLargerCheck(keyvalue[0], keyvalue[1]), primitiveexpression)
+    elif ('*' in primitiveexpression):
+        return (RSEAll(), primitiveexpression)
     else:
         return (RSEAttributeEqualCheck(key=primitiveexpression), primitiveexpression)
 
@@ -214,6 +215,24 @@ def resolve_elements(self, session):
         pass
 
 
+class RSEAll(BaseExpressionElement):
+    """
+    Representation of all RSEs
+    """
+
+    def resolve_elements(self, session):
+        """
+        Inherited from :py:func:`BaseExpressionElement.resolve_elements`
+        """
+        output = list_rses(session=session)
+        if not output:
+            return (set(), {})
+        rse_dict = {}
+        for rse in output:
+            rse_dict[rse['id']] = rse
+        return (set([rse['id'] for rse in output]), rse_dict)
+
+
 class RSEAttributeEqualCheck(BaseExpressionElement):
     """
     Representation of an RSE Attribute with Equal Check


Test patch:
diff --git a/lib/rucio/tests/test_rse_expression_parser.py b/lib/rucio/tests/test_rse_expression_parser.py
--- a/lib/rucio/tests/test_rse_expression_parser.py
+++ b/lib/rucio/tests/test_rse_expression_parser.py
@@ -7,6 +7,7 @@
 #
 # Authors:
 # - Martin Barisits, <martin.barisits@cern.ch>, 2013-2017
+# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2019
 
 from random import choice
 from string import ascii_uppercase, digits, ascii_lowercase
@@ -96,6 +97,11 @@ def test_attribute_reference(self):
         """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE attribute reference """
         assert_equal([t_rse['id'] for t_rse in rse_expression_parser.parse_expression("%s=uk" % self.attribute)], [self.rse4_id])
 
+    def test_all_rse(self):
+        """ RSE_EXPRESSION_PARSER (CORE) Test reference on all RSE """
+        all_rses = rse.list_rses()
+        assert_equal(sorted(rse_expression_parser.parse_expression("*")), sorted(all_rses))
+
     def test_tag_reference(self):
         """ RSE_EXPRESSION_PARSER (CORE) Test simple RSE tag reference """
         assert_equal(sorted([t_rse['id'] for t_rse in rse_expression_parser.parse_expression(self.tag1)]), sorted([self.rse1_id, self.rse2_id, self.rse3_id]))
@@ -130,6 +136,10 @@ def test_complicated_expression_2(self):
         assert_equal(sorted([t_rse['id'] for t_rse in rse_expression_parser.parse_expression("(((((%s))))|%s=us)&%s|(%s=at|%s=de)" %
                                                                                              (self.tag1, self.attribute, self.tag2, self.attribute, self.attribute))]), sorted([self.rse1_id, self.rse2_id, self.rse5_id]))
 
+    def test_complicated_expression_3(self):
+        """ RSE_EXPRESSION_PARSER (CORE) Test some complicated expression 3"""
+        assert_equal(sorted([t_rse['id'] for t_rse in rse_expression_parser.parse_expression("(*)&%s=at" % self.attribute)]), sorted([self.rse1_id]))
+
     @staticmethod
     def test_list_on_availability():
         """ RSE_EXPRESSION_PARSER (CORE) List rses based on availability filter"""