Repository: rucio/rucio

Issue:
clients: add option to create non-deterministic RSEs
Motivation
----------

Currently it's impossible to create an RSE with the nondet flag.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -933,10 +933,8 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
                                                     'type': tmp_pfn[4]['type'] if tmp_pfn[1] == 'zip' else str(rse_type),
                                                     'volatile': tmp_pfn[4]['volatile'] if tmp_pfn[1] == 'zip' else volatile,
                                                     'domain': tmp_pfn[1],
-                                                    'priority': tmp_pfn[2]}
-                        # instruct the client that the archive needs to be manually extracted
-                        if tmp_pfn[3]:
-                            file['pfns'][tmp_pfn[0]]['client_extract'] = True
+                                                    'priority': tmp_pfn[2],
+                                                    'client_extract': tmp_pfn[3]}
                 else:
                     # quick exit, but don't forget to set the total order for the priority
                     # --> exploit that L(AN) comes before W(AN) before Z(IP) alphabetically
@@ -966,10 +964,8 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
                                                     'type': tmp_pfn[4]['type'] if tmp_pfn[1] == 'zip' else str(rse_type),
                                                     'volatile': tmp_pfn[4]['volatile'] if tmp_pfn[1] == 'zip' else volatile,
                                                     'domain': tmp_pfn[1],
-                                                    'priority': tmp_pfn[2]}
-                        # instruct the client that the archive needs to be manually extracted
-                        if tmp_pfn[3]:
-                            file['pfns'][tmp_pfn[0]]['client_extract'] = True
+                                                    'priority': tmp_pfn[2],
+                                                    'client_extract': tmp_pfn[3]}
 
     # set the total order for the priority
     # --> exploit that L(AN) comes before W(AN) before Z(IP) alphabetically
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -19,6 +19,7 @@
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2017
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2018
+# - Martin Barisits <martin.barisits@cern.ch>, 2018
 
 from datetime import datetime
 from json import dumps, loads
@@ -359,7 +360,10 @@ def POST(self):
                     dictreplica = {}
                     for replica in rfile['pfns'].keys():
                         replicas.append(replica)
-                        dictreplica[replica] = (rfile['pfns'][replica]['domain'], rfile['pfns'][replica]['priority'], rfile['pfns'][replica]['rse'])
+                        dictreplica[replica] = (rfile['pfns'][replica]['domain'],
+                                                rfile['pfns'][replica]['priority'],
+                                                rfile['pfns'][replica]['rse'],
+                                                rfile['pfns'][replica]['client_extract'])
 
                     yield ' <file name="' + rfile['name'] + '">\n'
                     yield '  <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
@@ -391,6 +395,7 @@ def POST(self):
                         yield '   <url location="' + str(dictreplica[replica][2]) \
                             + '" domain="' + str(dictreplica[replica][0]) \
                             + '" priority="' + str(dictreplica[replica][1]) \
+                            + '" client_extract="' + str(dictreplica[replica][3]).lower() \
                             + '">' + replica + '</url>\n'
                         idx += 1
                         if limit and limit == idx:


Test patch:
diff --git a/lib/rucio/tests/test_replica_sorting.py b/lib/rucio/tests/test_replica_sorting.py
--- a/lib/rucio/tests/test_replica_sorting.py
+++ b/lib/rucio/tests/test_replica_sorting.py
@@ -14,6 +14,7 @@
 #
 # Authors:
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018
+# - Martin Barisits <martin.barisits@cern.ch>, 2018
 
 from nose.tools import assert_equal, assert_in, assert_not_in
 
@@ -155,11 +156,11 @@ def test_replica_sorting(self):
                                    schemes=['root', 'gsiftp', 'davs'],
                                    metalink=True,
                                    client_location={'site': 'APERTURE'})
-        assert_in('domain="lan" priority="1">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml)
-        assert_in('domain="lan" priority="2">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml)
-        assert_in('domain="wan" priority="3">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml)
-        assert_in('domain="wan" priority="4">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0', ml)
-        assert_in('domain="wan" priority="5">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="lan" priority="1" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="lan" priority="2" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="3" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="4" client_extract="false">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="5" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml)
         assert_not_in('priority="6"', ml)
 
         ml = self.rc.list_replicas(dids=[{'scope': 'mock',
@@ -168,11 +169,11 @@ def test_replica_sorting(self):
                                    schemes=['root', 'gsiftp', 'davs'],
                                    metalink=True,
                                    client_location={'site': 'BLACKMESA'})
-        assert_in('domain="lan" priority="1">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml)
-        assert_in('domain="lan" priority="2">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml)
-        assert_in('domain="wan" priority="3">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml)
-        assert_in('domain="wan" priority="4">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml)
-        assert_in('domain="wan" priority="5">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="lan" priority="1" client_extract="false">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="lan" priority="2" client_extract="false">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="3" client_extract="false">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="4" client_extract="false">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="5" client_extract="false">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0', ml)
         assert_not_in('priority="6"', ml)
 
         # TODO: intractable until RSE sorting is enabled