Repository: rucio/rucio

Issue:
list_replicas with rse_Expression and resolve_archive makes 2 files out of 1
Motivation
----------

```
$ rucio -H https://rucio-lb-int.cern.ch get --rse UAM-LCG2_DATADISK  --resolve-archives  data15_13TeV:DRAW_RPVLL.11258124._000142.pool.root.1
2018-07-17 09:39:52,928    INFO    Processing 1 item(s) for input
2018-07-17 09:39:53,089    INFO    Using 2 threads to download 2 files
2018-07-17 09:39:53,090    INFO    Thread 1/2: Preparing download of data15_13TeV:DRAW_RPVLL.11258124._000142.pool.root.1
2018-07-17 09:39:53,093    INFO    Thread 2/2: Preparing download of data15_13TeV:DRAW_RPVLL.11258124._000142.pool.root.1
2018-07-17 09:39:53,156    INFO    Thread 1/2: Trying to download with srm from MPPMU_DATADISK: data15_13TeV:DRAW_RPVLL.11258124._000142.pool.root.1
2018-07-17 09:39:53,251    INFO    Thread 2/2: Trying to download with srm from UAM-LCG2_DATADISK: data15_13TeV:DRAW_RPVLL.11258124._000142.pool.root.1
2018-07-17 09:39:54,580    INFO    Thread 1/2: File data15_13TeV:DRAW_RPVLL.11258124._000142.pool.root.1 successfully downloaded. 2.419 MB in 1.22 seconds = 1.98 MBps
2018-07-17 09:39:57,384    INFO    Thread 2/2: File data15_13TeV:DRAW_RPVLL.11258124._000142.pool.root.1 successfully downloaded. 137.346 MB in 3.68 seconds = 37.32 MBps
----------------------------------
Download summary
----------------------------------------
DID data15_13TeV:DRAW_RPVLL.11258124._000142.pool.root.1
Total files :                                 2
Downloaded files :                            2
Files already found locally :                 0
Files that cannot be downloaded :             0


-rwxr-xr-x. 1 walkerr zp   2418784 Jul 17 09:39 DRAW_RPVLL.14655879._000001.zip.1
-rwxr-xr-x. 1 walkerr zp 137346472 Jul 17 09:39 DRAW_RPVLL.14657167._000002.zip.1
```
For some reason when --rse and --resolve-archives is used, out of the 1 file list_replicas makes two. 



Discussion:
@mlassnig not sure if you have seen this ticket, would be great if you can have a look.
ok, figured it out. zipped replicas which are not part of the rse_expression are not correctly removed in the response to the client.

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -737,7 +737,7 @@ def _list_replicas_for_files(file_clause, state_clause, files, rse_clause, sessi
 
 
 def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
-                   schemes, files, rse_clause, client_location, domain,
+                   schemes, files, rse_clause, rse_expression, client_location, domain,
                    sign_urls, signature_lifetime, constituents, session):
 
     files = [dataset_clause and _list_replicas_for_datasets(dataset_clause, state_clause, rse_clause, session),
@@ -766,10 +766,12 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
             domain = deepcopy(original_domain)
 
             # if the file is a constituent, find the available archives and add them to the list of possible PFNs
+            # taking into account the original rse_expression
             if '%s:%s' % (scope, name) in constituents:
                 archive_result = list_replicas(dids=constituents['%s:%s' % (scope, name)],
                                                schemes=schemes, client_location=client_location,
                                                domain=domain, sign_urls=sign_urls,
+                                               rse_expression=rse_expression,
                                                signature_lifetime=signature_lifetime,
                                                session=session)
 
@@ -778,6 +780,12 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
                 # an additional empty pfn to the client
                 if rse is None:
                     for archive in archive_result:
+
+                        # RSE expression might limit the archives we're allowed to use
+                        # if it's empty due to non-matching RSE expression we must skip
+                        if archive['pfns'] == {}:
+                            continue
+
                         for tmp_archive in archive['pfns']:
                             archive['pfns'][tmp_archive]['domain'] = 'zip'
                             # at this point we don't know the protocol of the parent, so we have to peek
@@ -793,7 +801,7 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
 
                 # now, repeat the procedure slightly different in case if there are replicas available
                 # and make the archive just an additional available PFN
-                available_archives = [r['pfns'] for r in archive_result]
+                available_archives = [r['pfns'] for r in archive_result if r['pfns'] != {}]
                 for archive in available_archives:
                     for archive_pfn in archive:
                         # at this point we don't know the protocol of the parent, so we have to peek
@@ -1021,7 +1029,7 @@ def list_replicas(dids, schemes=None, unavailable=False, request_id=None,
         for rse in parse_expression(expression=rse_expression, session=session):
             rse_clause.append(models.RSEFileAssociation.rse_id == rse['id'])
     for f in _list_replicas(dataset_clause, file_clause, state_clause, pfns,
-                            schemes, files, rse_clause, client_location, domain,
+                            schemes, files, rse_clause, rse_expression, client_location, domain,
                             sign_urls, signature_lifetime, constituents, session):
         yield f
 


Test patch:
diff --git a/lib/rucio/tests/test_archive.py b/lib/rucio/tests/test_archive.py
--- a/lib/rucio/tests/test_archive.py
+++ b/lib/rucio/tests/test_archive.py
@@ -87,7 +87,6 @@ def test_list_archive_contents_transparently(self):
 
         res = [r['pfns'] for r in self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in files_with_replicas],
                                                         resolve_archives=True)]
-        print res
         assert_equal(len(res), 2)
         assert_equal(len(res[0]), 2)
         assert_equal(len(res[1]), 2)
@@ -109,3 +108,57 @@ def test_list_archive_contents_transparently(self):
             assert_in('weighted.storage.cube.zip?xrdcl.unzip=norep-', r.keys()[0])
 
         del_rse(rse)
+
+    def test_list_archive_contents_at_rse(self):
+        """ ARCHIVE (CORE): Transparent archive listing at RSE """
+
+        scope = 'mock'
+
+        rse1 = 'APERTURE_%s' % rse_name_generator()
+        add_rse(rse1)
+        add_protocol(rse1, {'scheme': 'root',
+                            'hostname': 'root.aperture.com',
+                            'port': 1409,
+                            'prefix': '//test/chamber/',
+                            'impl': 'rucio.rse.protocols.xrootd.Default',
+                            'domains': {
+                                'lan': {'read': 1, 'write': 1, 'delete': 1},
+                                'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+
+        rse2 = 'BLACKMESA_%s' % rse_name_generator()
+        add_rse(rse2)
+        add_protocol(rse2, {'scheme': 'root',
+                            'hostname': 'root.blackmesa.com',
+                            'port': 1409,
+                            'prefix': '//lambda/complex/',
+                            'impl': 'rucio.rse.protocols.xrootd.Default',
+                            'domains': {
+                                'lan': {'read': 1, 'write': 1, 'delete': 1},
+                                'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+
+        # register archive
+        archive1 = {'scope': scope, 'name': 'cube.1.zip', 'type': 'FILE', 'bytes': 2596, 'adler32': 'beefdead'}
+        archive2 = {'scope': scope, 'name': 'cube.2.zip', 'type': 'FILE', 'bytes': 5432, 'adler32': 'deadbeef'}
+        add_replicas(rse=rse1, files=[archive1], account='root')
+        add_replicas(rse=rse2, files=[archive2], account='root')
+
+        # archived files with replicas
+        archived_file = [{'scope': scope, 'name': 'zippedfile-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE',
+                          'bytes': 4322, 'adler32': 'beefbeef'} for i in xrange(2)]
+        self.dc.add_files_to_archive(scope=scope, name=archive1['name'], files=archived_file)
+        self.dc.add_files_to_archive(scope=scope, name=archive2['name'], files=archived_file)
+
+        res = [r['pfns'] for r in self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in archived_file],
+                                                        rse_expression=rse1,
+                                                        resolve_archives=True)]
+
+        res = self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in archived_file], metalink=True, rse_expression=rse1, resolve_archives=True)
+        assert_in('APERTURE', res)
+        assert_not_in('BLACKMESA', res)
+
+        res = self.rc.list_replicas(dids=[{'scope': scope, 'name': f['name']} for f in archived_file], metalink=True, rse_expression=rse2, resolve_archives=True)
+        assert_in('BLACKMESA', res)
+        assert_not_in('APERTURE', res)
+
+        del_rse(rse1)
+        del_rse(rse2)