Repository: rucio/rucio

Issue:
Issue in client_extract download 
Motivation
----------



Modification
------------





Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -1314,20 +1314,22 @@ def _prepare_items_for_download(self, did_to_options, merged_items_with_sources,
                 # dont move from mixed list to pure list
                 continue
 
-            # remove mixed cea id from mixed map and add it to pure map
-            cea_id_mixed_to_fiids.pop(cea_id_mixed)
+            # first add cea_id to pure map so it can be removed from mixed map later
             cea_id_pure_to_fiids.setdefault(cea_id_mixed, set()).update(fiids_mixed)
 
-            # move file items from mixed list to pure list
-            for fiid_mixed in fiids_mixed:
+            # now update all file_item mixed/pure maps
+            for fiid_mixed in list(fiids_mixed):
                 file_item = fiid_to_file_item[fiid_mixed]
+                # add cea id to file_item pure map
                 file_item.setdefault('cea_ids_pure', set()).add(cea_id_mixed)
 
-                # remove references from file to mixed archive and from mixed archive to file
-                for cea_id_mixed in file_item.pop('cea_ids_mixed'):
-                    cea_id_mixed_to_fiids[cea_id_mixed].discard(fiid_mixed)
-                    if not len(cea_id_mixed_to_fiids[cea_id_mixed]):
-                        cea_id_mixed_to_fiids.pop(cea_id_mixed)
+                # remove file item mixed map and
+                # remove references from all other mixed archives to file_item
+                for cea_id_mixed2 in file_item.pop('cea_ids_mixed'):
+                    cea_id_mixed_to_fiids[cea_id_mixed2].remove(fiid_mixed)
+
+            # finally remove cea_id from mixed map
+            cea_id_mixed_to_fiids.pop(cea_id_mixed)
 
         for file_item in all_file_items:
             cea_ids_pure = file_item.get('cea_ids_pure', set())
diff --git a/lib/rucio/clis/daemons/reaper/reaper.py b/lib/rucio/clis/daemons/reaper/reaper.py
--- a/lib/rucio/clis/daemons/reaper/reaper.py
+++ b/lib/rucio/clis/daemons/reaper/reaper.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Vincent Garonne, <vgaronne@gmail.com>, 2012-2018
+# - Vincent Garonne, <vgaronne@gmail.com>, 2012-2019
 # - Wen Guan, <wguan.icedew@gmail.com>, 2014
 # - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
 
@@ -23,6 +23,7 @@
 
 import argparse
 import signal
+import sys
 
 from rucio.daemons.reaper.reaper import run, stop
 
@@ -72,11 +73,19 @@ def get_parser():
     return parser
 
 
-def main():
+def main(argv=None):
+    """
+    The main reaper method called by the command.
 
+    :param argv: Command-line arguments. Default to  sys.argv if not set.
+    """
     signal.signal(signal.SIGTERM, stop)
+
+    if argv is None:
+        argv = sys.argv[1:]
+
     parser = get_parser()
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     try:
         run(total_workers=args.total_workers, chunk_size=args.chunk_size, greedy=args.greedy,
             once=args.run_once, scheme=args.scheme, rses=args.rses, threads_per_worker=args.threads_per_worker,
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -170,7 +170,9 @@ def GET(self, scope, name):
                             break
                     yield ' </file>\n'
 
-            # don't forget to send the metalink footer
+            # ensure complete metalink
+            if __first and metalink:
+                yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
             if metalink:
                 yield '</metalink>\n'
 
@@ -454,7 +456,9 @@ def POST(self):
                             break
                     yield ' </file>\n'
 
-            # don't forget to send the metalink footer
+            # ensure complete metalink
+            if __first and metalink:
+                yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
             if metalink:
                 yield '</metalink>\n'
 


Test patch:
diff --git a/lib/rucio/tests/test_reaper.py b/lib/rucio/tests/test_reaper.py
--- a/lib/rucio/tests/test_reaper.py
+++ b/lib/rucio/tests/test_reaper.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2013-2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,21 +13,23 @@
 # limitations under the License.
 #
 # Authors:
-# - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
+# - Vincent Garonne <vgaronne@gmail.com>, 2013-2019
 # - Martin Barisits <martin.barisits@cern.ch>, 2016
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
+import nose.tools
 
-from rucio.common.utils import generate_uuid
+from rucio.common.utils import execute, generate_uuid
 from rucio.core import rse as rse_core
 from rucio.core import replica as replica_core
-from rucio.daemons.reaper.reaper import reaper
+
+from rucio.clis.daemons.reaper.reaper import main
 
 
 def test_reaper():
     """ REAPER (DAEMON): Test the reaper daemon."""
     nb_files = 30
     file_size = 2147483648  # 2G
-    for i in range(nb_files):
+    for _ in range(nb_files):
         replica_core.add_replica(rse='MOCK', scope='data13_hip',
                                  name='lfn' + generate_uuid(), bytes=file_size,
                                  account='root', adler32=None, md5=None)
@@ -36,6 +38,10 @@ def test_reaper():
     rse_core.set_rse_limits(rse='MOCK', name='MinFreeSpace', value=10737418240)
     rse_core.set_rse_limits(rse='MOCK', name='MaxBeingDeletedFiles', value=10)
 
-    rses = [rse_core.get_rse('MOCK'), ]
-    reaper(once=True, rses=rses)
-    reaper(once=True, rses=rses)
+    argv = ['--run-once', '--rses', 'MOCK']
+    main(argv)
+    # Test the rucio-reaper console script
+    cmd = 'rucio-reaper ' + ' '.join(argv)
+    exitcode, out, err = execute(cmd)
+    print(cmd, out, err)
+    nose.tools.assert_equal(exitcode, 0)