Repository: rucio/rucio

Issue:
Regression in nrandom hadling for rucio download
Motivation
----------
```
rucio download --nrandom 
```
Doesn't randomly pick files in a dataset/container anymore.

The regression was introduced in 8e08799653a0330daba27ccfddc10e8ad199a6e6 which re-used the "item" variable name for an internal loop. Resulting it to be overriden and item['nrandom'] doesn't return the correct value anymore. 

Modification
------------
nrandom should work again.



Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -1110,13 +1110,13 @@ def _get_sources(self, merged_items, resolve_archives=True):
 
             # filtering out tape sources
             if self.is_tape_excluded:
-                for item in file_items:
-                    sources = item['sources']
-                    for src in item['sources']:
+                for file_item in file_items:
+                    sources = file_item['sources']
+                    for src in file_item['sources']:
                         if src in tape_rses:
                             sources.remove(src)
                     if not sources:
-                        logger(logging.WARNING, 'Requested did {} has only replicas on tape. No files will be download.'.format(item['did']))
+                        logger(logging.WARNING, 'Requested did {} has only replicas on tape. No files will be download.'.format(file_item['did']))
 
             nrandom = item.get('nrandom')
             if nrandom:


Test patch:
diff --git a/lib/rucio/tests/temp_factories.py b/lib/rucio/tests/temp_factories.py
--- a/lib/rucio/tests/temp_factories.py
+++ b/lib/rucio/tests/temp_factories.py
@@ -178,11 +178,23 @@ def cleanup(self, session=None):
         for rse_id, dids in dids_by_rse.items():
             replica_core.delete_replicas(rse_id=rse_id, files=dids, session=session)
 
-    def upload_test_file(self, rse_name, scope=None, name=None, path=None, return_full_item=False):
+    def _sanitize_or_set_scope(self, scope):
         if not scope:
             scope = self.default_scope
         elif isinstance(scope, str):
             scope = InternalScope(scope, vo=self.vo)
+        return scope
+
+    def make_dataset(self, scope=None, name=None):
+        scope = self._sanitize_or_set_scope(scope)
+        if not name:
+            name = 'dataset_%s' % generate_uuid()
+        did = {'scope': scope, 'name': name}
+        self.client.add_dataset(scope.external, name)
+        return did
+
+    def upload_test_file(self, rse_name, scope=None, name=None, path=None, return_full_item=False):
+        scope = self._sanitize_or_set_scope(scope)
         if not path:
             path = file_generator()
         if not name:
diff --git a/lib/rucio/tests/test_download.py b/lib/rucio/tests/test_download.py
--- a/lib/rucio/tests/test_download.py
+++ b/lib/rucio/tests/test_download.py
@@ -33,6 +33,7 @@
 from rucio.client.downloadclient import DownloadClient
 from rucio.common.exception import InputValidationError, NoFilesDownloaded
 from rucio.common.utils import generate_uuid
+from rucio.core import did as did_core
 from rucio.rse import rsemanager as rsemgr
 from rucio.rse.protocols.posix import Default as PosixProtocol
 
@@ -372,3 +373,22 @@ def test_trace_copy_out_and_checksum_validation(vo, rse_factory, file_factory, d
         traces = []
         download_client.download_dids([{'did': did_str, 'base_dir': tmp_dir, 'ignore_checksum': True}], traces_copy_out=traces)
         assert len(traces) == 1 and traces[0]['clientState'] == 'DONE'
+
+
+def test_norandom_respected(rse_factory, file_factory, download_client, root_account):
+    rse, _ = rse_factory.make_posix_rse()
+    did1 = file_factory.upload_test_file(rse)
+    did2 = file_factory.upload_test_file(rse)
+    dataset = file_factory.make_dataset()
+    did_core.attach_dids(dids=[did1, did2], account=root_account, **dataset)
+
+    dataset_did_str = '%s:%s' % (dataset['scope'], dataset['name'])
+
+    with TemporaryDirectory() as tmp_dir:
+        nrandom = 1
+        result = download_client.download_dids([{'did': dataset_did_str, 'nrandom': nrandom, 'base_dir': tmp_dir}])
+        assert len(result) == nrandom
+
+        nrandom = 2
+        result = download_client.download_dids([{'did': dataset_did_str, 'nrandom': nrandom, 'base_dir': tmp_dir}])
+        assert len(result) == nrandom