Repository: dandi/dandi-cli

Issue:
dandi download error
User reported error, I cannot reproduce:

```
dandi download https://gui.dandiarchive.org/#/file-browser/folder/5e72840a3da50caa9adb0489
...
2020-04-14 15:19:52,862 [       INFO] Updating dandiset.yaml from obtained dandiset metadat
Error: 'generator' object is not subscriptable
```

Which is strange because this command works for me, as well as:

```
dandi download https://dandiarchive.org/dandiset/000009/draft
```

The user is on a Windows machine. I'll update as I gather more info from the user.

Comments:

Patch:
diff --git a/dandi/download.py b/dandi/download.py
index b06e5080..1a09713a 100644
--- a/dandi/download.py
+++ b/dandi/download.py
@@ -5,7 +5,7 @@ import urllib.parse as up
 from . import girder, get_logger
 from .consts import dandiset_metadata_file
 from .dandiset import Dandiset
-from .utils import flattened
+from .utils import flatten, flattened
 
 lgr = get_logger()
 
@@ -150,7 +150,7 @@ def download(
 
     # there might be multiple asset_ids, e.g. if multiple files were selected etc,
     # so we will traverse all of them
-    files = flattened(
+    files = flatten(
         _get_asset_files(
             asset_id_, asset_type, output_dir, client, authenticate, existing, recursive
         )
diff --git a/dandi/utils.py b/dandi/utils.py
index 89bbd628..c29c8e06 100644
--- a/dandi/utils.py
+++ b/dandi/utils.py
@@ -1,4 +1,5 @@
 import datetime
+import inspect
 import itertools
 import os
 import os.path as op
@@ -178,15 +179,18 @@ def ensure_datetime(t, strip_tzinfo=False, tz=None):
 #
 # Generic
 #
-def flattened(it):
-    """Return a list with all items flattened if list or tuple"""
-    items = []
+def flatten(it):
+    """Yield items flattened if list, tuple or a generator"""
     for i in it:
-        if isinstance(i, (list, tuple)):
-            items.extend(flattened(i))
+        if isinstance(i, (list, tuple)) or inspect.isgenerator(i):
+            yield from flattened(i)
         else:
-            items.append(i)
-    return items
+            yield i
+
+
+def flattened(it):
+    """Return list with items flattened if list, tuple or a generator"""
+    return list(flatten(it))
 
 
 def updated(d, update):
@@ -345,10 +349,8 @@ def yaml_dump(rec):
     5.1: https://github.com/yaml/pyyaml/pull/256
     """
     import yaml
-    return yaml.safe_dump(
-        rec,
-        default_flow_style=False
-    )
+
+    return yaml.safe_dump(rec, default_flow_style=False)
 
 
 #


Test patch:
diff --git a/dandi/support/tests/test_cache.py b/dandi/support/tests/test_cache.py
index b307760c..5bb5c40d 100644
--- a/dandi/support/tests/test_cache.py
+++ b/dandi/support/tests/test_cache.py
@@ -153,14 +153,39 @@ def test_memoize_path(cache, tmp_path):
     check_new_memoread(1, "Content")
 
 
-def test_memoize_path_persist():
+def test_memoize_path_persist(tmp_path):
     from subprocess import run, PIPE
 
-    cache = PersistentCache(name=op.basename(__file__))
+    cache_name = op.basename(tmp_path)
+    script = op.join(tmp_path, "script.py")
+    with open(script, "w") as f:
+        f.write(
+            f"""\
+from os.path import basename
+from dandi.support.cache import PersistentCache
+cache = PersistentCache(name="{cache_name}")
+
+@cache.memoize_path
+def func(path):
+    print("Running %s." % basename(path), end="")
+    return "DONE"
+
+print(func(r"{script}"))
+"""
+        )
+
+    cache = PersistentCache(name=cache_name)
     cache.clear()
 
-    outputs = [run([sys.executable, __file__], stdout=PIPE) for i in range(3)]
-    assert outputs[0].stdout.strip().decode() == f"Running on {__file__}.DONE"
+    outputs = [
+        run([sys.executable, script], stdout=PIPE, stderr=PIPE) for i in range(3)
+    ]
+    print("Full outputs: %s" % repr(outputs))
+    if b"File name too long" in outputs[0].stderr:
+        # must be running during conda build which blows up paths with
+        # _placehold_ers
+        pytest.skip("seems to be running on conda and hitting the limits")
+    assert outputs[0].stdout.strip().decode() == f"Running script.py.DONE"
     for o in outputs[1:]:
         assert o.stdout.strip().decode() == f"DONE"
 
@@ -198,15 +223,3 @@ def test_memoize_path_tokens(tmp_path, cache, cache_tokens):
     # They both are independent, so both will cause a new readout
     check_new_memoread(memoread, 0, "content")
     check_new_memoread(memoread_tokens, 0, "content")
-
-
-if __name__ == "__main__":
-    infile = __file__
-    cache = PersistentCache(name=op.basename(infile))
-
-    @cache.memoize_path
-    def func(path):
-        print(f"Running on {path}.", end="")
-        return "DONE"
-
-    print(func(infile))
diff --git a/dandi/tests/test_utils.py b/dandi/tests/test_utils.py
index 8ea853b7..8347521d 100644
--- a/dandi/tests/test_utils.py
+++ b/dandi/tests/test_utils.py
@@ -7,6 +7,8 @@ from ..utils import (
     ensure_datetime,
     ensure_strtime,
     find_files,
+    flatten,
+    flattened,
     get_utcnow_datetime,
     is_same_time,
     on_windows,
@@ -106,3 +108,17 @@ def test_time_samples(t):
     assert is_same_time(
         ensure_datetime(t), "2018-09-27 00:29:17-00:00", tollerance=0
     )  # exactly the same
+
+
+def test_flatten():
+    assert inspect.isgenerator(flatten([1]))
+    # flattened is just a list() around flatten
+    assert flattened([1, [2, 3, [4]], 5, (i for i in range(2))]) == [
+        1,
+        2,
+        3,
+        4,
+        5,
+        0,
+        1,
+    ]