Repository: dandi/dandi-cli

Issue:
dandi download might generate "suboptimal" yaml
```
$> dandi download https://gui.dandiarchive.org/\#/folder/5e6d855776569eb93f451e50
2020-03-16 23:49:38,770 [    INFO] Downloading folder with id 5e6d855776569eb93f451e50 from https://girder.dandiarchive.org/
2020-03-16 23:49:38,885 [    INFO] Traversing remote dandisets (000002) recursively and downloading them locally
2020-03-16 23:49:38,885 [    INFO] Updating fdandiset.yaml from obtained dandiset metadata
(dev3) 3 10975.....................................:Mon 16 Mar 2020 11:49:39 PM EDT:.
smaug:/mnt/datasets/dandi
$> cat 000002/dandiset.yaml                                           
# DO NOT EDIT this file manually.
# It can be edied online and obtained from the dandiarchive.
# It also gets updated using dandi organize
{description: 'Activity in the mouse anterior lateral motor cortex (ALM) instructs
    directional movements, often seconds before movement initiation. It is unknown
    whether this preparatory activity is localized to ALM or widely distributed within
    motor cortex. Here we imaged activity across motor cortex while mice performed
    a whisker-based object localization task with a delayed, directional licking response.
    During tactile sensation and the delay epoch, object location was represented
    in motor cortex areas that are medial and posterior relative to ALM, including
    vibrissal motor cortex.', identifier: '000002', name: Anticipatory Activity in
    Mouse Motor Cortex}
$> dandi --version
0.4.1+2.g21b8505

```

but on my laptop -- it is all good... So likely it is a version of yaml and/or parameters to it.  We need `dandi wtf` (hence new #57).  

on bad host:
```
smaug:/mnt/datasets/dandi
$> welp yaml
PATH       : /usr/lib/python3/dist-packages/yaml/__init__.py
SRC PATH   : /usr/lib/python3/dist-packages/yaml/__init__.py
VERSION    : Not found
__version__: '3.13'
PACKAGE    : python3-yaml
ii  python3-yaml   3.13-2       amd64        YAML parser and emitter for Python3
```
on good:
```
$> welp yaml   
PATH       : /usr/lib/python3/dist-packages/yaml/__init__.py
SRC PATH   : /usr/lib/python3/dist-packages/yaml/__init__.py
VERSION    : Not found
__version__: '5.3'
PACKAGE    : python3-yaml
ii  python3-yaml   5.3-1        amd64        YAML parser and emitter for Python3
python3-yaml:
  Installed: 5.3-1
  Candidate: 5.3-2
  Version table:
     5.3-2 900
        900 http://deb.debian.org/debian bullseye/main amd64 Packages
        600 http://http.debian.net/debian sid/main amd64 Packages
 *** 5.3-1 100
        100 /var/lib/dpkg/status
```

I feel that I had such issue somewhere but now it is too late to try to remember on how to overcome.


Comments:

Patch:
diff --git a/dandi/dandiset.py b/dandi/dandiset.py
index 97a33a56..c6c5a5e1 100644
--- a/dandi/dandiset.py
+++ b/dandi/dandiset.py
@@ -4,7 +4,7 @@ import yaml
 from pathlib import Path
 
 from .consts import dandiset_metadata_file
-from .utils import find_parent_directory_containing
+from .utils import find_parent_directory_containing, yaml_dump
 
 from . import get_logger
 
@@ -52,7 +52,7 @@ class Dandiset(object):
 # It can be edied online and obtained from the dandiarchive.
 # It also gets updated using dandi organize
 """
-        yaml_rec = yaml.dump(meta)
+        yaml_rec = yaml_dump(meta)
         return header + yaml_rec
 
     def update_metadata(self, meta):
diff --git a/dandi/download.py b/dandi/download.py
index 0771a344..396c09f7 100644
--- a/dandi/download.py
+++ b/dandi/download.py
@@ -186,7 +186,7 @@ def download(
             dandiset_path = op.join(output_dir, e["path"])
             dandiset_yaml = op.join(dandiset_path, dandiset_metadata_file)
             lgr.info(
-                f"Updating f{dandiset_metadata_file} from obtained dandiset metadata"
+                f"Updating {dandiset_metadata_file} from obtained dandiset metadata"
             )
             if op.lexists(dandiset_yaml):
                 if existing != "overwrite":
diff --git a/dandi/utils.py b/dandi/utils.py
index a437d3ad..89bbd628 100644
--- a/dandi/utils.py
+++ b/dandi/utils.py
@@ -224,6 +224,7 @@ def find_files(
     regex,
     paths=os.curdir,
     exclude=None,
+    exclude_dotfiles=True,
     exclude_vcs=True,
     exclude_datalad=False,
     dirs=False,
@@ -268,6 +269,7 @@ def find_files(
                     regex,
                     paths=path,
                     exclude=exclude,
+                    exclude_dotfiles=exclude_dotfiles,
                     exclude_vcs=exclude_vcs,
                     exclude_datalad=exclude_datalad,
                     dirs=dirs,
@@ -286,6 +288,8 @@ def find_files(
     for dirpath, dirnames, filenames in os.walk(paths):
         names = (dirnames + filenames) if dirs else filenames
         # TODO: might want to uniformize on windows to use '/'
+        if exclude_dotfiles:
+            names = (n for n in names if not n.startswith("."))
         paths = (op.join(dirpath, name) for name in names)
         for path in filter(re.compile(regex).search, paths):
             if not exclude_path(path):
@@ -333,6 +337,20 @@ def find_parent_directory_containing(filename, path=None):
         path = path.parent  # go up
 
 
+def yaml_dump(rec):
+    """Consistent dump into yaml
+
+    Of primary importance is default_flow_style=False
+    to assure proper formatting on versions of pyyaml before
+    5.1: https://github.com/yaml/pyyaml/pull/256
+    """
+    import yaml
+    return yaml.safe_dump(
+        rec,
+        default_flow_style=False
+    )
+
+
 #
 # Borrowed from DataLad (MIT license)
 #


Test patch:
diff --git a/dandi/tests/test_utils.py b/dandi/tests/test_utils.py
index 7543caaf..8ea853b7 100644
--- a/dandi/tests/test_utils.py
+++ b/dandi/tests/test_utils.py
@@ -43,6 +43,35 @@ def test_find_files():
     # TODO: more tests
 
 
+def test_find_files_dotfiles(tmpdir):
+    tmpsubdir = tmpdir.mkdir("subdir")
+    for p in (".dot.nwb", "regular", ".git"):
+        for f in (tmpdir / p, tmpsubdir / p):
+            f.write_text("", "utf-8")
+
+    def relpaths(paths):
+        return sorted(op.relpath(p, tmpdir) for p in paths)
+
+    regular = ["regular", op.join("subdir", "regular")]
+    dotfiles = [".dot.nwb", op.join("subdir", ".dot.nwb")]
+    vcs = [".git", op.join("subdir", ".git")]
+
+    ff = find_files(".*", tmpdir)
+    assert relpaths(ff) == regular
+
+    ff = find_files(".*", tmpdir, exclude_dotfiles=False)
+    # we still exclude VCS
+    assert relpaths(ff) == sorted(regular + dotfiles)
+
+    # current VCS are also dot files
+    ff = find_files(".*", tmpdir, exclude_vcs=False)
+    assert relpaths(ff) == regular
+
+    # current VCS are also dot files
+    ff = find_files(".*", tmpdir, exclude_vcs=False, exclude_dotfiles=False)
+    assert relpaths(ff) == sorted(regular + dotfiles + vcs)
+
+
 def test_times_manipulations():
     t0 = get_utcnow_datetime()
     t0_isoformat = ensure_strtime(t0)