Repository: dandi/dandi-cli

Issue:
hidden resource fork files break organize command on mac with external drive
Nac users with an external drive are likely to have hidden "resource fork" files that have the prefix "`._`". These files are invisible and can normally be safely ignored, but they break the organize command. These files are difficult for mac users to identify, as they are hidden in Finder, and only findable with `ls -a` on the command line. I think it would be better to ignore them on organize. If not, then maybe we could throw an error right away, instead of waiting for all of the meta-data to be gathered (for 18 minutes in this case) before throwing the error.

```bash
(base) Bens-MacBook-Pro-2:dandi_staging bendichter$ dandi organize -d 000008 /Volumes/easystore5T/data/Tolias/nwb -f symlink
2020-03-17 14:43:47,494 [    INFO] Loading metadata from 1319 files
[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.
[Parallel(n_jobs=-1)]: Done   2 tasks      | elapsed:    9.1s
[Parallel(n_jobs=-1)]: Done   9 tasks      | elapsed:   10.4s
...
[Parallel(n_jobs=-1)]: Done 1285 tasks      | elapsed: 18.1min
[Parallel(n_jobs=-1)]: Done 1319 out of 1319 | elapsed: 18.6min finished
2020-03-17 15:02:25,252 [ WARNING] Completely empty record for /Volumes/easystore5T/data/Tolias/nwb/._20171204_sample_2.nwb
Error: 1 out of 1319 files were found not containing all necessary metadata: /Volumes/easystore5T/data/Tolias/nwb/._20171204_sample_2.nwb
```

Discussion:

Patch:
diff --git a/dandi/utils.py b/dandi/utils.py
index a437d3ad..421dafb2 100644
--- a/dandi/utils.py
+++ b/dandi/utils.py
@@ -224,6 +224,7 @@ def find_files(
     regex,
     paths=os.curdir,
     exclude=None,
+    exclude_dotfiles=True,
     exclude_vcs=True,
     exclude_datalad=False,
     dirs=False,
@@ -268,6 +269,7 @@ def find_files(
                     regex,
                     paths=path,
                     exclude=exclude,
+                    exclude_dotfiles=exclude_dotfiles,
                     exclude_vcs=exclude_vcs,
                     exclude_datalad=exclude_datalad,
                     dirs=dirs,
@@ -286,6 +288,8 @@ def find_files(
     for dirpath, dirnames, filenames in os.walk(paths):
         names = (dirnames + filenames) if dirs else filenames
         # TODO: might want to uniformize on windows to use '/'
+        if exclude_dotfiles:
+            names = (n for n in names if not n.startswith("."))
         paths = (op.join(dirpath, name) for name in names)
         for path in filter(re.compile(regex).search, paths):
             if not exclude_path(path):


Test patch:
diff --git a/dandi/tests/test_utils.py b/dandi/tests/test_utils.py
index 7543caaf..8ea853b7 100644
--- a/dandi/tests/test_utils.py
+++ b/dandi/tests/test_utils.py
@@ -43,6 +43,35 @@ def test_find_files():
     # TODO: more tests
 
 
+def test_find_files_dotfiles(tmpdir):
+    tmpsubdir = tmpdir.mkdir("subdir")
+    for p in (".dot.nwb", "regular", ".git"):
+        for f in (tmpdir / p, tmpsubdir / p):
+            f.write_text("", "utf-8")
+
+    def relpaths(paths):
+        return sorted(op.relpath(p, tmpdir) for p in paths)
+
+    regular = ["regular", op.join("subdir", "regular")]
+    dotfiles = [".dot.nwb", op.join("subdir", ".dot.nwb")]
+    vcs = [".git", op.join("subdir", ".git")]
+
+    ff = find_files(".*", tmpdir)
+    assert relpaths(ff) == regular
+
+    ff = find_files(".*", tmpdir, exclude_dotfiles=False)
+    # we still exclude VCS
+    assert relpaths(ff) == sorted(regular + dotfiles)
+
+    # current VCS are also dot files
+    ff = find_files(".*", tmpdir, exclude_vcs=False)
+    assert relpaths(ff) == regular
+
+    # current VCS are also dot files
+    ff = find_files(".*", tmpdir, exclude_vcs=False, exclude_dotfiles=False)
+    assert relpaths(ff) == sorted(regular + dotfiles + vcs)
+
+
 def test_times_manipulations():
     t0 = get_utcnow_datetime()
     t0_isoformat = ensure_strtime(t0)