Repository: bids-standard/pybids (python)

Issue:
MacOS Finder `._*` hidden metadata files cause pybids to crash
## Example
<details><summary><code>UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 37: invalid start byte</code> … <code>while trying to decode JSON from file […]/._sub-PA069_ses-V1W1_task-poke_run-2_bold.json</code></summary>

```Python
Traceback (most recent call last):
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/bids/layout/index.py", line 303, in load_json
    return json.load(handle)
  File "/opt/conda/envs/sdcflows/lib/python3.10/json/__init__.py", line 293, in load
    return loads(fp.read(),
  File "/opt/conda/envs/sdcflows/lib/python3.10/codecs.py", line 322, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 37: invalid start byte

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/opt/conda/envs/sdcflows/bin/sdcflows", line 8, in <module>
    sys.exit(main())
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/sdcflows/cli/main.py", line 39, in main
    parse_args(argv)
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/sdcflows/cli/parser.py", line 281, in parse_args
    config.from_dict(vars(opts))
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/sdcflows/config.py", line 589, in from_dict
    execution.load(settings)
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/sdcflows/config.py", line 249, in load
    cls.init()
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/sdcflows/config.py", line 476, in init
    cls._layout = BIDSLayout(
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/bids/layout/layout.py", line 177, in __init__
    _indexer(self)
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/bids/layout/index.py", line 154, in __call__
    self._index_metadata()
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/bids/layout/index.py", line 415, in _index_metadata
    file_md.update(pl())
  File "/opt/conda/envs/sdcflows/lib/python3.10/site-packages/bids/layout/index.py", line 305, in load_json
    raise OSError(
OSError: Error occurred while trying to decode JSON from file /ocean/projects/med220004p/shared/data_raw/vannucci/bids_raw/sub-PA069/ses-V1W1/func/._sub-PA069_ses-V1W1_task-poke_run-2_bold.json
```

</details>

## Proposed Solution

I think [these types of files (`._*` and `.DS_Store`)](https://lists.apple.com/archives/applescript-users/2006/Jun/msg00180.html) can be safely ignored.

## Context

In analyzing someone else's read-only (to me) data, I hit this issue. I worked around it by creating a symlinked recreation of the data directory without the MacOS hidden metadata files, but I don't think that should have been necessary.

Pull request:
fix: Ensure dotfiles are always filtered
Dotfiles are ignored by default, but custom ignore patterns override the default ignores. This change ensures that dotfiles are always ignored in either case.

Regression test reproduces #1069 and patch fixes it.

Closes #1069.

Patch:
diff --git a/src/bids/layout/validation.py b/src/bids/layout/validation.py
index 50732811..cbf4def9 100644
--- a/src/bids/layout/validation.py
+++ b/src/bids/layout/validation.py
@@ -32,9 +32,12 @@ EXAMPLE_DERIVATIVES_DESCRIPTION = {
 
 DEFAULT_LOCATIONS_TO_IGNORE = {
     re.compile(r"^/(code|models|sourcedata|stimuli)"),
-    re.compile(r'/\.'),
 }
 
+ALWAYS_IGNORE = (
+    re.compile(r'/\.'),  # dotfiles should never be indexed
+)
+
 def absolute_path_deprecation_warning():
     warnings.warn("The absolute_paths argument will be removed from PyBIDS "
                   "in 0.14. You can easily access the relative path of "
@@ -156,9 +159,11 @@ def _sort_patterns(patterns, root):
 
 def validate_indexing_args(ignore, force_index, root):
     if ignore is None:
-        ignore = list(
-            DEFAULT_LOCATIONS_TO_IGNORE - set(force_index or [])
-        )
+        ignore = DEFAULT_LOCATIONS_TO_IGNORE - set(force_index or [])
+
+    ignore = list(ignore)
+
+    ignore.extend(ALWAYS_IGNORE)
 
     # root has already been validated to be a directory
     ignore = _sort_patterns(ignore, root)


Test patch:
diff --git a/src/bids/layout/tests/conftest.py b/src/bids/layout/tests/conftest.py
index d5959045..6451f893 100644
--- a/src/bids/layout/tests/conftest.py
+++ b/src/bids/layout/tests/conftest.py
@@ -1,4 +1,5 @@
 from os.path import join
+import shutil
 
 import pytest
 
@@ -101,3 +102,10 @@ def layout_synthetic(tests_dir, request, db_dir):
 def layout_synthetic_nodb(tests_dir, request, db_dir):
     path = tests_dir / 'data' / 'synthetic'
     return BIDSLayout(path, derivatives=True)
+
+
+@pytest.fixture
+def temporary_dataset(tmp_path, tests_dir):
+    path = tests_dir / 'data' / 'ds005'
+    shutil.copytree(path, tmp_path / 'ds005')
+    return tmp_path / 'ds005'
diff --git a/src/bids/layout/tests/test_layout.py b/src/bids/layout/tests/test_layout.py
index c0c7089a..737db4a1 100644
--- a/src/bids/layout/tests/test_layout.py
+++ b/src/bids/layout/tests/test_layout.py
@@ -1170,3 +1170,38 @@ def test_symlinks_in_path(tests_dir, tmp_path):
     os.symlink(src_sub, link_sub)
 
     assert "Subjects: 1 | Sessions: 2 | Runs: 2" in str(BIDSLayout(tmp_path / "7t_trt"))
+
+
+def test_ignore_dotfiles(temporary_dataset):
+    arbitrary_dotfile = temporary_dataset / '.dotfile'
+    ds_store = temporary_dataset / 'sub-01' / '.DS_Store'
+    osx_companion_file = temporary_dataset / '._task-mixedgamblestask_bold.json'
+
+    arbitrary_dotfile.touch()
+    ds_store.touch()
+    osx_companion_file.touch()
+
+    # Default behavior
+    layout = BIDSLayout(temporary_dataset, validate=False)
+    assert str(temporary_dataset / 'dataset_description.json') in layout.files
+    assert str(arbitrary_dotfile) not in layout.files
+    assert str(ds_store) not in layout.files
+    assert str(osx_companion_file) not in layout.files
+
+    # Explicit ignores do not disable dotfile filtering
+    indexer = BIDSLayoutIndexer(ignore=['some_ignore'])
+    layout = BIDSLayout(temporary_dataset, validate=False, indexer=indexer)
+    assert str(temporary_dataset / 'dataset_description.json') in layout.files
+    assert str(arbitrary_dotfile) not in layout.files
+    assert str(ds_store) not in layout.files
+    assert str(osx_companion_file) not in layout.files
+
+
+def test_empty_directory(temporary_dataset):
+    anat = temporary_dataset / 'sub-01' / 'anat'
+    shutil.rmtree(anat)
+    anat.mkdir()
+
+    layout = BIDSLayout(temporary_dataset)
+
+    assert layout.get(subject='01', datatype='anat') == []