Repository: mne-tools/mne-bids

Issue:
mne_bids.get_entity_vals broken
@hoechenberger @adam2392  this commit:

https://github.com/mne-tools/mne-bids/commit/893f9be2e277f1fa52c685f9dede8dcc469c4684

broke the mne bids pipeline where we use this option written in the doc:

```
    root : path-like
        Path to the "root" directory from which to start traversing to gather
        BIDS entities from file- and folder names. This will commonly be the
        BIDS root, but it may also be a subdirectory inside of a BIDS dataset,
        e.g., the ``sub-X`` directory of a hypothetical subject ``X``.
```

see the "may also be a subdirectory inside of a BIDS dataset,"

what was the issue with the other behavior? 
mne_bids.get_entity_vals broken
@hoechenberger @adam2392  this commit:

https://github.com/mne-tools/mne-bids/commit/893f9be2e277f1fa52c685f9dede8dcc469c4684

broke the mne bids pipeline where we use this option written in the doc:

```
    root : path-like
        Path to the "root" directory from which to start traversing to gather
        BIDS entities from file- and folder names. This will commonly be the
        BIDS root, but it may also be a subdirectory inside of a BIDS dataset,
        e.g., the ``sub-X`` directory of a hypothetical subject ``X``.
```

see the "may also be a subdirectory inside of a BIDS dataset,"

what was the issue with the other behavior? 


Discussion:
Oh. The issue was / is that we want to exclude certain subdirectories like derivatives, sourcedata etc. 

Previously we silently ignored derivatives

But I see the issue you're having 

Let's have a call?
I had a call with @agramfort and will propose a fix later today.
Oh. The issue was / is that we want to exclude certain subdirectories like derivatives, sourcedata etc. 

Previously we silently ignored derivatives

But I see the issue you're having 

Let's have a call?
I had a call with @agramfort and will propose a fix later today.

Resolving patch:
diff --git a/mne_bids/path.py b/mne_bids/path.py
--- a/mne_bids/path.py
+++ b/mne_bids/path.py
@@ -1512,7 +1512,8 @@ def get_entity_vals(root, entity_key, *, ignore_subjects='emptyroom',
                     ignore_processings=None, ignore_spaces=None,
                     ignore_acquisitions=None, ignore_splits=None,
                     ignore_modalities=None, ignore_datatypes=None,
-                    with_key=False, verbose=None):
+                    ignore_dirs=('derivatives', 'sourcedata'), with_key=False,
+                    verbose=None):
     """Get list of values associated with an `entity_key` in a BIDS dataset.
 
     BIDS file names are organized by key-value pairs called "entities" [1]_.
@@ -1538,28 +1539,33 @@ def get_entity_vals(root, entity_key, *, ignore_subjects='emptyroom',
 
     entity_key : str
         The name of the entity key to search for.
-    ignore_subjects : str | iterable | None
+    ignore_subjects : str | array-like of str | None
         Subject(s) to ignore. By default, entities from the ``emptyroom``
         mock-subject are not returned. If ``None``, include all subjects.
-    ignore_sessions : str | iterable | None
+    ignore_sessions : str | array-like of str | None
         Session(s) to ignore. If ``None``, include all sessions.
-    ignore_tasks : str | iterable | None
+    ignore_tasks : str | array-like of str | None
         Task(s) to ignore. If ``None``, include all tasks.
-    ignore_runs : str | iterable | None
+    ignore_runs : str | array-like of str | None
         Run(s) to ignore. If ``None``, include all runs.
-    ignore_processings : str | iterable | None
+    ignore_processings : str | array-like of str | None
         Processing(s) to ignore. If ``None``, include all processings.
-    ignore_spaces : str | iterable | None
+    ignore_spaces : str | array-like of str | None
         Space(s) to ignore. If ``None``, include all spaces.
-    ignore_acquisitions : str | iterable | None
+    ignore_acquisitions : str | array-like of str | None
         Acquisition(s) to ignore. If ``None``, include all acquisitions.
-    ignore_splits : str | iterable | None
+    ignore_splits : str | array-like of str | None
         Split(s) to ignore. If ``None``, include all splits.
-    ignore_modalities : str | iterable | None
+    ignore_modalities : str | array-like of str | None
         Modalities(s) to ignore. If ``None``, include all modalities.
-    ignore_datatypes : str | iterable | None
+    ignore_datatypes : str | array-like of str | None
         Datatype(s) to ignore. If ``None``, include all datatypes (i.e.
         ``anat``, ``ieeg``, ``eeg``, ``meg``, ``func``, etc.)
+    ignore_dirs : str | array-like of str | None
+        Directories nested directly within ``root`` to ignore. If ``None``,
+        include all directories in the search.
+
+        .. versionadded:: 0.9
     with_key : bool
         If ``True``, returns the full entity with the key and the value. This
         will for example look like ``['sub-001', 'sub-002']``.
@@ -1599,7 +1605,7 @@ def get_entity_vals(root, entity_key, *, ignore_subjects='emptyroom',
         need_dir=True,
         name='Root directory'
     )
-    root = Path(root)
+    root = Path(root).expanduser()
 
     entities = ('subject', 'task', 'session', 'run', 'processing', 'space',
                 'acquisition', 'split', 'suffix')
@@ -1621,11 +1627,26 @@ def get_entity_vals(root, entity_key, *, ignore_subjects='emptyroom',
     ignore_splits = _ensure_tuple(ignore_splits)
     ignore_modalities = _ensure_tuple(ignore_modalities)
 
+    ignore_dirs = _ensure_tuple(ignore_dirs)
+    existing_ignore_dirs = [
+        root / d for d in ignore_dirs
+        if (root / d).exists() and (root / d).is_dir()
+    ]
+    ignore_dirs = _ensure_tuple(existing_ignore_dirs)
+
     p = re.compile(r'{}-(.*?)_'.format(entity_long_abbr_map[entity_key]))
     values = list()
-    filenames = root.glob(f'sub-*/**/*{entity_long_abbr_map[entity_key]}-*_*')
+    filenames = root.glob(f'**/*{entity_long_abbr_map[entity_key]}-*_*')
 
     for filename in filenames:
+        # Skip ignored directories
+        # XXX In Python 3.9, we can use Path.is_relative_to() here
+        if any([
+            str(filename).startswith(str(ignore_dir))
+            for ignore_dir in ignore_dirs
+        ]):
+            continue
+
         if ignore_datatypes and filename.parent.name in ignore_datatypes:
             continue
         if ignore_subjects and any([filename.stem.startswith(f'sub-{s}_')


Test patch:
diff --git a/mne_bids/tests/test_path.py b/mne_bids/tests/test_path.py
--- a/mne_bids/tests/test_path.py
+++ b/mne_bids/tests/test_path.py
@@ -125,6 +125,13 @@ def test_get_entity_vals(entity, expected_vals, kwargs, return_bids_test_dir):
         assert entities == [f'{entity_long_to_short[entity]}-{val}'
                             for val in expected_vals]
 
+        # Test without ignoring the derivatives dir
+        entities = get_entity_vals(
+            root=bids_root, entity_key=entity, **kwargs, ignore_dirs=None
+        )
+        if entity not in ('acquisition', 'run'):
+            assert 'deriv' in entities
+
     # Clean up
     shutil.rmtree(deriv_path)