Repository: bids-standard/pybids

Issue:
Add get() argument for filtering on defined (but arbitrary) entity names
We used to have the ability to filter files using `BIDSLayout.get()` on metadata fields that are defined in sidecars, but can have any value at all. This went away in the SQLAlchemy refactor, but would be much easier to re-implement now, and would be quite useful (e.g., see #509 for a concrete use case).

We could introduce a `defined_entities` argument that takes a string or list, and filters out any files that lack a defined value for at least one of the named entities. Alternatively, we could consider allowing `True` to mean "any defined value", which would nicely complement the current `[entity]=None` convention (meaning, entity must not be defined), but is pretty sketchy in that it would  ambiguous any time an entity has boolean dtype in the DB (I don't think this can actually happen right now, but in theory it could as the spec grows.)

EDIT: on second thought, the latter approach pretty much precludes having entities with boolean dtypes (since searching for `True` would then still return `False` values!), so I think the former is the way to go.
Add get() argument for filtering on defined (but arbitrary) entity names
We used to have the ability to filter files using `BIDSLayout.get()` on metadata fields that are defined in sidecars, but can have any value at all. This went away in the SQLAlchemy refactor, but would be much easier to re-implement now, and would be quite useful (e.g., see #509 for a concrete use case).

We could introduce a `defined_entities` argument that takes a string or list, and filters out any files that lack a defined value for at least one of the named entities. Alternatively, we could consider allowing `True` to mean "any defined value", which would nicely complement the current `[entity]=None` convention (meaning, entity must not be defined), but is pretty sketchy in that it would  ambiguous any time an entity has boolean dtype in the DB (I don't think this can actually happen right now, but in theory it could as the spec grows.)

EDIT: on second thought, the latter approach pretty much precludes having entities with boolean dtypes (since searching for `True` would then still return `False` values!), so I think the former is the way to go.


Discussion:
We could also just create a special enum:

```
class Query(Enum):
    NONE = auto()
    ANY = auto()
```

And then make everything literal *except* `[entity]=Query.<FLAG>`.

Edit: For consistency, we'd probably want to have a deprecation period where `None` was treated like `Query.NONE`.
That sounds reasonable. I don't really mind treating `None` like `Query.NONE` indefinitely; it's not like we have another use for it, and it saves users an import.

One other nice thing about using enums is we could potentially also add values for `FILENAME` and `METADATA` that requires the entity to be defined in the corresponding source.
We could also just create a special enum:

```
class Query(Enum):
    NONE = auto()
    ANY = auto()
```

And then make everything literal *except* `[entity]=Query.<FLAG>`.

Edit: For consistency, we'd probably want to have a deprecation period where `None` was treated like `Query.NONE`.
That sounds reasonable. I don't really mind treating `None` like `Query.NONE` indefinitely; it's not like we have another use for it, and it saves users an import.

One other nice thing about using enums is we could potentially also add values for `FILENAME` and `METADATA` that requires the entity to be defined in the corresponding source.

Resolving patch:
diff --git a/bids/layout/__init__.py b/bids/layout/__init__.py
--- a/bids/layout/__init__.py
+++ b/bids/layout/__init__.py
@@ -1,4 +1,4 @@
-from .layout import BIDSLayout, add_config_paths, parse_file_entities
+from .layout import BIDSLayout, add_config_paths, parse_file_entities, Query
 from .models import (BIDSFile, BIDSImageFile, BIDSDataFile, BIDSJSONFile,
                      Config, Entity, Tag)
 # Backwards compatibility
@@ -15,5 +15,6 @@
     "BIDSJSONFile",
     "Config",
     "Entity",
-    "Tag"
+    "Tag",
+    "Query"
 ]
diff --git a/bids/layout/layout.py b/bids/layout/layout.py
--- a/bids/layout/layout.py
+++ b/bids/layout/layout.py
@@ -9,6 +9,7 @@
 import copy
 import warnings
 import sqlite3
+import enum
 
 import sqlalchemy as sa
 from sqlalchemy.orm import joinedload
@@ -452,6 +453,8 @@ def _sanitize_query_dtypes(self, entities):
         # Fail silently because the DB may still know how to reconcile
         # type differences.
         for name, val in entities.items():
+            if isinstance(val, enum.Enum):
+                continue
             try:
                 if isinstance(val, (list, tuple)):
                     entities[name] = [ents[name]._astype(v) for v in val]
@@ -764,7 +767,12 @@ def get(self, return_type='object', target=None, scope='all',
             Any optional key/values to filter the entities on.
             Keys are entity names, values are regexes to filter on. For
             example, passing filters={'subject': 'sub-[12]'} would return
-            only files that match the first two subjects.
+            only files that match the first two subjects. In addition to
+            ordinary data types, the following enums are defined (in the 
+            Query class):
+                * Query.NONE: The named entity must not be defined.
+                * Query.ANY: the named entity must be defined, but can have any
+                    value.
 
         Returns
         -------
@@ -773,19 +781,9 @@ def get(self, return_type='object', target=None, scope='all',
 
         Notes
         -----
-        * In pybids 0.7.0, some keywords were changed. Namely: 'type'
-          becomes 'suffix', 'modality' becomes 'datatype', 'acq' becomes
-          'acquisition' and 'mod' becomes 'modality'. Using the wrong version
-          could result in get() silently returning wrong or no results. See
-          the changelog for more details.
         * In pybids 0.9.0, the 'extensions' argument has been removed in
           favor of the 'extension' entity.
         """
-        # Warn users still expecting 0.6 behavior
-        if 'type' in filters:
-            raise ValueError("As of pybids 0.7.0, the 'type' argument has been"
-                             " replaced with 'suffix'.")
-
         if 'extensions' in filters:
             filters['extension'] = filters.pop('extensions')
             warnings.warn("In pybids 0.9.0, the 'extensions' filter was "
@@ -924,11 +922,12 @@ def _build_file_query(self, **kwargs):
             for name, val in filters.items():
                 if isinstance(val, (list, tuple)) and len(val) == 1:
                     val = val[0]
-
-                if val is None:
-                    drop = query.filter(
-                        BIDSFile.tags.any(entity_name=name))
-                    query = query.except_(drop)
+                if val is None or isinstance(val, enum.Enum):
+                    name_clause = query.filter(BIDSFile.tags.any(entity_name=name))
+                    if val == Query.ANY:
+                        query = name_clause
+                    else:
+                        query = query.except_(name_clause)
                     continue
 
                 if regex:
@@ -1446,3 +1445,9 @@ def write_contents_to_file(self, entities, path_patterns=None,
         write_contents_to_file(path, contents=contents, link_to=link_to,
                                content_mode=content_mode, conflicts=conflicts,
                                root=self.root)
+
+
+class Query(enum.Enum):
+    """Enums for use with BIDSLayout.get()."""
+    NONE = 1 # Entity must not be present
+    ANY = 2  # Entity must be defined, but with an arbitrary value


Test patch:
diff --git a/bids/layout/tests/test_layout.py b/bids/layout/tests/test_layout.py
--- a/bids/layout/tests/test_layout.py
+++ b/bids/layout/tests/test_layout.py
@@ -10,7 +10,8 @@
 import pytest
 
 import bids
-from bids.layout import BIDSLayout, parse_file_entities, add_config_paths
+from bids.layout import (BIDSLayout, parse_file_entities, add_config_paths,
+                         Query)
 from bids.layout.models import (BIDSFile, BIDSImageFile, Entity, Config,
                                 FileAssociation)
 from bids.tests import get_test_data_path
@@ -226,18 +227,25 @@ def test_get_return_type_dir(layout_7t_trt, layout_7t_trt_relpath):
     assert target == res2
 
 
-def test_get_val_none(layout_7t_trt):
+@pytest.mark.parametrize("acq", [None, Query.NONE])
+def test_get_val_none(layout_7t_trt, acq):
     t1w_files = layout_7t_trt.get(subject='01', ses='1', suffix='T1w')
     assert len(t1w_files) == 1
     assert 'acq' not in t1w_files[0].path
-    t1w_files = layout_7t_trt.get(subject='01', ses='1', suffix='T1w', acquisition=None)
+    t1w_files = layout_7t_trt.get(subject='01', ses='1', suffix='T1w', acquisition=acq)
     assert len(t1w_files) == 1
-    t1w_files = layout_7t_trt.get(subject='01', ses='1', suffix='T1w', acquisition=[None])
-    assert len(t1w_files) == 1
-    bold_files = layout_7t_trt.get(subject='01', ses='1', suffix='bold', acquisition=None)
+    bold_files = layout_7t_trt.get(subject='01', ses='1', suffix='bold', acquisition=acq)
     assert len(bold_files) == 0
 
 
+def test_get_val_enum_any(layout_7t_trt):
+    t1w_files = layout_7t_trt.get(subject='01', ses='1', suffix='T1w', acquisition=Query.ANY)
+    assert not t1w_files
+    bold_files = layout_7t_trt.get(subject='01', ses='1', run=1, suffix='bold',
+                                   acquisition=Query.ANY)
+    assert len(bold_files) == 3
+
+
 def test_get_return_sorted(layout_7t_trt):
     bids_files = layout_7t_trt.get(target='subject')
     paths = [r.path for r in bids_files]