Repository: mne-tools/mne-python

Issue:
Missing values in epochs metadata are not handled correctly
I often have a column in my epochs metadata which indicates if a given response was correct or not. I thought that a boolean data type was appropriate, but because I also have missing values (if no response was given) I use the [`"boolean"` nullable dtype](https://pandas.pydata.org/docs/user_guide/boolean.html). However, indexing with such a dtype throws an error (even though the corresponding query works):

```python
import mne
import numpy as np
import pandas as pd
from random import choices

n_epochs, n_chans, n_samples = 100, 32, 1001
correct = pd.Series(choices([True, False, None], k=n_epochs), dtype="boolean")
metadata = pd.DataFrame({"correct": correct})
rng = np.random.default_rng()
epochs = mne.EpochsArray(
    data=rng.standard_normal(size=(n_epochs, n_chans, n_samples)),
    info=mne.create_info(n_chans, 500, "eeg"),
    metadata=metadata
)

metadata.query("correct")  # this works
epochs["correct"]  # this doesn't work
```

The error prints a long stack trace:

<details>

```
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File ~/Repositories/mne-python/mne/utils/mixin.py:242, in GetEpochsMixin._keys_to_idx(self, keys)
    238 try:
    239     # Assume it's a condition name
    240     return np.where(np.any(
    241         np.array([self.events[:, 2] == self.event_id[k]
--> 242                   for k in match_event_names(self.event_id, keys)]),
    243         axis=0))[0]
    244 except KeyError as err:
    245     # Could we in principle use metadata with these Epochs and keys?

File ~/Repositories/mne-python/mne/event.py:1493, in match_event_names(event_names, keys, on_missing)
   1492 if not matches:
-> 1493     _on_missing(
   1494         on_missing=on_missing,
   1495         msg=f'Event name "{key}" could not be found. The following events '
   1496             f'are present in the data: {", ".join(event_names)}',
   1497         error_klass=KeyError
   1498     )
   1500 matches = sorted(set(matches))  # deduplicate if necessary

File ~/Repositories/mne-python/mne/utils/check.py:901, in _on_missing(on_missing, msg, name, error_klass)
    900 if on_missing == 'raise':
--> 901     raise error_klass(msg)
    902 elif on_missing == 'warn':

KeyError: 'Event name "correct" could not be found. The following events are present in the data: 1'

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
Input In [40], in <cell line: 1>()
----> 1 epochs["correct"]

File ~/Repositories/mne-python/mne/utils/mixin.py:144, in GetEpochsMixin.__getitem__(self, item)
     78 def __getitem__(self, item):
     79     """Return an Epochs object with a copied subset of epochs.
     80 
     81     Parameters
   (...)
    142        .. versionadded:: 0.16
    143     """
--> 144     return self._getitem(item)

File ~/Repositories/mne-python/mne/utils/mixin.py:195, in GetEpochsMixin._getitem(***failed resolving arguments***)
    192 self._data = inst._data = data
    193 del self
--> 195 select = inst._item_to_select(item)
    196 has_selection = hasattr(inst, 'selection')
    197 if has_selection:

File ~/Repositories/mne-python/mne/utils/mixin.py:153, in GetEpochsMixin._item_to_select(self, item)
    150 # Convert string to indices
    151 if isinstance(item, (list, tuple)) and len(item) > 0 and \
    152         isinstance(item[0], str):
--> 153     select = self._keys_to_idx(item)
    154 elif isinstance(item, slice):
    155     select = item

File ~/Repositories/mne-python/mne/utils/mixin.py:262, in GetEpochsMixin._keys_to_idx(self, keys)
    259         msg += (' The epochs.metadata Pandas query did not '
    260                 'yield any results: %s' % (exp.args[0],))
    261     else:
--> 262         return np.where(mask)[0]
    263 else:
    264     # If not, warn this might be a problem
    265     msg += (' The epochs.metadata Pandas query could not '
    266             'be performed, consider installing Pandas.')

File <__array_function__ internals>:180, in where(*args, **kwargs)

File pandas/_libs/missing.pyx:382, in pandas._libs.missing.NAType.__bool__()

TypeError: boolean value of NA is ambiguous
```

</details>

Is this something we should support? Or should I just go back to using an `object` dtype column (which would mean more verbose indexing such as `epochs["correct == 'True'"]`)?



Discussion:
> Is this something we should support?

I agree it would be nice if this just worked
metadata.query("correct")  # this works

does not work for me.

That's strange. I just checked and it works for me. The important thing is the `dtype="boolean"` argument, which is different from the NumPy-based `"bool"`. Maybe it requires the latest pandas (I have 1.4.2)?
I have the latest pandas

> Message ID: ***@***.***>
>

Can you post your error? Can anyone else run the example and reproduce either my behavior or the error?
```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/var/folders/97/27l9b459769fxdczmngx97xr0000gn/T/ipykernel_49033/179100806.py in <module>
     14 )
     15 
---> 16 metadata.query("correct")

~/miniconda3/lib/python3.8/site-packages/pandas/core/frame.py in query(self, expr, inplace, **kwargs)
   4109         kwargs["level"] = kwargs.pop("level", 0) + 1
   4110         kwargs["target"] = None
-> 4111         res = self.eval(expr, **kwargs)
   4112 
   4113         try:

~/miniconda3/lib/python3.8/site-packages/pandas/core/frame.py in eval(self, expr, inplace, **kwargs)
   4238         kwargs["resolvers"] = tuple(kwargs.get("resolvers", ())) + resolvers
   4239 
-> 4240         return _eval(expr, inplace=inplace, **kwargs)
   4241 
   4242     def select_dtypes(self, include=None, exclude=None) -> DataFrame:

~/miniconda3/lib/python3.8/site-packages/pandas/core/computation/eval.py in eval(expr, parser, engine, truediv, local_dict, global_dict, resolvers, level, target, inplace)
    353         eng = ENGINES[engine]
    354         eng_inst = eng(parsed_expr)
--> 355         ret = eng_inst.evaluate()
    356 
    357         if parsed_expr.assigner is None:

~/miniconda3/lib/python3.8/site-packages/pandas/core/computation/engines.py in evaluate(self)
     78 
     79         # make sure no names in resolvers and locals/globals clash
---> 80         res = self._evaluate()
     81         return reconstruct_object(
     82             self.result_type, res, self.aligned_axes, self.expr.terms.return_type

~/miniconda3/lib/python3.8/site-packages/pandas/core/computation/engines.py in _evaluate(self)
    119         scope = env.full_scope
    120         _check_ne_builtin_clash(self.expr)
--> 121         return ne.evaluate(s, local_dict=scope)
    122 
    123 

~/miniconda3/lib/python3.8/site-packages/numexpr/necompiler.py in evaluate(ex, local_dict, global_dict, out, order, casting, **kwargs)
    819 
    820     # Create a signature
--> 821     signature = [(name, getType(arg)) for (name, arg) in
    822                  zip(names, arguments)]
    823 

~/miniconda3/lib/python3.8/site-packages/numexpr/necompiler.py in <listcomp>(.0)
    819 
    820     # Create a signature
--> 821     signature = [(name, getType(arg)) for (name, arg) in
    822                  zip(names, arguments)]
    823 

~/miniconda3/lib/python3.8/site-packages/numexpr/necompiler.py in getType(a)
    701     if kind == 'S':
    702         return bytes
--> 703     raise ValueError("unknown type %s" % a.dtype.name)
    704 
    705 

ValueError: unknown type object
```
I know what the pb is. You don't have numexpr installed on your machine and I do.

`metadata.query("correct", engine="python")`

works
and now I see:

`KeyError: 'Event name "correct == True" could not be found. The following events are present in the data: 1'`

but the problem is that bool cannot have NaN is native Python or in numpy. So the fix I see is to detect the NaN in `mask` after doing `mask = self.metadata.eval(keys[0], engine='python').values` in mixin.py and make sure the NaN are considered as False
I would not assume that missing values are automatically `False`. In fact, they should be neither `True` nor `False` because they are simply missing. This is also what the pandas query (apparently with the Python engine) does.
but the masking operation is to select what epochs to keep. So what do you
do with NaN you keep them by default?

> Message ID: ***@***.***>
>

No, I drop them by default. My expectation is:

- `metadata.query("correct")`: rows where column `correct` is `True`
- `metadata.query("not correct")`: rows where column `correct` is `False`
- `metadata.query("correct.isna()")`: rows where column `correct` is `<NA>`

Or am I completely wrong?

I could see a problem with our indexing: `epochs["correct"]` normally picks out epochs with the `correct` label, so this is not a query. I don't know how this ambiguity is currently handled or how we should handle it. We could enforce a more explicit query syntax for this, i.e. `epochs["correct == True"]`, but this would be rather unusual (and it currently does not work).
we use eval internally and not query. We could switch to query to get the
kept indices and use this for selection.

can you give it a try?

> Message ID: ***@***.***>
>


Resolving patch:
diff --git a/mne/utils/mixin.py b/mne/utils/mixin.py
--- a/mne/utils/mixin.py
+++ b/mne/utils/mixin.py
@@ -254,12 +254,15 @@ def _keys_to_idx(self, keys):
                 self._check_metadata(metadata=md)
                 try:
                     # Try metadata
-                    mask = self.metadata.eval(keys[0], engine='python').values
+                    vals = self.metadata.reset_index().query(
+                        keys[0],
+                        engine='python'
+                    ).index.values
                 except Exception as exp:
                     msg += (' The epochs.metadata Pandas query did not '
                             'yield any results: %s' % (exp.args[0],))
                 else:
-                    return np.where(mask)[0]
+                    return vals
             else:
                 # If not, warn this might be a problem
                 msg += (' The epochs.metadata Pandas query could not '


Test patch:
diff --git a/mne/tests/test_epochs.py b/mne/tests/test_epochs.py
--- a/mne/tests/test_epochs.py
+++ b/mne/tests/test_epochs.py
@@ -2969,7 +2969,7 @@ def test_default_values():
 @requires_pandas
 def test_metadata(tmp_path):
     """Test metadata support with pandas."""
-    from pandas import DataFrame
+    from pandas import DataFrame, Series, NA
 
     data = np.random.randn(10, 2, 2000)
     chs = ['a', 'b']
@@ -3154,6 +3154,23 @@ def test_metadata(tmp_path):
         assert len(epochs) == 1
         assert len(epochs.metadata) == 1
 
+    # gh-10705: support boolean columns
+    metadata = DataFrame(
+        {"A": Series([True, True, True, False, False, NA], dtype="boolean")}
+    )
+    rng = np.random.default_rng()
+    epochs = mne.EpochsArray(
+        data=rng.standard_normal(size=(6, 8, 500)),
+        info=mne.create_info(8, 250, "eeg"),
+        event_id={"A": 1},
+        metadata=metadata
+    )
+
+    assert len(epochs["A"]) == 6  # epochs of event type A
+    assert len(epochs["A == True"]) == 3  # epochs for which column A == True
+    assert len(epochs["not A"]) == 2  # epochs for which column A == False
+    assert len(epochs["A.isna()"]) == 1  # epochs for NA in column A
+
 
 def assert_metadata_equal(got, exp):
     """Assert metadata are equal."""