Repository: mne-tools/mne-python

Issue:
FIX: metadata index behavior does not comply with pandas
the raw index is reset by default. This makes the user lose track of the original indices of the metadata. #bad

```python
import mne
import numpy as np
import pandas as pd
epo = mne.Epochs(mne.io.RawArray(np.random.randn(10, 1000),
                                 mne.create_info(10, 1000.)),
                 events=np.array([[100, 0, 1], [200, 0, 1], [300, 0, 1]]),
                 tmin=-.050, tmax=.100,
                 metadata=pd.DataFrame([dict(original_key=idx) for idx in range(3)]))

epo[1:].metadata  # the raw index is reset by default. 
```


Discussion:
[#wetalkedaboutthis](https://github.com/mne-tools/mne-python/pull/4414#issuecomment-331449259)

[#rtfm](https://mne-tools.github.io/dev/auto_examples/preprocessing/plot_metadata_query.html) (though maybe we should make it more prominent)

#amidoingthisright

... basically we lose it during I/O and it would take some thinking (i.e., building something custom on top of the I/O that Pandas natively provides) to get it to be saved. Can you use `epochs.selection` or something else? Can we just advise people to add a column of `np.arange` if they really want the original indices, or is this too much of a hack?
> #wetalkedaboutthis

Maybe, my memory isn't great.

In any case, this remains a very bad design an can cause major analytical errors; might as well have a separate Dataframe.

How about we assign a `_index` column on the fly to store and regenerate the right indices? Then would just have to indicate to the subjects that this column name shouldn't be touched.

```python
pd.DataFrame([dict(trial=ii) for ii in range(10)], index=range(1, 11))
```
Can you check to see if it's redundant with `epochs.selection`? If so we can just use that perhaps
> Can you check to see if it's redundant with epochs.selection? If so we can just use that perhaps

yes, that seems to be doing the job. So we'd just need to do `pd.DataFrame(metadata, index= epochs.selection)` when subselecting epochs (`epochs[idx:]`), reading or writing.


yes. can we close?
We still need to create the fix. Specifically, anytime we internally read/write/modify the metadata, we need to pass the `epochs.selection` to the metadata index. 

(I don't have the bandwidth right now, sorry)

Resolving patch:
diff --git a/examples/preprocessing/plot_metadata_query.py b/examples/preprocessing/plot_metadata_query.py
--- a/examples/preprocessing/plot_metadata_query.py
+++ b/examples/preprocessing/plot_metadata_query.py
@@ -63,8 +63,11 @@ def plot_query_results(query):
 # First we'll create our metadata object. This should be a
 # :class:`pandas.DataFrame` with each row corresponding to an event.
 #
-# .. warning:: The Dataframe Index can change during MNE I/O operations, so
-#              do not rely on it to query your metadata.
+# .. warning:: Do not set or change the Dataframe index of ``epochs.metadata``.
+#              It will be controlled by MNE to mirror ``epochs.selection``.
+#              Also, while some inplace operations on ``epochs.metadata`` are
+#              possible, do not manually drop or add rows, as this will
+#              create inconsistency between the metadata and actual data.
 
 metadata = {'event_time': events[:, 0] / raw.info['sfreq'],
             'trial_number': range(len(events)),
diff --git a/mne/epochs.py b/mne/epochs.py
--- a/mne/epochs.py
+++ b/mne/epochs.py
@@ -280,11 +280,15 @@ def __init__(self, info, data, events, event_id=None, tmin=-0.2, tmax=0.5,
                         pass
 
             values = list(self.event_id.values())
-            selected = np.in1d(events[:, 2], values)
+            selected = np.where(np.in1d(events[:, 2], values))[0]
             if selection is None:
-                self.selection = np.where(selected)[0]
+                selection = selected
             else:
-                self.selection = selection
+                selection = np.array(selection, int)
+            if selection.shape != (len(selected),):
+                raise ValueError('selection must be shape %s got shape %s'
+                                 % (selected.shape, selection.shape))
+            self.selection = selection
             if drop_log is None:
                 self.drop_log = [list() if k in self.selection else ['IGNORED']
                                  for k in range(len(events))]
@@ -396,13 +400,10 @@ def __init__(self, info, data, events, event_id=None, tmin=-0.2, tmax=0.5,
                 self._data[ii] = np.dot(self._projector, epoch)
         self._filename = str(filename) if filename is not None else filename
 
-    @property
-    def metadata(self):
-        """Get a copy of the metadata."""
-        return deepcopy(self._metadata)
-
-    @metadata.setter
-    def metadata(self, metadata):
+    def _check_metadata(self, metadata=None, reset_index=False):
+        """Check metadata consistency."""
+        # reset_index=False will not copy!
+        metadata = self.metadata if metadata is None else metadata
         if metadata is not None:
             pd = _check_pandas_installed(strict=False)
             if pd is not False:
@@ -413,13 +414,30 @@ def metadata(self, metadata):
                     raise ValueError('metadata must have the same number of '
                                      'rows (%d) as events (%d)'
                                      % (len(metadata), len(self.events)))
-                metadata = metadata.reset_index(drop=True)  # makes a copy, too
-                n_col = metadata.shape[1]
+                if reset_index:
+                    metadata = metadata.reset_index(drop=True)  # makes a copy
+                    metadata.index = self.selection
             else:
                 if not isinstance(metadata, list):
                     raise TypeError('metdata must be a list, got %s'
                                     % (type(metadata),))
-                metadata = deepcopy(metadata)
+                if reset_index:
+                    metadata = deepcopy(metadata)
+        return metadata
+
+    @property
+    def metadata(self):
+        """Get the metadata."""
+        return self._metadata
+
+    @metadata.setter
+    @verbose
+    def metadata(self, metadata, verbose=None):
+        metadata = self._check_metadata(metadata, reset_index=True)
+        if metadata is not None:
+            if _check_pandas_installed(strict=False):
+                n_col = metadata.shape[1]
+            else:
                 n_col = len(metadata[0])
             n_col = ' with %d columns' % n_col
         else:
@@ -1105,14 +1123,15 @@ def drop(self, indices, reason='USER', verbose=None):
 
         if indices.dtype == bool:
             indices = np.where(indices)[0]
+        try_idx = np.where(indices < 0, indices + len(self.events), indices)
 
-        out_of_bounds = (indices < 0) | (indices >= len(self.events))
+        out_of_bounds = (try_idx < 0) | (try_idx >= len(self.events))
         if out_of_bounds.any():
             first = indices[out_of_bounds][0]
             raise IndexError("Epoch index %d is out of bounds" % first)
-        keep = np.setdiff1d(np.arange(len(self.events)), indices)
+        keep = np.setdiff1d(np.arange(len(self.events)), try_idx)
         self._getitem(keep, reason, copy=False, drop_event_id=False)
-        count = len(indices)
+        count = len(try_idx)
         logger.info('Dropped %d epoch%s' % (count, _pl(count)))
         return self
 
@@ -1214,12 +1233,7 @@ def _get_data(self, out=True, verbose=None):
             logger.info("%d bad epochs dropped" % (n_events - len(good_idx)))
 
             # Now update our properties
-            if len(good_idx) == 0:  # silly fix for old numpy index error
-                self.selection = np.array([], int)
-                self.events = np.empty((0, 3))
-            else:
-                self.selection = self.selection[good_idx]
-                self.events = np.atleast_2d(self.events[good_idx])
+            self._getitem(good_idx, None, copy=False, drop_event_id=False)
 
             # adjust the data size if there is a reason to (output or update)
             if out or self.preload:
@@ -1381,8 +1395,9 @@ def _keys_to_idx(self, keys):
             pd = _check_pandas_installed(strict=False)
             # See if the query can be done
             if pd is not False:
+                self._check_metadata()
                 try:
-                    # Try metadata matching
+                    # Try metadata
                     mask = self.metadata.eval(keys[0], engine='python').values
                 except Exception as exp:
                     msg += (' The epochs.metadata Pandas query did not '
@@ -1461,24 +1476,32 @@ def _getitem(self, item, reason='IGNORED', copy=True, drop_event_id=True):
             item = [item]
 
         # Convert string to indices
-        if isinstance(item, (list, tuple)) and \
+        if isinstance(item, (list, tuple)) and len(item) > 0 and \
                 isinstance(item[0], string_types):
             select = epochs._keys_to_idx(item)
+        elif isinstance(item, slice):
+            select = item
         else:
-            select = item if isinstance(item, slice) else np.atleast_1d(item)
+            select = np.atleast_1d(item)
+            if len(select) == 0:
+                select = np.array([], int)
 
         key_selection = epochs.selection[select]
-        for k in np.setdiff1d(epochs.selection, key_selection):
-            epochs.drop_log[k] = [reason]
+        if reason is not None:
+            for k in np.setdiff1d(epochs.selection, key_selection):
+                epochs.drop_log[k] = [reason]
         epochs.selection = key_selection
         epochs.events = np.atleast_2d(epochs.events[select])
         if epochs.metadata is not None:
             pd = _check_pandas_installed(strict=False)
             if pd is not False:
                 metadata = epochs.metadata.iloc[select]
+                metadata.index = epochs.selection
             else:
                 metadata = np.array(epochs.metadata, 'object')[select].tolist()
-            epochs.metadata = metadata  # will reset the index for us
+
+            # will reset the index for us
+            BaseEpochs.metadata.fset(epochs, metadata, verbose=False)
         if epochs.preload:
             # ensure that each Epochs instance owns its own data so we can
             # resize later if necessary
@@ -1544,7 +1567,8 @@ def copy(self):
         new._raw = raw
         return new
 
-    def save(self, fname, split_size='2GB'):
+    @verbose
+    def save(self, fname, split_size='2GB', verbose=True):
         """Save epochs in a fif file.
 
         Parameters
@@ -1560,6 +1584,10 @@ def save(self, fname, split_size='2GB'):
             Note: Due to FIFF file limitations, the maximum split size is 2GB.
 
             .. versionadded:: 0.10.0
+        verbose : bool, str, int, or None
+            If not None, override default verbose level (see
+            :func:`mne.verbose` and :ref:`Logging documentation <tut_logging>`
+            for more).
 
         Notes
         -----
@@ -1873,8 +1901,7 @@ class Epochs(BaseEpochs):
         subsets of data, see :meth:`mne.Epochs.__getitem__`.
         When a subset of the epochs is created in this (or any other
         supported) manner, the metadata object is subsetted in the same manner.
-        MNE does not guarantee that the Dataframe Index is consistent, so do
-        not rely on it, but only on the columns.
+        MNE will modify the row indices to match ``epochs.selection``.
 
         .. versionadded:: 0.16
     verbose : bool, str, int, or None
@@ -2047,6 +2074,11 @@ class EpochsArray(BaseEpochs):
     metadata : instance of pandas.DataFrame | None
         See :class:`mne.Epochs` docstring for details.
 
+        .. versionadded:: 0.16
+    selection : ndarray | None
+        The selection compared to the original set of epochs.
+        Can be None to use ``np.arange(len(events))``.
+
         .. versionadded:: 0.16
     verbose : bool, str, int, or None
         If not None, override default verbose level (see :func:`mne.verbose`
@@ -2074,7 +2106,7 @@ class EpochsArray(BaseEpochs):
     def __init__(self, data, info, events=None, tmin=0, event_id=None,
                  reject=None, flat=None, reject_tmin=None,
                  reject_tmax=None, baseline=None, proj=True,
-                 on_missing='error', metadata=None,
+                 on_missing='error', metadata=None, selection=None,
                  verbose=None):  # noqa: D102
         dtype = np.complex128 if np.any(np.iscomplex(data)) else np.float64
         data = np.asanyarray(data, dtype=dtype)
@@ -2096,12 +2128,11 @@ def __init__(self, data, info, events=None, tmin=0, event_id=None,
         tmax = (data.shape[2] - 1) / info['sfreq'] + tmin
         if event_id is None:  # convert to int to make typing-checks happy
             event_id = dict((str(e), int(e)) for e in np.unique(events[:, 2]))
-        super(EpochsArray, self).__init__(info, data, events, event_id, tmin,
-                                          tmax, baseline, reject=reject,
-                                          flat=flat, reject_tmin=reject_tmin,
-                                          reject_tmax=reject_tmax, decim=1,
-                                          metadata=metadata,
-                                          proj=proj, on_missing=on_missing)
+        super(EpochsArray, self).__init__(
+            info, data, events, event_id, tmin, tmax, baseline, reject=reject,
+            flat=flat, reject_tmin=reject_tmin, reject_tmax=reject_tmax,
+            decim=1, metadata=metadata, selection=selection, proj=proj,
+            on_missing=on_missing)
         if len(events) != np.in1d(self.events[:, 2],
                                   list(self.event_id.values())).sum():
             raise ValueError('The events must only contain event numbers from '
diff --git a/tutorials/plot_metadata_epochs.py b/tutorials/plot_metadata_epochs.py
--- a/tutorials/plot_metadata_epochs.py
+++ b/tutorials/plot_metadata_epochs.py
@@ -8,7 +8,7 @@
 For related uses of :class:`mne.Epochs`, see the starting tutorial
 :ref:`sphx_glr_auto_tutorials_plot_object_epochs.py`.
 
-Sometimes you've got a more complex trials structure that cannot be easily
+Sometimes you may have a complex trial structure that cannot be easily
 summarized as a set of unique integers. In this case, it may be useful to use
 the ``metadata`` attribute of :class:`mne.Epochs` objects. This must be a
 :class:`pandas.DataFrame` where each row corresponds to an epoch, and each


Test patch:
diff --git a/mne/tests/test_epochs.py b/mne/tests/test_epochs.py
--- a/mne/tests/test_epochs.py
+++ b/mne/tests/test_epochs.py
@@ -30,7 +30,8 @@
     bootstrap, equalize_epoch_counts, combine_event_ids, add_channels_epochs,
     EpochsArray, concatenate_epochs, BaseEpochs, average_movements)
 from mne.utils import (_TempDir, requires_pandas, run_tests_if_main,
-                       requires_version, _check_pandas_installed)
+                       requires_version, _check_pandas_installed,
+                       catch_logging)
 from mne.chpi import read_head_pos, head_pos_to_trans_rot_t
 
 from mne.io import RawArray, read_raw_fif
@@ -1733,7 +1734,7 @@ def test_drop_epochs():
 
     # Bound checks
     assert_raises(IndexError, epochs.drop, [len(epochs.events)])
-    assert_raises(IndexError, epochs.drop, [-1])
+    assert_raises(IndexError, epochs.drop, [-len(epochs.events) - 1])
     assert_raises(ValueError, epochs.drop, [[1, 2], [3, 4]])
 
     # Test selection attribute
@@ -2012,6 +2013,10 @@ def test_array_epochs():
                   event_id)
     assert_raises(ValueError, EpochsArray, data, info, events, tmin,
                   dict(a=1))
+    assert_raises(ValueError, EpochsArray, data, info, events, tmin,
+                  selection=[1])
+    # should be fine
+    EpochsArray(data, info, events, tmin, selection=np.arange(len(events)) + 5)
 
     # saving
     temp_fname = op.join(tempdir, 'test-epo.fif')
@@ -2226,8 +2231,11 @@ def test_metadata():
     event_id = {'zero': 0, 'one': 1}
     epochs = EpochsArray(data, info, metadata=meta,
                          events=events, event_id=event_id)
+    indices = np.arange(len(epochs))  # expected indices
+    assert_array_equal(epochs.metadata.index, indices)
 
     assert len(epochs[[1, 2]].events) == len(epochs[[1, 2]].metadata)
+    assert_array_equal(epochs[[1, 2]].metadata.index, indices[[1, 2]])
     assert len(epochs['one']) == 5
 
     # Construction
@@ -2261,17 +2269,29 @@ def test_metadata():
     with pytest.raises(KeyError):
         epochs['blah == "yo"']
 
+    assert_array_equal(epochs.selection, indices)
+    epochs.drop(0)
+    assert_array_equal(epochs.selection, indices[1:])
+    assert_array_equal(epochs.metadata.index, indices[1:])
+    epochs.drop([0, -1])
+    assert_array_equal(epochs.selection, indices[2:-1])
+    assert_array_equal(epochs.metadata.index, indices[2:-1])
+    assert_array_equal(len(epochs), 7)  # originally 10
+
     # I/O
     # Make sure values don't change with I/O
     tempdir = _TempDir()
     temp_fname = op.join(tempdir, 'tmp-epo.fif')
     temp_one_fname = op.join(tempdir, 'tmp-one-epo.fif')
-    epochs.save(temp_fname)
+    with catch_logging() as log:
+        epochs.save(temp_fname, verbose=True)
+    assert log.getvalue() == ''  # assert no junk from metadata setting
     epochs_read = read_epochs(temp_fname, preload=True)
     assert_metadata_equal(epochs.metadata, epochs_read.metadata)
     epochs_arr = EpochsArray(epochs._data, epochs.info, epochs.events,
                              tmin=0, event_id=epochs.event_id,
-                             metadata=epochs.metadata)
+                             metadata=epochs.metadata,
+                             selection=epochs.selection)
     assert_metadata_equal(epochs.metadata, epochs_arr.metadata)
 
     with pytest.raises(TypeError):  # Needs to be a dataframe
@@ -2322,6 +2342,28 @@ def test_metadata():
     assert_metadata_equal(epochs_one_nopandas_read.metadata,
                           epochs_one.metadata)
 
+    # gh-4820
+    raw_data = np.random.randn(10, 1000)
+    info = mne.create_info(10, 1000.)
+    raw = mne.io.RawArray(raw_data, info)
+    events = [[0, 0, 1], [100, 0, 1], [200, 0, 1], [300, 0, 1]]
+    metadata = DataFrame([dict(idx=idx) for idx in range(len(events))])
+    epochs = mne.Epochs(raw, events=events, tmin=-.050, tmax=.100,
+                        metadata=metadata)
+    epochs.drop_bad()
+    assert len(epochs) == len(epochs.metadata)
+
+    # gh-4821
+    epochs.metadata['new_key'] = 1
+    assert_array_equal(epochs['new_key == 1'].get_data(),
+                       epochs.get_data())
+    # ensure bad user changes break things
+    epochs.metadata.drop(epochs.metadata.index[2], inplace=True)
+    assert len(epochs.metadata) == len(epochs) - 1
+    with pytest.raises(ValueError,
+                       match='metadata must have the same number of rows .*'):
+        epochs['new_key == 1']
+
 
 def assert_metadata_equal(got, exp):
     """Assert metadata are equal."""