Repository: mne-tools/mne-python

Issue:
BUG: read_epochs for split files has incorrect events
I have a large epochs file, that gets split into two files when I call epochs.save()
When I load it again using mne.read_epochs(), the events corresponding to the second file are incorrect, offset by [last epoch from the first file + tmax + 10s].

The bug comes from _concatenate_epochs.

This breaks the relationship between the raw file and the events list, which (aside from causing downstream problems with my code) can lead to an error in epochs.get_data if the data were loaded with preload=False.


Discussion:
can you share a code snippet to replicate usings sample data? you can make
random events if it's just for testing.

`
import os.path as op
import mne

data_path = op.join(mne.datasets.sample.data_path(), 'MEG', 'sample')
raw = mne.io.read_raw_fif(op.join(data_path, 'sample_audvis_raw.fif'), preload=True)

events = mne.make_fixed_length_events(raw,1)
epochs = mne.Epochs(raw, events)
epochs.save('test-epo.fif',split_size='100MB')

epochs2 = mne.read_epochs('test-epo.fif',preload=False)
epochs2.get_data() # throws error
assert(all([e1==e2 for e1,e2 in zip(epochs.events[:,0],epochs2.events[:,0])])) # assertion error
`
I don't think @larsoner is good enough to fix this :)

I see what you did there :)

Yes I can try hopefully next week

Resolving patch:
diff --git a/mne/epochs.py b/mne/epochs.py
--- a/mne/epochs.py
+++ b/mne/epochs.py
@@ -2590,7 +2590,7 @@ def __init__(self, fname, proj=True, preload=True,
 
         (info, data, events, event_id, tmin, tmax, metadata, baseline,
          selection, drop_log, _) = \
-            _concatenate_epochs(ep_list, with_data=preload)
+            _concatenate_epochs(ep_list, with_data=preload, add_offset=False)
         # we need this uniqueness for non-preloaded data to work properly
         if len(np.unique(events[:, 0])) != len(events):
             raise RuntimeError('Event time samples were not unique')
@@ -2777,7 +2777,7 @@ def _compare_epochs_infos(info1, info2, ind):
                          'runs to a common head position.' % ind)
 
 
-def _concatenate_epochs(epochs_list, with_data=True):
+def _concatenate_epochs(epochs_list, with_data=True, add_offset=True):
     """Auxiliary function for concatenating epochs."""
     if not isinstance(epochs_list, (list, tuple)):
         raise TypeError('epochs_list must be a list or tuple, got %s'
@@ -2821,7 +2821,8 @@ def _concatenate_epochs(epochs_list, with_data=True):
             data.append(epochs.get_data())
         evs = epochs.events.copy()
         # add offset
-        evs[:, 0] += events_offset
+        if add_offset:
+            evs[:, 0] += events_offset
         # Update offset for the next iteration.
         # offset is the last epoch + tmax + 10 second
         events_offset += (np.max(evs[:, 0]) +


Test patch:
diff --git a/mne/tests/test_epochs.py b/mne/tests/test_epochs.py
--- a/mne/tests/test_epochs.py
+++ b/mne/tests/test_epochs.py
@@ -838,6 +838,27 @@ def test_read_write_epochs():
         assert_equal(epochs.get_data().shape[-1], 1)
 
 
+def test_split_saving():
+    """Test saving split epochs."""
+    # See gh-5102
+    tempdir = _TempDir()
+    raw = mne.io.RawArray(np.random.RandomState(0).randn(100, 10000),
+                          mne.create_info(100, 1000.))
+    events = mne.make_fixed_length_events(raw, 1)
+    epochs = mne.Epochs(raw, events)
+    epochs_data = epochs.get_data()
+    fname = op.join(tempdir, 'test-epo.fif')
+    epochs.save(fname, split_size='1MB')
+    assert op.isfile(fname)
+    assert op.isfile(fname[:-4] + '-1.fif')
+    assert op.isfile(fname[:-4] + '-2.fif')
+    assert not op.isfile(fname[:-4] + '-3.fif')
+    for preload in (True, False):
+        epochs2 = mne.read_epochs(fname, preload=preload)
+        assert_allclose(epochs2.get_data(), epochs_data)
+        assert_array_equal(epochs.events, epochs2.events)
+
+
 def test_epochs_proj():
     """Test handling projection (apply proj in Raw or in Epochs)."""
     tempdir = _TempDir()