Repository: mne-tools/mne-python

Issue:
ENH: concatenate_epochs can be more memory efficient
Hello,

I was trying to concatenate 2 epochs with mne.concatenate_epochs, but python return memory error related to numpy array.

Last_epochs_standard = mne.concatenate_epochs([epochs1,epochs2])
Loading data for 33684 events and 1688 original time points ...
Loading data for 47154 events and 1688 original time points ...
Traceback (most recent call last):
  File "<input>", line 1, in <module>
  File "/home/wvxxxx/anaconda3/envs/Nex/lib/python3.7/site-packages/mne/epochs.py", line 2723, in concatenate_epochs
    add_offset=add_offset))
  File "/home/wvxxxx/anaconda3/envs/Nex/lib/python3.7/site-packages/mne/epochs.py", line 2683, in _concatenate_epochs
    data = np.concatenate(data, axis=0)
  File "<__array_function__ internals>", line 6, in concatenate
MemoryError: Unable to allocate array with shape (80838, 32, 1688) and data type float64

 The data size is the problem with mne.concatenate. Does it have another way to proceed ?

Best.

William






Discussion:
yes it's a memory issue.

do you have an idea of the size of memory you would need to do the
operation?

>

Added this to #6380 so that we can keep track of the places we want to add `preload=str` support in one place, closing this (but feel free to continue discussion)
Not really. I have 63Gio in my actual computer. Each epoch data are not greater than 8go size.
> Each epoch data are not greater than 8go size.

On disk they are stored in single precision (4 bytes) but in memory they are stored as double (8 bytes). So:
```
>>> mne.utils.sizeof_fmt(80838 * 32 * 1688 * 8)
'32.53 GB'
```
And if each is preloaded to begin with, you have to double this requirement (> 64 GB).
I read already data with preload=False. 
Ahh, it's probably then because our code does (effectively) `epochs.get_data()` and then allocates then assigns. We can probably make this a bit more efficient by iterating over the two `epochs` objects rather than calling `get_data` (which is probably what we do) so that only one epoch per time ends up in memory. This will also only work if we know `len(epochs)` ahead of time, which is an additional complication (for non-preloaded ones we'll need to call `drop_bad` on both then iterate) but could be workable.

Resolving patch:
diff --git a/mne/epochs.py b/mne/epochs.py
--- a/mne/epochs.py
+++ b/mne/epochs.py
@@ -2943,7 +2943,10 @@ def _concatenate_epochs(epochs_list, with_data=True, add_offset=True):
             raise TypeError('epochs_list[%d] must be an instance of Epochs, '
                             'got %s' % (ei, type(epochs)))
     out = epochs_list[0]
-    data = [out.get_data()] if with_data else None
+    offsets = [0]
+    if with_data:
+        out.drop_bad()
+        offsets.append(len(out))
     events = [out.events]
     metadata = [out.metadata]
     baseline, tmin, tmax = out.baseline, out.tmin, out.tmax
@@ -2955,7 +2958,7 @@ def _concatenate_epochs(epochs_list, with_data=True, add_offset=True):
     # offset is the last epoch + tmax + 10 second
     events_offset = (np.max(out.events[:, 0]) +
                      int((10 + tmax) * epochs.info['sfreq']))
-    for ii, epochs in enumerate(epochs_list[1:]):
+    for ii, epochs in enumerate(epochs_list[1:], 1):
         _compare_epochs_infos(epochs.info, info, ii)
         if not np.allclose(epochs.times, epochs_list[0].times):
             raise ValueError('Epochs must have same times')
@@ -2974,7 +2977,8 @@ def _concatenate_epochs(epochs_list, with_data=True, add_offset=True):
                                             epochs.event_id[key]))
 
         if with_data:
-            data.append(epochs.get_data())
+            epochs.drop_bad()
+            offsets.append(len(epochs))
         evs = epochs.events.copy()
         # add offset
         if add_offset:
@@ -3004,8 +3008,17 @@ def _concatenate_epochs(epochs_list, with_data=True, add_offset=True):
             metadata = pd.concat(metadata)
         else:  # dict of dicts
             metadata = sum(metadata, list())
+    assert len(offsets) == (len(epochs_list) if with_data else 0) + 1
+    data = None
     if with_data:
-        data = np.concatenate(data, axis=0)
+        offsets = np.cumsum(offsets)
+        for start, stop, epochs in zip(offsets[:-1], offsets[1:], epochs_list):
+            this_data = epochs.get_data()
+            if data is None:
+                data = np.empty(
+                    (offsets[-1], len(out.ch_names), len(out.times)),
+                    dtype=this_data.dtype)
+            data[start:stop] = this_data
     return (info, data, events, event_id, tmin, tmax, metadata, baseline,
             selection, drop_log, verbose)
 


Test patch:
diff --git a/mne/tests/test_epochs.py b/mne/tests/test_epochs.py
--- a/mne/tests/test_epochs.py
+++ b/mne/tests/test_epochs.py
@@ -2420,7 +2420,7 @@ def test_concatenate_epochs():
     assert_equal(epochs_conc.drop_log, epochs.drop_log * 2)
 
     epochs2 = epochs.copy().load_data()
-    with pytest.raises(ValueError, match='nchan.*must match'):
+    with pytest.raises(ValueError, match=r"epochs\[1\].info\['nchan'\] must"):
         concatenate_epochs(
             [epochs, epochs2.copy().drop_channels(epochs2.ch_names[:1])])