Repository: mne-tools/mne-python

Issue:
Memory-Mapping Preload Fails When Adding Channels
When data is memory-mapped to a file, there is a memory error in:

e-packages/mne/channels/channels.py", line 877, in add_channels
    data = np.concatenate(data, axis=con_axis)
MemoryError



Discussion:
A [minimal reproducible example](http://sscce.org/) on [gist](https://gist.github.com/) would be very much appreciated
https://gist.github.com/alexrockhill/f2ddbae467e8e5728db48834f2601e0d

Here is a gist that could work. There is some difficulty because the memory error depends on the RAM of the machine and also to create a RawArray, one must be able to load the RawArray into memory, so there must be a workaround where many raw files are made and loaded into memory-mapped files.
We could probably handle this more gracefully. It might be possible to resize the memmapped array. If not, then we could make a new one, possibly via param.

Resolving patch:
diff --git a/mne/channels/channels.py b/mne/channels/channels.py
--- a/mne/channels/channels.py
+++ b/mne/channels/channels.py
@@ -8,6 +8,7 @@
 
 import os
 import os.path as op
+import sys
 
 import numpy as np
 from scipy import sparse
@@ -837,6 +838,11 @@ def add_channels(self, add_list, force_update_info=False):
         See Also
         --------
         drop_channels
+
+        Notes
+        -----
+        If ``self`` is a Raw instance that has been preloaded into a
+        :obj:`numpy.memmap` instance, the memmap will be resized.
         """
         # avoid circular imports
         from ..io import BaseRaw, _merge_info
@@ -868,14 +874,31 @@ def add_channels(self, add_list, force_update_info=False):
                 raise AssertionError('All data dimensions except channels '
                                      'must match, got %s != %s'
                                      % (shapes[0], shape))
+        del shapes
 
         # Create final data / info objects
-        data = np.concatenate(data, axis=con_axis)
         infos = [self.info] + [inst.info for inst in add_list]
         new_info = _merge_info(infos, force_update_to_first=force_update_info)
 
         # Now update the attributes
-        self._data = data
+        if isinstance(self._data, np.memmap) and con_axis == 0 and \
+                sys.platform != 'darwin':  # resizing not available--no mremap
+            # Use a resize and fill in other ones
+            out_shape = (sum(d.shape[0] for d in data),) + data[0].shape[1:]
+            n_bytes = np.prod(out_shape) * self._data.dtype.itemsize
+            self._data.flush()
+            self._data.base.resize(n_bytes)
+            self._data = np.memmap(self._data.filename, mode='r+',
+                                   dtype=self._data.dtype, shape=out_shape)
+            assert self._data.shape == out_shape
+            assert self._data.nbytes == n_bytes
+            offset = len(data[0])
+            for d in data[1:]:
+                this_len = len(d)
+                self._data[offset:offset + this_len] = d
+                offset += this_len
+        else:
+            self._data = np.concatenate(data, axis=con_axis)
         self.info = new_info
         if isinstance(self, BaseRaw):
             self._cals = np.concatenate([getattr(inst, '_cals')
@@ -1101,7 +1124,7 @@ def _ch_neighbor_connectivity(ch_names, neighbors):
 
     for neigh in neighbors:
         if (not isinstance(neigh, list) and
-           not all(isinstance(c, string_types) for c in neigh)):
+                not all(isinstance(c, string_types) for c in neigh)):
             raise ValueError('`neighbors` must be a list of lists of str')
 
     ch_connectivity = np.eye(len(ch_names), dtype=bool)


Test patch:
diff --git a/mne/io/fiff/tests/test_raw_fiff.py b/mne/io/fiff/tests/test_raw_fiff.py
--- a/mne/io/fiff/tests/test_raw_fiff.py
+++ b/mne/io/fiff/tests/test_raw_fiff.py
@@ -8,6 +8,7 @@
 from functools import partial
 import itertools as itt
 import os.path as op
+import sys
 
 import numpy as np
 from numpy.testing import (assert_array_almost_equal, assert_array_equal,
@@ -20,7 +21,8 @@
 from mne.io import RawArray, concatenate_raws, read_raw_fif
 from mne.io.tests.test_raw import _test_concat, _test_raw_reader
 from mne import (concatenate_events, find_events, equalize_channels,
-                 compute_proj_raw, pick_types, pick_channels, create_info)
+                 compute_proj_raw, pick_types, pick_channels, create_info,
+                 pick_info)
 from mne.utils import (_TempDir, requires_pandas, object_diff,
                        requires_mne, run_subprocess, run_tests_if_main)
 from mne.externals.six.moves import zip, cPickle as pickle
@@ -660,7 +662,7 @@ def test_io_complex():
 
     for di, dtype in enumerate(dtypes):
         imag_rand = np.array(1j * rng.randn(data_orig.shape[0],
-                             data_orig.shape[1]), dtype)
+                                            data_orig.shape[1]), dtype)
 
         raw_cp = raw.copy()
         raw_cp._data = np.array(raw_cp._data, dtype)
@@ -1478,4 +1480,46 @@ def test_equalize_channels():
         assert_equal(ch_names, e.ch_names)
 
 
+def test_memmap(tmpdir):
+    """Test some interesting memmapping cases."""
+    # concatenate_raw
+    memmaps = [op.join(str(tmpdir), str(ii)) for ii in range(3)]
+    raw_0 = read_raw_fif(test_fif_fname, preload=memmaps[0])
+    assert raw_0._data.filename == memmaps[0]
+    raw_1 = read_raw_fif(test_fif_fname, preload=memmaps[1])
+    assert raw_1._data.filename == memmaps[1]
+    raw_0.append(raw_1, preload=memmaps[2])
+    assert raw_0._data.filename == memmaps[2]
+    # add_channels
+    orig_data = raw_0[:][0]
+    new_ch_info = pick_info(raw_0.info, [0])
+    new_ch_info['chs'][0]['ch_name'] = 'foo'
+    new_ch_info._update_redundant()
+    new_data = np.linspace(0, 1, len(raw_0.times))[np.newaxis]
+    ch = RawArray(new_data, new_ch_info)
+    raw_0.add_channels([ch])
+    if sys.platform == 'darwin':
+        assert not hasattr(raw_0._data, 'filename')
+    else:
+        assert raw_0._data.filename == memmaps[2]
+    assert_allclose(orig_data, raw_0[:-1][0], atol=1e-7)
+    assert_allclose(new_data, raw_0[-1][0], atol=1e-7)
+
+    # now let's see if .copy() actually works; it does, but eventually
+    # we should make it optionally memmap to a new filename rather than
+    # create an in-memory version (filename=None)
+    raw_0 = read_raw_fif(test_fif_fname, preload=memmaps[0])
+    assert raw_0._data.filename == memmaps[0]
+    assert raw_0._data[:1, 3:5].all()
+    raw_1 = raw_0.copy()
+    assert isinstance(raw_1._data, np.memmap)
+    assert raw_1._data.filename is None
+    raw_0._data[:] = 0.
+    assert not raw_0._data.any()
+    assert raw_1._data[:1, 3:5].all()
+    # other things like drop_channels and crop work but do not use memmapping,
+    # eventually we might want to add support for some of these as users
+    # require them.
+
+
 run_tests_if_main()