Repository: mne-tools/mne-python

Issue:
memory usage with apply_inverse_epochs and return_generator
If I generate stcs with 

``` python
stcs = apply_inverse_epochs(epochs, inverse_operator, lambda2,
        method, None, pick_ori="normal",return_generator=True)
```

and then iterator over the stcs I see accumulating ram usage (measured via top) for each iteration.

Interestingly it depends on what parts of the stc I access for instance

``` python
for stc in stcs:
    print stc.data.shape[0]
```

accumulates ram.

where as

``` python
for stc in stcs:
    print stc.times[0]
```

doesn't.

Is this expected? It is causing me issues going through a 270 sec raw file in 1 second epochs.

this is the script I was using to look at it. https://gist.github.com/bloyl/6799492#file-test_returngenerators-py

Thanks
Luke



Discussion:
Accessing `stc.data.shape[0]` causes the used memory to increase because the inverse is applied delayed, i.e., `stc.data` is only computed when you actually access it. If you just want to access the shape, you can use `stc.shape`, which is the same as `stc.data.shape`, but doesn't cause `stc.data` to be computed.

However, the memory shouldn't accumulate when you go through the loop (`stc` from the previous loop iteration should always be freed). Can you check if the same thing happens when you put a `del stc` line inside the loop? 

@mluessi Memory accumulates on my box if I don't delete. That's interesting. I would not have expected this assuming that the memory would be freed once I reassign the identifier.

Yes, this is very strange indeed..

It's even stranger. On closer inspection it even accumulates when deleting and setting IPython cache size to 1 ...

Maybe it has something to do with the subclassed array that is used to make the on-demand computation of `stc.data` work. I will investigate.

I  guess the critical question is whether what ratchets up here is strictly related to the epochs. I know this is working correctly on big data sets. E.g. our source decoding example would not have fit into memory on a smaller computer without this trick (producing Malloc errors).

Is there anything that I can do to help this along? I dont know of any good memory profilers for python but if anyone has any suggestions (and pointers) I'd be happy to take alook.
--Luke

Hej, check out this memory profiler which is really very helpful: https://github.com/fabianp/memory_profiler

Here's an example how it can be used:
https://gist.github.com/dengemann/6687214

Thanks. I will take a look.

In a related issue, it looks like even if `return_generator=False`, the STCs aren't actually computed until, say, `stcs[0].data` is accessed. I know because doing `stcs = apply_inverse_epochs(..., return_generator=False)` is almost instantaneous, while `[stc.data for stc in stcs]` takes a long time. Is this the desired behavior, in other words, always doing delayed computation, or is this also a bug (my intuition)?

ping @mluessi 

ping @mluessi or whoever feels like taking a look. This should be fixed for 0.7.


Resolving patch:
diff --git a/mne/source_estimate.py b/mne/source_estimate.py
--- a/mne/source_estimate.py
+++ b/mne/source_estimate.py
@@ -345,48 +345,6 @@ def _make_stc(data, vertices, tmin=None, tstep=None, subject=None):
     return stc
 
 
-class _NotifyArray(np.ndarray):
-    """Array class that executes a callback when it is modified
-    """
-    def __new__(cls, input_array, modify_callback=None):
-        obj = np.asarray(input_array).view(cls)
-        obj.modify_callback = modify_callback
-        return obj
-
-    def __array_finalize__(self, obj):
-        if obj is None:
-            # an empty constructor was used
-            return
-
-        # try to copy the callback
-        self.modify_callback = getattr(obj, 'modify_callback', None)
-
-    def _modified_(self):
-        """Execute the callback if it is set"""
-        if self.modify_callback is not None:
-            self.modify_callback()
-
-    def __getattribute__(self, name):
-        # catch ndarray methods that modify the array inplace
-        if name in ['fill', 'itemset', 'resize', 'sort']:
-            self._modified_()
-
-        return object.__getattribute__(self, name)
-
-    def __setitem__(self, item, value):
-        self._modified_()
-        np.ndarray.__setitem__(self, item, value)
-
-    def __array_wrap__(self, out_arr, context=None):
-        # this method is called whenever a numpy ufunc (+, +=..) is called
-        # the last entry in context is the array that receives the result
-        if (context is not None and len(context[1]) == 3
-                and context[1][2] is self):
-            self._modified_()
-
-        return np.ndarray.__array_wrap__(self, out_arr, context)
-
-
 def _verify_source_estimate_compat(a, b):
     """Make sure two SourceEstimates are compatible for arith. operations"""
     compat = False
@@ -481,20 +439,17 @@ def __init__(self, data, vertices=None, tmin=None, tstep=None,
         self.verbose = verbose
         self._kernel = kernel
         self._sens_data = sens_data
+        self._kernel_removed = False
         self.times = None
         self._update_times()
         self.subject = _check_subject(None, subject, False)
 
     def _remove_kernel_sens_data_(self):
-        """Remove kernel and sensor space data
-
-        Note: self._data is also computed if it is None
+        """Remove kernel and sensor space data and compute self._data
         """
         if self._kernel is not None or self._sens_data is not None:
-            # we can no longer use the kernel and sens_data
-            logger.info('STC data modified: removing kernel and sensor data')
-            if self._data is None:
-                self._data = np.dot(self._kernel, self._sens_data)
+            self._kernel_removed = True
+            self._data = np.dot(self._kernel, self._sens_data)
             self._kernel = None
             self._sens_data = None
 
@@ -517,7 +472,6 @@ def crop(self, tmin=None, tmax=None):
 
         if self._kernel is not None and self._sens_data is not None:
             self._sens_data = self._sens_data[:, mask]
-            self._data = None  # will be recomputed when data is accessed
         else:
             self._data = self._data[:, mask]
 
@@ -564,11 +518,9 @@ def resample(self, sfreq, npad=100, window='boxcar', n_jobs=1,
     @property
     def data(self):
         if self._data is None:
-            # compute the solution the first time the data is accessed
-            # return a "notify array", so we can later remove the kernel
-            # and sensor data if the user modifies self._data
-            self._data = _NotifyArray(np.dot(self._kernel, self._sens_data),
-                modify_callback=self._remove_kernel_sens_data_)
+            # compute the solution the first time the data is accessed and
+            # remove the kernel and sensor data
+            self._remove_kernel_sens_data_()
         return self._data
 
     @property
@@ -788,6 +740,10 @@ def transform_data(self, transform_fun, fun_args=None,
             fun_args = tuple()
 
         if self._kernel is None and self._sens_data is None:
+            if self._kernel_removed:
+                warnings.warn('Performance can be improved by not accessing '
+                              'the data attribute before calling this method.')
+
             # transform source space data directly
             data_t = transform_fun(self.data[idx, tmin_idx:tmax_idx],
                                    *fun_args, **kwargs)


Test patch:
diff --git a/mne/tests/test_source_estimate.py b/mne/tests/test_source_estimate.py
--- a/mne/tests/test_source_estimate.py
+++ b/mne/tests/test_source_estimate.py
@@ -453,31 +453,6 @@ def test_transform():
     assert_array_equal(stc.data, data_t)
 
 
-def test_notify_array_source_estimate():
-    """Test that modifying the stc data removes the kernel and sensor data"""
-    # make up some data
-    n_sensors, n_vertices, n_times = 10, 20, 4
-    kernel = np.random.randn(n_vertices, n_sensors)
-    sens_data = np.random.randn(n_sensors, n_times)
-    vertices = np.arange(n_vertices)
-
-    stc = VolSourceEstimate((kernel, sens_data), vertices=vertices,
-                            tmin=0., tstep=1.)
-
-    assert_true(stc._data is None)
-    assert_true(stc._kernel is not None)
-    assert_true(stc._sens_data is not None)
-
-    # now modify the data in some way
-    data_half = stc.data[:, n_times / 2:]
-    data_half[0] = 1.0
-    data_half.fill(1.0)
-
-    # the kernel and sensor data can no longer be used: they have been removed
-    assert_true(stc._kernel is None)
-    assert_true(stc._sens_data is None)
-
-
 @requires_sklearn
 def test_spatio_temporal_tris_connectivity():
     """Test spatio-temporal connectivity from triangles"""