Repository: mne-tools/mne-python

Issue:
Using `.crop()` on mff dataset leads to errors later on
The problem happens when doing `events = mne.find_events(raw)` or `raw.load_data()` when the raw file was previously cropped, for example:
```python
import mne

fname = 'some_file.mff'
raw = mne.io.read_raw_egi(fname)
raw.crop(tmin=2.5)
```

now both:
```python
events = mne.find_events(raw)
```
and:
```python
raw.load_data()
```
lead to the same error. Both errors are the same starting from `_read_segment_file` but I paste both below.
for `mne.find_events(raw)`:
```python-traceback
<c:\src\mne-python\mne\externals\decorator.py:decorator-gen-129> in find_events(raw, stim_channel, output, consecutive, min_duration, shortest_event, mask, uint_cast, mask_type, initial_event, verbose)

c:\src\mne-python\mne\utils\_logging.py in wrapper(*args, **kwargs)
     87             with use_log_level(verbose_level):
     88                 return function(*args, **kwargs)
---> 89         return function(*args, **kwargs)
     90     return FunctionMaker.create(
     91         function, 'return decfunc(%(signature)s)',

c:\src\mne-python\mne\event.py in find_events(raw, stim_channel, output, consecutive, min_duration, shortest_event, mask, uint_cast, mask_type, initial_event, verbose)
    684     if len(picks) == 0:
    685         raise ValueError('No stim channel found to extract event triggers.')
--> 686     data, _ = raw[picks, :]
    687 
    688     events_list = []

c:\src\mne-python\mne\io\base.py in __getitem__(self, item)
    887             data = self._read_segment(start=start, stop=stop, sel=sel,
    888                                       projector=self._projector,
--> 889                                       verbose=self.verbose)
    890         times = self.times[start:stop]
    891         return data, times

c:\src\mne-python\mne\io\base.py in _read_segment(self, start, stop, sel, data_buffer, projector, verbose)
    541             self._read_segment_file(data[:, this_sl], idx, fi,
    542                                     int(start_file), int(stop_file),
--> 543                                     cals, mult)
    544             offset += n_read
    545         return data

c:\src\mne-python\mne\io\egi\egimff.py in _read_segment_file(self, data, idx, fi, start, stop, cals, mult)
    508             # Start reading samples
    509             while samples_to_read > 0:
--> 510                 this_block_info = _block_r(fid)
    511                 if this_block_info is not None:
    512                     current_block_info = this_block_info

c:\src\mne-python\mne\io\egi\general.py in _block_r(fid)
    141 def _block_r(fid):
    142     """Read meta data."""
--> 143     if np.fromfile(fid, dtype=np.dtype('i4'), count=1)[0] != 1:  # not metadata
    144         return None
    145     header_size = np.fromfile(fid, dtype=np.dtype('i4'), count=1)[0]

IndexError: index 0 is out of bounds for axis 0 with size 0
```

for `raw.load_data()`:
```python-traceback
<c:\src\mne-python\mne\externals\decorator.py:decorator-gen-132> in load_data(self, verbose)

c:\src\mne-python\mne\utils\_logging.py in wrapper(*args, **kwargs)
     87             with use_log_level(verbose_level):
     88                 return function(*args, **kwargs)
---> 89         return function(*args, **kwargs)
     90     return FunctionMaker.create(
     91         function, 'return decfunc(%(signature)s)',

c:\src\mne-python\mne\io\base.py in load_data(self, verbose)
    631         """
    632         if not self.preload:
--> 633             self._preload_data(True)
    634         return self
    635 

<c:\src\mne-python\mne\externals\decorator.py:decorator-gen-133> in _preload_data(self, preload, verbose)

c:\src\mne-python\mne\utils\_logging.py in wrapper(*args, **kwargs)
     87             with use_log_level(verbose_level):
     88                 return function(*args, **kwargs)
---> 89         return function(*args, **kwargs)
     90     return FunctionMaker.create(
     91         function, 'return decfunc(%(signature)s)',

c:\src\mne-python\mne\io\base.py in _preload_data(self, preload, verbose)
    641         logger.info('Reading %d ... %d  =  %9.3f ... %9.3f secs...' %
    642                     (0, len(self.times) - 1, 0., self.times[-1]))
--> 643         self._data = self._read_segment(data_buffer=data_buffer)
    644         assert len(self._data) == self.info['nchan']
    645         self.preload = True

c:\src\mne-python\mne\io\base.py in _read_segment(self, start, stop, sel, data_buffer, projector, verbose)
    541             self._read_segment_file(data[:, this_sl], idx, fi,
    542                                     int(start_file), int(stop_file),
--> 543                                     cals, mult)
    544             offset += n_read
    545         return data

c:\src\mne-python\mne\io\egi\egimff.py in _read_segment_file(self, data, idx, fi, start, stop, cals, mult)
    508             # Start reading samples
    509             while samples_to_read > 0:
--> 510                 this_block_info = _block_r(fid)
    511                 if this_block_info is not None:
    512                     current_block_info = this_block_info

c:\src\mne-python\mne\io\egi\general.py in _block_r(fid)
    141 def _block_r(fid):
    142     """Read meta data."""
--> 143     if np.fromfile(fid, dtype=np.dtype('i4'), count=1)[0] != 1:  # not metadata
    144         return None
    145     header_size = np.fromfile(fid, dtype=np.dtype('i4'), count=1)[0]

IndexError: index 0 is out of bounds for axis 0 with size 0
```

I'm on master, but had this problem 3 months ago too.
`preload=True` when reading eliminates the issue, but it would be nice if `preload` was not necessary.


Discussion:
This suggests that 1) `_test_raw_reader` is not complete enough (or `egimff` does not use it!) and 2) there is some bug with the `egimff` reader.
I'm having exactly the same problem when trying to crop mff file to multiple segments.
Some segments are valid even without preloading data.
Adjusting tmin/tmax values can produce a valid segment.
With "preload=True" all segments are valid.
The error for invalid segments is: IndexError('index 0 is out of bounds for axis 0 with size 0',)
I fear you'll need to dig into the code unless the original authors of this
can find the time to help.

>

There are 2 small errors egimff in _read_segment_file function.
i'm unable to write test unit and make PR but i'll more than happy to explain them if anybody is willing to do that
Can you share a code snippet and file to replicate the problem on a different machine ?

Here is some more info:

1st problem is that when offset is calculated from start sample  - it is never reseted to every additional data block is being read from that offset. That's wrong, because just 1st block will have some offset and additional blocks should be read whole - ie from beginning.

here is snippet from egimff.py:
```
# Compute indexes
samples_read = block_data.shape[1]
if offset_samples > 0:
	# First block read, skip to the offset:
	block_data = block_data[:, offset_samples:]
	samples_read = samples_read - offset_samples
	offset_samples = 0 # added, as offset should be reset, so next blocks will be read from beginning
```
this exact code is repeated once more for PNS data, so we need to correct 2nd occurrence too.

2nd problem is when reading  events:
```
# take into account events
if len(egi_events) > 0:
	e_chs = egi_events[:, s_start:s_end] # this is wrong, because we ignore starting offset
	block_data = np.vstack([block_data, e_chs])
```
the line should be:
```
	e_chs = egi_events[:, start+s_start:start+s_end] # we read events from same offset as data
```

so crop will fail on any raw if data is not preloaded , because it will read all blocks from same offset (not just the 1st one)  and egi_events will be read from 1st sample

I'll provide some code snippet later but it can be tested on any raw file.

here is a code snippet that works for me. Tell me how to break it:

import os.path as op

from mne.datasets import testing
from mne.io import read_raw_egi

data_path = testing.data_path(download=False)
egi_fname_mff = op.join(data_path, 'EGI', 'test_egi.mff')
raw = read_raw_egi(egi_fname_mff, include=None, preload=False)
raw.crop(2, 5)
raw.crop(0, 2)
raw.load_data()

it works, but the data is corrupted

try this: 
raw.crop(17.5, 20.5)
raw.load_data()

this shouldn't work

Resolving patch:
diff --git a/mne/io/egi/egimff.py b/mne/io/egi/egimff.py
--- a/mne/io/egi/egimff.py
+++ b/mne/io/egi/egimff.py
@@ -522,6 +522,7 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
                     # First block read, skip to the offset:
                     block_data = block_data[:, offset_samples:]
                     samples_read = samples_read - offset_samples
+                    offset_samples = 0
                 if samples_to_read < samples_read:
                     # Last block to read, skip the last samples
                     block_data = block_data[:, :samples_to_read]
@@ -532,7 +533,7 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
 
                 # take into account events
                 if len(egi_events) > 0:
-                    e_chs = egi_events[:, s_start:s_end]
+                    e_chs = egi_events[:, start + s_start:start + s_end]
                     block_data = np.vstack([block_data, e_chs])
 
                 data_view = data[:n_data1_channels, s_start:s_end]
@@ -602,6 +603,7 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
                         # First block read, skip to the offset:
                         block_data = block_data[:, offset_samples:]
                         samples_read = samples_read - offset_samples
+                        offset_samples = 0
 
                     if samples_to_read < samples_read:
                         # Last block to read, skip the last samples


Test patch:
diff --git a/mne/io/egi/tests/test_egi.py b/mne/io/egi/tests/test_egi.py
--- a/mne/io/egi/tests/test_egi.py
+++ b/mne/io/egi/tests/test_egi.py
@@ -185,4 +185,16 @@ def test_io_egi_pns_mff_bug():
         assert_array_equal(mat_data, raw_data)
 
 
+@requires_testing_data
+def test_io_egi_crop_no_preload():
+    """Test crop non-preloaded EGI MFF data (BUG)."""
+    egi_fname_mff = op.join(data_path(), 'EGI', 'test_egi.mff')
+    raw = read_raw_egi(egi_fname_mff, preload=False)
+    raw.crop(17.5, 20.5)
+    raw.load_data()
+    raw_preload = read_raw_egi(egi_fname_mff, preload=True)
+    raw_preload.crop(17.5, 20.5)
+    raw_preload.load_data()
+    assert_allclose(raw._data, raw_preload._data)
+
 run_tests_if_main()