Repository: mne-tools/mne-python

Issue:
split fiff io error
I am encountering a few cases where `_read_raw_file` returns an `IndexError` when trying to read in the latter portion of split `.fif` files.  From Trace...

> if directory[first].kind == FIFF.FIFF_DATA_SKIP:
> IndexError: list index out of range

What exactly is this about? and anyone know how I can get around it?



Discussion:
That sounds like a bad bug we should fix for 0.12. I think I have a dataset I can try to replicate it on. But just in case, can you make sure you're on latest master, make a short script to reproduce the error, email it to me, and I'll take a look tomorrow.


Resolving patch:
diff --git a/mne/io/base.py b/mne/io/base.py
--- a/mne/io/base.py
+++ b/mne/io/base.py
@@ -1362,10 +1362,12 @@ def save(self, fname, picks=None, tmin=0, tmax=None, buffer_size_sec=10,
         #   Convert to samples
         start = int(np.floor(tmin * self.info['sfreq']))
 
+        # "stop" is the first sample *not* to save, so we need +1's here
         if tmax is None:
-            stop = self.last_samp + 1 - self.first_samp
+            stop = np.inf
         else:
-            stop = int(np.floor(tmax * self.info['sfreq']))
+            stop = self.time_as_index(float(tmax), use_rounding=True)[0] + 1
+        stop = min(stop, self.last_samp - self.first_samp + 1)
         buffer_size = self._get_buffer_size(buffer_size_sec)
 
         # write the raw file
@@ -1989,6 +1991,11 @@ def _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start,
                split_size, part_idx, prev_fname):
     """Write raw file with splitting
     """
+    # we've done something wrong if we hit this
+    n_times_max = len(raw.times)
+    if start >= stop or stop > n_times_max:
+        raise RuntimeError('Cannot write raw file with no data: %s -> %s '
+                           '(max: %s) requested' % (start, stop, n_times_max))
 
     if part_idx > 0:
         # insert index in filename
@@ -1999,7 +2006,8 @@ def _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start,
     logger.info('Writing %s' % use_fname)
 
     fid, cals = _start_writing_raw(use_fname, info, picks, data_type,
-                                   reset_range)
+                                   reset_range, raw.annotations)
+    use_picks = slice(None) if picks is None else picks
 
     first_samp = raw.first_samp + start
     if first_samp != 0:
@@ -2015,16 +2023,18 @@ def _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start,
         write_int(fid, FIFF.FIFF_REF_FILE_NUM, part_idx - 1)
         end_block(fid, FIFF.FIFFB_REF)
 
-    pos_prev = None
+    pos_prev = fid.tell()
+    if pos_prev > split_size:
+        raise ValueError('file is larger than "split_size" after writing '
+                         'measurement information, you must use a larger '
+                         'value for split size: %s plus enough bytes for '
+                         'the chosen buffer_size' % pos_prev)
+    next_file_buffer = 2 ** 20  # extra cushion for last few post-data tags
     for first in range(start, stop, buffer_size):
-        last = first + buffer_size
-        if last >= stop:
-            last = stop + 1
-
-        if picks is None:
-            data, times = raw[:, first:last]
-        else:
-            data, times = raw[picks, first:last]
+        # Write blocks <= buffer_size in size
+        last = min(first + buffer_size, stop)
+        data, times = raw[use_picks, first:last]
+        assert len(times) == last - first
 
         if projector is not None:
             data = np.dot(projector, data)
@@ -2035,23 +2045,26 @@ def _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start,
                         '[done]')
             break
         logger.info('Writing ...')
-
-        if pos_prev is None:
-            pos_prev = fid.tell()
-
         _write_raw_buffer(fid, data, cals, fmt, inv_comp)
 
         pos = fid.tell()
         this_buff_size_bytes = pos - pos_prev
-        if this_buff_size_bytes > split_size / 2:
-            raise ValueError('buffer size is too large for the given split'
-                             'size: decrease "buffer_size_sec" or increase'
-                             '"split_size".')
-        if pos > split_size:
-            warn('file is larger than "split_size"')
+        overage = pos - split_size + next_file_buffer
+        if overage > 0:
+            # This should occur on the first buffer write of the file, so
+            # we should mention the space required for the meas info
+            raise ValueError(
+                'buffer size (%s) is too large for the given split size (%s) '
+                'by %s bytes after writing info (%s) and leaving enough space '
+                'for end tags (%s): decrease "buffer_size_sec" or increase '
+                '"split_size".' % (this_buff_size_bytes, split_size, overage,
+                                   pos_prev, next_file_buffer))
 
         # Split files if necessary, leave some space for next file info
-        if pos >= split_size - this_buff_size_bytes - 2 ** 20:
+        # make sure we check to make sure we actually *need* another buffer
+        # with the "and" check
+        if pos >= split_size - this_buff_size_bytes - next_file_buffer and \
+                first + buffer_size < stop:
             next_fname, next_idx = _write_raw(
                 fname, raw, info, picks, fmt,
                 data_type, reset_range, first + buffer_size, stop, buffer_size,
@@ -2069,18 +2082,6 @@ def _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start,
 
         pos_prev = pos
 
-    if raw.annotations is not None:
-        start_block(fid, FIFF.FIFFB_MNE_ANNOTATIONS)
-        write_float(fid, FIFF.FIFF_MNE_BASELINE_MIN, raw.annotations.onset)
-        write_float(fid, FIFF.FIFF_MNE_BASELINE_MAX,
-                    raw.annotations.duration + raw.annotations.onset)
-        # To allow : in description, they need to be replaced for serialization
-        write_name_list(fid, FIFF.FIFF_COMMENT, [d.replace(':', ';') for d in
-                                                 raw.annotations.description])
-        if raw.annotations.orig_time is not None:
-            write_double(fid, FIFF.FIFF_MEAS_DATE, raw.annotations.orig_time)
-        end_block(fid, FIFF.FIFFB_MNE_ANNOTATIONS)
-
     logger.info('Closing %s [done]' % use_fname)
     if info.get('maxshield', False):
         end_block(fid, FIFF.FIFFB_SMSH_RAW_DATA)
@@ -2092,7 +2093,7 @@ def _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start,
 
 
 def _start_writing_raw(name, info, sel=None, data_type=FIFF.FIFFT_FLOAT,
-                       reset_range=True):
+                       reset_range=True, annotations=None):
     """Start write raw data in file
 
     Data will be written in float
@@ -2110,6 +2111,8 @@ def _start_writing_raw(name, info, sel=None, data_type=FIFF.FIFFT_FLOAT,
         5 (FIFFT_DOUBLE), 16 (FIFFT_DAU_PACK16), or 3 (FIFFT_INT) for raw data.
     reset_range : bool
         If True, the info['chs'][k]['range'] parameter will be set to unity.
+    annotations : instance of Annotations or None
+        The annotations to write.
 
     Returns
     -------
@@ -2119,12 +2122,12 @@ def _start_writing_raw(name, info, sel=None, data_type=FIFF.FIFFT_FLOAT,
         calibration factors.
     """
     #
-    #    Measurement info
+    # Measurement info
     #
     info = pick_info(info, sel)
 
     #
-    #  Create the file and save the essentials
+    # Create the file and save the essentials
     #
     fid = start_file(name)
     start_block(fid, FIFF.FIFFB_MEAS)
@@ -2144,6 +2147,21 @@ def _start_writing_raw(name, info, sel=None, data_type=FIFF.FIFFT_FLOAT,
 
     write_meas_info(fid, info, data_type=data_type, reset_range=reset_range)
 
+    #
+    # Annotations
+    #
+    if annotations is not None:
+        start_block(fid, FIFF.FIFFB_MNE_ANNOTATIONS)
+        write_float(fid, FIFF.FIFF_MNE_BASELINE_MIN, annotations.onset)
+        write_float(fid, FIFF.FIFF_MNE_BASELINE_MAX,
+                    annotations.duration + annotations.onset)
+        # To allow : in description, they need to be replaced for serialization
+        write_name_list(fid, FIFF.FIFF_COMMENT, [d.replace(':', ';') for d in
+                                                 annotations.description])
+        if annotations.orig_time is not None:
+            write_double(fid, FIFF.FIFF_MEAS_DATE, annotations.orig_time)
+        end_block(fid, FIFF.FIFFB_MNE_ANNOTATIONS)
+
     #
     # Start the raw data
     #
diff --git a/mne/io/fiff/raw.py b/mne/io/fiff/raw.py
--- a/mne/io/fiff/raw.py
+++ b/mne/io/fiff/raw.py
@@ -169,6 +169,28 @@ def _read_raw_file(self, fname, allow_maxshield, preload, compensation,
 
             info, meas = read_meas_info(fid, tree, clean_bads=True)
 
+            annotations = None
+            annot_data = dir_tree_find(tree, FIFF.FIFFB_MNE_ANNOTATIONS)
+            if len(annot_data) > 0:
+                annot_data = annot_data[0]
+                for k in range(annot_data['nent']):
+                    kind = annot_data['directory'][k].kind
+                    pos = annot_data['directory'][k].pos
+                    orig_time = None
+                    tag = read_tag(fid, pos)
+                    if kind == FIFF.FIFF_MNE_BASELINE_MIN:
+                        onset = tag.data
+                    elif kind == FIFF.FIFF_MNE_BASELINE_MAX:
+                        duration = tag.data - onset
+                    elif kind == FIFF.FIFF_COMMENT:
+                        description = tag.data.split(':')
+                        description = [d.replace(';', ':') for d in
+                                       description]
+                    elif kind == FIFF.FIFF_MEAS_DATE:
+                        orig_time = float(tag.data)
+                annotations = Annotations(onset, duration, description,
+                                          orig_time)
+
             #   Locate the data of interest
             raw_node = dir_tree_find(meas, FIFF.FIFFB_RAW_DATA)
             if len(raw_node) == 0:
@@ -211,6 +233,7 @@ def _read_raw_file(self, fname, allow_maxshield, preload, compensation,
                 tag = read_tag(fid, directory[first].pos)
                 first_samp = int(tag.data)
                 first += 1
+                _check_entry(first, nent)
 
             #   Omit initial skip
             if directory[first].kind == FIFF.FIFF_DATA_SKIP:
@@ -218,38 +241,17 @@ def _read_raw_file(self, fname, allow_maxshield, preload, compensation,
                 tag = read_tag(fid, directory[first].pos)
                 first_skip = int(tag.data)
                 first += 1
+                _check_entry(first, nent)
 
             raw = _RawShell()
             raw.filename = fname
             raw.first_samp = first_samp
+            raw.annotations = annotations
 
             #   Go through the remaining tags in the directory
             raw_extras = list()
             nskip = 0
             orig_format = None
-            annotations = None
-
-            annot_data = dir_tree_find(tree, FIFF.FIFFB_MNE_ANNOTATIONS)
-            if len(annot_data) > 0:
-                annot_data = annot_data[0]
-                for k in range(annot_data['nent']):
-                    kind = annot_data['directory'][k].kind
-                    pos = annot_data['directory'][k].pos
-                    orig_time = None
-                    tag = read_tag(fid, pos)
-                    if kind == FIFF.FIFF_MNE_BASELINE_MIN:
-                        onset = tag.data
-                    elif kind == FIFF.FIFF_MNE_BASELINE_MAX:
-                        duration = tag.data - onset
-                    elif kind == FIFF.FIFF_COMMENT:
-                        description = tag.data.split(':')
-                        description = [d.replace(';', ':') for d in
-                                       description]
-                    elif kind == FIFF.FIFF_MEAS_DATE:
-                        orig_time = float(tag.data)
-                annotations = Annotations(onset, duration, description,
-                                          orig_time)
-            raw.annotations = annotations
 
             for k in range(first, nent):
                 ent = directory[k]
@@ -465,6 +467,12 @@ def fix_mag_coil_types(self):
         return self
 
 
+def _check_entry(first, nent):
+    """Helper to sanity check entries"""
+    if first >= nent:
+        raise IOError('Could not read data, perhaps this is a corrupt file')
+
+
 def read_raw_fif(fnames, allow_maxshield=False, preload=False,
                  proj=False, compensation=None, add_eeg_ref=True,
                  verbose=None):


Test patch:
diff --git a/mne/io/array/tests/test_array.py b/mne/io/array/tests/test_array.py
--- a/mne/io/array/tests/test_array.py
+++ b/mne/io/array/tests/test_array.py
@@ -32,7 +32,7 @@ def test_array_raw():
     """
     import matplotlib.pyplot as plt
     # creating
-    raw = Raw(fif_fname).crop(2, 5)
+    raw = Raw(fif_fname).crop(2, 5, copy=False)
     data, times = raw[:, :]
     sfreq = raw.info['sfreq']
     ch_names = [(ch[4:] if 'STI' not in ch else ch)
diff --git a/mne/io/brainvision/tests/test_brainvision.py b/mne/io/brainvision/tests/test_brainvision.py
--- a/mne/io/brainvision/tests/test_brainvision.py
+++ b/mne/io/brainvision/tests/test_brainvision.py
@@ -37,9 +37,11 @@
 def test_brainvision_data_filters():
     """Test reading raw Brain Vision files
     """
-    raw = _test_raw_reader(read_raw_brainvision,
-                           vhdr_fname=vhdr_highpass_path, montage=montage,
-                           eog=eog)
+    with warnings.catch_warnings(record=True) as w:  # event parsing
+        raw = _test_raw_reader(
+            read_raw_brainvision, vhdr_fname=vhdr_highpass_path,
+            montage=montage, eog=eog)
+    assert_true(all('parse triggers that' in str(ww.message) for ww in w))
 
     assert_equal(raw.info['highpass'], 0.1)
     assert_equal(raw.info['lowpass'], 250.)
@@ -51,9 +53,11 @@ def test_brainvision_data():
     assert_raises(IOError, read_raw_brainvision, vmrk_path)
     assert_raises(ValueError, read_raw_brainvision, vhdr_path, montage,
                   preload=True, scale="foo")
-    raw_py = _test_raw_reader(read_raw_brainvision,
-                              vhdr_fname=vhdr_path, montage=montage, eog=eog)
-
+    with warnings.catch_warnings(record=True) as w:  # event parsing
+        raw_py = _test_raw_reader(
+            read_raw_brainvision, vhdr_fname=vhdr_path, montage=montage,
+            eog=eog)
+    assert_true(all('parse triggers that' in str(ww.message) for ww in w))
     assert_true('RawBrainVision' in repr(raw_py))
 
     assert_equal(raw_py.info['highpass'], 0.)
diff --git a/mne/io/fiff/tests/test_raw_fiff.py b/mne/io/fiff/tests/test_raw_fiff.py
--- a/mne/io/fiff/tests/test_raw_fiff.py
+++ b/mne/io/fiff/tests/test_raw_fiff.py
@@ -243,7 +243,7 @@ def test_multiple_files():
     """
     # split file
     tempdir = _TempDir()
-    raw = Raw(fif_fname).crop(0, 10)
+    raw = Raw(fif_fname).crop(0, 10, copy=False)
     raw.load_data()
     raw.load_data()  # test no operation
     split_size = 3.  # in seconds
@@ -261,6 +261,9 @@ def test_multiple_files():
         fname = op.join(tempdir, 'test_raw_split-%d_raw.fif' % ri)
         raw.save(fname, tmin=tmins[ri], tmax=tmaxs[ri])
         raws[ri] = Raw(fname)
+        assert_equal(len(raws[ri].times),
+                     int(round((tmaxs[ri] - tmins[ri]) *
+                               raw.info['sfreq'])) + 1)  # + 1 b/c inclusive
     events = [find_events(r, stim_channel='STI 014') for r in raws]
     last_samps = [r.last_samp for r in raws]
     first_samps = [r.first_samp for r in raws]
@@ -269,6 +272,7 @@ def test_multiple_files():
     assert_raises(ValueError, concatenate_raws, raws, True, events[1:])
     all_raw_1, events1 = concatenate_raws(raws, preload=False,
                                           events_list=events)
+    assert_allclose(all_raw_1.times, raw.times)
     assert_equal(raw.first_samp, all_raw_1.first_samp)
     assert_equal(raw.last_samp, all_raw_1.last_samp)
     assert_allclose(raw[:, :][0], all_raw_1[:, :][0])
@@ -362,6 +366,9 @@ def test_split_files():
     """
     tempdir = _TempDir()
     raw_1 = Raw(fif_fname, preload=True)
+    # Test a very close corner case
+    raw_crop = raw_1.copy().crop(0, 1., copy=False)
+
     assert_allclose(raw_1.info['buffer_size_sec'], 10., atol=1e-2)  # samp rate
     split_fname = op.join(tempdir, 'split_raw.fif')
     raw_1.save(split_fname, buffer_size_sec=1.0, split_size='10MB')
@@ -383,6 +390,64 @@ def test_split_files():
     assert_array_equal(data_1, data_2)
     assert_array_equal(times_1, times_2)
 
+    # test the case where we only end up with one buffer to write
+    # (GH#3210). These tests rely on writing meas info and annotations
+    # taking up a certain number of bytes, so if we change those functions
+    # somehow, the numbers below for e.g. split_size might need to be
+    # adjusted.
+    raw_crop = raw_1.copy().crop(0, 5, copy=False)
+    try:
+        raw_crop.save(split_fname, split_size='1MB',  # too small a size
+                      buffer_size_sec=1., overwrite=True)
+    except ValueError as exp:
+        assert_true('after writing measurement information' in str(exp), exp)
+    try:
+        raw_crop.save(split_fname,
+                      split_size=3002276,  # still too small, now after Info
+                      buffer_size_sec=1., overwrite=True)
+    except ValueError as exp:
+        assert_true('too large for the given split size' in str(exp), exp)
+    # just barely big enough here; the right size to write exactly one buffer
+    # at a time so we hit GH#3210 if we aren't careful
+    raw_crop.save(split_fname, split_size='4.5MB',
+                  buffer_size_sec=1., overwrite=True)
+    raw_read = read_raw_fif(split_fname)
+    assert_allclose(raw_crop[:][0], raw_read[:][0], atol=1e-20)
+
+    # Check our buffer arithmetic
+
+    # 1 buffer required
+    raw_crop = raw_1.copy().crop(0, 1, copy=False)
+    raw_crop.save(split_fname, buffer_size_sec=1., overwrite=True)
+    raw_read = read_raw_fif(split_fname)
+    assert_equal(len(raw_read._raw_extras[0]), 1)
+    assert_equal(raw_read._raw_extras[0][0]['nsamp'], 301)
+    assert_allclose(raw_crop[:][0], raw_read[:][0])
+    # 2 buffers required
+    raw_crop.save(split_fname, buffer_size_sec=0.5, overwrite=True)
+    raw_read = read_raw_fif(split_fname)
+    assert_equal(len(raw_read._raw_extras[0]), 2)
+    assert_equal(raw_read._raw_extras[0][0]['nsamp'], 151)
+    assert_equal(raw_read._raw_extras[0][1]['nsamp'], 150)
+    assert_allclose(raw_crop[:][0], raw_read[:][0])
+    # 2 buffers required
+    raw_crop.save(split_fname,
+                  buffer_size_sec=1. - 1.01 / raw_crop.info['sfreq'],
+                  overwrite=True)
+    raw_read = read_raw_fif(split_fname)
+    assert_equal(len(raw_read._raw_extras[0]), 2)
+    assert_equal(raw_read._raw_extras[0][0]['nsamp'], 300)
+    assert_equal(raw_read._raw_extras[0][1]['nsamp'], 1)
+    assert_allclose(raw_crop[:][0], raw_read[:][0])
+    raw_crop.save(split_fname,
+                  buffer_size_sec=1. - 2.01 / raw_crop.info['sfreq'],
+                  overwrite=True)
+    raw_read = read_raw_fif(split_fname)
+    assert_equal(len(raw_read._raw_extras[0]), 2)
+    assert_equal(raw_read._raw_extras[0][0]['nsamp'], 299)
+    assert_equal(raw_read._raw_extras[0][1]['nsamp'], 2)
+    assert_allclose(raw_crop[:][0], raw_read[:][0])
+
 
 def test_load_bad_channels():
     """Test reading/writing of bad channels
@@ -444,7 +509,7 @@ def test_io_raw():
             assert_equal(desc1, desc2)
 
     # Let's construct a simple test for IO first
-    raw = Raw(fif_fname).crop(0, 3.5)
+    raw = Raw(fif_fname).crop(0, 3.5, copy=False)
     raw.load_data()
     # put in some data that we know the values of
     data = rng.randn(raw._data.shape[0], raw._data.shape[1])
@@ -473,7 +538,7 @@ def test_io_raw():
                             if ch_names[k][0] == 'M']
         n_channels = 100
         meg_channels_idx = meg_channels_idx[:n_channels]
-        start, stop = raw.time_as_index([0, 5])
+        start, stop = raw.time_as_index([0, 5], use_rounding=True)
         data, times = raw[meg_channels_idx, start:(stop + 1)]
         meg_ch_names = [ch_names[k] for k in meg_channels_idx]
 
@@ -710,7 +775,7 @@ def test_preload_modify():
 def test_filter():
     """Test filtering (FIR and IIR) and Raw.apply_function interface
     """
-    raw = Raw(fif_fname).crop(0, 7)
+    raw = Raw(fif_fname).crop(0, 7, copy=False)
     raw.load_data()
     sig_dec = 11
     sig_dec_notch = 12