Repository: mne-tools/mne-python

Issue:
preload_data is only for RawFIF
I was just working with a RawEDF and I went to call `raw.preload_data`, but I noticed that this method didn't exist for this Raw class. It's currently only implemented for RawFIF. I also saw that there is a new (to me at least) parameter in `_read_segment` called `data_buffer` that is being used whereas the other modules don't have this. should we use this method's use?

Discussion:
Eric89GXL: Okay, this has some potentially nice (up to 2x) memory-saving changes for EDF, but I don't think we can get the same benefit for KIT (at least not easily). This unifies `preload_data` and how `preload` is handled across raw types, and allows `data_buffer` to be used for EDF and KIT as well as FIF.

Ready for review from my end. @teonlamont please look closely to make sure it makes sense. Tests still pass, and I've been relying on those being correct to make sure things all worked properly.
agramfort: @teonlamont please take a close look

Patch:
diff --git a/mne/io/base.py b/mne/io/base.py
index 9a3c83be2..578109de7 100644
--- a/mne/io/base.py
+++ b/mne/io/base.py
@@ -215,22 +215,39 @@ class _BaseRaw(ProjMixin, ContainsMixin, PickDropChannelsMixin,
 
     Subclasses must provide the following methods:
 
-        * _read_segment(start, stop, sel, projector, verbose)
+        * _read_segment(start, stop, sel, data_buffer, projector, verbose)
           (only needed for types that support on-demand disk reads)
     """
     @verbose
-    def __init__(self, info, data=None,
+    def __init__(self, info, preload=False,
                  first_samps=(0,), last_samps=None,
                  filenames=(), rawdirs=(),
                  comp=None, orig_comp_grade=None,
-                 orig_format='double', verbose=None):
-        # some functions (e.g., filtering) only work w/64-bit data
-        if data is not None:
-            if data.dtype not in (np.float64, np.complex128):
+                 orig_format='double',
+                 verbose=None):
+        # wait until the end to preload data, but triage here
+        if isinstance(preload, np.ndarray):
+            # some functions (e.g., filtering) only work w/64-bit data
+            if preload.dtype not in (np.float64, np.complex128):
                 raise RuntimeError('datatype must be float64 or complex128, '
-                                   'not %s' % data.dtype)
+                                   'not %s' % preload.dtype)
+            self._data = preload
+            self.preload = True
+            self._last_samps = np.array([self._data.shape[1] - 1])
+            load_from_disk = False
+        else:
+            if last_samps is None:
+                raise ValueError('last_samps must be given unless preload is '
+                                 'an ndarray')
+            if preload is False:
+                self.preload = False
+                load_from_disk = False
+            elif preload is not True and not isinstance(preload, string_types):
+                raise ValueError('bad preload: %s' % preload)
+            else:
+                load_from_disk = True
+            self._last_samps = np.array(last_samps)
         self.info = info
-        self._data = data
         cals = np.empty(info['nchan'])
         for k in range(info['nchan']):
             cals[k] = info['chs'][k]['range'] * info['chs'][k]['cal']
@@ -240,19 +257,70 @@ class _BaseRaw(ProjMixin, ContainsMixin, PickDropChannelsMixin,
         self.comp = comp
         self._orig_comp_grade = orig_comp_grade
         self._filenames = list(filenames)
-        self.preload = True if data is not None else False
         self._first_samps = np.array(first_samps)
         self.orig_format = orig_format
-        if data is not None:
-            self._last_samps = np.array([self._data.shape[1] - 1])
-        else:
-            self._last_samps = np.array(last_samps)
         self._projectors = list()
         self._projector = None
+        # If we have True or a string, actually do the preloading
+        if load_from_disk:
+            self._preload_data(preload)
 
-    def _read_segment(start, stop, sel, projector, verbose):
+    def _read_segment(start, stop, sel, data_buffer, projector, verbose):
+        """Read a chunk of raw data
+
+        Parameters
+        ----------
+        start : int, (optional)
+            first sample to include (first is 0). If omitted, defaults to the
+            first sample in data.
+        stop : int, (optional)
+            First sample to not include.
+            If omitted, data is included to the end.
+        sel : array, optional
+            Indices of channels to select.
+        data_buffer : array or str, optional
+            numpy array to fill with data read, must have the correct shape.
+            If str, a np.memmap with the correct data type will be used
+            to store the data.
+        projector : array
+            SSP operator to apply to the data.
+        verbose : bool, str, int, or None
+            If not None, override default verbose level (see mne.verbose).
+
+        Returns
+        -------
+        data : array, [channels x samples]
+           the data matrix (channels x samples).
+        times : array, [samples]
+            returns the time values corresponding to the samples.
+        """
         raise NotImplementedError
 
+    @verbose
+    def preload_data(self, verbose=None):
+        """Preload raw data
+
+        Parameters
+        ----------
+        verbose : bool, str, int, or None
+            If not None, override default verbose level (see mne.verbose).
+
+        Notes
+        -----
+        This function will preload raw data if it was not already preloaded.
+        If data were already preloaded, it will do nothing.
+        """
+        if not self.preload:
+            self._preload_data(True)
+
+    def _preload_data(self, preload):
+        """This function actually preloads the data"""
+        data_buffer = preload if isinstance(preload, string_types) else None
+        self._data = self._read_segment(data_buffer=data_buffer)[0]
+        assert len(self._data) == self.info['nchan']
+        self.preload = True
+        self.close()
+
     @property
     def first_samp(self):
         return self._first_samps[0]
@@ -1344,7 +1412,6 @@ class _BaseRaw(ProjMixin, ContainsMixin, PickDropChannelsMixin,
         raws : list, or Raw instance
             list of Raw instances to concatenate to the current instance
             (in order), or a single raw instance to concatenate.
-
         preload : bool, str, or None (default None)
             Preload data into memory for data manipulation and faster indexing.
             If True, the data will be preloaded into memory (fast, requires
@@ -1414,7 +1481,8 @@ class _BaseRaw(ProjMixin, ContainsMixin, PickDropChannelsMixin,
     def close(self):
         """Clean up the object.
 
-        Does nothing for now.
+        Does nothing for objects that close their file descriptors.
+        Things like RawFIF will override this method.
         """
         pass
 
diff --git a/mne/io/edf/edf.py b/mne/io/edf/edf.py
index 5a8eb8e5f..700849294 100644
--- a/mne/io/edf/edf.py
+++ b/mne/io/edf/edf.py
@@ -60,9 +60,12 @@ class RawEDF(_BaseRaw):
         -1 corresponds to the last channel.
         If None, the annotation channel is not used.
         Note: this is overruled by the annotation file if specified.
-    preload : bool
-        If True, all data are loaded at initialization.
-        If False, data are not read until save.
+    preload : bool or str (default False)
+        Preload data into memory for data manipulation and faster indexing.
+        If True, the data will be preloaded into memory (fast, requires
+        large amount of memory). If preload is a string, preload is the
+        file name of a memory-mapped file which is used to store the data
+        on the hard drive (slower, requires less memory).
     verbose : bool, str, int, or None
         If not None, override default verbose level (see mne.verbose).
 
@@ -89,18 +92,9 @@ class RawEDF(_BaseRaw):
         # Raw attributes
         last_samps = [self._edf_info['nsamples'] - 1]
         super(RawEDF, self).__init__(
-            info, last_samps=last_samps, orig_format='int', verbose=verbose)
-
-        if preload:
-            self.preload = preload
-            logger.info('Reading raw data from %s...' % input_fname)
-            self._data, _ = self._read_segment()
-            assert len(self._data) == self.info['nchan']
-            self._last_samps = np.array([self._data.shape[1] - 1])
-            logger.info('    Range : %d ... %d =  %9.3f ... %9.3f secs'
-                        % (self.first_samp, self.last_samp,
-                           float(self.first_samp) / self.info['sfreq'],
-                           float(self.last_samp) / self.info['sfreq']))
+            info, preload, last_samps=last_samps, orig_format='int',
+            verbose=verbose)
+
         logger.info('Ready.')
 
     def __repr__(self):
@@ -110,50 +104,27 @@ class RawEDF(_BaseRaw):
              "n_channels x n_times : %s x %s" % (n_chan, data_range))
         return "<RawEDF  |  %s>" % ', '.join(s)
 
-    def _read_segment(self, start=0, stop=None, sel=None, verbose=None,
-                      projector=None):
-        """Read a chunk of raw data
-
-        Parameters
-        ----------
-        start : int, (optional)
-            first sample to include (first is 0). If omitted, defaults to the
-            first sample in data.
-        stop : int, (optional)
-            First sample to not include.
-            If omitted, data is included to the end.
-        sel : array, optional
-            Indices of channels to select.
-        projector : array
-            SSP operator to apply to the data.
-        verbose : bool, str, int, or None
-            If not None, override default verbose level (see mne.verbose).
-
-        Returns
-        -------
-        data : array, [channels x samples]
-           the data matrix (channels x samples).
-        times : array, [samples]
-            returns the time values corresponding to the samples.
-        """
+    @verbose
+    def _read_segment(self, start=0, stop=None, sel=None, data_buffer=None,
+                      projector=None, verbose=None):
+        """Read a chunk of raw data"""
         from scipy.interpolate import interp1d
         if sel is None:
-            sel = list(range(self.info['nchan']))
-        elif len(sel) == 1 and sel[0] == 0 and start == 0 and stop == 1:
-            return (666, 666)
+            sel = np.arange(self.info['nchan'])
         if projector is not None:
             raise NotImplementedError('Currently does not handle projections.')
         if stop is None:
             stop = self.last_samp + 1
         elif stop > self.last_samp + 1:
             stop = self.last_samp + 1
+        sel = np.array(sel)
 
         #  Initial checks
         start = int(start)
         stop = int(stop)
 
         n_samps = self._edf_info['n_samps']
-        max_samp = self._edf_info['max_samp']
+        buf_len = self._edf_info['max_samp']
         sfreq = self.info['sfreq']
         n_chan = self.info['nchan']
         data_size = self._edf_info['data_size']
@@ -165,8 +136,8 @@ class RawEDF(_BaseRaw):
         subtype = self._edf_info['subtype']
 
         # this is used to deal with indexing in the middle of a sampling period
-        blockstart = int(floor(float(start) / max_samp) * max_samp)
-        blockstop = int(ceil(float(stop) / max_samp) * max_samp)
+        blockstart = int(floor(float(start) / buf_len) * buf_len)
+        blockstop = int(ceil(float(stop) / buf_len) * buf_len)
         if blockstop > self.last_samp:
             blockstop = self.last_samp + 1
 
@@ -188,100 +159,179 @@ class RawEDF(_BaseRaw):
         picks = [stim_channel, tal_channel]
         offsets[picks] = 0
 
-        with open(self.info['filename'], 'rb') as fid:
+        # set up output array
+        data_shape = (len(sel), stop - start)
+        if isinstance(data_buffer, np.ndarray):
+            if data_buffer.shape != data_shape:
+                raise ValueError('data_buffer has incorrect shape')
+            data = data_buffer
+        else:
+            data = np.empty(data_shape, dtype=float)
+
+        read_size = blockstop - blockstart
+        this_data = np.empty((len(sel), buf_len))
+        """
+        Consider this example:
+
+        tmin, tmax = (2, 27)
+        read_size = 30
+        buf_len = 10
+        sfreq = 1.
+
+                        +---------+---------+---------+
+        File structure: |  buf0   |   buf1  |   buf2  |
+                        +---------+---------+---------+
+        File time:      0        10        20        30
+                        +---------+---------+---------+
+        Requested time:   2                       27
+
+                        |                             |
+                    blockstart                    blockstop
+                          |                        |
+                        start                    stop
+
+        We need 27 - 2 = 25 samples (per channel) to store our data, and
+        we need to read from 3 buffers (30 samples) to get all of our data.
+
+        On all reads but the first, the data we read starts at
+        the first sample of the buffer. On all reads but the last,
+        the data we read ends on the last sample of the buffer.
+
+        We call this_data the variable that stores the current buffer's data,
+        and data the variable that stores the total output.
+
+        On the first read, we need to do this::
+
+            >>> data[0:buf_len-2] = this_data[2:buf_len]
+
+        On the second read, we need to do::
+
+            >>> data[1*buf_len-2:2*buf_len-2] = this_data[0:buf_len]
+
+        On the final read, we need to do::
+
+            >>> data[2*buf_len-2:3*buf_len-2-3] = this_data[0:buf_len-3]
+
+        """
+        with open(self.info['filename'], 'rb', buffering=0) as fid:
             # extract data
-            fid.seek(data_offset)
-            buffer_size = blockstop - blockstart
-            pointer = blockstart * n_chan * data_size
-            fid.seek(data_offset + pointer)
-            datas = np.empty((n_chan, buffer_size), dtype=float)
-            blocks = int(ceil(float(buffer_size) / max_samp))
-            for i in range(blocks):
-                data = np.empty((n_chan, max_samp), dtype=np.int32)
+            fid.seek(data_offset + blockstart * n_chan * data_size)
+            n_blk = int(ceil(float(read_size) / buf_len))
+            start_offset = start - blockstart
+            end_offset = blockstop - stop
+            for bi in range(n_blk):
+                # Triage start (sidx) and end (eidx) indices for
+                # data (d) and read (r)
+                if bi == 0:
+                    d_sidx = 0
+                    r_sidx = start_offset
+                else:
+                    d_sidx = bi * buf_len - start_offset
+                    r_sidx = 0
+                if bi == n_blk - 1:
+                    d_eidx = data_shape[1]
+                    r_eidx = buf_len - end_offset
+                else:
+                    d_eidx = (bi + 1) * buf_len - start_offset
+                    r_eidx = buf_len
+                n_buf_samp = r_eidx - r_sidx
+                count = 0
                 for j, samp in enumerate(n_samps):
                     # bdf data: 24bit data
-                    if subtype in ('24BIT', 'bdf'):
-                        ch_data = np.fromfile(fid, dtype=np.uint8,
-                                              count=samp * data_size)
-                        ch_data = ch_data.reshape(-1, 3).astype(np.int32)
-                        ch_data = ((ch_data[:, 0]) +
-                                   (ch_data[:, 1] << 8) +
-                                   (ch_data[:, 2] << 16))
-                        # 24th bit determines the sign
-                        ch_data[ch_data >= (1 << 23)] -= (1 << 24)
-                    # edf data: 16bit data
+                    if j not in sel:
+                        fid.seek(samp * data_size, 1)
+                        continue
+                    if samp == buf_len:
+                        # use faster version with skips built in
+                        if r_sidx > 0:
+                            fid.seek(r_sidx * data_size, 1)
+                        ch_data = _read_ch(fid, subtype, n_buf_samp, data_size)
+                        if r_eidx < buf_len:
+                            fid.seek((buf_len - r_eidx) * data_size, 1)
                     else:
-                        ch_data = np.fromfile(fid, dtype='<i2', count=samp)
-                    if j == tal_channel:
-                        # don't resample tal_channel,
-                        # pad with zeros instead.
-                        n_missing = int(max_samp - samp)
-                        ch_data = np.hstack([ch_data,
-                                             [0] * n_missing])
-                    elif j == stim_channel and samp < max_samp:
-                        if annot and annotmap or \
-                                tal_channel is not None:
-                            # don't bother with resampling the stim ch
-                            # because it gets overwritten later on.
-                            ch_data = np.zeros(max_samp)
+                        # read in all the data and triage appropriately
+                        ch_data = _read_ch(fid, subtype, samp, data_size)
+                        if j == tal_channel:
+                            # don't resample tal_channel,
+                            # pad with zeros instead.
+                            n_missing = int(buf_len - samp)
+                            ch_data = np.hstack([ch_data, [0] * n_missing])
+                            ch_data = ch_data[r_sidx:r_eidx]
+                        elif j == stim_channel:
+                            if annot and annotmap or \
+                                    tal_channel is not None:
+                                # don't bother with resampling the stim ch
+                                # because it gets overwritten later on.
+                                ch_data = np.zeros(n_buf_samp)
+                            else:
+                                warnings.warn('Interpolating stim channel.'
+                                              ' Events may jitter.')
+                                oldrange = np.linspace(0, 1, samp + 1, True)
+                                newrange = np.linspace(0, 1, buf_len, False)
+                                newrange = newrange[r_sidx:r_eidx]
+                                ch_data = interp1d(
+                                    oldrange, np.append(ch_data, 0),
+                                    kind='zero')(newrange)
                         else:
-                            warnings.warn('Interpolating stim channel.'
-                                          ' Events may jitter.')
-                            oldrange = np.linspace(0, 1, samp + 1,
-                                                   True)
-                            newrange = np.linspace(0, 1, max_samp,
-                                                   False)
-                            ch_data = interp1d(
-                                oldrange, np.append(ch_data, 0),
-                                kind='zero')(newrange)
-                    elif samp != max_samp:
-                        ch_data = resample(x=ch_data, up=max_samp, down=samp,
-                                           npad=0)
-                    data[j] = ch_data
-                start_pt = int(max_samp * i)
-                stop_pt = int(start_pt + max_samp)
-                datas[:, start_pt:stop_pt] = data
-        datas *= gains.T
-        datas += offsets
+                            ch_data = resample(ch_data, buf_len, samp,
+                                               npad=0)[r_sidx:r_eidx]
+                    this_data[count, :n_buf_samp] = ch_data
+                    count += 1
+                data[:, d_sidx:d_eidx] = this_data[:, :n_buf_samp]
+        data *= gains.T[sel]
+        data += offsets[sel]
 
         if stim_channel is not None:
+            stim_channel_idx = np.where(sel == stim_channel)[0][0]
             if annot and annotmap:
-                datas[stim_channel] = 0
                 evts = _read_annot(annot, annotmap, sfreq, self.last_samp)
-                datas[stim_channel, :evts.size] = evts[start:stop]
+                stim = evts[start:stop]
             elif tal_channel is not None:
-                evts = _parse_tal_channel(datas[tal_channel])
+                tal_channel_idx = np.where(sel == tal_channel)[0][0]
+                evts = _parse_tal_channel(data[tal_channel_idx])
                 self._edf_info['events'] = evts
 
                 unique_annots = sorted(set([e[2] for e in evts]))
                 mapping = dict((a, n + 1) for n, a in enumerate(unique_annots))
 
-                datas[stim_channel] = 0
+                stim = np.zeros(read_size)
                 for t_start, t_duration, annotation in evts:
                     evid = mapping[annotation]
                     n_start = int(t_start * sfreq)
                     n_stop = int(t_duration * sfreq) + n_start - 1
                     # make sure events without duration get one sample
                     n_stop = n_stop if n_stop > n_start else n_start + 1
-                    if any(datas[stim_channel][n_start:n_stop]):
+                    if any(stim[n_start:n_stop]):
                         raise NotImplementedError('EDF+ with overlapping '
                                                   'events not supported.')
-                    datas[stim_channel][n_start:n_stop] = evid
+                    stim[n_start:n_stop] = evid
             else:
-                # Allows support for up to 16-bit trigger values
-                mask = 2 ** 16 - 1
-                stim = np.array(datas[stim_channel], int)
-                mask = mask * np.ones(stim.shape, int)
-                stim = np.bitwise_and(stim, mask)
-                datas[stim_channel] = stim
-        datastart = start - blockstart
-        datastop = stop - blockstart
-        datas = datas[sel, datastart:datastop]
+                # Allows support for up to 16-bit trigger values (2 ** 16 - 1)
+                stim = np.bitwise_and(data[stim_channel_idx].astype(int),
+                                      65535)
+            data[stim_channel_idx, :] = \
+                stim[start - blockstart:stop - blockstart]
 
         logger.info('[done]')
         times = np.arange(start, stop, dtype=float) / self.info['sfreq']
-
-        return datas, times
+        return data, times
+
+
+def _read_ch(fid, subtype, samp, data_size):
+    """Helper to read a number of samples for a single channel"""
+    if subtype in ('24BIT', 'bdf'):
+        ch_data = np.fromfile(fid, dtype=np.uint8,
+                              count=samp * data_size)
+        ch_data = ch_data.reshape(-1, 3).astype(np.int32)
+        ch_data = ((ch_data[:, 0]) +
+                   (ch_data[:, 1] << 8) +
+                   (ch_data[:, 2] << 16))
+        # 24th bit determines the sign
+        ch_data[ch_data >= (1 << 23)] -= (1 << 24)
+    # edf data: 16bit data
+    else:
+        ch_data = np.fromfile(fid, dtype='<i2', count=samp)
+    return ch_data
 
 
 def _parse_tal_channel(tal_channel_data):
@@ -324,46 +374,7 @@ def _parse_tal_channel(tal_channel_data):
 
 def _get_edf_info(fname, stim_channel, annot, annotmap, tal_channel,
                   eog, misc, preload):
-    """Extracts all the information from the EDF+,BDF file.
-
-    Parameters
-    ----------
-    fname : str
-        Raw EDF+,BDF file to be read.
-    stim_channel : str | int | None
-        The channel name or channel index (starting at 0).
-        -1 corresponds to the last channel.
-        If None, there will be no stim channel added.
-    annot : str | None
-        Path to annotation file.
-        If None, no derived stim channel will be added (for files requiring
-        annotation file to interpret stim channel).
-    annotmap : str | None
-        Path to annotation map file containing mapping from label to trigger.
-        Must be specified if annot is not None.
-    tal_channel : int | None
-        The channel index (starting at 0).
-        Index of the channel containing EDF+ annotations.
-        -1 corresponds to the last channel.
-        If None, the annotation channel is not used.
-        Note: this is overruled by the annotation file if specified.
-    eog : list of str | None
-        Names of channels that should be designated EOG channels. Names should
-        correspond to the electrodes in the edf file. Default is None.
-    misc : list of str | None
-        Names of channels that should be designated MISC channels. Names
-        should correspond to the electrodes in the edf file. Default is None.
-    preload : bool
-        If True, all data are loaded at initialization.
-        If False, data are not read until save.
-
-    Returns
-    -------
-    info : instance of Info
-        The measurement info.
-    edf_info : dict
-        A dict containing all the EDF+,BDF  specific parameters.
-    """
+    """Extracts all the information from the EDF+,BDF file"""
 
     if eog is None:
         eog = []
@@ -632,9 +643,12 @@ def read_raw_edf(input_fname, montage=None, eog=None, misc=None,
         -1 corresponds to the last channel.
         If None, the annotation channel is not used.
         Note: this is overruled by the annotation file if specified.
-    preload : bool
-        If True, all data are loaded at initialization.
-        If False, data are not read until save.
+    preload : bool or str (default False)
+        Preload data into memory for data manipulation and faster indexing.
+        If True, the data will be preloaded into memory (fast, requires
+        large amount of memory). If preload is a string, preload is the
+        file name of a memory-mapped file which is used to store the data
+        on the hard drive (slower, requires less memory).
     verbose : bool, str, int, or None
         If not None, override default verbose level (see mne.verbose).
 
diff --git a/mne/io/fiff/raw.py b/mne/io/fiff/raw.py
index 6d3027fcd..9894f8513 100644
--- a/mne/io/fiff/raw.py
+++ b/mne/io/fiff/raw.py
@@ -114,7 +114,7 @@ class RawFIF(_BaseRaw):
         _check_raw_compatibility(raws)
 
         super(RawFIF, self).__init__(
-            copy.deepcopy(raws[0].info), None,
+            copy.deepcopy(raws[0].info), False,
             [r.first_samp for r in raws], [r.last_samp for r in raws],
             [r.filename for r in raws], [r._rawdir for r in raws],
             copy.deepcopy(raws[0].comp), raws[0]._orig_comp_grade,
@@ -135,36 +135,6 @@ class RawFIF(_BaseRaw):
         if proj:
             self.apply_proj()
 
-    def _preload_data(self, preload):
-        """This function actually preloads the data"""
-        if isinstance(preload, string_types):
-            # we will use a memmap: preload is a filename
-            data_buffer = preload
-        else:
-            data_buffer = None
-
-        self._data = self._read_segment(data_buffer=data_buffer)[0]
-        self.preload = True
-        # close files once data are preloaded
-        self.close()
-
-    @verbose
-    def preload_data(self, verbose=None):
-        """Preload raw data
-
-        Parameters
-        ----------
-        verbose : bool, str, int, or None
-            If not None, override default verbose level (see mne.verbose).
-
-        Notes
-        -----
-        This function will preload raw data if it was not already preloaded.
-        If data were already preloaded, it will do nothing.
-        """
-        if not self.preload:
-            self._preload_data(True)
-
     @verbose
     def _read_raw_file(self, fname, allow_maxshield, preload, compensation,
                        do_check_fname=True, verbose=None):
@@ -376,36 +346,10 @@ class RawFIF(_BaseRaw):
 
         return raw, next_fname
 
+    @verbose
     def _read_segment(self, start=0, stop=None, sel=None, data_buffer=None,
-                      verbose=None, projector=None):
-        """Read a chunk of raw data
-
-        Parameters
-        ----------
-        start : int, (optional)
-            first sample to include (first is 0). If omitted, defaults to the
-            first sample in data.
-        stop : int, (optional)
-            First sample to not include.
-            If omitted, data is included to the end.
-        sel : array, optional
-            Indices of channels to select.
-        data_buffer : array or str, optional
-            numpy array to fill with data read, must have the correct shape.
-            If str, a np.memmap with the correct data type will be used
-            to store the data.
-        verbose : bool, str, int, or None
-            If not None, override default verbose level (see mne.verbose).
-        projector : array
-            SSP operator to apply to the data.
-
-        Returns
-        -------
-        data : array, [channels x samples]
-           the data matrix (channels x samples).
-        times : array, [samples]
-            returns the time values corresponding to the samples.
-        """
+                      projector=None, verbose=None):
+        """Read a chunk of raw data"""
         #  Initial checks
         start = int(start)
         stop = self.n_times if stop is None else min([int(stop), self.n_times])
diff --git a/mne/io/kit/kit.py b/mne/io/kit/kit.py
index 9ccb87ccb..8de71ac9a 100644
--- a/mne/io/kit/kit.py
+++ b/mne/io/kit/kit.py
@@ -63,9 +63,12 @@ class RawKIT(_BaseRaw):
     stimthresh : float
         The threshold level for accepting voltage changes in KIT trigger
         channels as a trigger event. If None, stim must also be set to None.
-    preload : bool
-        If True, all data are loaded at initialization.
-        If False, data are not read until save.
+    preload : bool or str (default False)
+        Preload data into memory for data manipulation and faster indexing.
+        If True, the data will be preloaded into memory (fast, requires
+        large amount of memory). If preload is a string, preload is the
+        file name of a memory-mapped file which is used to store the data
+        on the hard drive (slower, requires less memory).
     verbose : bool, str, int, or None
         If not None, override default verbose level (see mne.verbose).
 
@@ -90,9 +93,9 @@ class RawKIT(_BaseRaw):
         logger.info('Creating Info structure...')
 
         last_samps = [self._kit_info['n_samples'] - 1]
+        self._set_stimchannels(info, stim)
         super(RawKIT, self).__init__(
-            info, last_samps=last_samps, verbose=verbose)
-        self._set_stimchannels(stim)
+            info, preload, last_samps=last_samps, verbose=verbose)
 
         if isinstance(mrk, list):
             mrk = [read_mrk(marker) if isinstance(marker, string_types)
@@ -103,19 +106,9 @@ class RawKIT(_BaseRaw):
             self.info['dig'] = dig_points
             self.info['dev_head_t'] = dev_head_t
         elif (mrk is not None or elp is not None or hsp is not None):
-            err = ("mrk, elp and hsp need to be provided as a group (all or "
-                   "none)")
-            raise ValueError(err)
+            raise ValueError('mrk, elp and hsp need to be provided as a group '
+                             '(all or none)')
 
-        if preload:
-            self.preload = preload
-            logger.info('Reading raw data from %s...' % input_fname)
-            self._data, _ = self._read_segment()
-            assert self._data.shape == (self.info['nchan'], self.last_samp + 1)
-
-            logger.info('    Range : %d ... %d =  %9.3f ... %9.3f secs'
-                        % (self.first_samp, self.last_samp,
-                           self.times[0], self.times[-1]))
         logger.info('Ready.')
 
     def __repr__(self):
@@ -151,7 +144,7 @@ class RawKIT(_BaseRaw):
 
         return stim_ch
 
-    def _set_stimchannels(self, stim='<'):
+    def _set_stimchannels(self, info, stim='<'):
         """Specify how the trigger channel is synthesized from analog channels.
 
         Has to be done before loading data. For a RawKIT instance that has been
@@ -160,6 +153,8 @@ class RawKIT(_BaseRaw):
 
         Parameters
         ----------
+        info : instance of MeasInfo
+            The measurement info.
         stim : list of int | '<' | '>'
             Can be submitted as list of trigger channels.
             If a list is not specified, the default triggers extracted from
@@ -171,7 +166,7 @@ class RawKIT(_BaseRaw):
         """
         if stim is not None:
             if isinstance(stim, str):
-                picks = pick_types(self.info, meg=False, ref_meg=False,
+                picks = pick_types(info, meg=False, ref_meg=False,
                                    misc=True, exclude=[])[:8]
                 if stim == '<':
                     stim = picks[::-1]
@@ -181,16 +176,16 @@ class RawKIT(_BaseRaw):
                     raise ValueError("stim needs to be list of int, '>' or "
                                      "'<', not %r" % str(stim))
             elif np.max(stim) >= self._kit_info['nchan']:
-                msg = ("Tried to set stim channel %i, but sqd file only has %i"
-                       " channels" % (np.max(stim), self._kit_info['nchan']))
-                raise ValueError(msg)
+                raise ValueError('Tried to set stim channel %i, but sqd file '
+                                 'only has %i channels'
+                                 % (np.max(stim), self._kit_info['nchan']))
             # modify info
-            self.info['nchan'] = self._kit_info['nchan'] + 1
+            info['nchan'] = self._kit_info['nchan'] + 1
             ch_name = 'STI 014'
             chan_info = {}
             chan_info['cal'] = KIT.CALIB_FACTOR
-            chan_info['logno'] = self.info['nchan']
-            chan_info['scanno'] = self.info['nchan']
+            chan_info['logno'] = info['nchan']
+            chan_info['scanno'] = info['nchan']
             chan_info['range'] = 1.0
             chan_info['unit'] = FIFF.FIFF_UNIT_NONE
             chan_info['unit_mul'] = 0
@@ -198,50 +193,27 @@ class RawKIT(_BaseRaw):
             chan_info['coil_type'] = FIFF.FIFFV_COIL_NONE
             chan_info['loc'] = np.zeros(12)
             chan_info['kind'] = FIFF.FIFFV_STIM_CH
-            self.info['chs'].append(chan_info)
-            self.info['ch_names'].append(ch_name)
+            info['chs'].append(chan_info)
+            info['ch_names'].append(ch_name)
         if self.preload:
             err = "Can't change stim channel after preloading data"
             raise NotImplementedError(err)
 
         self._kit_info['stim'] = stim
 
-    def _read_segment(self, start=0, stop=None, sel=None, verbose=None,
-                      projector=None):
-        """Read a chunk of raw data
-
-        Parameters
-        ----------
-        start : int, (optional)
-            first sample to include (first is 0). If omitted, defaults to the
-            first sample in data.
-        stop : int, (optional)
-            First sample to not include.
-            If omitted, data is included to the end.
-        sel : array, optional
-            Indices of channels to select.
-        projector : array
-            SSP operator to apply to the data.
-        verbose : bool, str, int, or None
-            If not None, override default verbose level (see mne.verbose).
-
-        Returns
-        -------
-        data : array, [channels x samples]
-           the data matrix (channels x samples).
-        times : array, [samples]
-            returns the time values corresponding to the samples.
-        """
+    @verbose
+    def _read_segment(self, start=0, stop=None, sel=None, data_buffer=None,
+                      projector=None, verbose=None):
+        """Read a chunk of raw data"""
         if sel is None:
-            sel = list(range(self.info['nchan']))
-        elif len(sel) == 1 and sel[0] == 0 and start == 0 and stop == 1:
-            return (666, 666)
+            sel = np.arange(self.info['nchan'])
         if projector is not None:
             raise NotImplementedError('Currently does not handle projections.')
         if stop is None:
             stop = self.last_samp + 1
         elif stop > self.last_samp + 1:
             stop = self.last_samp + 1
+        sel = np.array(sel)
 
         #  Initial checks
         start = int(start)
@@ -294,6 +266,14 @@ class RawKIT(_BaseRaw):
             data = np.vstack((data, stim_ch))
         data = data[sel]
 
+        # This maybe should be refactored to actually save memory...
+        data_shape = data.shape
+        if isinstance(data_buffer, np.ndarray):
+            if data_buffer.shape != data_shape:
+                raise ValueError('data_buffer has incorrect shape')
+            data_buffer[...] = data
+            data = data_buffer
+
         logger.info('[done]')
         times = np.arange(start, stop, dtype=float) / self.info['sfreq']
 
@@ -413,9 +393,8 @@ class EpochsKIT(EpochsArray):
 
         for key, val in event_id.items():
             if val not in events[:, 2]:
-                msg = ('No matching events found for %s '
-                       '(event id %i)' % (key, val))
-                raise ValueError(msg)
+                raise ValueError('No matching events found for %s '
+                                 '(event id %i)' % (key, val))
 
         data = self._read_data()
         assert data.shape == (self._kit_info['n_epochs'], self.info['nchan'],
@@ -520,8 +499,8 @@ def _set_dig_kit(mrk, elp, hsp, auto_decimate=True):
         msg = ("The selected head shape contained {n_in} points, which is "
                "more than recommended ({n_rec}), and was automatically "
                "downsampled to {n_new} points. The preferred way to "
-               "downsample is using FastScan.")
-        msg = msg.format(n_in=n_pts, n_rec=KIT.DIG_POINTS, n_new=n_new)
+               "downsample is using FastScan."
+               ).format(n_in=n_pts, n_rec=KIT.DIG_POINTS, n_new=n_new)
         logger.warning(msg)
 
     if isinstance(elp, string_types):


Test patch:
diff --git a/mne/io/brainvision/tests/test_brainvision.py b/mne/io/brainvision/tests/test_brainvision.py
index 5131d978a..13762077d 100644
--- a/mne/io/brainvision/tests/test_brainvision.py
+++ b/mne/io/brainvision/tests/test_brainvision.py
@@ -47,6 +47,7 @@ def test_brainvision_data():
     assert_raises(TypeError, read_raw_brainvision, vhdr_path, montage,
                   preload=True, scale="0")
     raw_py = read_raw_brainvision(vhdr_path, montage, eog=eog, preload=True)
+    raw_py.preload_data()  # currently does nothing
 
     assert_equal(raw_py.info['highpass'], 0.)
     assert_equal(raw_py.info['lowpass'], 250.)
diff --git a/mne/io/bti/tests/test_bti.py b/mne/io/bti/tests/test_bti.py
index 914233034..f6dc30465 100644
--- a/mne/io/bti/tests/test_bti.py
+++ b/mne/io/bti/tests/test_bti.py
@@ -53,6 +53,7 @@ def test_read_pdf():
 def test_crop():
     """ Test crop raw """
     raw = read_raw_bti(pdf_fnames[0], config_fnames[0], hs_fnames[0])
+    raw.preload_data()  # currently does nothing
     y, t = raw[:]
     t0, t1 = 0.25 * t[-1], 0.75 * t[-1]
     mask = (t0 <= t) * (t <= t1)
diff --git a/mne/io/edf/tests/test_edf.py b/mne/io/edf/tests/test_edf.py
index 326a8ea51..f2959c3e2 100644
--- a/mne/io/edf/tests/test_edf.py
+++ b/mne/io/edf/tests/test_edf.py
@@ -47,9 +47,6 @@ def test_bdf_data():
     picks = pick_types(raw_py.info, meg=False, eeg=True, exclude='bads')
     data_py, _ = raw_py[picks]
 
-    print(raw_py)  # to test repr
-    print(raw_py.info)  # to test Info repr
-
     # this .mat was generated using the EEG Lab Biosemi Reader
     raw_eeglab = io.loadmat(bdf_eeglab_path)
     raw_eeglab = raw_eeglab['data'] * 1e-6  # data are stored in microvolts
@@ -118,13 +115,20 @@ def test_read_segment():
     assert_allclose(data1, data11, rtol=1e-6)
     assert_array_almost_equal(times1, times11)
     assert_equal(sorted(raw1.info.keys()), sorted(raw11.info.keys()))
-
-    raw2 = read_raw_edf(edf_path, stim_channel=None, preload=True)
-    raw2_file = op.join(tempdir, 'test2-raw.fif')
-    raw2.save(raw2_file, overwrite=True)
-    data2, times2 = raw2[:139, :]
-    assert_allclose(data1, data2, rtol=1e-6)
-    assert_array_equal(times1, times2)
+    data2, times2 = raw1[0, 0:1]
+    assert_array_equal(data2[0], data1[0, 0:1])
+    assert_array_equal(times2, times1[0:1])
+
+    buffer_fname = op.join(tempdir, 'buffer')
+    for preload in (buffer_fname, True, False):  # false here means "delayed"
+        raw2 = read_raw_edf(edf_path, stim_channel=None, preload=preload)
+        if preload is False:
+            raw2.preload_data()
+        raw2_file = op.join(tempdir, 'test2-raw.fif')
+        raw2.save(raw2_file, overwrite=True)
+        data2, times2 = raw2[:139, :]
+        assert_allclose(data1, data2, rtol=1e-6)
+        assert_array_equal(times1, times2)
 
     raw1 = Raw(raw1_file, preload=True)
     raw2 = Raw(raw2_file, preload=True)
diff --git a/mne/io/egi/tests/test_egi.py b/mne/io/egi/tests/test_egi.py
index b12e59c38..4faf0535b 100644
--- a/mne/io/egi/tests/test_egi.py
+++ b/mne/io/egi/tests/test_egi.py
@@ -26,7 +26,8 @@ def test_io_egi():
     tempdir = _TempDir()
     with warnings.catch_warnings(record=True) as w:
         warnings.simplefilter('always', category=RuntimeWarning)
-        read_raw_egi(egi_fname, include=None)
+        raw = read_raw_egi(egi_fname, include=None)
+        raw.preload_data()  # currently does nothing
         assert_equal(len(w), 1)
         assert_true(w[0].category == RuntimeWarning)
         msg = 'Did not find any event code with more than one event.'
diff --git a/mne/io/kit/tests/test_kit.py b/mne/io/kit/tests/test_kit.py
index b3fe8a434..43f711f2c 100644
--- a/mne/io/kit/tests/test_kit.py
+++ b/mne/io/kit/tests/test_kit.py
@@ -96,12 +96,25 @@ def test_read_segment():
                         preload=True)
     raw2_file = op.join(tempdir, 'test2-raw.fif')
     raw2.save(raw2_file, buffer_size_sec=.1, overwrite=True)
+    data1, times1 = raw1[0, 0:1]
+
     raw1 = Raw(raw1_file, preload=True)
     raw2 = Raw(raw2_file, preload=True)
     assert_array_equal(raw1._data, raw2._data)
+    data2, times2 = raw2[0, 0:1]
+    assert_array_almost_equal(data1, data2)
+    assert_array_almost_equal(times1, times2)
     raw3 = read_raw_kit(sqd_path, mrk_path, elp_path, hsp_path, stim='<',
                         preload=True)
     assert_array_almost_equal(raw1._data, raw3._data)
+    raw4 = read_raw_kit(sqd_path, mrk_path, elp_path, hsp_path, stim='<',
+                        preload=False)
+    raw4.preload_data()
+    buffer_fname = op.join(tempdir, 'buffer')
+    assert_array_almost_equal(raw1._data, raw4._data)
+    raw5 = read_raw_kit(sqd_path, mrk_path, elp_path, hsp_path, stim='<',
+                        preload=buffer_fname)
+    assert_array_almost_equal(raw1._data, raw5._data)
 
 
 def test_ch_loc():