Repository: mne-tools/mne-python

Issue:
Add event_id to read_epochs or selectively read
If you only want to load a subset of your epochs from disk, suppplying an event_id could save on memory. currently, it loads all the epochs, and then you can do a selection. for long experiments, and large numbers of epochs, this is not ideal. the event id would find the indices in the events and from there, it can index and load just those epochs. 

one thing that I am not certain of is whether all the epochs would still be temporarily stored in memory. if that's the case, is there any way to read only select epochs?

cc: @Eric89GXL 



Discussion:
This would be solved by the more general solution of adding general
on-access reads, so that's my preferred solution. I might be able to do it
next week.

are we planning to deal with the epochs reading on demand in this cycle?

I might be able to do it. Would that supercede this?

yeah, i believe so, on-demand then event_id capability.


Resolving patch:
diff --git a/mne/epochs.py b/mne/epochs.py
--- a/mne/epochs.py
+++ b/mne/epochs.py
@@ -8,7 +8,7 @@
 #
 # License: BSD (3-clause)
 
-import copy as cp
+from copy import deepcopy
 import warnings
 import json
 import inspect
@@ -24,7 +24,7 @@
 from .io.meas_info import read_meas_info, write_meas_info, _merge_info
 from .io.open import fiff_open, _get_next_fname
 from .io.tree import dir_tree_find
-from .io.tag import read_tag
+from .io.tag import read_tag, read_tag_info
 from .io.constants import FIFF
 from .io.pick import (pick_types, channel_indices_by_type, channel_type,
                       pick_channels, pick_info)
@@ -149,7 +149,8 @@ def __init__(self, info, data, events, event_id, tmin, tmax,
                  picks=None, name='Unknown', reject=None, flat=None,
                  decim=1, reject_tmin=None, reject_tmax=None, detrend=None,
                  add_eeg_ref=True, proj=True, on_missing='error',
-                 preload_at_end=False, verbose=None):
+                 preload_at_end=False, selection=None, drop_log=None,
+                 verbose=None):
 
         self.verbose = verbose
         self.name = name
@@ -198,9 +199,15 @@ def __init__(self, info, data, events, event_id, tmin, tmax,
 
             values = list(self.event_id.values())
             selected = in1d(events[:, 2], values)
-            self.selection = np.where(selected)[0]
-            self.drop_log = [list() if k in self.selection else ['IGNORED']
-                             for k in range(len(events))]
+            if selection is None:
+                self.selection = np.where(selected)[0]
+            else:
+                self.selection = selection
+            if drop_log is None:
+                self.drop_log = [list() if k in self.selection else ['IGNORED']
+                                 for k in range(len(events))]
+            else:
+                self.drop_log = drop_log
             events = events[selected]
             n_events = len(events)
             if n_events > 1:
@@ -293,18 +300,11 @@ def __init__(self, info, data, events, event_id, tmin, tmax,
         self._reject_setup(reject, flat)
 
         # do the rest
-        if proj not in [True, 'delayed', False]:
-            raise ValueError(r"'proj' must either be 'True', 'False' or "
-                             "'delayed'")
-
-        # proj is on when applied in Raw
-        proj = proj or (raw is not None and raw.proj)
+        valid_proj = [True, 'delayed', False]
+        if proj not in valid_proj:
+            raise ValueError('"proj" must be one of %s, not %s'
+                             % (valid_proj, proj))
         if proj == 'delayed':
-            if self.reject is None:
-                raise RuntimeError('The delayed SSP mode was requested '
-                                   'but no rejection parameters are present. '
-                                   'Please add rejection parameters before '
-                                   'using this option.')
             self._do_delayed_proj = True
             logger.info('Entering delayed SSP mode.')
         else:
@@ -563,7 +563,7 @@ def iter_evoked(self):
         while True:
             data, event_id = self.next(True)
             tmin = self.times[0]
-            info = cp.deepcopy(self.info)
+            info = deepcopy(self.info)
 
             yield EvokedArray(data, info, tmin, comment=str(event_id))
 
@@ -711,7 +711,7 @@ def _compute_mean_or_stderr(self, picks, mode='ave'):
             # convert to stderr if requested, could do in one pass but do in
             # two (slower) in case there are large numbers
             if _do_std:
-                data_mean = cp.copy(data)
+                data_mean = data.copy()
                 data.fill(0.)
                 for e in self:
                     data += (e - data_mean) ** 2
@@ -724,7 +724,7 @@ def _compute_mean_or_stderr(self, picks, mode='ave'):
             data /= np.sqrt(n_events)
         kind = _aspect_rev.get(str(_aspect_kind), 'Unknown')
 
-        info = cp.deepcopy(self.info)
+        info = deepcopy(self.info)
         evoked = EvokedArray(data, info, tmin=self.times[0],
                              comment=self.name, nave=n_events, kind=kind,
                              verbose=self.verbose)
@@ -1448,7 +1448,7 @@ def copy(self):
         """Return copy of Epochs instance"""
         raw = self._raw
         del self._raw
-        new = cp.deepcopy(self)
+        new = deepcopy(self)
         self._raw = raw
         new._raw = raw
         return new
@@ -1737,7 +1737,10 @@ def __init__(self, raw, events, event_id, tmin, tmax, baseline=(None, 0),
         if not isinstance(raw, _BaseRaw):
             raise ValueError('The first argument to `Epochs` must be an '
                              'instance of `mne.io.Raw`')
-        info = cp.deepcopy(raw.info)
+        info = deepcopy(raw.info)
+
+        # proj is on when applied in Raw
+        proj = proj or raw.proj
 
         # call _BaseEpochs constructor
         super(Epochs, self).__init__(info, None, events, event_id, tmin, tmax,
@@ -2063,7 +2066,8 @@ def _is_good(e, ch_names, channel_type_idx, reject, flat, full_report=False,
 
 
 @verbose
-def _read_one_epoch_file(f, tree, fname, proj, add_eeg_ref, verbose):
+def _read_one_epoch_file(f, tree, fname, preload):
+    """Helper to read a single FIF file"""
 
     with f as fid:
         #   Read the measurement info
@@ -2086,6 +2090,7 @@ def _read_one_epoch_file(f, tree, fname, proj, add_eeg_ref, verbose):
         # Now find the data in the block
         name = None
         data = None
+        data_tag = None
         bmin, bmax = None, None
         baseline = None
         selection = None
@@ -2103,8 +2108,10 @@ def _read_one_epoch_file(f, tree, fname, proj, add_eeg_ref, verbose):
                 tag = read_tag(fid, pos)
                 name = tag.data
             elif kind == FIFF.FIFF_EPOCH:
-                tag = read_tag(fid, pos)
-                data = tag.data.astype(np.float)
+                # delay reading until later
+                fid.seek(pos, 0)
+                data_tag = read_tag_info(fid)
+                data_tag.pos = pos
             elif kind == FIFF.FIFF_MNE_BASELINE_MIN:
                 tag = read_tag(fid, pos)
                 bmin = float(tag.data)
@@ -2121,7 +2128,7 @@ def _read_one_epoch_file(f, tree, fname, proj, add_eeg_ref, verbose):
         if bmin is not None or bmax is not None:
             baseline = (bmin, bmax)
 
-        nsamp = last - first + 1
+        n_samp = last - first + 1
         logger.info('    Found the data of interest:')
         logger.info('        t = %10.2f ... %10.2f ms (%s)'
                     % (1000 * first / info['sfreq'],
@@ -2130,48 +2137,44 @@ def _read_one_epoch_file(f, tree, fname, proj, add_eeg_ref, verbose):
             logger.info('        %d CTF compensation matrices available'
                         % len(info['comps']))
 
-        # Read the data
-        if data is None:
+        # Inspect the data
+        if data_tag is None:
             raise ValueError('Epochs data not found')
-        if data.shape[2] != nsamp:
+        epoch_shape = (len(info['ch_names']), n_samp)
+        expected = len(events) * np.prod(epoch_shape)
+        if data_tag.size // 4 - 4 != expected:  # 32-bit floats stored
             raise ValueError('Incorrect number of samples (%d instead of %d)'
-                             % (data.shape[2], nsamp))
+                             % (data_tag.size // 4, expected))
+
+        # Calibration factors
+        cals = np.array([[info['chs'][k]['cal'] *
+                          info['chs'][k].get('scale', 1.0)]
+                         for k in range(info['nchan'])], np.float64)
 
-        # Calibrate
-        cals = np.array([info['chs'][k]['cal'] *
-                         info['chs'][k].get('scale', 1.0)
-                         for k in range(info['nchan'])])
-        data *= cals[np.newaxis, :, np.newaxis]
+        # Read the data
+        if preload:
+            data = read_tag(fid, data_tag.pos).data.astype(np.float64)
+            data *= cals[np.newaxis, :, :]
 
         # Put it all together
         tmin = first / info['sfreq']
         tmax = last / info['sfreq']
         event_id = (dict((str(e), e) for e in np.unique(events[:, 2]))
                     if mappings is None else mappings)
-        # here we ignore missing events, since users should already be
-        # aware of missing events if they have saved data that way
-        epochs = _BaseEpochs(info, data, events, event_id, tmin, tmax,
-                             baseline, name=name, on_missing='ignore',
-                             verbose=verbose)
-        activate = False if epochs._do_delayed_proj else proj
-        epochs._projector, epochs.info = setup_proj(info, add_eeg_ref,
-                                                    activate=activate)
-
         # In case epochs didn't have a FIFF.FIFFB_MNE_EPOCHS_SELECTION tag
         # (version < 0.8):
         if selection is None:
-            selection = np.arange(len(epochs))
+            selection = np.arange(len(events))
         if drop_log is None:
             drop_log = [[] for _ in range(len(epochs))]  # noqa, analysis:ignore
 
-        epochs.selection = selection
-        epochs.drop_log = drop_log
-        epochs._bad_dropped = True
-    return epochs
+    return (info, data, data_tag, events, event_id, tmin, tmax, baseline, name,
+            selection, drop_log, epoch_shape, cals)
 
 
 @verbose
-def read_epochs(fname, proj=True, add_eeg_ref=True, verbose=None):
+def read_epochs(fname, proj=True, add_eeg_ref=True, preload=True,
+                verbose=None):
     """Read epochs from a fif file
 
     Parameters
@@ -2192,6 +2195,9 @@ def read_epochs(fname, proj=True, add_eeg_ref=True, verbose=None):
     add_eeg_ref : bool
         If True, an EEG average reference will be added (unless one
         already exists).
+    preload : bool
+        If True, read all epochs from disk immediately. If False, epochs will
+        be read on demand.
     verbose : bool, str, int, or None
         If not None, override default verbose level (see mne.verbose).
         Defaults to raw.verbose.
@@ -2201,25 +2207,144 @@ def read_epochs(fname, proj=True, add_eeg_ref=True, verbose=None):
     epochs : instance of Epochs
         The epochs
     """
-    check_fname(fname, 'epochs', ('-epo.fif', '-epo.fif.gz'))
-
-    fnames = [fname]
-    epochs = list()
-    for fname in fnames:
-        logger.info('Reading %s ...' % fname)
-        fid, tree, _ = fiff_open(fname)
-        next_fname = _get_next_fname(fid, fname, tree)
-        epoch = _read_one_epoch_file(fid, tree, fname, proj, add_eeg_ref,
-                                     verbose)
-        epochs.append(epoch)
-
-        if next_fname is not None:
-            fnames.append(next_fname)
-
-    epochs = _concatenate_epochs(epochs, read_file=True)
-    epochs._bad_dropped = True
+    return EpochsFIF(fname, proj, add_eeg_ref, preload, verbose)
 
-    return epochs
+
+class _RawContainer(object):
+    def __init__(self, fid, data_tag, event_samps, epoch_shape, cals):
+        self.fid = fid
+        self.data_tag = data_tag
+        self.event_samps = event_samps
+        self.epoch_shape = epoch_shape
+        self.cals = cals
+        self.proj = False
+
+    def __del__(self):
+        self.fid.close()
+
+
+class EpochsFIF(_BaseEpochs):
+    """Epochs read from disk
+
+    Parameters
+    ----------
+    fname : str
+        The name of the file, which should end with -epo.fif or -epo.fif.gz.
+    proj : bool | 'delayed'
+        Apply SSP projection vectors. If proj is 'delayed' and reject is not
+        None the single epochs will be projected before the rejection
+        decision, but used in unprojected state if they are kept.
+        This way deciding which projection vectors are good can be postponed
+        to the evoked stage without resulting in lower epoch counts and
+        without producing results different from early SSP application
+        given comparable parameters. Note that in this case baselining,
+        detrending and temporal decimation will be postponed.
+        If proj is False no projections will be applied which is the
+        recommended value if SSPs are not used for cleaning the data.
+    add_eeg_ref : bool
+        If True, an EEG average reference will be added (unless one
+        already exists).
+    preload : bool
+        If True, read all epochs from disk immediately. If False, epochs will
+        be read on demand.
+    verbose : bool, str, int, or None
+        If not None, override default verbose level (see mne.verbose).
+        Defaults to raw.verbose.
+
+    See Also
+    --------
+    mne.Epochs
+    mne.epochs.combine_event_ids
+    mne.Epochs.equalize_event_counts
+    """
+    @verbose
+    def __init__(self, fname, proj=True, add_eeg_ref=True, preload=True,
+                 verbose=None):
+        check_fname(fname, 'epochs', ('-epo.fif', '-epo.fif.gz'))
+
+        fnames = [fname]
+        ep_list = list()
+        raw = list()
+        for fname in fnames:
+            logger.info('Reading %s ...' % fname)
+            fid, tree, _ = fiff_open(fname)
+            next_fname = _get_next_fname(fid, fname, tree)
+            (info, data, data_tag, events, event_id, tmin, tmax, baseline,
+             name, selection, drop_log, epoch_shape, cals) = \
+                _read_one_epoch_file(fid, tree, fname, preload)
+            # here we ignore missing events, since users should already be
+            # aware of missing events if they have saved data that way
+            epoch = _BaseEpochs(
+                info, data, events, event_id, tmin, tmax, baseline,
+                on_missing='ignore', selection=selection, drop_log=drop_log,
+                add_eeg_ref=False, proj=False, verbose=False)
+            ep_list.append(epoch)
+            if not preload:
+                # store everything we need to index back to the original data
+                raw.append(_RawContainer(fiff_open(fname)[0], data_tag,
+                                         events[:, 0].copy(), epoch_shape,
+                                         cals))
+
+            if next_fname is not None:
+                fnames.append(next_fname)
+
+        (info, data, events, event_id, tmin, tmax, baseline, selection,
+         drop_log, _) = _concatenate_epochs(ep_list, with_data=preload)
+        # we need this uniqueness for non-preloaded data to work properly
+        if len(np.unique(events[:, 0])) != len(events):
+            raise RuntimeError('Event time samples were not unique')
+
+        # correct the drop log
+        assert len(drop_log) % len(fnames) == 0
+        step = len(drop_log) // len(fnames)
+        offsets = np.arange(step, len(drop_log) + 1, step)
+        for i1, i2 in zip(offsets[:-1], offsets[1:]):
+            other_log = drop_log[i1:i2]
+            for k, (a, b) in enumerate(zip(drop_log, other_log)):
+                    if a == ['IGNORED'] and b != ['IGNORED']:
+                        drop_log[k] = b
+        drop_log = drop_log[:step]
+
+        # call _BaseEpochs constructor
+        super(EpochsFIF, self).__init__(
+            info, data, events, event_id, tmin, tmax, baseline, raw=raw,
+            name=name, proj=proj, add_eeg_ref=add_eeg_ref,
+            preload_at_end=False, on_missing='ignore', selection=selection,
+            drop_log=drop_log, verbose=verbose)
+        # use the private property instead of drop_bad_epochs so that epochs
+        # are not all read from disk for preload=False
+        self._bad_dropped = True
+
+    @verbose
+    def _get_epoch_from_raw(self, idx, verbose=None):
+        """Load one epoch from disk"""
+        # Find the right file and offset to use
+        event_samp = self.events[idx, 0]
+        for raw in self._raw:
+            idx = np.where(raw.event_samps == event_samp)[0]
+            if len(idx) == 1:
+                idx = idx[0]
+                size = np.prod(raw.epoch_shape) * 4
+                offset = idx * size
+                break
+        else:
+            # read the correct subset of the data
+            raise RuntimeError('Correct epoch could not be found, please '
+                               'contact mne-python developers')
+        # the following is equivalent to this, but faster:
+        #
+        # >>> data = read_tag(raw.fid, raw.data_tag.pos).data.astype(float)
+        # >>> data *= raw.cals[np.newaxis, :, :]
+        # >>> data = data[idx]
+        #
+        # Eventually this could be refactored in io/tag.py if other functions
+        # could make use of it
+
+        raw.fid.seek(raw.data_tag.pos + offset + 16, 0)  # 16 = Tag header
+        data = np.fromstring(raw.fid.read(size), '>f4').astype(np.float64)
+        data.shape = raw.epoch_shape
+        data *= raw.cals
+        return data
 
 
 def bootstrap(epochs, random_state=None):
@@ -2367,49 +2492,47 @@ def _compare_epochs_infos(info1, info2, ind):
         raise ValueError('SSP projectors in epochs files must be the same')
 
 
-def _concatenate_epochs(epochs_list, read_file=False):
+def _concatenate_epochs(epochs_list, with_data=True):
     """Auxiliary function for concatenating epochs."""
     out = epochs_list[0]
-    data = [out.get_data()]
+    data = [out.get_data()] if with_data else None
     events = [out.events]
-    drop_log = cp.deepcopy(out.drop_log)
-    event_id = cp.deepcopy(out.event_id)
+    baseline, tmin, tmax = out.baseline, out.tmin, out.tmax
+    info = deepcopy(out.info)
+    verbose = out.verbose
+    drop_log = deepcopy(out.drop_log)
+    event_id = deepcopy(out.event_id)
     selection = out.selection
     for ii, epochs in enumerate(epochs_list[1:]):
-        _compare_epochs_infos(epochs.info, epochs_list[0].info, ii)
+        _compare_epochs_infos(epochs.info, info, ii)
         if not np.array_equal(epochs.times, epochs_list[0].times):
             raise ValueError('Epochs must have same times')
 
-        if epochs.baseline != epochs_list[0].baseline:
+        if epochs.baseline != baseline:
             raise ValueError('Baseline must be same for all epochs')
 
-        data.append(epochs.get_data())
+        if with_data:
+            data.append(epochs.get_data())
         events.append(epochs.events)
         selection = np.concatenate((selection, epochs.selection))
-        if read_file:
-            for k, (a, b) in enumerate(zip(drop_log, epochs.drop_log)):
-                if a == ['IGNORED'] and b != ['IGNORED']:
-                    drop_log[k] = b
-        else:
-            drop_log.extend(epochs.drop_log)
+        drop_log.extend(epochs.drop_log)
         event_id.update(epochs.event_id)
     events = np.concatenate(events, axis=0)
-    # do not do this if epochs read from disk are being concatenated
-    if read_file is False:
-        events[:, 0] = np.arange(len(events))  # arbitrary after concat
-
-    baseline = epochs_list[0].baseline
-    out = _BaseEpochs(out.info, np.concatenate(data, axis=0), events, event_id,
-                      out.tmin, out.tmax, baseline=baseline, add_eeg_ref=False,
-                      proj=False, verbose=out.verbose, on_missing='ignore')
-    # We previously only set the drop log here, but we also need to set the
-    # selection, too
-    if read_file is False:
-        selection = np.where([len(d) == 0 for d in drop_log])[0]
-
-    assert len(selection) == len(out.drop_log)
-    out.selection = selection
-    out.drop_log = drop_log
+    if with_data:
+        data = np.concatenate(data, axis=0)
+    return (info, data, events, event_id, tmin, tmax, baseline, selection,
+            drop_log, verbose)
+
+
+def _finish_concat(info, data, events, event_id, tmin, tmax, baseline,
+                   selection, drop_log, verbose):
+    """Helper to finish concatenation for epochs not read from disk"""
+    events[:, 0] = np.arange(len(events))  # arbitrary after concat
+    selection = np.where([len(d) == 0 for d in drop_log])[0]
+    out = _BaseEpochs(info, data, events, event_id, tmin, tmax,
+                      baseline=baseline, add_eeg_ref=False,
+                      selection=selection, drop_log=drop_log,
+                      proj=False, on_missing='ignore', verbose=verbose)
     out.drop_bad_epochs()
     return out
 
@@ -2431,4 +2554,4 @@ def concatenate_epochs(epochs_list):
     -----
     .. versionadded:: 0.9.0
     """
-    return _concatenate_epochs(epochs_list)
+    return _finish_concat(*_concatenate_epochs(epochs_list))
diff --git a/mne/io/tag.py b/mne/io/tag.py
--- a/mne/io/tag.py
+++ b/mne/io/tag.py
@@ -204,9 +204,10 @@ def read_tag(fid, pos=None, shape=None, rlims=None):
         If tuple, the shape of the stored matrix. Only to be used with
         data stored as a vector (not implemented for matrices yet).
     rlims : tuple | None
-        If tuple, the first and last rows to retrieve. Note that data are
-        assumed to be stored row-major in the file. Only to be used with
-        data stored as a vector (not implemented for matrices yet).
+        If tuple, the first (inclusive) and last (exclusive) rows to retrieve.
+        Note that data are assumed to be stored row-major in the file. Only to
+        be used with data stored as a vector (not implemented for matrices
+        yet).
 
     Returns
     -------


Test patch:
diff --git a/mne/tests/test_epochs.py b/mne/tests/test_epochs.py
--- a/mne/tests/test_epochs.py
+++ b/mne/tests/test_epochs.py
@@ -48,7 +48,7 @@
 
 
 def _get_data():
-    raw = io.Raw(raw_fname, add_eeg_ref=False)
+    raw = io.Raw(raw_fname, add_eeg_ref=False, proj=False)
     events = read_events(event_name)
     picks = pick_types(raw.info, meg=True, eeg=True, stim=True,
                        ecg=True, eog=True, include=['STI 014'],
@@ -378,6 +378,7 @@ def test_read_write_epochs():
     baseline = (None, 0)
     epochs = Epochs(raw, events, event_id, tmin, tmax, picks=picks,
                     baseline=baseline, preload=True)
+    epochs_orig = epochs.copy()
     epochs_no_bl = Epochs(raw, events, event_id, tmin, tmax, picks=picks,
                           baseline=None, preload=True)
     assert_true(epochs_no_bl.baseline is None)
@@ -397,7 +398,6 @@ def test_read_write_epochs():
                            eog=True, exclude='bads')
     eog_ch_names = [raw.ch_names[k] for k in eog_picks]
     epochs.drop_channels(eog_ch_names)
-    epochs_no_bl.drop_channels(eog_ch_names)
     assert_true(len(epochs.info['chs']) == len(epochs.ch_names) ==
                 epochs.get_data().shape[1])
     data_no_eog = epochs.get_data()
@@ -433,100 +433,141 @@ def test_read_write_epochs():
     assert_true(n_dec_min <= n_dec <= n_dec_min + 1)
     assert_true(evoked_dec.info['sfreq'] == evoked.info['sfreq'] / 4)
 
-    # test IO
-    epochs.save(temp_fname)
-    epochs_no_bl.save(temp_fname_no_bl)
-    epochs_read = read_epochs(temp_fname)
-    epochs_no_bl_read = read_epochs(temp_fname_no_bl)
-    assert_raises(ValueError, epochs.apply_baseline, baseline=[1, 2, 3])
-    epochs_no_bl_read.apply_baseline(baseline)
-    assert_true(epochs_no_bl_read.baseline == baseline)
-    assert_true(str(epochs_read).startswith('<Epochs'))
-
-    assert_array_equal(epochs_no_bl_read.times, epochs.times)
-    assert_array_almost_equal(epochs_read.get_data(), epochs.get_data())
-    assert_array_almost_equal(epochs.get_data(), epochs_no_bl_read.get_data())
-    assert_array_equal(epochs_read.times, epochs.times)
-    assert_array_almost_equal(epochs_read.average().data, evoked.data)
-    assert_equal(epochs_read.proj, epochs.proj)
-    bmin, bmax = epochs.baseline
-    if bmin is None:
-        bmin = epochs.times[0]
-    if bmax is None:
-        bmax = epochs.times[-1]
-    baseline = (bmin, bmax)
-    assert_array_almost_equal(epochs_read.baseline, baseline)
-    assert_array_almost_equal(epochs_read.tmin, epochs.tmin, 2)
-    assert_array_almost_equal(epochs_read.tmax, epochs.tmax, 2)
-    assert_equal(epochs_read.event_id, epochs.event_id)
-
-    epochs.event_id.pop('1')
-    epochs.event_id.update({'a:a': 1})  # test allow for ':' in key
-    epochs.save(op.join(tempdir, 'foo-epo.fif'))
-    epochs_read2 = read_epochs(op.join(tempdir, 'foo-epo.fif'))
-    assert_equal(epochs_read2.event_id, epochs.event_id)
-
-    # add reject here so some of the epochs get dropped
-    epochs = Epochs(raw, events, event_id, tmin, tmax, picks=picks,
-                    baseline=(None, 0), reject=reject)
-    epochs.save(temp_fname)
-    # ensure bad events are not saved
-    epochs_read3 = read_epochs(temp_fname)
-    assert_array_equal(epochs_read3.events, epochs.events)
-    data = epochs.get_data()
-    assert_true(epochs_read3.events.shape[0] == data.shape[0])
+    # Test event access on non-preloaded data (#2345)
 
-    # test copying loaded one (raw property)
-    epochs_read4 = epochs_read3.copy()
-    assert_array_almost_equal(epochs_read4.get_data(), data)
-    # test equalizing loaded one (drop_log property)
-    epochs_read4.equalize_event_counts(epochs.event_id)
+    # due to reapplication of the proj matrix, this is our quality limit
+    # for some tests
+    tols = dict(atol=1e-3, rtol=1e-20)
 
-    epochs.drop_epochs([1, 2], reason='can we recover orig ID?')
-    epochs.save(temp_fname)
-    epochs_read5 = read_epochs(temp_fname)
-    assert_array_equal(epochs_read5.selection, epochs.selection)
-    assert_equal(len(epochs_read5.selection), len(epochs_read5.events))
-    assert_array_equal(epochs_read5.drop_log, epochs.drop_log)
-
-    # Test that one can drop channels on read file
-    epochs_read5.drop_channels(epochs_read5.ch_names[:1])
-
-    # test warnings on bad filenames
+    raw, events, picks = _get_data()
+    events[::2, 1] = 1
+    events[1::2, 2] = 2
+    event_ids = dict(a=1, b=2)
+    for proj in (True, 'delayed', False):
+        epochs = Epochs(raw, events, event_ids, tmin, tmax, picks=picks,
+                        baseline=(None, 0), proj=proj, reject=reject,
+                        add_eeg_ref=True)
+        data1 = epochs.get_data()
+        data2 = epochs.apply_proj().get_data()
+        assert_allclose(data1, data2, **tols)
+        epochs.save(temp_fname)
+        epochs_read = read_epochs(temp_fname, preload=False)
+        assert_allclose(epochs.get_data(), epochs_read.get_data(), **tols)
+        assert_allclose(epochs['a'].get_data(),
+                        epochs_read['a'].get_data(), **tols)
+        assert_allclose(epochs['b'].get_data(),
+                        epochs_read['b'].get_data(), **tols)
+
+    # ensure we don't leak file descriptors
+    epochs_read = read_epochs(temp_fname, preload=False)
+    epochs_copy = epochs_read.copy()
+    del epochs_read
+    epochs_copy.get_data()
     with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter('always')
-        epochs_badname = op.join(tempdir, 'test-bad-name.fif.gz')
-        epochs.save(epochs_badname)
-        read_epochs(epochs_badname)
-    assert_true(len(w) == 2)
+        del epochs_copy
+    assert_equal(len(w), 0)
 
-    # test loading epochs with missing events
-    epochs = Epochs(raw, events, dict(foo=1, bar=999), tmin, tmax, picks=picks,
-                    on_missing='ignore')
-    epochs.save(temp_fname)
-    epochs_read = read_epochs(temp_fname)
-    assert_allclose(epochs.get_data(), epochs_read.get_data())
-    assert_array_equal(epochs.events, epochs_read.events)
-    assert_equal(set(epochs.event_id.keys()),
-                 set(text_type(x) for x in epochs_read.event_id.keys()))
+    # test IO
+    for preload in (False, True):
+        epochs = epochs_orig.copy()
+        epochs.save(temp_fname)
+        epochs_no_bl.save(temp_fname_no_bl)
+        epochs_read = read_epochs(temp_fname, preload=preload)
+        epochs_no_bl.save(temp_fname_no_bl)
+        epochs_read = read_epochs(temp_fname)
+        epochs_no_bl_read = read_epochs(temp_fname_no_bl)
+        assert_raises(ValueError, epochs.apply_baseline, baseline=[1, 2, 3])
+        epochs_no_bl_read.apply_baseline(baseline)
+        assert_true(epochs_no_bl_read.baseline == baseline)
+        assert_true(str(epochs_read).startswith('<Epochs'))
+
+        assert_array_equal(epochs_no_bl_read.times, epochs.times)
+        assert_array_almost_equal(epochs_read.get_data(), epochs.get_data())
+        assert_array_almost_equal(epochs.get_data(),
+                                  epochs_no_bl_read.get_data())
+        assert_array_equal(epochs_read.times, epochs.times)
+        assert_array_almost_equal(epochs_read.average().data, evoked.data)
+        assert_equal(epochs_read.proj, epochs.proj)
+        bmin, bmax = epochs.baseline
+        if bmin is None:
+            bmin = epochs.times[0]
+        if bmax is None:
+            bmax = epochs.times[-1]
+        baseline = (bmin, bmax)
+        assert_array_almost_equal(epochs_read.baseline, baseline)
+        assert_array_almost_equal(epochs_read.tmin, epochs.tmin, 2)
+        assert_array_almost_equal(epochs_read.tmax, epochs.tmax, 2)
+        assert_equal(epochs_read.event_id, epochs.event_id)
+
+        epochs.event_id.pop('1')
+        epochs.event_id.update({'a:a': 1})  # test allow for ':' in key
+        epochs.save(op.join(tempdir, 'foo-epo.fif'))
+        epochs_read2 = read_epochs(op.join(tempdir, 'foo-epo.fif'),
+                                   preload=preload)
+        assert_equal(epochs_read2.event_id, epochs.event_id)
+
+        # add reject here so some of the epochs get dropped
+        epochs = Epochs(raw, events, event_id, tmin, tmax, picks=picks,
+                        baseline=(None, 0), reject=reject)
+        epochs.save(temp_fname)
+        # ensure bad events are not saved
+        epochs_read3 = read_epochs(temp_fname, preload=preload)
+        assert_array_equal(epochs_read3.events, epochs.events)
+        data = epochs.get_data()
+        assert_true(epochs_read3.events.shape[0] == data.shape[0])
+
+        # test copying loaded one (raw property)
+        epochs_read4 = epochs_read3.copy()
+        assert_array_almost_equal(epochs_read4.get_data(), data)
+        # test equalizing loaded one (drop_log property)
+        epochs_read4.equalize_event_counts(epochs.event_id)
+
+        epochs.drop_epochs([1, 2], reason='can we recover orig ID?')
+        epochs.save(temp_fname)
+        epochs_read5 = read_epochs(temp_fname, preload=preload)
+        assert_array_equal(epochs_read5.selection, epochs.selection)
+        assert_equal(len(epochs_read5.selection), len(epochs_read5.events))
+        assert_array_equal(epochs_read5.drop_log, epochs.drop_log)
 
-    # test saving split epoch files
-    epochs.save(temp_fname, split_size='7MB')
-    epochs_read = read_epochs(temp_fname)
-    assert_allclose(epochs.get_data(), epochs_read.get_data())
-    assert_array_equal(epochs.events, epochs_read.events)
-    assert_array_equal(epochs.selection, epochs_read.selection)
-    assert_equal(epochs.drop_log, epochs_read.drop_log)
-
-    # Test that having a single time point works
-    epochs.preload_data()
-    epochs.crop(0, 0, copy=False)
-    assert_equal(len(epochs.times), 1)
-    assert_equal(epochs.get_data().shape[-1], 1)
-    epochs.save(temp_fname)
-    epochs_read = read_epochs(temp_fname)
-    assert_equal(len(epochs_read.times), 1)
-    assert_equal(epochs.get_data().shape[-1], 1)
+        if preload:
+            # Test that one can drop channels on read file
+            epochs_read5.drop_channels(epochs_read5.ch_names[:1])
+
+        # test warnings on bad filenames
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always')
+            epochs_badname = op.join(tempdir, 'test-bad-name.fif.gz')
+            epochs.save(epochs_badname)
+            read_epochs(epochs_badname, preload=preload)
+        assert_true(len(w) == 2)
+
+        # test loading epochs with missing events
+        epochs = Epochs(raw, events, dict(foo=1, bar=999), tmin, tmax,
+                        picks=picks, on_missing='ignore')
+        epochs.save(temp_fname)
+        epochs_read = read_epochs(temp_fname, preload=preload)
+        assert_allclose(epochs.get_data(), epochs_read.get_data(), **tols)
+        assert_array_equal(epochs.events, epochs_read.events)
+        assert_equal(set(epochs.event_id.keys()),
+                     set(text_type(x) for x in epochs_read.event_id.keys()))
+
+        # test saving split epoch files
+        epochs.save(temp_fname, split_size='7MB')
+        epochs_read = read_epochs(temp_fname, preload=preload)
+        assert_allclose(epochs.get_data(), epochs_read.get_data(), **tols)
+        assert_array_equal(epochs.events, epochs_read.events)
+        assert_array_equal(epochs.selection, epochs_read.selection)
+        assert_equal(epochs.drop_log, epochs_read.drop_log)
+
+        # Test that having a single time point works
+        epochs.preload_data()
+        epochs.crop(0, 0, copy=False)
+        assert_equal(len(epochs.times), 1)
+        assert_equal(epochs.get_data().shape[-1], 1)
+        epochs.save(temp_fname)
+        epochs_read = read_epochs(temp_fname, preload=preload)
+        assert_equal(len(epochs_read.times), 1)
+        assert_equal(epochs.get_data().shape[-1], 1)
 
 
 def test_epochs_proj():
@@ -1254,9 +1295,6 @@ def test_epochs_proj_mixin():
     # wrong proj argument
     assert_raises(ValueError, Epochs, raw, events[:4], event_id, tmin, tmax,
                   picks=picks, baseline=(None, 0), proj='crazy')
-    # delayed without reject params
-    assert_raises(RuntimeError, Epochs, raw, events[:4], event_id, tmin, tmax,
-                  picks=picks, baseline=(None, 0), proj='delayed', reject=None)
 
     for preload in [True, False]:
         epochs = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks,