Repository: mne-tools/mne-python

Issue:
WIP fix brainvision event parser
Closes #6267


Discussion:

Resolving patch:
diff --git a/mne/annotations.py b/mne/annotations.py
--- a/mne/annotations.py
+++ b/mne/annotations.py
@@ -784,7 +784,9 @@ def _select_annotations_based_on_description(descriptions, event_id, regexp):
 
     event_id_ = dict()
     dropped = []
-    for desc in descriptions:
+    # Iterate over the sorted descriptions so that the Counter mapping
+    # is slightly less arbitrary
+    for desc in sorted(descriptions):
         if desc in event_id_:
             continue
 
@@ -813,22 +815,33 @@ def _select_annotations_based_on_description(descriptions, event_id, regexp):
 
 
 @verbose
-def events_from_annotations(raw, event_id=None, regexp=None, use_rounding=True,
-                            chunk_duration=None, verbose=None):
+def events_from_annotations(raw, event_id="auto", regexp=None,
+                            use_rounding=True, chunk_duration=None,
+                            verbose=None):
     """Get events and event_id from an Annotations object.
 
     Parameters
     ----------
     raw : instance of Raw
         The raw data for which Annotations are defined.
-    event_id : dict | callable | None
-        Dictionary of string keys and integer values as used in mne.Epochs
-        to map annotation descriptions to integer event codes. Only the
-        keys present will be mapped and the annotations with other descriptions
-        will be ignored. Otherwise, a callable that provides an integer given
-        a string or that returns None for an event to ignore.
-        If None, all descriptions of annotations are mapped
-        and assigned arbitrary unique integer values.
+    event_id : dict | callable | None | 'auto'
+        Can be:
+
+        - **dict**: map descriptions (keys) to integer event codes (values).
+          Only the descriptions present will be mapped, others will be ignored.
+        - **callable**: must take a string input and returns an integer event
+          code or None to ignore it.
+        - **None**: Map descriptions to unique integer values based on their
+          ``sorted`` order.
+        - **'auto' (default)**: prefer a raw-format-specific parser:
+
+          - Brainvision: map stimulus events to their integer part; response
+            events to integer part + 1000; optic events to integer part + 2000;
+            'SyncStatus/Sync On' to 99998; 'New Segment/' to 99999;
+            all others like ``None`` with an offset of 10000.
+          - Other raw formats: Behaves like None.
+
+          .. versionadded:: 0.18
     regexp : str | None
         Regular expression used to filter the annotations whose
         descriptions is a match.
@@ -856,6 +869,9 @@ def events_from_annotations(raw, event_id=None, regexp=None, use_rounding=True,
 
     annotations = raw.annotations
 
+    if event_id == 'auto':
+        event_id = getattr(raw, '_get_auto_event_id', lambda: None)()
+
     event_sel, event_id_ = _select_annotations_based_on_description(
         annotations.description, event_id=event_id, regexp=regexp)
 
diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py
--- a/mne/io/brainvision/brainvision.py
+++ b/mne/io/brainvision/brainvision.py
@@ -132,6 +132,11 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
                     block[:n_data_ch, ii] = [float(l) for l in line]
             _mult_cal_one(data, block, idx, cals, mult)
 
+    @classmethod
+    def _get_auto_event_id(cls):
+        """Return default ``event_id`` behavior for Brainvision."""
+        return _BVEventParser()
+
 
 def _read_segments_c(raw, data, idx, fi, start, stop, cals, mult):
     """Read chunk of vectorized raw data."""
@@ -831,3 +836,36 @@ def read_raw_brainvision(vhdr_fname, montage=None,
     return RawBrainVision(vhdr_fname=vhdr_fname, montage=montage, eog=eog,
                           misc=misc, scale=scale, preload=preload,
                           stim_channel=stim_channel, verbose=verbose)
+
+
+_BV_EVENT_IO_OFFSETS = {'Stimulus/S': 0, 'Response/R': 1000, 'Optic/O': 2000}
+_OTHER_ACCEPTED_MARKERS = {
+    'New Segment/': 99999, 'SyncStatus/Sync On': 99998
+}
+_OTHER_OFFSET = 10001  # where to start "unknown" event_ids
+
+
+class _BVEventParser(object):
+    """Parse standard brainvision events, accounting for non-standard ones."""
+
+    def __init__(self):
+        self.other_event_ids = dict()
+
+    def __call__(self, description):
+        """Parse BrainVision event codes (like `Stimulus/S 11`) to ints."""
+        offsets = _BV_EVENT_IO_OFFSETS
+
+        maybe_digit = description[-3:].strip()
+        kind = description[:-3]
+        if maybe_digit.isdigit() and kind in offsets:
+            code = int(maybe_digit) + offsets[kind]
+        elif description in _OTHER_ACCEPTED_MARKERS:
+            code = _OTHER_ACCEPTED_MARKERS[description]
+        else:
+            if description not in self.other_event_ids:
+                # Each time a new one is added, len(self.other_event_ids)
+                # will grow, so implicitly this is a counter for us
+                self.other_event_ids[description] = \
+                    _OTHER_OFFSET + len(self.other_event_ids)
+            code = self.other_event_ids[description]
+        return code


Test patch:
diff --git a/mne/io/brainvision/tests/test_brainvision.py b/mne/io/brainvision/tests/test_brainvision.py
--- a/mne/io/brainvision/tests/test_brainvision.py
+++ b/mne/io/brainvision/tests/test_brainvision.py
@@ -4,7 +4,6 @@
 #
 # License: BSD (3-clause)
 
-import inspect
 import os.path as op
 from os import unlink
 import shutil
@@ -21,9 +20,9 @@
 from mne.io import read_raw_fif, read_raw_brainvision
 from mne.io.tests.test_raw import _test_raw_reader
 from mne.datasets import testing
+from mne.annotations import events_from_annotations
 
-FILE = inspect.getfile(inspect.currentframe())
-data_dir = op.join(op.dirname(op.abspath(FILE)), 'data')
+data_dir = op.join(op.dirname(__file__), 'data')
 vhdr_path = op.join(data_dir, 'test.vhdr')
 vmrk_path = op.join(data_dir, 'test.vmrk')
 eeg_path = op.join(data_dir, 'test.eeg')
@@ -436,23 +435,6 @@ def test_brainvision_neuroone_export():
 def test_read_vmrk_annotations():
     """Test load brainvision annotations."""
     sfreq = 1000.0
-    annotations = read_annotations(vmrk_path, sfreq=sfreq)
-    assert annotations.orig_time == 1384359243.794231
-    expected = np.array([0, 486., 496., 1769., 1779., 3252., 3262., 4935.,
-                         4945., 5999., 6619., 6629., 7629., 7699.]) / sfreq
-    description = ['New Segment/',
-                   'Stimulus/S253', 'Stimulus/S255', 'Stimulus/S254',
-                   'Stimulus/S255', 'Stimulus/S254', 'Stimulus/S255',
-                   'Stimulus/S253', 'Stimulus/S255', 'Response/R255',
-                   'Stimulus/S254', 'Stimulus/S255',
-                   'SyncStatus/Sync On', 'Optic/O  1']
-    assert_array_almost_equal(annotations.onset,
-                              expected, decimal=7)
-    assert_array_equal(annotations.description, description)
-
-    # Test automatic detection of sfreq from header file
-    annotations_auto = read_annotations(vmrk_path)
-    assert_array_equal(annotations.onset, annotations_auto.onset)
 
     # Test vmrk file without annotations
     # delete=False is for Windows compatibility
@@ -462,11 +444,72 @@ def test_read_vmrk_annotations():
         for item in head:
             temp.write(item)
         temp.seek(0)
-        annotations = read_annotations(temp.name, sfreq=sfreq)
+        read_annotations(temp.name, sfreq=sfreq)
     try:
         temp.close()
         unlink(temp.name)
     except FileNotFoundError:
         pass
 
+
+@testing.requires_testing_data
+def test_read_vhdr_annotations_and_events():
+    """Test load brainvision annotations and parse them to events."""
+    sfreq = 1000.0
+    expected_orig_time = 1384359243.794231
+    expected_onset_latency = np.array(
+        [0, 486., 496., 1769., 1779., 3252., 3262., 4935., 4945., 5999., 6619.,
+         6629., 7629., 7699.]
+    )
+    expected_annot_description = [
+        'New Segment/', 'Stimulus/S253', 'Stimulus/S255', 'Stimulus/S254',
+        'Stimulus/S255', 'Stimulus/S254', 'Stimulus/S255', 'Stimulus/S253',
+        'Stimulus/S255', 'Response/R255', 'Stimulus/S254', 'Stimulus/S255',
+        'SyncStatus/Sync On', 'Optic/O  1'
+    ]
+    expected_events = np.stack([
+        expected_onset_latency,
+        np.zeros_like(expected_onset_latency),
+        [99999, 253, 255, 254, 255, 254, 255, 253, 255, 1255, 254, 255, 99998,
+         2001],
+    ]).astype('int64').T
+    expected_event_id = {'New Segment/': 99999, 'Stimulus/S253': 253,
+                         'Stimulus/S255': 255, 'Stimulus/S254': 254,
+                         'Response/R255': 1255, 'SyncStatus/Sync On': 99998,
+                         'Optic/O  1': 2001}
+
+    raw = read_raw_brainvision(vhdr_path, eog=eog)
+
+    # validate annotations
+    assert raw.annotations.orig_time == expected_orig_time
+    assert_allclose(raw.annotations.onset, expected_onset_latency / sfreq)
+    assert_array_equal(raw.annotations.description, expected_annot_description)
+
+    # validate event extraction
+    events, event_id = events_from_annotations(raw)
+    assert_array_equal(events, expected_events)
+    assert event_id == expected_event_id
+
+    # validate that None gives us a sorted list
+    expected_none_event_id = {desc: idx + 1 for idx, desc in enumerate(sorted(
+        event_id.keys()))}
+    events, event_id = events_from_annotations(raw, event_id=None)
+    assert event_id == expected_none_event_id
+
+    # Add some custom ones, plus a 2-digit one
+    s_10 = 'Stimulus/S 10'
+    raw.annotations.append([1, 2, 3], 10, ['ZZZ', s_10, 'YYY'])
+    expected_event_id.update(YYY=10001, ZZZ=10002)  # others starting at 10001
+    expected_event_id[s_10] = 10
+    _, event_id = events_from_annotations(raw)
+    assert event_id == expected_event_id
+
+
+@testing.requires_testing_data
+def test_automatic_vmrk_sfreq_recovery():
+    """Test proper sfreq inference by checking the onsets."""
+    assert_array_equal(read_annotations(vmrk_path, sfreq='auto'),
+                       read_annotations(vmrk_path, sfreq=1000.0))
+
+
 run_tests_if_main()
diff --git a/mne/io/edf/tests/test_edf.py b/mne/io/edf/tests/test_edf.py
--- a/mne/io/edf/tests/test_edf.py
+++ b/mne/io/edf/tests/test_edf.py
@@ -298,8 +298,8 @@ def test_load_generator(fname, recwarn):
                 'sine 50 Hz']
     assert raw.get_data().shape == (11, 120000)
     assert raw.ch_names == ch_names
-    assert event_id == {'RECORD START': 1, 'REC STOP': 2}
-    assert_array_equal(events, [[0, 0, 1], [120000, 0, 2]])
+    assert event_id == {'RECORD START': 2, 'REC STOP': 1}
+    assert_array_equal(events, [[0, 0, 2], [120000, 0, 1]])
 
 
 @pytest.mark.parametrize('EXPECTED, test_input', [
diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py
--- a/mne/tests/test_annotations.py
+++ b/mne/tests/test_annotations.py
@@ -548,6 +548,11 @@ def test_events_from_annot_in_raw_objects():
     assert_array_equal(events[:, 0], events3[:, 0])
     assert set(event_id.keys()) == set(event_id3.keys())
 
+    # ensure that these actually got sorted properly
+    expected_event_id = {
+        desc: idx + 1 for idx, desc in enumerate(sorted(event_id.keys()))}
+    assert event_id3 == expected_event_id
+
     first = np.unique(events3[:, 2])
     second = np.arange(1, len(event_id) + 1, 1).astype(first.dtype)
     assert_array_equal(first, second)