Repository: mne-tools/mne-python

Issue:
GDF events
Check this MWE. `mne.events_from_annotations` is not able to pick up the different events and they all get mapped to `'GDF event'`. Plus I'm not sure if we should expose `mne.io.find_edf_events` nor `edf_raw.find_edf_events()` because that would now work with `raw.find_edf_events()`. (and I would expect the same methods, no matter which reader did I use to load `raw`)


MWE
```py
"""
Disclaimer this MWE uses data from http://www.bbci.de/competition/iv/#dataset2a
"""
import mne
import numpy as np
from os import path as op

tmpdir = mne.utils._TempDir()
DROPBOX_FILE_URL = "https://www.dropbox.com/s/nn6dzc5cc7riz7c/A01E.gdf?dl=1"
fname = op.join(tmpdir, 'A01E.gdf')

mne.utils._fetch_file(url=DROPBOX_FILE_URL, file_name=fname,
                      hash_='ec3de227cd3bf1760accca0e3269590e')

raw = mne.io.read_raw_edf(fname)

# Regular workflow
events_A, event_id_A = mne.events_from_annotations(raw)
print(raw.annotations)
print(events_A.shape)
print(event_id_A)

# whatever
def my_events_from_annotations(edf_raw):
    _, onset, label, duration, _ = edf_raw.find_edf_events()
    events = np.stack([onset, duration, label], axis=1)
    event_id = {'GDF event {}'.format(chr(65 + idx)): l
                for idx, l in enumerate(set(label))}
    return events, event_id

events_B, event_id_B = my_events_from_annotations(raw)

print(events_B.shape)
print(event_id_B)

np.testing.assert_array_equal(events_A[:, 0], events_B[:, 0])
```
output
```<Annotations  |  595 segments : GDF event (595), orig_time : 2005-01-19 12:00:00>
(595, 3)
{'GDF event': 1}
(595, 3)
{'GDF event A': 768, 'GDF event B': 783, 'GDF event C': 1072, 'GDF event D': 276, 'GDF event E': 277, 'GDF event F': 32766, 'GDF event G': 1023}
```

ping: @jaeilepp


Discussion:
We should not have `find_edf_events` in 0.18 (or maybe 0.19). In 0.18, all events people need should now be either in `STIM` channel (*not* synthesized), or in annotations. If they aren't in either place, we have a bug.
cc: @cbrnr do you know the data? your name appears in the website as organizer
@larsoner `find_edf_events` was introduced in `0.15` and we did not think of removing it in `0.17`.

Now my question is what would we expect from `events_from_annotations`? check the usage of events ids here
https://github.com/robintibor/braindecode/blob/master/braindecode/datasets/bcic_iv_2a.py#L49

I'm not sure where those ids come from.

Resolving patch:
diff --git a/mne/io/edf/_utils.py b/mne/io/edf/_utils.py
new file mode 100644
--- /dev/null
+++ b/mne/io/edf/_utils.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""Helper functions for EDF, EDF+, BDF converters to FIF."""
+
+# Authors: Teon Brooks <teon.brooks@gmail.com>
+#          Martin Billinger <martin.billinger@tugraz.at>
+#          Nicolas Barascud <nicolas.barascud@ens.fr>
+#          Stefan Appelhoff <stefan.appelhoff@mailbox.org>
+#          Joan Massich <mailsik@gmail.com>
+#
+# License: BSD (3-clause)
+
+import re
+from ...utils import hashfunc
+
+
+def _load_gdf_events_lut(fname, md5):
+    if hashfunc(fname, hash_type='md5') != md5:
+        raise ValueError("File %s is corrupted. mdf5 hashes don't match." %
+                         fname)
+
+    # load the stuff
+    with open(fname, 'r') as fh:
+        elements = [line for line in fh if not line.startswith("#")]
+
+    event_id, event_name = list(), list()
+    for elem in elements:
+        event_id_i, *event_name_i = elem.split('\t')
+        event_id.append(int(event_id_i, 0))
+        clean_name = re.sub('[ \t]+', ' ', ' '.join(event_name_i))
+        clean_name = re.sub('\n', '', clean_name)
+        event_name.append(clean_name)
+
+    return dict(zip(event_id, event_name))
diff --git a/mne/io/edf/edf.py b/mne/io/edf/edf.py
--- a/mne/io/edf/edf.py
+++ b/mne/io/edf/edf.py
@@ -15,6 +15,7 @@
 import re
 
 import numpy as np
+import os.path as op
 
 from ...utils import verbose, logger, warn
 from ..utils import _blk_read_lims
@@ -22,10 +23,18 @@
 from ..meas_info import _empty_info, _unique_channel_names, DATE_NONE
 from ..constants import FIFF
 from ...filter import resample
-from ...utils import copy_function_doc_to_method_doc
-from ...annotations import Annotations
+from ...utils import copy_function_doc_to_method_doc, deprecated
+from ...annotations import Annotations, events_from_annotations
+from ._utils import _load_gdf_events_lut
 
 
+GDF_EVENT_ENCODES_FILE = op.join(op.dirname(__file__), 'gdf_encodes.txt')
+GDF_EVENTS_LUT = _load_gdf_events_lut(fname=GDF_EVENT_ENCODES_FILE,
+                                      md5='12134a9be7e0bfa5941e95f8bfd330f7')
+
+
+@deprecated('find_edf_events is deprecated in 0.18, and will be removed'
+            ' in 0.19. Please use `mne.events_from_annotations` instead')
 def find_edf_events(raw):
     """Get original EDF events as read from the header.
 
@@ -65,7 +74,7 @@ def find_edf_events(raw):
     events : ndarray
         The events as they are in the file header.
     """
-    return raw.find_edf_events()
+    return events_from_annotations(raw)
 
 
 class RawEDF(BaseRaw):
@@ -176,34 +185,19 @@ def __init__(self, input_fname, montage, eog=None, misc=None,
             verbose=verbose)
 
         # Read annotations from file and set it
-        annot = None
+        onset, duration, desc = list(), list(), list()
         ext = os.path.splitext(input_fname)[1][1:].lower()
         if ext in ('gdf'):
-            events = edf_info.get('events', None)
-            # Annotations in GDF: events are stored as the following
-            # list: `events = [n_events, pos, typ, chn, dur]` where pos is the
-            # latency, dur is the duration in samples. They both are
-            # numpy.ndarray
-            if events is not None and events[1].shape[0] > 0:
-                # For whatever reason, typ has the same content as pos
-                # therefore we set an arbitrary description
-                desc = 'GDF event'
-                annot = Annotations(onset=events[1] / self.info['sfreq'],
-                                    duration=events[4] / self.info['sfreq'],
-                                    description=desc,
-                                    orig_time=None)
+            onset, duration, desc = _get_annotations_gdf(edf_info,
+                                                         self.info['sfreq'])
         elif len(edf_info['tal_idx']) > 0:
             # Read TAL data exploiting the header info (no regexp)
             tal_data = self._read_segment_file([], [], 0, 0, int(self.n_times),
                                                None, None)
             onset, duration, desc = _read_annotations_edf(tal_data[0])
 
-            # in EDF, annotations are relative to first_samp
-            annot = Annotations(onset=onset, duration=duration,
-                                description=desc, orig_time=None)
-
-        if annot is not None:
-            self.set_annotations(annot)
+        self.set_annotations(Annotations(onset=onset, duration=duration,
+                                         description=desc, orig_time=None))
 
     @verbose
     def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
@@ -328,8 +322,10 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
         return tal_data
 
     @copy_function_doc_to_method_doc(find_edf_events)
+    @deprecated('find_edf_events is deprecated in 0.18, and will be removed'
+                ' in 0.19. Please use `mne.events_from_annotations` instead')
     def find_edf_events(self):
-        return self._raw_extras[0]['events']
+        return events_from_annotations(self)
 
 
 def _read_ch(fid, subtype, samp, dtype_byte, dtype=None):
@@ -1223,3 +1219,20 @@ def _get_edf_default_event_id(descriptions):
     mapping = dict((a, n) for n, a in
                    enumerate(sorted(set(descriptions)), start=1))
     return mapping
+
+
+def _get_annotations_gdf(edf_info, sfreq):
+    onset, duration, desc = list(), list(), list()
+    events = edf_info.get('events', None)
+    # Annotations in GDF: events are stored as the following
+    # list: `events = [n_events, pos, typ, chn, dur]` where pos is the
+    # latency, dur is the duration in samples. They both are
+    # numpy.ndarray
+    if events is not None and events[1].shape[0] > 0:
+        onset = events[1] / sfreq
+        duration = events[4] / sfreq
+        desc = [GDF_EVENTS_LUT[key]
+                if key in GDF_EVENTS_LUT else 'Unknown'
+                for key in events[2]]
+
+    return onset, duration, desc
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -85,6 +85,7 @@ def package_tree(pkgroot):
                                 op.join('gui', 'help', '*.json'),
                                 op.join('html', '*.js'),
                                 op.join('html', '*.css'),
-                                op.join('io', 'artemis123', 'resources', '*.csv')
+                                op.join('io', 'artemis123', 'resources', '*.csv'),
+                                op.join('io', 'edf', 'gdf_encodes.txt')
                                 ]},
           scripts=['bin/mne'])


Test patch:
diff --git a/mne/datasets/sleep_physionet/tests/test_physionet.py b/mne/datasets/sleep_physionet/tests/test_physionet.py
--- a/mne/datasets/sleep_physionet/tests/test_physionet.py
+++ b/mne/datasets/sleep_physionet/tests/test_physionet.py
@@ -124,6 +124,7 @@ def test_sleep_physionet_age(physionet_tmpdir, mocker):
 @requires_good_network
 @requires_pandas
 @requires_version('xlrd', '0.9')
+@pytest.mark.skip(reason="Broken with new pandas 0.24 and xlrd")
 def test_run_update_temazepam_records(tmpdir):
     """Test Sleep Physionet URL handling."""
     import pandas as pd
diff --git a/mne/io/edf/tests/test_edf.py b/mne/io/edf/tests/test_edf.py
--- a/mne/io/edf/tests/test_edf.py
+++ b/mne/io/edf/tests/test_edf.py
@@ -26,6 +26,7 @@
 from mne.io.edf.edf import _get_edf_default_event_id
 from mne.io.edf.edf import _read_annotations_edf
 from mne.io.edf.edf import _read_ch
+from mne.io.edf.edf import find_edf_events
 from mne.io.pick import channel_indices_by_type
 from mne.annotations import events_from_annotations, read_annotations
 from mne.io.meas_info import _kind_dict as _KIND_DICT
@@ -197,6 +198,16 @@ def test_to_data_frame():
         assert_array_equal(df.values[:, 0], raw._data[0] * 1e13)
 
 
+def test_find_edf_events_deprecation():
+    """Test find_edf_events deprecation."""
+    raw = read_raw_edf(edf_path)
+    with pytest.deprecated_call(match="find_edf_events"):
+        raw.find_edf_events()
+
+    with pytest.deprecated_call(match="find_edf_events"):
+        find_edf_events(raw)
+
+
 def test_read_raw_edf_stim_channel_input_parameters():
     """Test edf raw reader deprecation."""
     _MSG = "`read_raw_edf` is not supposed to trigger a deprecation warning"
diff --git a/mne/io/edf/tests/test_gdf.py b/mne/io/edf/tests/test_gdf.py
--- a/mne/io/edf/tests/test_gdf.py
+++ b/mne/io/edf/tests/test_gdf.py
@@ -15,7 +15,8 @@
 from mne.io.meas_info import DATE_NONE
 from mne.io.tests.test_raw import _test_raw_reader
 from mne.utils import run_tests_if_main
-from mne import pick_types, find_events
+from mne import pick_types, find_events, events_from_annotations
+from mne.io.edf.edf import GDF_EVENTS_LUT
 
 data_path = testing.data_path(download=False)
 gdf1_path = op.join(data_path, 'GDF', 'test_gdf_1.25')
@@ -30,9 +31,10 @@ def test_gdf_data():
     data, _ = raw[picks]
 
     # Test Status is added as event
-    EXPECTED_EVS_ONSETS = raw._raw_extras[0]['events'][1][::2]
-    evs = raw.find_edf_events()
-    assert_array_equal(evs[1][::2], EXPECTED_EVS_ONSETS)
+    EXPECTED_EVS_ONSETS = raw._raw_extras[0]['events'][1]
+    evs, evs_id = events_from_annotations(raw)
+    assert_array_equal(evs[:, 0], EXPECTED_EVS_ONSETS)
+    assert evs_id == {'Unknown': 1}
 
     # this .npy was generated using the official biosig python package
     raw_biosig = np.load(gdf1_path + '_biosig.npy')
@@ -77,4 +79,9 @@ def test_gdf2_data():
                      eog=None, misc=None)
 
 
+def test_gdf_events_lut():
+    """Test something about GDF_EVENTS_LUT to make sure it has not changed."""
+    assert len(GDF_EVENTS_LUT) == 200
+
+
 run_tests_if_main()