Repository: mne-tools/mne-python

Issue:
raw.plot() annotations can't be dragged to extend anymore
Using master, dragging an edge from an existing annotation doesn't extend it anymore. Strangely, there must be something else going on (maybe related to a matplotlib update?), because annotation mode (i.e. pressing "a" in the raw viz window) doesn't work before commit https://github.com/mne-tools/mne-python/commit/dba99f19e24ed3e59f72bb8eaff6174c104568ca (I get an `IndexError` in `matplotlib/gridspec.py`) when the annotation window is created. Something in that commit seemed to fix that (I don't know what), but even though this got magically fixed somehow, annotations can't be dragged anymore.


Discussion:
OK, it seems like the change related to the MPL update is here: https://github.com/mne-tools/mne-python/commit/8270c1840ea821d8307ba2e16e3f334f8913f01f. I'm trying to spot the issue right now.
https://github.com/mne-tools/mne-python/commit/79959786786311aedd20da272fb4de94a20f16ef introduced this bug. Now checking to find the culprit.

Resolving patch:
diff --git a/mne/viz/raw.py b/mne/viz/raw.py
--- a/mne/viz/raw.py
+++ b/mne/viz/raw.py
@@ -1019,6 +1019,9 @@ def _plot_raw_traces(params, color, bad_color, event_lines=None,
             # "remove" lines
             lines[ii].set_xdata([])
             lines[ii].set_ydata([])
+
+    params['ax'].texts = []   # delete event and annotation texts
+
     # deal with event lines
     if params['event_times'] is not None:
         # find events in the time window
@@ -1050,7 +1053,6 @@ def _plot_raw_traces(params, color, bad_color, event_lines=None,
                 line.set_xdata([])
                 line.set_ydata([])
 
-        params['ax'].texts = []   # delete event and annotation texts
         # don't add event numbers for more than 50 visible events
         if len(event_times) <= 50:
             for ev_time, ev_num in zip(event_times, event_nums):
diff --git a/mne/viz/utils.py b/mne/viz/utils.py
--- a/mne/viz/utils.py
+++ b/mne/viz/utils.py
@@ -1867,12 +1867,9 @@ def _annotate_select(vmin, vmax, params):
     active_idx = _get_active_radiobutton(params['fig_annotation'].radio)
     description = params['fig_annotation'].radio.labels[active_idx].get_text()
     if raw.annotations is None:
-        annot = Annotations([onset], [duration], [description])
-        raw.annotations = annot
-    else:
-        _merge_annotations(onset, onset + duration, description,
-                           raw.annotations)
-
+        raw.annotations = Annotations([], [], [])
+    _merge_annotations(onset, onset + duration, description,
+                       raw.annotations)
     _plot_annotations(params['raw'], params)
     params['plot_fun']()
 
@@ -1898,8 +1895,9 @@ def _plot_annotations(raw, params):
         params['ax_hscroll'].fill_betweenx(
             (0., 1.), annot_start, annot_end, alpha=0.3,
             color=params['segment_colors'][dscr])
-    # Adjust half a sample backward to make it clear what is included
-    params['segments'] = np.array(segments) - 0.5 / raw.info['sfreq']
+    # Do not adjust half a sample backward (even though this would make it
+    # clearer what is included) because this breaks click-drag functionality
+    params['segments'] = np.array(segments)
     params['annot_description'] = descriptions
 
 
@@ -1914,14 +1912,16 @@ def _setup_annotation_colors(params):
     else:
         descriptions = list()
     color_keys = np.union1d(descriptions, params['added_label'])
-    color_cycle = cycle(np.delete(COLORS, 2))  # no red
-    for _ in np.intersect1d(list(color_keys), list(segment_colors.keys())):
-        next(color_cycle)
+    color_cycle = cycle(np.delete(COLORS, COLORS.index('r')))  # no red
+    for key, color in segment_colors.items():
+        assert color in COLORS
+        if color != 'r' and key in color_keys:
+            next(color_cycle)
     for idx, key in enumerate(color_keys):
         if key in segment_colors:
             continue
         elif key.lower().startswith('bad') or key.lower().startswith('edge'):
-            segment_colors[key] = 'red'
+            segment_colors[key] = 'r'
         else:
             segment_colors[key] = next(color_cycle)
     params['segment_colors'] = segment_colors
@@ -1947,24 +1947,38 @@ def _annotations_closed(event, params):
 
 def _on_hover(event, params):
     """Handle hover event."""
+    from matplotlib.patheffects import Stroke, Normal
     if (event.button is not None or
             event.inaxes != params['ax'] or event.xdata is None):
         return
     for coll in params['ax'].collections:
         if coll.contains(event)[0]:
-            path = coll.get_paths()[-1]
-            mn = min(path.vertices[:, 0])
-            mx = max(path.vertices[:, 0])
+            path = coll.get_paths()
+            assert len(path) == 1
+            path = path[0]
+            color = coll.get_edgecolors()[0]
+            mn = path.vertices[:, 0].min()
+            mx = path.vertices[:, 0].max()
+            # left/right line
             x = mn if abs(event.xdata - mn) < abs(event.xdata - mx) else mx
+            mask = path.vertices[:, 0] == x
             ylim = params['ax'].get_ylim()
+
+            def drag_callback(x0):
+                path.vertices[mask, 0] = x0
+
             if params['segment_line'] is None:
                 modify_callback = partial(_annotation_modify, params=params)
-                line = params['ax'].plot([x, x], ylim, color='r',
-                                         linewidth=3, picker=5.)[0]
-                dl = DraggableLine(line, modify_callback)
+                line = params['ax'].plot([x, x], ylim, color=color,
+                                         linewidth=2., picker=5.)[0]
+                dl = DraggableLine(line, modify_callback, drag_callback)
                 params['segment_line'] = dl
             else:
                 params['segment_line'].set_x(x)
+                params['segment_line'].drag_callback = drag_callback
+            line = params['segment_line'].line
+            pe = [Stroke(linewidth=4, foreground=color, alpha=0.5), Normal()]
+            line.set_path_effects(pe if line.contains(event)[0] else pe[1:])
             params['vertline_t'].set_text('%.3f' % x)
             params['ax_vertline'].set_data(0,
                                            np.array(params['ax'].get_ylim()))
@@ -2153,7 +2167,7 @@ def _connection_line(x, fig, sourceax, targetax, y=1.,
                   clip_on=False)
 
 
-class DraggableLine:
+class DraggableLine(object):
     """Custom matplotlib line for moving around by drag and drop.
 
     Parameters
@@ -2164,11 +2178,12 @@ class DraggableLine:
         Callback to call when line is released.
     """
 
-    def __init__(self, line, callback):  # noqa: D102
+    def __init__(self, line, modify_callback, drag_callback):  # noqa: D102
         self.line = line
         self.press = None
         self.x0 = line.get_xdata()[0]
-        self.callback = callback
+        self.modify_callback = modify_callback
+        self.drag_callback = drag_callback
         self.cidpress = self.line.figure.canvas.mpl_connect(
             'button_press_event', self.on_press)
         self.cidrelease = self.line.figure.canvas.mpl_connect(
@@ -2198,6 +2213,7 @@ def on_motion(self, event):
         x0, y0, xpress, ypress = self.press
         dx = event.xdata - xpress
         self.line.set_xdata(x0 + dx)
+        self.drag_callback((x0 + dx)[0])
         self.line.figure.canvas.draw()
 
     def on_release(self, event):
@@ -2206,7 +2222,7 @@ def on_release(self, event):
             return
         self.press = None
         self.line.figure.canvas.draw()
-        self.callback(self.x0, event.xdata)
+        self.modify_callback(self.x0, event.xdata)
         self.x0 = event.xdata
 
     def remove(self):


Test patch:
diff --git a/mne/viz/tests/test_raw.py b/mne/viz/tests/test_raw.py
--- a/mne/viz/tests/test_raw.py
+++ b/mne/viz/tests/test_raw.py
@@ -6,14 +6,15 @@
 import os.path as op
 import warnings
 import itertools
+from distutils.version import LooseVersion
 
-from numpy.testing import assert_raises, assert_equal
+from numpy.testing import assert_raises, assert_allclose
 
 from mne import read_events, pick_types, Annotations
 from mne.datasets import testing
 from mne.io import read_raw_fif, read_raw_ctf
 from mne.utils import requires_version, run_tests_if_main
-from mne.viz.utils import _fake_click, _annotation_radio_clicked
+from mne.viz.utils import _fake_click, _annotation_radio_clicked, _sync_onset
 from mne.viz import plot_raw, plot_sensors
 
 # Set our plotters to test mode
@@ -47,14 +48,27 @@ def _get_events():
     return read_events(event_name)
 
 
-def _annotation_helper(raw):
+def _annotation_helper(raw, events=False):
     """Helper for testing interactive annotations."""
     import matplotlib.pyplot as plt
+    # Some of our checks here require modern mpl to work properly
+    mpl_good_enough = LooseVersion(matplotlib.__version__) >= '2.0'
     n_anns = 0 if raw.annotations is None else len(raw.annotations.onset)
+    plt.close('all')
 
-    fig = raw.plot()
+    if events:
+        events = np.array([[raw.first_samp + 100, 0, 1],
+                           [raw.first_samp + 300, 0, 3]])
+        n_events = len(events)
+    else:
+        events = None
+        n_events = 0
+    fig = raw.plot(events=events)
+    assert len(plt.get_fignums()) == 1
     data_ax = fig.axes[0]
     fig.canvas.key_press_event('a')  # annotation mode
+    assert len(plt.get_fignums()) == 2
+    assert len(fig.axes[0].texts) == n_anns + n_events
     # modify description
     ann_fig = plt.gcf()
     for key in ' test':
@@ -69,6 +83,15 @@ def _annotation_helper(raw):
     _fake_click(fig, data_ax, [1., 1.], xform='data', button=1, kind='press')
     _fake_click(fig, data_ax, [5., 1.], xform='data', button=1, kind='motion')
     _fake_click(fig, data_ax, [5., 1.], xform='data', button=1, kind='release')
+    assert len(raw.annotations.onset) == n_anns + 1
+    assert len(raw.annotations.duration) == n_anns + 1
+    assert len(raw.annotations.description) == n_anns + 1
+    assert raw.annotations.description[n_anns] == 'BAD_ test'
+    assert len(fig.axes[0].texts) == n_anns + 1 + n_events
+    onset = raw.annotations.onset[n_anns]
+    want_onset = _sync_onset(raw, 1., inverse=True)
+    assert_allclose(onset, want_onset)
+    assert_allclose(raw.annotations.duration[n_anns], 4.)
     # hover event
     _fake_click(fig, data_ax, [4.5, 1.], xform='data', button=None,
                 kind='motion')
@@ -79,27 +102,48 @@ def _annotation_helper(raw):
     _fake_click(fig, data_ax, [2.5, 1.], xform='data', button=1, kind='motion')
     _fake_click(fig, data_ax, [2.5, 1.], xform='data', button=1,
                 kind='release')
+    if mpl_good_enough:
+        assert raw.annotations.onset[n_anns] == onset
+        assert_allclose(raw.annotations.duration[n_anns], 1.5)
     # modify annotation from beginning
     _fake_click(fig, data_ax, [1., 1.], xform='data', button=1, kind='press')
-    _fake_click(fig, data_ax, [1.1, 1.], xform='data', button=1, kind='motion')
-    _fake_click(fig, data_ax, [1.1, 1.], xform='data', button=1,
+    _fake_click(fig, data_ax, [0.5, 1.], xform='data', button=1, kind='motion')
+    _fake_click(fig, data_ax, [0.5, 1.], xform='data', button=1,
                 kind='release')
-    assert_equal(len(raw.annotations.onset), n_anns + 1)
-    assert_equal(len(raw.annotations.duration), n_anns + 1)
-    assert_equal(len(raw.annotations.description), n_anns + 1)
-    assert_equal(raw.annotations.description[n_anns], 'BAD_ test')
+    if mpl_good_enough:
+        assert_allclose(raw.annotations.onset[n_anns], onset - 0.5, atol=1e-10)
+        assert_allclose(raw.annotations.duration[n_anns], 2.0)
+    assert len(raw.annotations.onset) == n_anns + 1
+    assert len(raw.annotations.duration) == n_anns + 1
+    assert len(raw.annotations.description) == n_anns + 1
+    assert raw.annotations.description[n_anns] == 'BAD_ test'
+    assert len(fig.axes[0].texts) == n_anns + 1 + n_events
+    fig.canvas.key_press_event('shift+right')
+    assert len(fig.axes[0].texts) == 0
+    fig.canvas.key_press_event('shift+left')
+    assert len(fig.axes[0].texts) == n_anns + 1 + n_events
 
     # draw another annotation merging the two
     _fake_click(fig, data_ax, [5.5, 1.], xform='data', button=1, kind='press')
     _fake_click(fig, data_ax, [2., 1.], xform='data', button=1, kind='motion')
     _fake_click(fig, data_ax, [2., 1.], xform='data', button=1, kind='release')
     # delete the annotation
+    assert len(raw.annotations.onset) == n_anns + 1
+    assert len(raw.annotations.duration) == n_anns + 1
+    assert len(raw.annotations.description) == n_anns + 1
+    if mpl_good_enough:
+        assert_allclose(raw.annotations.onset[n_anns], onset - 0.5, atol=1e-10)
+        assert_allclose(raw.annotations.duration[n_anns], 5.0)
+    assert len(fig.axes[0].texts) == n_anns + 1 + n_events
+    # Delete
     _fake_click(fig, data_ax, [1.5, 1.], xform='data', button=3, kind='press')
     fig.canvas.key_press_event('a')  # exit annotation mode
-
-    assert_equal(len(raw.annotations.onset), n_anns)
-    assert_equal(len(raw.annotations.duration), n_anns)
-    assert_equal(len(raw.annotations.description), n_anns)
+    assert len(raw.annotations.onset) == n_anns
+    assert len(fig.axes[0].texts) == n_anns + n_events
+    fig.canvas.key_press_event('shift+right')
+    assert len(fig.axes[0].texts) == 0
+    fig.canvas.key_press_event('shift+left')
+    assert len(fig.axes[0].texts) == n_anns + n_events
     plt.close('all')
 
 
@@ -219,13 +263,12 @@ def test_plot_annotations():
     """Test annotation mode of the plotter."""
     raw = _get_raw()
     raw.info['lowpass'] = 10.
-    with warnings.catch_warnings(record=True):  # matplotlib
-        _annotation_helper(raw)
+    _annotation_helper(raw)
+    _annotation_helper(raw, events=True)
 
     with warnings.catch_warnings(record=True):  # cut off
         raw.annotations = Annotations([42], [1], 'test', raw.info['meas_date'])
-    with warnings.catch_warnings(record=True):  # matplotlib
-        _annotation_helper(raw)
+    _annotation_helper(raw)
 
 
 @requires_version('scipy', '0.10')
@@ -283,7 +326,7 @@ def test_plot_raw_psd():
         for dB, estimate in itertools.product((True, False),
                                               ('power', 'amplitude')):
             raw.plot_psd(average=True, dB=dB, estimate=estimate)
-    assert_equal(len(w), 4)
+    assert len(w) == 4
     # test reject_by_annotation
     raw = _get_raw()
     raw.annotations = Annotations([1, 5], [3, 3], ['test', 'test'])
@@ -321,7 +364,7 @@ def test_plot_sensors():
     # Click with no sensors
     _fake_click(fig, ax, (0., 0.), xform='data')
     _fake_click(fig, ax, (0, 0.), xform='data', kind='release')
-    assert_equal(len(fig.lasso.selection), 0)
+    assert len(fig.lasso.selection) == 0
 
     # Lasso with 1 sensor
     _fake_click(fig, ax, (-0.5, 0.5), xform='data')
@@ -330,12 +373,12 @@ def test_plot_sensors():
     _fake_click(fig, ax, (0., 0.), xform='data', kind='motion')
     fig.canvas.key_press_event('control')
     _fake_click(fig, ax, (-0.5, 0.), xform='data', kind='release')
-    assert_equal(len(fig.lasso.selection), 1)
+    assert len(fig.lasso.selection) == 1
 
     _fake_click(fig, ax, (-0.09, -0.43), xform='data')  # single selection
-    assert_equal(len(fig.lasso.selection), 2)
+    assert len(fig.lasso.selection) == 2
     _fake_click(fig, ax, (-0.09, -0.43), xform='data')  # deselect
-    assert_equal(len(fig.lasso.selection), 1)
+    assert len(fig.lasso.selection) == 1
     plt.close('all')