Repository: mne-tools/mne-python

Issue:
xDAWN filters selected along incorrect axis.
It appears to me that there is a bug in the ``Xdawn`` code that incorrectly chooses filters. Filters should be in the rows of the corresponding array [0].
Specifically, in https://github.com/mne-tools/mne-python/blob/master/mne/preprocessing/xdawn.py#L477 filters and patterns are transposed. However, ``_fit_xdawn`` already correctly puts filters and patterns in rows of the array. https://github.com/mne-tools/mne-python/blob/master/mne/preprocessing/xdawn.py#L195 .


#### Steps to reproduce

Since I was a bit skeptical about the finding, a few arguments that the current implementation seems to be incorrect.
 1. Since eigenvectors are normalized in ``_fit_xdawn`` to unit norm, this should hold for rows in the filter matrix. However, the norm of the columns and not the rows of the filter matrix correspond to 1. In the ``plot_decoding_xdawn_eeg`` example,
```
In[19]: np.mean(np.isclose(np.linalg.norm(clf.steps[0][1].filters_['aud_l'], axis=0), 1))
Out[19]: 1.0
In[20]: np.mean(np.isclose(np.linalg.norm(clf.steps[0][1].filters_['aud_l'], axis=1), 1))
Out[20]: 0.0
```
 2. the current code fails if not the complete decomposition is stored (when the internal ``n_components`` does not correspond to the cannel code, https://github.com/mne-tools/mne-python/blob/master/mne/preprocessing/xdawn.py#L462).
 3. performance is *substantially* better using the correct selection.
    For example, running the ``plot_decoding_xdawn_eeg`` example on ``master`` yields
```
              precision    recall  f1-score   support

       aud_l       0.51      0.58      0.55        72
       aud_r       0.52      0.47      0.49        73
       vis_l       0.72      0.79      0.75        73
       vis_r       0.78      0.67      0.72        70

    accuracy                           0.63       288
   macro avg       0.63      0.63      0.63       288
weighted avg       0.63      0.63      0.63       288
```

    Running it instead with the transposed filters results in 

```
                  precision    recall  f1-score   support
       aud_l       0.79      0.69      0.74        72
       aud_r       0.72      0.81      0.76        73
       vis_l       0.99      0.97      0.98        73
       vis_r       0.96      0.97      0.96        70
    accuracy                           0.86       288
   macro avg       0.86      0.86      0.86       288
weighted avg       0.86      0.86      0.86       288
```
    

I'll create a pull request to address this.

[0] e.g., during transform, rows are selected https://github.com/mne-tools/mne-python/blob/master/mne/preprocessing/xdawn.py#L509



Discussion:

Resolving patch:
diff --git a/examples/decoding/plot_decoding_xdawn_eeg.py b/examples/decoding/plot_decoding_xdawn_eeg.py
--- a/examples/decoding/plot_decoding_xdawn_eeg.py
+++ b/examples/decoding/plot_decoding_xdawn_eeg.py
@@ -32,7 +32,7 @@
 from sklearn.metrics import classification_report, confusion_matrix
 from sklearn.preprocessing import MinMaxScaler
 
-from mne import io, pick_types, read_events, Epochs
+from mne import io, pick_types, read_events, Epochs, EvokedArray
 from mne.datasets import sample
 from mne.preprocessing import Xdawn
 from mne.decoding import Vectorizer
@@ -49,6 +49,7 @@
 event_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw-eve.fif'
 tmin, tmax = -0.1, 0.3
 event_id = dict(aud_l=1, aud_r=2, vis_l=3, vis_r=4)
+n_filter = 3
 
 # Setup for reading the raw data
 raw = io.read_raw_fif(raw_fname, preload=True)
@@ -63,7 +64,7 @@
                 verbose=False)
 
 # Create classification pipeline
-clf = make_pipeline(Xdawn(n_components=3),
+clf = make_pipeline(Xdawn(n_components=n_filter),
                     Vectorizer(),
                     MinMaxScaler(),
                     LogisticRegression(penalty='l1', solver='liblinear',
@@ -101,3 +102,27 @@
 plt.ylabel('True label')
 plt.xlabel('Predicted label')
 plt.show()
+
+
+###############################################################################
+# The ``patterns_`` attribute of a fitted Xdawn instance (here from the last
+# cross-validation fold) can be used for visualization.
+
+fig, axes = plt.subplots(nrows=len(event_id), ncols=n_filter)
+fitted_xdawn = clf.steps[0][1]
+for i, (cur_class, cur_patterns) in enumerate(fitted_xdawn.patterns_.items()):
+    tmp_info = epochs.info.copy()
+    tmp_info['sfreq'] = 1.
+    pattern_evoked = EvokedArray(cur_patterns[:n_filter].T, tmp_info, tmin=0)
+    pattern_evoked.plot_topomap(
+        times=np.arange(n_filter), ch_type=None,
+        scalings=None,
+        time_format='{} / comp. %01d'.format(cur_class),
+        colorbar=False,
+        head_pos={'center': [0, 0]},
+        show_names=False,
+        axes=axes[i, :],
+        extrapolate='head',
+        show=False)
+fig.subplots_adjust(hspace=0.3)
+plt.show()
diff --git a/examples/preprocessing/plot_xdawn_denoising.py b/examples/preprocessing/plot_xdawn_denoising.py
--- a/examples/preprocessing/plot_xdawn_denoising.py
+++ b/examples/preprocessing/plot_xdawn_denoising.py
@@ -64,6 +64,10 @@
 # Plot image epoch before xdawn
 plot_epochs_image(epochs['vis_r'], picks=[230], vmin=-500, vmax=500)
 
+###############################################################################
+# Now, we estimate a set of xDAWN filters for the epochs (which contain only
+# the ``vis_r`` class).
+
 # Estimates signal covariance
 signal_cov = compute_raw_covariance(raw, picks=picks)
 
@@ -73,7 +77,10 @@
 # Fit xdawn
 xd.fit(epochs)
 
-# Denoise epochs
+###############################################################################
+# Epochs are denoised by calling ``apply``, which by default keeps only the
+# signal subspace corresponding to the first ``n_components`` specified in the
+# ``Xdawn`` constructor above.
 epochs_denoised = xd.apply(epochs)
 
 # Plot image epoch after Xdawn
diff --git a/mne/preprocessing/xdawn.py b/mne/preprocessing/xdawn.py
--- a/mne/preprocessing/xdawn.py
+++ b/mne/preprocessing/xdawn.py
@@ -137,6 +137,7 @@ def _fit_xdawn(epochs_data, y, n_components, reg=None, signal_cov=None,
     -------
     filters : array, shape (n_channels, n_channels)
         The Xdawn components used to decompose the data for each event type.
+        Each row corresponds to one component.
     patterns : array, shape (n_channels, n_channels)
         The Xdawn patterns used to restore the signals for each event type.
     evokeds : array, shape (n_class, n_components, n_times)
@@ -371,7 +372,8 @@ class Xdawn(_XdawnTransformer):
     ----------
     filters_ : dict of ndarray
         If fit, the Xdawn components used to decompose the data for each event
-        type, else empty.
+        type, else empty. For each event type, the filters are in the rows of
+        the corresponding array.
     patterns_ : dict of ndarray
         If fit, the Xdawn patterns used to restore the signals for each event
         type, else empty.
@@ -474,8 +476,8 @@ def fit(self, epochs, y=None):
         idx = np.argsort([value for _, value in epochs.event_id.items()])
         for eid, this_filter, this_pattern, this_evo in zip(
                 epochs.event_id, filters[idx], patterns[idx], evokeds[idx]):
-            self.filters_[eid] = this_filter.T
-            self.patterns_[eid] = this_pattern.T
+            self.filters_[eid] = this_filter
+            self.patterns_[eid] = this_pattern
             n_events = len(epochs[eid])
             evoked = EvokedArray(this_evo, use_info, tmin=epochs.tmin,
                                  comment=eid, nave=n_events)
@@ -625,7 +627,7 @@ def _pick_sources(self, data, include, exclude, eid):
         logger.info('Transforming to Xdawn space')
 
         # Apply unmixing
-        sources = np.dot(self.filters_[eid].T, data)
+        sources = np.dot(self.filters_[eid], data)
 
         if include not in (None, list()):
             mask = np.ones(len(sources), dtype=np.bool)
@@ -637,7 +639,7 @@ def _pick_sources(self, data, include, exclude, eid):
             sources[exclude_] = 0.
             logger.info('Zeroing out %i Xdawn components' % len(exclude_))
         logger.info('Inverse transforming to sensor space')
-        data = np.dot(self.patterns_[eid], sources)
+        data = np.dot(self.patterns_[eid].T, sources)
 
         return data
 


Test patch:
diff --git a/mne/preprocessing/tests/test_xdawn.py b/mne/preprocessing/tests/test_xdawn.py
--- a/mne/preprocessing/tests/test_xdawn.py
+++ b/mne/preprocessing/tests/test_xdawn.py
@@ -7,11 +7,14 @@
 import os.path as op
 import sys
 
-from numpy.testing import assert_array_equal, assert_array_almost_equal
+from numpy.testing import (assert_array_equal, assert_array_almost_equal,
+                           assert_allclose)
 import pytest
+from scipy import linalg, stats
 
 from mne import (Epochs, read_events, pick_types, compute_raw_covariance,
                  create_info, EpochsArray)
+from mne.decoding import Vectorizer
 from mne.io import read_raw_fif
 from mne.utils import (requires_sklearn, run_tests_if_main, check_version,
                        _get_numpy_libs)
@@ -73,6 +76,8 @@ def test_xdawn_fit():
     evoked = epochs['cond2'].average()
     assert_array_equal(evoked.data, xd.evokeds_['cond2'].data)
 
+    assert_allclose(np.linalg.norm(xd.filters_['cond2'], axis=1), 1)
+
     # ========== with signal cov provided ====================
     # Provide covariance object
     signal_cov = compute_raw_covariance(raw, picks=picks)
@@ -242,8 +247,8 @@ def test_XdawnTransformer():
 
     xdt = _XdawnTransformer()
     xdt.fit(X, y)
-    assert_array_almost_equal(xd.filters_['cond2'][:, :2],
-                              xdt.filters_.reshape(2, 2, 8)[0].T)
+    assert_array_almost_equal(xd.filters_['cond2'][:2, :],
+                              xdt.filters_.reshape(2, 2, 8)[0])
 
     # Transform testing
     xdt.transform(X[1:, ...])  # different number of epochs
@@ -262,4 +267,93 @@ def test_XdawnTransformer():
     pytest.raises(ValueError, xdt.inverse_transform, 42)
 
 
+def _simulate_erplike_mixed_data(n_epochs=100, n_channels=10):
+    rng = np.random.RandomState(42)
+    tmin, tmax = 0., 1.
+    sfreq = 100.
+    informative_ch_idx = 0
+
+    y = rng.randint(0, 2, n_epochs)
+    n_times = int((tmax - tmin) * sfreq)
+    epoch_times = np.linspace(tmin, tmax, n_times)
+
+    target_template = 1e-6 * (epoch_times - tmax) * np.sin(
+        2 * np.pi * epoch_times)
+    nontarget_template = 0.7e-6 * (epoch_times - tmax) * np.sin(
+        2 * np.pi * (epoch_times - 0.1))
+
+    epoch_data = rng.randn(n_epochs, n_channels, n_times) * 5e-7
+    epoch_data[y == 0, informative_ch_idx, :] += nontarget_template
+    epoch_data[y == 1, informative_ch_idx, :] += target_template
+
+    mixing_mat = linalg.svd(rng.randn(n_channels, n_channels))[0]
+    mixed_epoch_data = np.dot(mixing_mat.T, epoch_data).transpose((1, 0, 2))
+
+    events = np.zeros((n_epochs, 3), dtype=int)
+    events[:, 0] = np.arange(0, n_epochs * n_times, n_times)
+    events[:, 2] = y
+
+    info = create_info(
+        ch_names=['C{:02d}'.format(i) for i in range(n_channels)],
+        ch_types=['eeg'] * n_channels,
+        sfreq=sfreq)
+    epochs = EpochsArray(mixed_epoch_data, info, events,
+                         tmin=tmin,
+                         event_id={'nt': 0, 't': 1})
+
+    return epochs, mixing_mat
+
+
+@requires_sklearn
+def test_xdawn_decoding_performance():
+    """Test decoding performance and extracted pattern on synthetic data."""
+    from sklearn.model_selection import KFold
+    from sklearn.pipeline import make_pipeline
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.preprocessing import MinMaxScaler
+    from sklearn.metrics import accuracy_score
+
+    n_xdawn_comps = 3
+    expected_accuracy = 0.98
+
+    epochs, mixing_mat = _simulate_erplike_mixed_data(n_epochs=100)
+    y = epochs.events[:, 2]
+
+    # results of Xdawn and _XdawnTransformer should match
+    xdawn_pipe = make_pipeline(
+        Xdawn(n_components=n_xdawn_comps),
+        Vectorizer(),
+        MinMaxScaler(),
+        LogisticRegression(solver='liblinear'))
+    xdawn_trans_pipe = make_pipeline(
+        _XdawnTransformer(n_components=n_xdawn_comps),
+        Vectorizer(),
+        MinMaxScaler(),
+        LogisticRegression(solver='liblinear'))
+
+    cv = KFold(n_splits=3, shuffle=False)
+    for pipe, X in (
+            (xdawn_pipe, epochs),
+            (xdawn_trans_pipe, epochs.get_data())):
+        predictions = np.empty_like(y, dtype=float)
+        for train, test in cv.split(X, y):
+            pipe.fit(X[train], y[train])
+            predictions[test] = pipe.predict(X[test])
+
+        cv_accuracy_xdawn = accuracy_score(y, predictions)
+        assert_allclose(cv_accuracy_xdawn, expected_accuracy, atol=0.01)
+
+        # for both event types, the first component should "match" the mixing
+        fitted_xdawn = pipe.steps[0][1]
+        if isinstance(fitted_xdawn, Xdawn):
+            relev_patterns = np.concatenate(
+                [comps[[0]] for comps in fitted_xdawn.patterns_.values()])
+        else:
+            relev_patterns = fitted_xdawn.patterns_[::n_xdawn_comps]
+
+        for i in range(len(relev_patterns)):
+            r, _ = stats.pearsonr(relev_patterns[i, :], mixing_mat[0, :])
+            assert np.abs(r) > 0.99
+
+
 run_tests_if_main()