Repository: mne-tools/mne-python

Issue:
MAINT: Beamformer comparisons
As discussed with @larsoner, we will add a test for LCMV and DICS to test against FieldTrip (i.e., against saved FieldTrip output). 



Discussion:
Steps:
- get ``.m`` file to run _unit-gain_, _unit-array_, and _unit-noise-gain_ beamformer in FieldTrip
- save FieldTrip output as ``.stc`` files
- use these files to test MNE-Python output against in new test

Sounds good to me

Resolving patch:
diff --git a/mne/beamformer/_compute_beamformer.py b/mne/beamformer/_compute_beamformer.py
--- a/mne/beamformer/_compute_beamformer.py
+++ b/mne/beamformer/_compute_beamformer.py
@@ -250,7 +250,7 @@ def _compute_beamformer(G, Cm, reg, n_orient, weight_norm, pick_ori,
                 elif inversion == 'matrix':
                     # Invert for all dipoles simultaneously using matrix
                     # inversion.
-                    norm = linalg.pinv(Ck, 0.1)
+                    norm = linalg.pinv2(Ck)
                 # Reapply source covariance after inversion
                 norm *= sk
                 norm *= sk[:, np.newaxis]
diff --git a/mne/beamformer/_dics.py b/mne/beamformer/_dics.py
--- a/mne/beamformer/_dics.py
+++ b/mne/beamformer/_dics.py
@@ -205,6 +205,11 @@ def make_dics(info, forward, csd, reg=0.05, label=None, pick_ori=None,
     # Determine how to normalize the leadfield
     if normalize_fwd:
         if inversion == 'single':
+            if weight_norm == 'unit-noise-gain':
+                raise ValueError('The computation of a unit-noise-gain '
+                                 'beamformer with inversion="single" is not '
+                                 'stable with depth normalization, set  '
+                                 'normalize_fwd to False.')
             combine_xyz = False
         else:
             combine_xyz = 'fro'
diff --git a/mne/datasets/utils.py b/mne/datasets/utils.py
--- a/mne/datasets/utils.py
+++ b/mne/datasets/utils.py
@@ -236,7 +236,7 @@ def _data_path(path=None, force_update=False, update_path=True, download=True,
     path = _get_path(path, key, name)
     # To update the testing or misc dataset, push commits, then make a new
     # release on GitHub. Then update the "releases" variable:
-    releases = dict(testing='0.63', misc='0.3')
+    releases = dict(testing='0.64', misc='0.3')
     # And also update the "hashes['testing']" variable below.
 
     # To update any other dataset, update the data archive itself (upload
@@ -318,7 +318,7 @@ def _data_path(path=None, force_update=False, update_path=True, download=True,
         sample='fc2d5b9eb0a144b1d6ba84dc3b983602',
         somato='77a7601948c9e38d2da52446e2eab10f',
         spm='9f43f67150e3b694b523a21eb929ea75',
-        testing='6efcfc1202020918afc28d2552d8d82d',
+        testing='c370726ee2d2c028e05ba19f0aeda338',
         multimodal='26ec847ae9ab80f58f204d09e2c08367',
         opm='370ad1dcfd5c47e029e692c85358a374',
         visual_92_categories=['74f50bbeb65740903eadc229c9fa759f',


Test patch:
diff --git a/mne/beamformer/tests/test_dics.py b/mne/beamformer/tests/test_dics.py
--- a/mne/beamformer/tests/test_dics.py
+++ b/mne/beamformer/tests/test_dics.py
@@ -33,6 +33,7 @@
                       'rostralmiddlefrontal-lh.label')
 
 
+@pytest.fixture(scope='module', params=[testing._pytest_param()])
 def _load_forward():
     """Load forward models."""
     fwd_free = mne.read_forward_solution(fname_fwd)
@@ -106,16 +107,15 @@ def _test_weight_norm(filters, norm=1):
 @pytest.mark.slowtest
 @testing.requires_testing_data
 @requires_h5py
-def test_make_dics(tmpdir):
+def test_make_dics(tmpdir, _load_forward):
     """Test making DICS beamformer filters."""
     # We only test proper handling of parameters here. Testing the results is
     # done in test_apply_dics_timeseries and test_apply_dics_csd.
 
-    fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward()
+    fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward
     epochs, _, csd, _ = _simulate_data(fwd_fixed)
     with pytest.raises(RuntimeError, match='several sensor types'):
-        make_dics(epochs.info, fwd_surf, csd, label=label, pick_ori=None,
-                  weight_norm='unit-noise-gain')
+        make_dics(epochs.info, fwd_surf, csd, label=label, pick_ori=None)
     epochs.pick_types(meg='grad')
 
     with pytest.raises(ValueError, match="Invalid value for the 'pick_ori'"):
@@ -146,6 +146,9 @@ def test_make_dics(tmpdir):
     with pytest.raises(NotImplementedError, match='implemented with pick_ori'):
         make_dics(epochs.info, fwd_free, csd, reduce_rank=True,
                   pick_ori='max-power', inversion='single')
+    with pytest.raises(ValueError, match='not stable with depth'):
+        make_dics(epochs.info, fwd_free, csd, weight_norm='unit-noise-gain',
+                  inversion='single', normalize_fwd=True)
 
     # Sanity checks on the returned filters
     n_freq = len(csd.frequencies)
@@ -156,7 +159,7 @@ def test_make_dics(tmpdir):
     n_channels = len(epochs.ch_names)
     # Test return values
     filters = make_dics(epochs.info, fwd_surf, csd, label=label, pick_ori=None,
-                        weight_norm='unit-noise-gain')
+                        weight_norm='unit-noise-gain', normalize_fwd=False)
     assert filters['weights'].shape == (n_freq, n_verts * n_orient, n_channels)
     assert np.iscomplexobj(filters['weights'])
     assert filters['csd'] == csd
@@ -168,7 +171,7 @@ def test_make_dics(tmpdir):
     assert filters['pick_ori'] is None
     assert filters['n_orient'] == n_orient
     assert filters['inversion'] == 'single'
-    assert filters['normalize_fwd']
+    assert not filters['normalize_fwd']
     assert filters['weight_norm'] == 'unit-noise-gain'
     assert 'DICS' in repr(filters)
     assert 'subject "sample"' in repr(filters)
@@ -180,14 +183,16 @@ def test_make_dics(tmpdir):
     # Test picking orientations. Also test weight norming under these different
     # conditions.
     filters = make_dics(epochs.info, fwd_surf, csd, label=label,
-                        pick_ori='normal', weight_norm='unit-noise-gain')
+                        pick_ori='normal', weight_norm='unit-noise-gain',
+                        normalize_fwd=False)
     n_orient = 1
     assert filters['weights'].shape == (n_freq, n_verts * n_orient, n_channels)
     assert filters['n_orient'] == n_orient
     _test_weight_norm(filters)
 
     filters = make_dics(epochs.info, fwd_surf, csd, label=label,
-                        pick_ori='max-power', weight_norm='unit-noise-gain')
+                        pick_ori='max-power', weight_norm='unit-noise-gain',
+                        normalize_fwd=False)
     n_orient = 1
     assert filters['weights'].shape == (n_freq, n_verts * n_orient, n_channels)
     assert filters['n_orient'] == n_orient
@@ -242,28 +247,22 @@ def test_make_dics(tmpdir):
     assert object_diff(filters, filters_read) == ''
 
 
-@pytest.mark.slowtest
-@testing.requires_testing_data
-def test_apply_dics_csd():
+def test_apply_dics_csd(_load_forward):
     """Test applying a DICS beamformer to a CSD matrix."""
-    fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward()
+    fwd_free, fwd_surf, fwd_fixed, _, label = _load_forward
     epochs, _, csd, source_vertno = _simulate_data(fwd_fixed)
     vertices = np.intersect1d(label.vertices, fwd_free['src'][0]['vertno'])
     source_ind = vertices.tolist().index(source_vertno)
     reg = 1  # Lots of regularization for our toy dataset
 
-    # Construct an identity "noise" CSD, which we will use to test the
-    # 'unit-noise-gain' setting.
-    csd_noise = csd.copy()
-    inds = np.triu_indices(csd.n_channels)
-    # Using [:, :] syntax for in-place broadcasting
-    csd_noise._data[:, :] = np.eye(csd.n_channels)[inds][:, np.newaxis]
-
     with pytest.raises(RuntimeError, match='several sensor types'):
         make_dics(epochs.info, fwd_free, csd)
     epochs.pick_types(meg='grad')
 
     # Try different types of forward models
+    assert label.hemi == 'lh'
+    assert vertices[source_ind] == source_vertno
+    rr_want = fwd_free['src'][0]['rr'][source_vertno]
     for fwd in [fwd_free, fwd_surf, fwd_fixed]:
         filters = make_dics(epochs.info, fwd, csd, label=label, reg=reg,
                             inversion='single')
@@ -271,46 +270,70 @@ def test_apply_dics_csd():
         assert f == [10, 20]
 
         # Did we find the true source at 20 Hz?
-        assert np.argmax(power.data[:, 1]) == source_ind
+        idx = np.argmax(power.data[:, 1])
+        rr_got = fwd_free['src'][0]['rr'][vertices[idx]]
+        dist = np.linalg.norm(rr_got - rr_want)
+        assert dist == 0.
 
         # Is the signal stronger at 20 Hz than 10?
         assert power.data[source_ind, 1] > power.data[source_ind, 0]
 
-    # Try picking different orientations and inversion modes
-    for pick_ori in [None, 'normal', 'max-power']:
-        for inversion in ['single', 'matrix']:
-            # Matrix inversion mode needs more regularization for this toy
-            # dataset.
-            if inversion == 'matrix':
-                reg_ = 5
-            else:
-                reg_ = reg
-
-            filters = make_dics(epochs.info, fwd_surf, csd, label=label,
-                                reg=reg_, pick_ori=pick_ori,
-                                inversion=inversion,
-                                weight_norm='unit-noise-gain')
-            power, f = apply_dics_csd(csd, filters)
-            assert f == [10, 20]
-            assert np.argmax(power.data[:, 1]) == source_ind
-            assert power.data[source_ind, 1] > power.data[source_ind, 0]
-
-            # Test unit-noise-gain weighting
-            noise_power, f = apply_dics_csd(csd_noise, filters)
-            assert np.allclose(noise_power.data, 1)
-
-            # Test filter with forward normalization instead of weight
-            # normalization
-            filters = make_dics(epochs.info, fwd_surf, csd, label=label,
-                                reg=reg_, pick_ori=pick_ori,
-                                inversion=inversion, weight_norm=None,
-                                normalize_fwd=True)
-            power, f = apply_dics_csd(csd, filters)
-            assert f == [10, 20]
-            assert np.argmax(power.data[:, 1]) == source_ind
-            assert power.data[source_ind, 1] > power.data[source_ind, 0]
 
-    # Test using a real-valued filter
+@pytest.mark.parametrize('pick_ori', [None, 'normal', 'max-power'])
+@pytest.mark.parametrize('inversion', ['single', 'matrix'])
+def test_apply_dics_ori_inv(_load_forward, pick_ori, inversion):
+    """Testpicking different orientations and inversion modes."""
+    fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward
+    epochs, _, csd, source_vertno = _simulate_data(fwd_fixed)
+    epochs.pick_types('grad')
+    vertices = np.intersect1d(label.vertices, fwd_free['src'][0]['vertno'])
+    source_ind = vertices.tolist().index(source_vertno)
+    rr_want = fwd_free['src'][0]['rr'][source_vertno]
+
+    reg_ = 5 if inversion == 'matrix' else 1
+    filters = make_dics(epochs.info, fwd_surf, csd, label=label,
+                        reg=reg_, pick_ori=pick_ori,
+                        inversion=inversion, normalize_fwd=False,
+                        weight_norm='unit-noise-gain')
+    power, f = apply_dics_csd(csd, filters)
+    assert f == [10, 20]
+    idx = np.argmax(power.data[:, 1])
+    rr_got = fwd_free['src'][0]['rr'][vertices[idx]]
+    dist = np.linalg.norm(rr_got - rr_want)
+    assert dist <= (0.03 if inversion == 'matrix' else 0.)
+    assert power.data[source_ind, 1] > power.data[source_ind, 0]
+
+    # Test unit-noise-gain weighting
+    csd_noise = csd.copy()
+    inds = np.triu_indices(csd.n_channels)
+    csd_noise._data[...] = np.eye(csd.n_channels)[inds][:, np.newaxis]
+    noise_power, f = apply_dics_csd(csd_noise, filters)
+    assert np.allclose(noise_power.data, 1)
+
+    # Test filter with forward normalization instead of weight
+    # normalization
+    filters = make_dics(epochs.info, fwd_surf, csd, label=label,
+                        reg=reg_, pick_ori=pick_ori,
+                        inversion=inversion, weight_norm=None,
+                        normalize_fwd=True)
+    power, f = apply_dics_csd(csd, filters)
+    assert f == [10, 20]
+    idx = np.argmax(power.data[:, 1])
+    rr_got = fwd_free['src'][0]['rr'][vertices[idx]]
+    dist = np.linalg.norm(rr_got - rr_want)
+    assert dist <= (0.035 if inversion == 'matrix' else 0.)
+    assert power.data[source_ind, 1] > power.data[source_ind, 0]
+
+
+def test_real(_load_forward):
+    """Test using a real-valued filter."""
+    fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward
+    epochs, _, csd, source_vertno = _simulate_data(fwd_fixed)
+    vertices = np.intersect1d(label.vertices, fwd_free['src'][0]['vertno'])
+    source_ind = vertices.tolist().index(source_vertno)
+    rr_want = fwd_free['src'][0]['rr'][source_vertno]
+    epochs.pick_types('grad')
+    reg = 1  # Lots of regularization for our toy dataset
     filters_real = make_dics(epochs.info, fwd_surf, csd, label=label, reg=reg,
                              real_filter=True)
     # Also test here that no warings are thrown - implemented to check whether
@@ -329,7 +352,10 @@ def test_apply_dics_csd():
                              reduce_rank=True)
     power, f = apply_dics_csd(csd, filters_real)
     assert f == [10, 20]
-    assert np.argmax(power.data[:, 1]) == source_ind
+    idx = np.argmax(power.data[:, 1])
+    rr_got = fwd_free['src'][0]['rr'][vertices[idx]]
+    dist = np.linalg.norm(rr_got - rr_want)
+    assert dist <= 0.02
     assert power.data[source_ind, 1] > power.data[source_ind, 0]
 
     # Test computing source power on a volume source space
@@ -347,12 +373,11 @@ def test_apply_dics_csd():
         apply_dics_csd(csd, filters_vol)
 
 
-@testing.requires_testing_data
 @pytest.mark.filterwarnings("ignore:The use of several sensor types with the"
                             ":RuntimeWarning")
-def test_apply_dics_timeseries():
+def test_apply_dics_timeseries(_load_forward):
     """Test DICS applied to timeseries data."""
-    fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward()
+    fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward
     epochs, evoked, csd, source_vertno = _simulate_data(fwd_fixed)
     vertices = np.intersect1d(label.vertices, fwd_free['src'][0]['vertno'])
     source_ind = vertices.tolist().index(source_vertno)
@@ -360,8 +385,8 @@ def test_apply_dics_timeseries():
 
     with pytest.raises(RuntimeError, match='several sensor types'):
         make_dics(evoked.info, fwd_surf, csd)
-
     evoked.pick_types(meg='grad')
+
     multiple_filters = make_dics(evoked.info, fwd_surf, csd, label=label,
                                  reg=reg)
 
@@ -445,10 +470,10 @@ def test_apply_dics_timeseries():
 
 @pytest.mark.slowtest
 @testing.requires_testing_data
-def test_tf_dics():
+def test_tf_dics(_load_forward):
     """Test 5D time-frequency beamforming based on DICS."""
-    fwd_free, fwd_surf, fwd_fixed, fwd_vol, label = _load_forward()
-    epochs, evoked, _, source_vertno = _simulate_data(fwd_fixed)
+    fwd_free, fwd_surf, fwd_fixed, _, label = _load_forward
+    epochs, _, _, source_vertno = _simulate_data(fwd_fixed)
     vertices = np.intersect1d(label.vertices, fwd_free['src'][0]['vertno'])
     source_ind = vertices.tolist().index(source_vertno)
     reg = 1  # Lots of regularization for our toy dataset
diff --git a/mne/beamformer/tests/test_external.py b/mne/beamformer/tests/test_external.py
new file mode 100644
--- /dev/null
+++ b/mne/beamformer/tests/test_external.py
@@ -0,0 +1,98 @@
+# Authors: Britta Westner <britta.wstnr@gmail.com>
+#
+# License: BSD (3-clause)
+
+# preparing everything:
+import mne
+import numpy as np
+import os.path as op
+from scipy.io import savemat
+
+from mne.datasets import testing
+from mne.beamformer import make_lcmv, apply_lcmv
+from mne.beamformer.tests.test_lcmv import _get_data
+from mne.utils import run_tests_if_main
+
+
+data_path = testing.data_path(download=False)
+ft_data_path = op.join(data_path, 'fieldtrip', 'beamformer')
+fname_raw = op.join(data_path, 'MEG', 'sample', 'sample_audvis_trunc_raw.fif')
+fname_cov = op.join(data_path, 'MEG', 'sample', 'sample_audvis_trunc-cov.fif')
+fname_fwd = op.join(data_path, 'MEG', 'sample',
+                    'sample_audvis_trunc-meg-eeg-oct-4-fwd.fif')
+fname_fwd_vol = op.join(data_path, 'MEG', 'sample',
+                        'sample_audvis_trunc-meg-vol-7-fwd.fif')
+fname_event = op.join(data_path, 'MEG', 'sample',
+                      'sample_audvis_trunc_raw-eve.fif')
+fname_label = op.join(data_path, 'MEG', 'sample', 'labels', 'Aud-lh.label')
+
+reject = dict(grad=4000e-13, mag=4e-12)
+
+
+def _get_bf_data(save_fieldtrip=False):
+    raw, epochs, evoked, data_cov, _, _, _, _, _, fwd = _get_data(proj=False)
+
+    if save_fieldtrip is True:
+        # raw needs to be saved with all channels and picked in FieldTrip
+        raw.save(op.join(ft_data_path, 'raw.fif'), overwrite=True)
+
+        # src (tris are not available in fwd['src'] once imported into MATLAB)
+        src = fwd['src'].copy()
+        mne.write_source_spaces(op.join(ft_data_path, 'src.fif'), src)
+
+    # pick gradiometers only:
+    epochs.pick_types(meg='grad')
+    evoked.pick_types(meg='grad')
+
+    # compute covariance matrix
+    data_cov = mne.compute_covariance(epochs, tmin=0.04, tmax=0.145,
+                                      method='empirical')
+
+    if save_fieldtrip is True:
+        # if the covariance matrix and epochs need resaving:
+        # data covariance:
+        cov_savepath = op.join(ft_data_path, 'sample_cov')
+        sample_cov = {'sample_cov': data_cov['data']}
+        savemat(cov_savepath, sample_cov)
+        # evoked data:
+        ev_savepath = op.join(ft_data_path, 'sample_evoked')
+        data_ev = {'sample_evoked': evoked.data}
+        savemat(ev_savepath, data_ev)
+
+    return evoked, data_cov, fwd
+
+
+@testing.requires_testing_data
+def test_lcmv_fieldtrip():
+    """Test LCMV vs fieldtrip output."""
+    evoked, data_cov, fwd = _get_bf_data()
+
+    # beamformer types to be tested: unit-gain (vector and scalar) and
+    # unit-noise-gain
+    bf_types = ['ug_vec', 'ug_scal', 'ung']
+    weight_norms = [None, None, 'unit-noise-gain']
+    pick_oris = [None, 'max-power', 'max-power']
+
+    for bf_type, weight_norm, pick_ori in zip(bf_types, weight_norms,
+                                              pick_oris):
+
+        # run the MNE-Python beamformer
+        filters = make_lcmv(evoked.info, fwd, data_cov=data_cov,
+                            noise_cov=None, pick_ori=pick_ori, reg=0.05,
+                            weight_norm=weight_norm)
+        stc_mne = apply_lcmv(evoked, filters)
+        # take the absolute value, since orientation is arbitrary by 180 degr.
+        stc_mne.data[:, :] = np.abs(stc_mne.data)
+
+        # load the FieldTrip output
+        ft_fname = op.join(ft_data_path, 'ft_source_' + bf_type + '-vol.stc')
+        stc_ft = mne.read_source_estimate(ft_fname)
+
+        # calculate the Pearson correlation between the source solutions:
+        pearson = np.corrcoef(np.concatenate(stc_mne.data),
+                              np.concatenate(stc_ft.data))
+
+        assert pearson[0, 1] >= 0.99
+
+
+run_tests_if_main()
diff --git a/mne/beamformer/tests/test_lcmv.py b/mne/beamformer/tests/test_lcmv.py
--- a/mne/beamformer/tests/test_lcmv.py
+++ b/mne/beamformer/tests/test_lcmv.py
@@ -208,7 +208,7 @@ def test_make_lcmv(tmpdir, reg, proj):
         max_stc = stc.data[idx]
         tmax = stc.times[np.argmax(max_stc)]
 
-        assert 0.09 < tmax < 0.12, tmax
+        assert 0.08 < tmax < 0.14, tmax
         assert 0.9 < np.max(max_stc) < 3., np.max(max_stc)
 
         if fwd is forward:
@@ -225,9 +225,9 @@ def test_make_lcmv(tmpdir, reg, proj):
             tmax = stc_normal.times[np.argmax(max_stc)]
 
             lower = 0.04 if proj else 0.025
-            assert lower < tmax < 0.13, tmax
+            assert lower < tmax < 0.14, tmax
             lower = 3e-7 if proj else 2e-7
-            assert lower < np.max(max_stc) < 5e-7, np.max(max_stc)
+            assert lower < np.max(max_stc) < 3e-6, np.max(max_stc)
 
             # No weight normalization was applied, so the amplitude of normal
             # orientation results should always be smaller than free
@@ -260,7 +260,7 @@ def test_make_lcmv(tmpdir, reg, proj):
             mean_stc_max_pow = \
                 stc_max_power.extract_label_time_course(label, fwd['src'],
                                                         mode='mean')
-            assert_array_less(np.abs(mean_stc - mean_stc_max_pow), 0.6)
+            assert_array_less(np.abs(mean_stc - mean_stc_max_pow), 1.0)
 
         # Test NAI weight normalization:
         filters = make_lcmv(evoked.info, fwd, data_cov, reg=reg,
@@ -748,12 +748,12 @@ def test_lcmv_reg_proj(proj, weight_norm):
         # None always represents something not normalized, reflecting channel
         # weights
         for stc in (stc_nocov, stc_cov):
-            assert_allclose(stc.data.std(), 1.4e-8, rtol=0.1)
+            assert_allclose(stc.data.std(), 2.8e-8, rtol=0.1)
     else:
         assert weight_norm == 'unit-noise-gain'
         # Channel scalings depend on presence of noise_cov
         assert_allclose(stc_nocov.data.std(), 5.3e-13, rtol=0.1)
-        assert_allclose(stc_cov.data.std(), 0.12, rtol=0.1)
+        assert_allclose(stc_cov.data.std(), 0.13, rtol=0.1)
 
 
 @pytest.mark.parametrize('reg, weight_norm, use_cov, depth, lower, upper', [
@@ -787,7 +787,7 @@ def test_localization_bias_fixed(bias_params_fixed, reg, weight_norm, use_cov,
 @pytest.mark.parametrize(
     'reg, pick_ori, weight_norm, use_cov, depth, lower, upper', [
         (0.05, 'vector', 'unit-noise-gain', True, None, 36, 39),
-        (0.05, 'vector', 'unit-noise-gain', False, None, 23, 25),
+        (0.05, 'vector', 'unit-noise-gain', False, None, 11, 13),
         (0.05, 'vector', 'nai', True, None, 36, 39),
         (0.05, 'vector', None, True, None, 12, 14),
         (0.05, 'vector', None, True, 0.8, 39, 43),