Repository: mne-tools/mne-python

Issue:
tfce not handling negative values
#### Describe the bug
Hello,
When performing sample permutation cluster t-tests (two tailed) with tfce on an array with negative values, I obtain only positive t values. The t values seem correct, but the sign is wrong (for some samples at least). I get exactly the same result when performing a test  on X and on -X. 

#### Steps to reproduce
```
import mne
import numpy as np
from mne.stats import permutation_cluster_1samp_test

#fake data: difference between two conditions for 3 subjects x 2 vertices x 2 times
X = np.array([[[-10, -5  ],
        [-2, -7]],

       [[-4, -5],
        [-8, -0]],

       [[-6, -3],
        [-4, -2]]])

n_permutations = 100
threshold = dict(start=0., step=0.5) 
tobs, clu, clu_pvalues, ho = permutation_cluster_1samp_test(X, n_permutations = n_permutations,
                                            threshold=threshold, tail=0)
print("tobs", tobs)

```

#### Expected results
Negative values in T obs.

#### Actual results
```
tobs [[3.00614489 4.25614489]
 [2.29903811 1.        ]]
```

#### Additional information
Platform:      Windows-10-10.0.18362-SP0
Python:        3.7.5 (default, Oct 31 2019, 15:18:51) [MSC v.1916 64 bit (AMD64)]
Executable:    C:\Users\fleur\Anaconda3\python.exe
CPU:           Intel64 Family 6 Model 142 Stepping 10, GenuineIntel: 8 cores
Memory:        7.9 GB

mne:           0.20.0
numpy:         1.17.4 {blas=mkl_rt, lapack=mkl_rt}
scipy:         1.3.2
matplotlib:    3.1.1 {backend=module://ipykernel.pylab.backend_inline}

sklearn:       0.22
numba:         0.46.0
nibabel:       2.5.1
cupy:          Not found
pandas:        0.25.3
dipy:          1.0.0
mayavi:        4.7.1 {qt_api=pyqt5, PyQt5=5.9.2}
pyvista:       0.22.4
vtk:           8.1.2


Discussion:

Resolving patch:
diff --git a/mne/stats/__init__.py b/mne/stats/__init__.py
--- a/mne/stats/__init__.py
+++ b/mne/stats/__init__.py
@@ -1,7 +1,8 @@
 """Functions for statistical analysis."""
 
 from .parametric import (f_threshold_mway_rm, f_mway_rm, f_oneway,
-                         _parametric_ci, ttest_1samp_no_p)
+                         _parametric_ci, ttest_1samp_no_p,
+                         ttest_ind_no_p)
 from .permutations import (permutation_t_test, _ci,
                            bootstrap_confidence_interval)
 from .cluster_level import (
diff --git a/mne/stats/cluster_level.py b/mne/stats/cluster_level.py
--- a/mne/stats/cluster_level.py
+++ b/mne/stats/cluster_level.py
@@ -384,7 +384,7 @@ def _find_clusters(x, threshold, tail=0, connectivity=None, max_step=1,
         elif tail == 1:
             stop = np.max(use_x)
         else:  # tail == 0
-            stop = np.max(np.abs(use_x))
+            stop = max(np.max(use_x), -np.min(use_x))
         del use_x
         thresholds = np.arange(threshold['start'], stop,
                                threshold['step'], float)
@@ -436,7 +436,7 @@ def _find_clusters(x, threshold, tail=0, connectivity=None, max_step=1,
                                                 ndimage)
                 clusters += out[0]
                 sums = np.concatenate((sums, out[1]))
-        if tfce is True:
+        if tfce:
             # the score of each point is the sum of the h^H * e^E for each
             # supporting section "rectangle" h x e.
             if ti == 0:
@@ -455,7 +455,7 @@ def _find_clusters(x, threshold, tail=0, connectivity=None, max_step=1,
                 else:
                     len_c = len(c)
                 scores[c] += h * (len_c ** e_power)
-    if tfce is True:
+    if tfce:
         # each point gets treated independently
         clusters = np.arange(x.size)
         if connectivity is None or connectivity is False:
@@ -892,9 +892,13 @@ def _permutation_cluster_test(X, threshold, n_permutations, tail, stat_fun,
                          partitions=partitions, t_power=t_power,
                          show_info=True)
     clusters, cluster_stats = out
+
+    # The stat should have the same shape as the samples
+    t_obs.shape = sample_shape
+
     # For TFCE, return the "adjusted" statistic instead of raw scores
     if isinstance(threshold, dict):
-        t_obs = cluster_stats.copy()
+        t_obs = cluster_stats.reshape(t_obs.shape) * np.sign(t_obs)
 
     logger.info('Found %d clusters' % len(clusters))
 
@@ -908,9 +912,6 @@ def _permutation_cluster_test(X, threshold, n_permutations, tail, stat_fun,
         if out_type == 'indices':
             clusters = _cluster_mask_to_indices(clusters)
 
-    # The stat should have the same shape as the samples
-    t_obs.shape = sample_shape
-
     # convert our seed to orders
     # check to see if we can do an exact test
     # (for a two-tailed test, we can exploit symmetry to just do half)
diff --git a/mne/stats/parametric.py b/mne/stats/parametric.py
--- a/mne/stats/parametric.py
+++ b/mne/stats/parametric.py
@@ -65,6 +65,55 @@ def ttest_1samp_no_p(X, sigma=0, method='relative'):
     return np.mean(X, axis=0) / np.sqrt(var / X.shape[0])
 
 
+def ttest_ind_no_p(a, b, equal_var=True, sigma=1e-3):
+    """Independent samples t-test with hat adjustment and no p calculation.
+
+    This is a modified version of :func:`scipy.stats.ttest_ind`. It operates
+    along the first axis.
+
+    Parameters
+    ----------
+    a : array-like
+        The first array.
+    b : array-like
+        The second array.
+    equal_var : bool
+        Assume equal variance. See :func:`scipy.stats.ttest_ind`.
+    sigma : float
+        The regularization. See :func:`ttest_1samp_no_p`.
+
+    Returns
+    -------
+    t : array
+        T values.
+    """
+    v1 = np.var(a, axis=0, ddof=1)
+    v2 = np.var(b, axis=0, ddof=1)
+    n1 = a.shape[0]
+    n2 = b.shape[0]
+    if equal_var:
+        df = n1 + n2 - 2.0
+        var = ((n1 - 1) * v1 + (n2 - 1) * v2) / df
+        var = var * (1.0 / n1 + 1.0 / n2)
+    else:
+        vn1 = v1 / n1
+        vn2 = v2 / n2
+        with np.errstate(divide='ignore', invalid='ignore'):
+            df = (vn1 + vn2)**2 / (vn1**2 / (n1 - 1) + vn2**2 / (n2 - 1))
+
+        # If df is undefined, variances are zero (assumes n1 > 0 & n2 > 0).
+        # Hence it doesn't matter what df is as long as it's not NaN.
+        df = np.where(np.isnan(df), 1, df)
+        var = vn1 + vn2
+    if sigma > 0:
+        var += sigma * np.max(var)
+    denom = np.sqrt(var)
+    d = np.mean(a, 0) - np.mean(b, 0)
+    with np.errstate(divide='ignore', invalid='ignore'):
+        t = np.divide(d, denom)
+    return t
+
+
 def f_oneway(*args):
     """Perform a 1-way ANOVA.
 


Test patch:
diff --git a/mne/stats/tests/test_cluster_level.py b/mne/stats/tests/test_cluster_level.py
--- a/mne/stats/tests/test_cluster_level.py
+++ b/mne/stats/tests/test_cluster_level.py
@@ -14,7 +14,7 @@
 
 from mne.fixes import has_numba
 from mne.parallel import _force_serial
-from mne.stats import cluster_level
+from mne.stats import cluster_level, ttest_ind_no_p
 from mne.stats.cluster_level import (permutation_cluster_test, f_oneway,
                                      permutation_cluster_1samp_test,
                                      spatio_temporal_cluster_test,
@@ -437,6 +437,45 @@ def test_cluster_permutation_with_connectivity(numba_conditional):
                     buffer_size=None)
 
 
+@pytest.mark.parametrize('threshold', [
+    0.1,
+    pytest.param(dict(start=0., step=0.5), id='TFCE'),
+])
+@pytest.mark.parametrize('kind', ('1samp', 'ind'))
+def test_permutation_cluster_signs(threshold, kind):
+    """Test cluster signs."""
+    # difference between two conditions for 3 subjects x 2 vertices x 2 times
+    X = np.array([[[-10, 5], [-2, -7]],
+                  [[-4, 5], [-8, -0]],
+                  [[-6, 3], [-4, -2]]], float)
+    want_signs = np.sign(np.mean(X, axis=0))
+    n_permutations = 1
+    if kind == '1samp':
+        func = permutation_cluster_1samp_test
+        stat_fun = ttest_1samp_no_p
+        use_X = X
+    else:
+        assert kind == 'ind'
+        func = permutation_cluster_test
+        stat_fun = ttest_ind_no_p
+        use_X = [X, np.random.RandomState(0).randn(*X.shape) * 0.1]
+    tobs, clu, clu_pvalues, _ = func(
+        use_X, n_permutations=n_permutations, threshold=threshold, tail=0,
+        stat_fun=stat_fun)
+    clu_signs = np.zeros(X.shape[1:])
+    used = np.zeros(X.shape[1:])
+    assert len(clu) == len(clu_pvalues)
+    for c, p in zip(clu, clu_pvalues):
+        assert not used[c].any()
+        assert len(np.unique(np.sign(tobs[c]))) == 1
+        clu_signs[c] = np.sign(tobs[c])[0]
+        used[c] = True
+    assert used.all()
+    assert clu_signs.all()
+    assert_array_equal(np.sign(tobs), want_signs)
+    assert_array_equal(clu_signs, want_signs)
+
+
 @requires_sklearn
 def test_permutation_connectivity_equiv(numba_conditional):
     """Test cluster level permutations with and without connectivity."""
diff --git a/mne/stats/tests/test_parametric.py b/mne/stats/tests/test_parametric.py
--- a/mne/stats/tests/test_parametric.py
+++ b/mne/stats/tests/test_parametric.py
@@ -1,9 +1,13 @@
+from functools import partial
 from itertools import product
 
 import pytest
-from numpy.testing import assert_array_almost_equal
+from numpy.testing import (assert_array_almost_equal, assert_allclose,
+                           assert_array_less)
 import numpy as np
+import scipy.stats
 
+import mne
 from mne.stats.parametric import (f_mway_rm, f_threshold_mway_rm,
                                   _map_effects)
 
@@ -112,3 +116,44 @@ def test_f_twoway_rm():
 
     fvals, _ = f_mway_rm(test_data, [8], 'A')
     assert_array_almost_equal(fvals, test_external['r_fvals_1way'], 5)
+
+
+@pytest.mark.parametrize('kind, kwargs', [
+    ('1samp', {}),
+    ('ind', {}),  # equal_var=True is the default
+    ('ind', dict(equal_var=True)),
+    ('ind', dict(equal_var=False)),
+])
+@pytest.mark.parametrize('sigma', (0., 1e-3,))
+@pytest.mark.parametrize('seed', [0, 42, 1337])
+def test_ttest_equiv(kind, kwargs, sigma, seed):
+    """Test t-test equivalence."""
+    rng = np.random.RandomState(seed)
+
+    def theirs(*a, **kw):
+        f = getattr(scipy.stats, 'ttest_%s' % (kind,))
+        if kind == '1samp':
+            func = partial(f, popmean=0, **kwargs)
+        else:
+            func = partial(f, **kwargs)
+        return func(*a, **kw)[0]
+
+    ours = partial(getattr(mne.stats, 'ttest_%s_no_p' % (kind,)),
+                   sigma=sigma, **kwargs)
+
+    X = rng.randn(3, 4, 5)
+    if kind == 'ind':
+        X = [X, rng.randn(30, 4, 5)]  # should differ based on equal_var
+        got = ours(*X)
+        want = theirs(*X)
+    else:
+        got = ours(X)
+        want = theirs(X)
+    if sigma == 0.:
+        assert_allclose(got, want, rtol=1e-7, atol=1e-6)
+    else:
+        assert not np.allclose(got, want, rtol=1e-7, atol=1e-6)
+        # should mostly be similar, but uniformly smaller because we add
+        # something to the divisor (var)
+        assert_allclose(got, want, rtol=2e-1, atol=1e-2)
+        assert_array_less(np.abs(got), np.abs(want))