Repository: mne-tools/mne-python

Issue:
possible bug in permutation_cluster_test
I noticed that [here](https://github.com/mne-tools/mne-python/blob/master/mne/stats/cluster_level.py#L933) the PPF is only defined once for both `within` and `between` tests.  Shouldn't the `between` test be using `scipy.stats.f.ppf` instead of `scipy.stats.t.ppf` on [this line](https://github.com/mne-tools/mne-python/blob/master/mne/stats/cluster_level.py#L954)?  Also shouldn't the degrees of freedom be `dfn=(n_groups - 1), dfd=(total_subjs - n_groups)` instead of `*n_samples_per_group`?




Discussion:
@larsoner git blame suggest you wrote this:

https://github.com/mne-tools/mne-python/blame/master/mne/stats/cluster_level.py#L954

thoughts?

@agramfort that's just a refactoring commit, but indeed it introduced using `t.ppf` rather than `f.ppf`, which is a bug.

If you traverse the `git blame` to the line's origin, though, you find that the DOF was added by someone named @agramfort, so maybe you can ask him for some insight into that bit :)

https://github.com/mne-tools/mne-python/commit/6c31a6d175e941eed5a76cf291583d42ab24f97f#diff-23fe117cda04337c2d4a6c0fe30f0c76R804

I don't know offhand and would need to look into the math, which I can do in about a week.
looks like this guy @agramfort was correct in 2013 but his test was not
careful enough :)

I think @drammock is suggesting the DOF is incorrect, is it?
is there a test we could come up with?

@agramfort I'm not sure what kind of test you mean...  it seems to me this is a simple question of whether the formula was correctly implemented.  Quoting from [wikipedia](https://en.wikipedia.org/wiki/F-test#Multiple-comparison_ANOVA_problems):

> This F-statistic follows the F-distribution with degrees of freedom d1=K−1 and d2=N−K under the null hypothesis [where K is the number of groups and N is the overall sample size].

A very simple example of how the current implementation is wrong is to give it more than 3 groups:
```
import numpy as np
from mne.stats import spatio_temporal_cluster_test
# 4 groups, 10 subjs/group, 50 timepoints, 1 vertex
fake_data = [np.random.randn(10, 50, 1) for _ in range(4)]
fvals, clusters, pvals, H0 = spatio_temporal_cluster_test(fake_data)
```
yields `TypeError: _parse_args() takes from 2 to 4 positional arguments but 5 were given` (because the `ppf` function is getting passed `*n_samples_per_group`, instead of just two DF args (`dfn` and `dfd`).
ok you made your point perfectly.

Any chance you can do the PR? at least start it?

thanks a lot


Resolving patch:
diff --git a/mne/stats/cluster_level.py b/mne/stats/cluster_level.py
old mode 100755
new mode 100644
--- a/mne/stats/cluster_level.py
+++ b/mne/stats/cluster_level.py
@@ -930,8 +930,8 @@ def get_progress_bar(seeds):
 def _check_fun(X, stat_fun, threshold, tail=0, kind='within'):
     """Check the stat_fun and threshold values."""
     from scipy import stats
-    ppf = stats.t.ppf
     if kind == 'within':
+        ppf = stats.t.ppf
         if threshold is None:
             if stat_fun is not None and stat_fun is not ttest_1samp_no_p:
                 warn('Automatic threshold is only valid for stat_fun=None '
@@ -945,15 +945,17 @@ def _check_fun(X, stat_fun, threshold, tail=0, kind='within'):
         stat_fun = ttest_1samp_no_p if stat_fun is None else stat_fun
     else:
         assert kind == 'between'
+        ppf = stats.f.ppf
         if threshold is None:
             if stat_fun is not None and stat_fun is not f_oneway:
                 warn('Automatic threshold is only valid for stat_fun=None '
                      '(or f_oneway), got %s' % (stat_fun,))
-            p_thresh = 0.05 / (1 + (tail == 0))
-            n_samples_per_group = [len(x) for x in X]
-            threshold = ppf(1. - p_thresh, *n_samples_per_group)
-            if np.sign(tail) < 0:
-                threshold = -threshold
+            elif tail != 1:
+                warn('Ignoring argument "tail", performing 1-tailed F-test')
+            p_thresh = 0.05
+            dfn = len(X) - 1
+            dfd = np.sum([len(x) for x in X]) - len(X)
+            threshold = ppf(1. - p_thresh, dfn, dfd)
             logger.info("Using a threshold of {:.6f}".format(threshold))
         stat_fun = f_oneway if stat_fun is None else stat_fun
     return stat_fun, threshold


Test patch:
diff --git a/mne/stats/tests/test_cluster_level.py b/mne/stats/tests/test_cluster_level.py
--- a/mne/stats/tests/test_cluster_level.py
+++ b/mne/stats/tests/test_cluster_level.py
@@ -13,7 +13,7 @@
 import pytest
 
 from mne.parallel import _force_serial
-from mne.stats.cluster_level import (permutation_cluster_test,
+from mne.stats.cluster_level import (permutation_cluster_test, f_oneway,
                                      permutation_cluster_1samp_test,
                                      spatio_temporal_cluster_test,
                                      spatio_temporal_cluster_1samp_test,
@@ -47,6 +47,42 @@ def _get_conditions():
     return condition1_1d, condition2_1d, condition1_2d, condition2_2d
 
 
+def test_thresholds():
+    """Test automatic threshold calculations."""
+    # within subjects
+    rng = np.random.RandomState(0)
+    X = rng.randn(10, 1, 1) + 0.08
+    want_thresh = -stats.t.ppf(0.025, len(X) - 1)
+    assert 0.03 < stats.ttest_1samp(X[:, 0, 0], 0)[1] < 0.05
+    my_fun = partial(ttest_1samp_no_p)
+    with catch_logging() as log:
+        with pytest.warns(RuntimeWarning, match='threshold is only valid'):
+            out = permutation_cluster_1samp_test(X, stat_fun=my_fun,
+                                                 verbose=True)
+    log = log.getvalue()
+    assert str(want_thresh)[:6] in log
+    assert len(out[1]) == 1  # 1 cluster
+    assert 0.03 < out[2] < 0.05
+    # between subjects
+    Y = rng.randn(10, 1, 1)
+    Z = rng.randn(10, 1, 1) - 0.7
+    X = [X, Y, Z]
+    want_thresh = stats.f.ppf(1. - 0.05, 2, sum(len(a) for a in X) - len(X))
+    p = stats.f_oneway(*X)[1]
+    assert 0.03 < p < 0.05
+    my_fun = partial(f_oneway)  # just to make the check fail
+    with catch_logging() as log:
+        with pytest.warns(RuntimeWarning, match='threshold is only valid'):
+            out = permutation_cluster_test(X, tail=1, stat_fun=my_fun,
+                                           verbose=True)
+    log = log.getvalue()
+    assert str(want_thresh)[:6] in log
+    assert len(out[1]) == 1  # 1 cluster
+    assert 0.03 < out[2] < 0.05
+    with pytest.warns(RuntimeWarning, match='Ignoring argument "tail"'):
+        permutation_cluster_test(X, tail=0)
+
+
 def test_cache_dir():
     """Test use of cache dir."""
     tempdir = _TempDir()
@@ -136,7 +172,7 @@ def test_cluster_permutation_test():
         assert_equal(np.sum(cluster_p_values < 0.05), 1)
 
         T_obs, clusters, cluster_p_values, hist = permutation_cluster_test(
-            [condition1, condition2], n_permutations=100, tail=0, seed=1,
+            [condition1, condition2], n_permutations=100, tail=1, seed=1,
             buffer_size=None)
         assert_equal(np.sum(cluster_p_values < 0.05), 1)
 
@@ -144,7 +180,7 @@ def test_cluster_permutation_test():
         buffer_size = condition1.shape[1] // 10
         T_obs, clusters, cluster_p_values_buff, hist =\
             permutation_cluster_test([condition1, condition2],
-                                     n_permutations=100, tail=0, seed=1,
+                                     n_permutations=100, tail=1, seed=1,
                                      n_jobs=2, buffer_size=buffer_size)
         assert_array_equal(cluster_p_values, cluster_p_values_buff)
 
@@ -338,8 +374,9 @@ def test_cluster_permutation_with_connectivity():
                       connectivity=connectivity, threshold=[])
 
         # wrong value for tail
-        pytest.raises(ValueError, spatio_temporal_func, X1d_3,
-                      connectivity=connectivity, tail=2)
+        with pytest.warns(None):  # sometimes ignoring tail
+            pytest.raises(ValueError, spatio_temporal_func, X1d_3,
+                          connectivity=connectivity, tail=2)
 
         # make sure it actually found a significant point
         out_connectivity_6 = spatio_temporal_func(X1d_3, n_permutations=50,