Repository: nilearn/nilearn

Issue:
permuted_ols only returns h0_fmax for first regressor
<!--Provide a brief description of the bug.-->
When there is more than one regressor in `tested_vars`, `permuted_ols` returns only the null distribution of maximum t-statistics for the first regressor.

Here is the relevant code:

https://github.com/nilearn/nilearn/blob/e7804ebe858349781674c0d2b5c78e27afb52fa1/nilearn/mass_univariate/permuted_least_squares.py#L512

<!--Please fill in the following information, to the best of your ability.-->
Nilearn version: `main` as of e7804ebe858349781674c0d2b5c78e27afb52fa1.

### Expected behavior

When `n_regressors` is >1, I would expect `h0_fmax` to have that number of rows.

### Actual behavior

`h0_fmax` is a 1D array, with one element per 

### Steps and code to reproduce bug

Here is some code, adapted from one of the permuted_ols tests, that should reproduce the bug.

```python
from sklearn.utils import check_random_state
from nilearn.mass_univariate import permuted_ols

random_state = 0
rng = check_random_state(random_state)

# design parameters
n_samples = 50
n_descriptors = 10
n_regressors = 2
n_covars = 2
n_perm = 10

# create design
target_vars = rng.randn(n_samples, n_descriptors)
tested_vars = rng.randn(n_samples, n_regressors)
confounding_vars = rng.randn(n_samples, n_covars)

neg_log10_pvals, own_scores, h0_fmax = permuted_ols(
    tested_vars,
    target_vars,
    confounding_vars,
    model_intercept=False,
    n_perm=n_perm,
    random_state=random_state,
)
assert own_scores.shape == (n_regressors, n_descriptors)
assert neg_log10_pvals.shape == (n_regressors, n_descriptors)
assert h0_fmax.shape == (n_regressors, n_perm)
# AssertionError, assert (10,) == (2, 10)
```



Discussion:
But the problem is that the permutation scheme we use is invalid if we are to test several covariates (the error rate will not be controlled jointly for the two or more tests if we do them concurrently, because the model will test against a global null).
I think we should document the problem clearly, making this  a documentation issue.

Should it just not support multiple `tested_vars` then? It seems like it should have a single `tested_var` and anything else should go into `confounding_vars`.
Sorry, my comment above was wrong, because test_vars are tested independently.
But then you're right, the output of the model is wrong, `p_values` should have shape `(n_regressors, n_descriptors)`

Resolving patch:
diff --git a/nilearn/mass_univariate/permuted_least_squares.py b/nilearn/mass_univariate/permuted_least_squares.py
--- a/nilearn/mass_univariate/permuted_least_squares.py
+++ b/nilearn/mass_univariate/permuted_least_squares.py
@@ -369,10 +369,14 @@ def permuted_ols(
         The ranks of the scores into the h0 distribution correspond to the
         p-values.
 
-    h0_fmax : array-like, shape=(n_perm, )
+    h0_fmax : array-like, shape=(n_regressors, n_perm)
         Distribution of the (max) t-statistic under the null hypothesis
         (obtained from the permutations). Array is sorted.
 
+        .. versionchanged:: 0.9.1
+
+            Return H0 for all regressors, instead of only the first one.
+
     References
     ----------
     .. [1] Anderson, M. J. & Robinson, J. (2001). Permutation tests for
@@ -530,4 +534,4 @@ def permuted_ols(
     if two_sided_test:
         scores_original_data = scores_original_data * sign_scores_original_data
 
-    return - np.log10(pvals), scores_original_data.T, h0_fmax[0]
+    return - np.log10(pvals), scores_original_data.T, h0_fmax


Test patch:
diff --git a/nilearn/mass_univariate/tests/test_permuted_least_squares.py b/nilearn/mass_univariate/tests/test_permuted_least_squares.py
--- a/nilearn/mass_univariate/tests/test_permuted_least_squares.py
+++ b/nilearn/mass_univariate/tests/test_permuted_least_squares.py
@@ -144,6 +144,7 @@ def test_permuted_ols_check_h0_noeffect_labelswap(random_state=0):
 
     # design parameters
     n_samples = 100
+    n_regressors = 1
 
     # create dummy design with no effect
     target_var = rng.randn(n_samples, 1)
@@ -169,38 +170,47 @@ def test_permuted_ols_check_h0_noeffect_labelswap(random_state=0):
         pval, orig_scores, h0 = permuted_ols(
             tested_var, target_var, model_intercept=False,
             n_perm=n_perm, two_sided_test=False, random_state=i)
-        assert_equal(h0.size, n_perm)
+        assert_equal(h0.shape, (n_regressors, n_perm))
+        h0_intercept = h0[0, :]
+
         # Kolmogorov-Smirnov test
-        kstest_pval = stats.kstest(h0, stats.t(n_samples - 1).cdf)[1]
+        kstest_pval = stats.kstest(h0_intercept, stats.t(n_samples - 1).cdf)[1]
         all_kstest_pvals.append(kstest_pval)
         mse = np.mean(
-            (stats.t(n_samples - 1).cdf(np.sort(h0))
-             - np.linspace(0, 1, h0.size + 1)[1:]) ** 2)
+            (stats.t(n_samples - 1).cdf(np.sort(h0_intercept))
+             - np.linspace(0, 1, h0_intercept.size + 1)[1:]) ** 2)
         all_mse.append(mse)
+
         ### Case no. 2: intercept in the model
         pval, orig_scores, h0 = permuted_ols(
             tested_var, target_var, model_intercept=True,
             n_perm=n_perm, two_sided_test=False, random_state=i)
         assert_array_less(pval, 1.)  # pval should not be significant
+        h0_intercept = h0[0, :]
+
         # Kolmogorov-Smirnov test
-        kstest_pval = stats.kstest(h0, stats.t(n_samples - 2).cdf)[1]
+        kstest_pval = stats.kstest(h0_intercept, stats.t(n_samples - 2).cdf)[1]
         all_kstest_pvals_intercept.append(kstest_pval)
         mse = np.mean(
-            (stats.t(n_samples - 2).cdf(np.sort(h0))
-             - np.linspace(0, 1, h0.size + 1)[1:]) ** 2)
+            (stats.t(n_samples - 2).cdf(np.sort(h0_intercept))
+             - np.linspace(0, 1, h0_intercept.size + 1)[1:]) ** 2)
         all_mse_intercept.append(mse)
+
         ### Case no. 3: intercept in the model, no centering of tested vars
         pval, orig_scores, h0 = permuted_ols(
             tested_var_not_centered, target_var, model_intercept=True,
             n_perm=n_perm, two_sided_test=False, random_state=i)
         assert_array_less(pval, 1.)  # pval should not be significant
+        h0_intercept = h0[0, :]
+
         # Kolmogorov-Smirnov test
-        kstest_pval = stats.kstest(h0, stats.t(n_samples - 2).cdf)[1]
+        kstest_pval = stats.kstest(h0_intercept, stats.t(n_samples - 2).cdf)[1]
         all_kstest_pvals_intercept2.append(kstest_pval)
         mse = np.mean(
-            (stats.t(n_samples - 2).cdf(np.sort(h0))
-             - np.linspace(0, 1, h0.size + 1)[1:]) ** 2)
+            (stats.t(n_samples - 2).cdf(np.sort(h0_intercept))
+             - np.linspace(0, 1, h0_intercept.size + 1)[1:]) ** 2)
         all_mse_intercept2.append(mse)
+
     all_kstest_pvals = np.array(all_kstest_pvals).reshape(
         (len(perm_ranges), -1))
     all_kstest_pvals_intercept = np.array(all_kstest_pvals_intercept).reshape(
@@ -227,10 +237,11 @@ def test_permuted_ols_check_h0_noeffect_signswap(random_state=0):
 
     # design parameters
     n_samples = 100
+    n_regressors = 1
 
     # create dummy design with no effect
     target_var = rng.randn(n_samples, 1)
-    tested_var = np.ones((n_samples, 1))
+    tested_var = np.ones((n_samples, n_regressors))
 
     # permuted OLS
     # We check that h0 is close to the theoretical distribution, which is
@@ -245,14 +256,17 @@ def test_permuted_ols_check_h0_noeffect_signswap(random_state=0):
         pval, orig_scores, h0 = permuted_ols(
             tested_var, target_var, model_intercept=False,
             n_perm=n_perm, two_sided_test=False, random_state=i)
-        assert_equal(h0.size, n_perm)
+        assert h0.shape == (n_regressors, n_perm)
+        h0_intercept = h0[0, :]
+
         # Kolmogorov-Smirnov test
-        kstest_pval = stats.kstest(h0, stats.t(n_samples).cdf)[1]
+        kstest_pval = stats.kstest(h0_intercept, stats.t(n_samples).cdf)[1]
         all_kstest_pvals.append(kstest_pval)
         mse = np.mean(
-            (stats.t(n_samples).cdf(np.sort(h0))
-             - np.linspace(0, 1, h0.size + 1)[1:]) ** 2)
+            (stats.t(n_samples).cdf(np.sort(h0_intercept))
+             - np.linspace(0, 1, h0_intercept.size + 1)[1:]) ** 2)
         all_mse.append(mse)
+
     all_kstest_pvals = np.array(all_kstest_pvals).reshape(
         (len(perm_ranges), -1))
     all_mse = np.array(all_mse).reshape((len(perm_ranges), -1))
@@ -407,6 +421,7 @@ def test_permuted_ols_nocovar_multivariate(random_state=0):
     n_samples = 50
     n_descriptors = 10
     n_regressors = 2
+    n_perm = 10
 
     # create design
     target_vars = rng.randn(n_samples, n_descriptors)
@@ -416,10 +431,16 @@ def test_permuted_ols_nocovar_multivariate(random_state=0):
     ref_scores = get_tvalue_with_alternative_library(tested_var, target_vars)
 
     # permuted OLS
-    _, own_scores, _ = permuted_ols(
-        tested_var, target_vars, model_intercept=False,
-        n_perm=0, random_state=random_state)
+    neg_log10_pvals, own_scores, h0_fmax = permuted_ols(
+        tested_var,
+        target_vars,
+        model_intercept=False,
+        n_perm=n_perm,
+        random_state=random_state,
+    )
     assert_array_almost_equal(ref_scores, own_scores, decimal=6)
+    assert neg_log10_pvals.shape == (n_regressors, n_descriptors)
+    assert h0_fmax.shape == (n_regressors, n_perm)
 
     ### Adds intercept (should be equivalent to centering variates)
     # permuted OLS