Repository: mne-tools/mne-python

Issue:
output of permutation_cluster_1samp_test seems wrong for out_type='indices'
Originally reported by @dimitriospantazis. Here is the code to reproduce (based on an MNE example):

```py
import numpy as np
import matplotlib.pyplot as plt

import mne
from mne.time_frequency import tfr_morlet
from mne.stats import permutation_cluster_1samp_test
from mne.datasets import sample

data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_raw.fif'
tmin, tmax, event_id = -0.3, 0.6, 1

# Setup for reading the raw data
raw = mne.io.read_raw_fif(raw_fname)
events = mne.find_events(raw, stim_channel='STI 014')

include = []
raw.info['bads'] += ['MEG 2443', 'EEG 053']  # bads + 2 more

# picks MEG gradiometers
picks = mne.pick_types(raw.info, meg='grad', eeg=False, eog=True,
                       stim=False, include=include, exclude='bads')

# Load condition 1
event_id = 1
epochs = mne.Epochs(raw, events, event_id, tmin, tmax, picks=picks,
                    baseline=(None, 0), preload=True,
                    reject=dict(grad=4000e-13, eog=150e-6))
# just use right temporal sensors for speed
epochs.pick_channels(mne.read_selection('Right-temporal'))
evoked = epochs.average()

decim = 5
freqs = np.arange(8, 40, 2)  # define frequencies of interest
sfreq = raw.info['sfreq']  # sampling in Hz
tfr_epochs = tfr_morlet(epochs, freqs, n_cycles=4., decim=decim,
                        average=False, return_itc=False, n_jobs=1)

# Baseline power
tfr_epochs.apply_baseline(mode='logratio', baseline=(-.100, 0))

# Crop in time to keep only what is between 0 and 400 ms
evoked.crop(-0.1, 0.4)
tfr_epochs.crop(-0.1, 0.4)

epochs_power = tfr_epochs.data

threshold = 3.
n_permutations = 50  # Warning: 50 is way too small for real-world analysis.
data = epochs_power[0][0]
for out_type in ['mask', 'indices']:
    T_obs, clusters, cluster_p_values, H0 = \
        permutation_cluster_1samp_test(data, n_permutations=n_permutations,
                                    threshold=threshold, tail=1,
                                    adjacency=None,
                                    out_type=out_type, verbose=True)
    print(clusters)
```

I get:

```
Computing cluster p-values
Done.
[(slice(2, 4, None),), (slice(46, 49, None),)]
stat_fun(H1): min=-7.417671 max=4.199342
Running initial clustering
Found 2 clusters
Permuting 49 times...
100%|██████████████████████████████████████|  : 49/49 [00:00<00:00, 1025.40it/s]
Computing cluster p-values
Done.
[(array([0]),), (array([0]),)]
```

In the mask format, you get two different clusters but in the other format you get the same cluster repeated twice.

Looking at the [source code](https://github.com/mne-tools/mne-python/blob/4df9ed838f1d56471d071308fb62b9ec749cb10d/mne/stats/cluster_level) the loop does look buggy to me.


Discussion:

Resolving patch:
diff --git a/mne/stats/cluster_level.py b/mne/stats/cluster_level.py
--- a/mne/stats/cluster_level.py
+++ b/mne/stats/cluster_level.py
@@ -557,11 +557,20 @@ def _cluster_indices_to_mask(components, n_tot):
     return components
 
 
-def _cluster_mask_to_indices(components):
+def _cluster_mask_to_indices(components, shape):
     """Convert to the old format of clusters, which were bool arrays."""
     for ci, c in enumerate(components):
-        if not isinstance(c, slice):
-            components[ci] = np.where(c)[0]
+        if isinstance(c, np.ndarray):  # mask
+            components[ci] = np.where(c.reshape(shape))
+        else:
+            assert isinstance(c, tuple), type(c)
+            c = list(c)  # tuple->list
+            for ii, cc in enumerate(c):
+                if isinstance(cc, slice):
+                    c[ii] = np.arange(cc.start, cc.stop)
+                else:
+                    c[ii] = np.where(cc)[0]
+            components[ci] = tuple(c)
     return components
 
 
@@ -918,7 +927,7 @@ def _permutation_cluster_test(X, threshold, n_permutations, tail, stat_fun,
     else:
         # ndimage outputs slices or boolean masks by default
         if out_type == 'indices':
-            clusters = _cluster_mask_to_indices(clusters)
+            clusters = _cluster_mask_to_indices(clusters, t_obs.shape)
 
     # convert our seed to orders
     # check to see if we can do an exact test
@@ -1087,7 +1096,7 @@ def permutation_cluster_test(
         no points are excluded.
     %(clust_stepdown)s
     %(clust_power_f)s
-    %(clust_out_none)s
+    %(clust_out)s
     %(clust_disjoint)s
     %(clust_buffer)s
     %(verbose)s
@@ -1147,7 +1156,7 @@ def permutation_cluster_1samp_test(
         no points are excluded.
     %(clust_stepdown)s
     %(clust_power_t)s
-    %(clust_out_none)s
+    %(clust_out)s
     %(clust_disjoint)s
     %(clust_buffer)s
     %(verbose)s
diff --git a/mne/utils/docs.py b/mne/utils/docs.py
--- a/mne/utils/docs.py
+++ b/mne/utils/docs.py
@@ -1753,22 +1753,15 @@
 docdict['clust_power_f'] = docdict['clust_power'].format('F')
 docdict['clust_out'] = """
 out_type : 'mask' | 'indices'
-    Output format of clusters. If ``'mask'``, returns boolean arrays the same
-    shape as the input data, with ``True`` values indicating locations that are
-    part of a cluster. If ``'indices'``, returns a list of lists, where each
-    sublist contains the indices of locations that together form a cluster.
-    Note that for large datasets, ``'indices'`` may use far less memory than
-    ``'mask'``. Default is ``'indices'``.
-"""
-docdict['clust_out_none'] = """
-out_type : 'mask' | 'indices'
-    Output format of clusters. If ``'mask'``, returns boolean arrays the same
-    shape as the input data, with ``True`` values indicating locations that are
-    part of a cluster. If ``'indices'``, returns a list of lists, where each
-    sublist contains the indices of locations that together form a cluster.
-    Note that for large datasets, ``'indices'`` may use far less memory than
-    ``'mask'``. The default translates to ``'mask'`` in version 0.21 but will
-    change to ``'indices'`` in version 0.22.
+    Output format of clusters within a list.
+    If ``'mask'``, returns a list of boolean arrays,
+    each with the same shape as the input data (or slices if the shape is 1D
+    and adjacency is None), with ``True`` values indicating locations that are
+    part of a cluster. If ``'indices'``, returns a list of tuple of ndarray,
+    where each ndarray contains the indices of locations that together form the
+    given cluster along the given dimension. Note that for large datasets,
+    ``'indices'`` may use far less memory than ``'mask'``.
+    Default is ``'indices'``.
 """
 docdict['clust_disjoint'] = """
 check_disjoint : bool


Test patch:
diff --git a/mne/stats/tests/test_cluster_level.py b/mne/stats/tests/test_cluster_level.py
--- a/mne/stats/tests/test_cluster_level.py
+++ b/mne/stats/tests/test_cluster_level.py
@@ -22,8 +22,7 @@
                                      spatio_temporal_cluster_test,
                                      spatio_temporal_cluster_1samp_test,
                                      ttest_1samp_no_p, summarize_clusters_stc)
-from mne.utils import (run_tests_if_main, catch_logging, check_version,
-                       requires_sklearn)
+from mne.utils import catch_logging, check_version, requires_sklearn
 
 
 @pytest.fixture(scope="function", params=('Numba', 'NumPy'))
@@ -688,4 +687,43 @@ def test_tfce_thresholds(numba_conditional):
             data, tail=1, out_type='mask', threshold=dict(start=1, step=-0.5))
 
 
-run_tests_if_main()
+# 1D gives slices, 2D+ gives boolean masks
+@pytest.mark.parametrize('shape', ((11,), (11, 3), (11, 1, 2)))
+@pytest.mark.parametrize('out_type', ('mask', 'indices'))
+@pytest.mark.parametrize('adjacency', (None, 'sparse'))
+def test_output_equiv(shape, out_type, adjacency):
+    """Test equivalence of output types."""
+    rng = np.random.RandomState(0)
+    n_subjects = 10
+    data = rng.randn(n_subjects, *shape)
+    data -= data.mean(axis=0, keepdims=True)
+    data[:, 2:4] += 2
+    data[:, 6:9] += 2
+    want_mask = np.zeros(shape, int)
+    want_mask[2:4] = 1
+    want_mask[6:9] = 2
+    if adjacency is not None:
+        assert adjacency == 'sparse'
+        adjacency = combine_adjacency(*shape)
+    clusters = permutation_cluster_1samp_test(
+        X=data, n_permutations=1, adjacency=adjacency, out_type=out_type)[1]
+    got_mask = np.zeros_like(want_mask)
+    for n, clu in enumerate(clusters, 1):
+        if out_type == 'mask':
+            if len(shape) == 1 and adjacency is None:
+                assert isinstance(clu, tuple)
+                assert len(clu) == 1
+                assert isinstance(clu[0], slice)
+            else:
+                assert isinstance(clu, np.ndarray)
+                assert clu.dtype == bool
+                assert clu.shape == shape
+            got_mask[clu] = n
+        else:
+            assert isinstance(clu, tuple)
+            for c in clu:
+                assert isinstance(c, np.ndarray)
+                assert c.dtype.kind == 'i'
+            assert out_type == 'indices'
+            got_mask[np.ix_(*clu)] = n
+    assert_array_equal(got_mask, want_mask)