Repository: mne-tools/mne-python

Issue:
`high is out of bounds for int64` in cluster permutation test
I've been getting an error in cluster permutation tests when applied on single-trial data, when the number of trials exceeds ~60

FWICT this is due to `max_perms` being too large in cluster_level.py : 
https://github.com/mne-tools/mne-python/blob/cd24596e14383dc978d754a51f8fe9fabbd76b21/mne/stats/cluster_level.py#L761-L768

I can reproduce the error when running [this MNE tutorial](https://github.com/mne-tools/mne-python/blob/master/tutorials/plot_stats_cluster_1samp_test_time_frequency.py), when rejecting no epochs (`Reject=None`). In that case `epochs_power` has shape (72, 16, 49), which isn't extreme, but I'm still getting the following error:

<details><summary>Log output</summary>

```python
Traceback (most recent call last):
  File "/Users/nicolas/Toolboxes/mne-python-nbara/mne/stats/cluster_level.py", line 1163, in permutation_cluster_1samp_test
    buffer_size=buffer_size)
  File "<string>", line 2, in _permutation_cluster_test
  File "/Users/nicolas/Toolboxes/mne-python-nbara/mne/utils.py", line 726, in verbose
    return function(*args, **kwargs)
  File "/Users/nicolas/Toolboxes/mne-python-nbara/mne/stats/cluster_level.py", line 768, in _permutation_cluster_test
    orders = rng.choice(max_perms, n_permutations - 1)
  File "mtrand.pyx", line 1147, in mtrand.RandomState.choice (numpy/random/mtrand/mtrand.c:18069)
  File "mtrand.pyx", line 975, in mtrand.RandomState.randint (numpy/random/mtrand/mtrand.c:16083)
ValueError: high is out of bounds for int64
```

</details>  
 


Discussion:
don't do 2**60 permutations. limit n_permutations to lets say 10000 for a
p-value threshold at 0.01

Mmh but doesn't `n_permutations` default to 1024 in `permutation_cluster_1samp_test` &co?

In the code snippet above, `max_perms` is only a function of the number of samples (in my case, trials)
@agramfort - the problem seems to be different, the realized premutations are chosen from all possible permutations (as an int that is later transformed to its binary representation) and it seems that if there are too many possible permutations `rng.choice` gives errors.
I see my "clever trick" is really a "bug" sometimes :)

We can go back to using resampling with replacement in the non-exact case. (My code changed it to be sampling without replacement.) Sound good? If so I can make a PR
you should not allow replacement in theory AFAIK

Okay I think there is another way to do it that should be safe enough

Resolving patch:
diff --git a/mne/stats/cluster_level.py b/mne/stats/cluster_level.py
--- a/mne/stats/cluster_level.py
+++ b/mne/stats/cluster_level.py
@@ -764,10 +764,32 @@ def _permutation_cluster_test(X, threshold, n_permutations, tail, stat_fun,
             # convert to binary array representation
             orders = np.arange(max_perms)
             extra = ' (exact test)'
-        else:
-            orders = rng.choice(max_perms, n_permutations - 1)
-        orders = [np.fromiter(np.binary_repr(s + 1, n_samples), dtype=int)
-                  for s in orders]
+            orders = [np.fromiter(np.binary_repr(s + 1, n_samples), dtype=int)
+                      for s in orders]
+        elif n_samples <= 20:  # fast way to do it for small(ish) n_samples
+            orders = rng.choice(max_perms, n_permutations - 1, replace=False)
+            orders = [np.fromiter(np.binary_repr(s + 1, n_samples), dtype=int)
+                      for s in orders]
+        else:  # n_samples >= 64
+            # Here we can just use the hash-table (w/collision detection)
+            # functionality of a dict to ensure uniqueness
+            orders = np.zeros((n_permutations - 1, n_samples), int)
+            hashes = {}
+            ii = 0
+            # in the symmetric case, we should never flip one of the subjects
+            # to prevent positive/negative equivalent collisions
+            use_samples = n_samples - (tail == 0)
+            while ii < n_permutations - 1:
+                signs = tuple((rng.rand(use_samples) < 0.5).astype(int))
+                if signs not in hashes:
+                    orders[ii, :use_samples] = signs
+                    if tail == 0 and rng.rand() < 0.5:
+                        # To undo the non-flipping of the last subject in the
+                        # tail == 0 case, half the time we use the positive
+                        # last subject, half the time negative last subject
+                        orders[ii] = 1 - orders[ii]
+                    hashes[signs] = None
+                    ii += 1
     else:
         do_perm_func = _do_permutations
         X_full = np.concatenate(X, axis=0)
@@ -1021,9 +1043,6 @@ def permutation_cluster_test(X, threshold=None, n_permutations=1024,
                                      buffer_size=buffer_size)
 
 
-permutation_cluster_test.__test__ = False
-
-
 @verbose
 def permutation_cluster_1samp_test(X, threshold=None, n_permutations=1024,
                                    tail=0, stat_fun=ttest_1samp_no_p,
@@ -1163,9 +1182,6 @@ def permutation_cluster_1samp_test(X, threshold=None, n_permutations=1024,
                                      buffer_size=buffer_size)
 
 
-permutation_cluster_1samp_test.__test__ = False
-
-
 @verbose
 def spatio_temporal_cluster_1samp_test(X, threshold=None,
                                        n_permutations=1024, tail=0,
@@ -1304,9 +1320,6 @@ def spatio_temporal_cluster_1samp_test(X, threshold=None,
     return out
 
 
-spatio_temporal_cluster_1samp_test.__test__ = False
-
-
 @verbose
 def spatio_temporal_cluster_test(X, threshold=1.67, n_permutations=1024,
                                  tail=0, stat_fun=f_oneway,
@@ -1421,9 +1434,6 @@ def spatio_temporal_cluster_test(X, threshold=1.67, n_permutations=1024,
     return out
 
 
-spatio_temporal_cluster_test.__test__ = False
-
-
 def _st_mask_from_s_inds(n_times, n_vertices, vertices, set_as=True):
     """Compute mask to apply to a spatio-temporal connectivity matrix.
 
diff --git a/mne/stats/permutations.py b/mne/stats/permutations.py
--- a/mne/stats/permutations.py
+++ b/mne/stats/permutations.py
@@ -127,6 +127,7 @@ def permutation_t_test(X, n_permutations=10000, tail=0, n_jobs=1,
     std0 = np.sqrt(X2 - mu0 ** 2) * dof_scaling  # get std with var splitting
     T_obs = np.mean(X, axis=0) / (std0 / sqrt(n_samples))
 
+    # XXX Someday this should be refactored with the cluster code
     if do_exact:
         perms = bin_perm_rep(n_samples, a=1, b=-1)[1:, :]
     else:
@@ -149,8 +150,6 @@ def permutation_t_test(X, n_permutations=10000, tail=0, n_jobs=1,
 
     return T_obs, p_values, H0
 
-permutation_t_test.__test__ = False  # for nosetests
-
 
 def _bootstrap_ci(arr, ci=.95, n_bootstraps=2000, statfun='mean',
                   random_state=None):


Test patch:
diff --git a/mne/stats/tests/test_cluster_level.py b/mne/stats/tests/test_cluster_level.py
--- a/mne/stats/tests/test_cluster_level.py
+++ b/mne/stats/tests/test_cluster_level.py
@@ -84,6 +84,18 @@ def test_cache_dir():
             del os.environ['MNE_MEMMAP_MIN_SIZE']
 
 
+def test_permutation_large_n_samples():
+    """Test that non-replacement works with large N."""
+    X = np.random.RandomState(0).randn(72, 1) + 1
+    for n_samples in (11, 72):
+        tails = (0, 1) if n_samples <= 20 else (0,)
+        for tail in tails:
+            H0 = permutation_cluster_1samp_test(
+                X[:n_samples], threshold=1e-4, tail=tail)[-1]
+            assert H0.shape == (1024,)
+            assert len(np.unique(H0)) >= 1024 - (H0 == 0).sum()
+
+
 def test_permutation_step_down_p():
     """Test cluster level permutations with step_down_p."""
     try: