Repository: nilearn/nilearn

Issue:
z-values from compute_contrast are inaccurate for p-values close to 1
Dear nistats team,
using the compute_contrast function, my colleague and I noticed that if one contrast and its exact inversion are plotted, there are some inconsistencies regarding negative z-scores.


> What version of Nistats are you using?

0.0.1b

> What were you trying to do?

use compute_contrast to make z-score maps

> What did you expect will happen?

If i have a contrast [+1 -1] and a contrast [-1 +1], the z-scores should be inverted, respectively

> What actually happened?

z-scores cannot go lower than -8.2, because the associated p-values (e.g. 0.99999999999999999) are represented as 1, while the corresponding p-values of the inverted contrast (e.g. 0.00000000000000001) are not.
See the following notebook which uses the example "2.1.2. Analysis of a single session, single subject fMRI dataset" from the nistats page to illustrate the issue:

[https://gist.github.com/mwegrzyn/9349842ee2abbff05df735a595bb239a](https://gist.github.com/mwegrzyn/9349842ee2abbff05df735a595bb239a)

Thank you.

Best regards,
Martin Wegrzyn


Discussion:
@bthirion @mwegrzyn I'll start working on this problem. Do you have any further suggestions besides what is reported [here](https://gist.github.com/mwegrzyn/9349842ee2abbff05df735a595bb239a)?
Dear @alpinho,

thank you very much for your message and for working on the issue. Right now, I don't have any further ideas. But if you have anything that you need help with, I would be happy to try and contribute something (maybe tests or documentation; I'm not too experienced with working on software, but am looking for opportunities to gain more experience). Thank you!

Best regards,
Martin

Resolving patch:
diff --git a/nilearn/_utils/glm.py b/nilearn/_utils/glm.py
--- a/nilearn/_utils/glm.py
+++ b/nilearn/_utils/glm.py
@@ -1,6 +1,6 @@
 """ Misc utilities for the library
 
-Authors: Bertrand Thirion, Matthew Brett, 2015
+Authors: Bertrand Thirion, Matthew Brett, Ana Luisa Pinho, 2020
 """
 import csv
 import glob
@@ -143,11 +143,41 @@ def _check_run_tables(run_imgs, tables_, tables_name):
     return tables_
 
 
-def z_score(pvalue):
-    """ Return the z-score corresponding to a given p-value.
+def z_score(pvalue, one_minus_pvalue=None):
+    """ Return the z-score(s) corresponding to certain p-value(s) and,
+    optionally, one_minus_pvalue(s) provided as inputs.
+
+    Parameters
+    ----------
+    pvalue: float or 1-d array shape=(n_pvalues,) computed using
+            the survival function
+
+    one_minus_pvalue: float or
+                      1-d array shape=(n_one_minus_pvalues,), optional;
+                      it shall take the value returned by
+                      /nilearn/glm/contrasts.py::one_minus_pvalue
+                      which computes the p_value using the
+                      cumulative distribution function,
+                      with n_one_minus_pvalues = n_pvalues
+
+    Returns
+    -------
+    z_scores: 1-d array shape=(n_z_scores,), with n_z_scores = n_pvalues
     """
-    pvalue = np.minimum(np.maximum(pvalue, 1.e-300), 1. - 1.e-16)
-    return norm.isf(pvalue)
+    pvalue = np.clip(pvalue, 1.e-300, 1. - 1.e-16)
+    z_scores_sf = norm.isf(pvalue)
+
+    if one_minus_pvalue is not None:
+        one_minus_pvalue = np.clip(one_minus_pvalue, 1.e-300, 1. - 1.e-16)
+        z_scores_cdf = norm.ppf(one_minus_pvalue)
+        z_scores = np.empty(pvalue.size)
+        use_cdf = z_scores_sf < 0
+        use_sf = np.logical_not(use_cdf)
+        z_scores[np.atleast_1d(use_cdf)] = z_scores_cdf[use_cdf]
+        z_scores[np.atleast_1d(use_sf)] = z_scores_sf[use_sf]
+    else:
+        z_scores = z_scores_sf
+    return z_scores
 
 
 def multiple_fast_inverse(a):
diff --git a/nilearn/glm/contrasts.py b/nilearn/glm/contrasts.py
--- a/nilearn/glm/contrasts.py
+++ b/nilearn/glm/contrasts.py
@@ -2,7 +2,7 @@
 This module is for contrast computation and operation on contrast to
 obtain fixed effect results.
 
-Author: Bertrand Thirion, Martin Perez-Guevara, 2016
+Author: Bertrand Thirion, Martin Perez-Guevara, Ana Luisa Pinho 2020
 """
 
 from warnings import warn
@@ -192,6 +192,7 @@ def __init__(self, effect, variance, dim=None, dof=DEF_DOFMAX,
         self.contrast_type = contrast_type
         self.stat_ = None
         self.p_value_ = None
+        self.one_minus_pvalue_ = None
         self.baseline = 0
         self.tiny = tiny
         self.dofmax = dofmax
@@ -236,8 +237,9 @@ def stat(self, baseline=0.0):
         return self.stat_
 
     def p_value(self, baseline=0.0):
-        """Return a parametric estimate of the p-value associated
-        with the null hypothesis: (H0) 'contrast equals baseline'
+        """Return a parametric estimate of the p-value associated with
+        the null hypothesis (H0): 'contrast equals baseline',
+        using the survival function
 
         Parameters
         ----------
@@ -262,6 +264,35 @@ def p_value(self, baseline=0.0):
         self.p_value_ = p_values
         return p_values
 
+    def one_minus_pvalue(self, baseline=0.0):
+        """Return a parametric estimate of the 1 - p-value associated with
+        the null hypothesis (H0): 'contrast equals baseline',
+        using the cumulative distribution function,
+        to ensure numerical stability
+
+        Parameters
+        ----------
+        baseline : float, optional
+            baseline value for the test statistic
+
+        Returns
+        -------
+        one_minus_pvalues : 1-d array, shape=(n_voxels,)
+            one_minus_pvalues, one per voxel
+        """
+        if self.stat_ is None or not self.baseline == baseline:
+            self.stat_ = self.stat(baseline)
+        # Valid conjunction as in Nichols et al, Neuroimage 25, 2005.
+        if self.contrast_type == 't':
+            one_minus_pvalues = sps.t.cdf(self.stat_,
+                                          np.minimum(self.dof, self.dofmax))
+        else:
+            assert self.contrast_type == 'F'
+            one_minus_pvalues = sps.f.cdf(self.stat_, self.dim,
+                                          np.minimum(self.dof, self.dofmax))
+        self.one_minus_pvalue_ = one_minus_pvalues
+        return one_minus_pvalues
+
     def z_score(self, baseline=0.0):
         """Return a parametric estimation of the z-score associated
         with the null hypothesis: (H0) 'contrast equals baseline'
@@ -279,9 +310,12 @@ def z_score(self, baseline=0.0):
         """
         if self.p_value_ is None or not self.baseline == baseline:
             self.p_value_ = self.p_value(baseline)
+        if self.one_minus_pvalue_ is None:
+            self.one_minus_pvalue_ = self.one_minus_pvalue(baseline)
 
         # Avoid inf values kindly supplied by scipy.
-        self.z_score_ = z_score(self.p_value_)
+        self.z_score_ = z_score(self.p_value_,
+                                one_minus_pvalue=self.one_minus_pvalue_)
         return self.z_score_
 
     def __add__(self, other):


Test patch:
diff --git a/nilearn/glm/tests/test_contrasts.py b/nilearn/glm/tests/test_contrasts.py
--- a/nilearn/glm/tests/test_contrasts.py
+++ b/nilearn/glm/tests/test_contrasts.py
@@ -4,7 +4,8 @@
 from sklearn.datasets import make_regression
 from sklearn.linear_model import LinearRegression
 
-from nilearn.glm.contrasts import (_compute_fixed_effect_contrast,
+from nilearn.glm.contrasts import (Contrast,
+                                   _compute_fixed_effect_contrast,
                                    _compute_fixed_effects_params,
                                    compute_contrast,
                                    expression_to_contrast_vector)
@@ -164,3 +165,11 @@ def test_low_level_fixed_effects():
                                                precision_weighted=True)
     assert_almost_equal(Xw, 1.2 * X1)
     assert_almost_equal(Vw, .8 * V1)
+
+
+def test_one_minus_pvalue():
+    effect = np.ones((1, 3))
+    variance = effect[0]
+    contrast = Contrast(effect, variance, contrast_type="t")
+    assert np.allclose(contrast.one_minus_pvalue(), 0.84, 1)
+    assert np.allclose(contrast.stat_, 1., 1)
diff --git a/nilearn/glm/tests/test_utils.py b/nilearn/glm/tests/test_utils.py
--- a/nilearn/glm/tests/test_utils.py
+++ b/nilearn/glm/tests/test_utils.py
@@ -4,18 +4,24 @@
 import numpy as np
 import pandas as pd
 import pytest
+import scipy.stats as sps
 import scipy.linalg as spl
+
 from nibabel.tmpdirs import InTemporaryDirectory
 from numpy.testing import assert_almost_equal, assert_array_almost_equal
 from scipy.stats import norm
 
-from nilearn._utils.data_gen import create_fake_bids_dataset
+from nilearn._utils.data_gen import (create_fake_bids_dataset,
+                                     generate_fake_fmri)
 from nilearn._utils.glm import (_check_and_load_tables,
                                 _check_list_length_match, _check_run_tables,
                                 full_rank, get_bids_files,
                                 get_design_from_fslmat, multiple_fast_inverse,
                                 multiple_mahalanobis, parse_bids_filename,
                                 positive_reciprocal, z_score)
+from nilearn.input_data import NiftiMasker
+from nilearn.glm.first_level import (FirstLevelModel,
+                                     make_first_level_design_matrix)
 
 
 def test_full_rank():
@@ -31,12 +37,103 @@ def test_full_rank():
 
 
 def test_z_score():
-    p = np.random.RandomState(42).uniform(size=10)
-    assert_array_almost_equal(norm.sf(z_score(p)), p)
-    # check the numerical precision
-    for p in [1.e-250, 1 - 1.e-16]:
-        assert_array_almost_equal(z_score(p), norm.isf(p))
-    assert_array_almost_equal(z_score(np.float32(1.e-100)), norm.isf(1.e-300))
+    # ################# Check z-scores computed from t-values #################
+    # Randomly draw samples from the standard Student’s t distribution
+    tval = np.random.RandomState(42).standard_t(10, size=10)
+    # Estimate the p-values using the Survival Function (SF)
+    pval = sps.t.sf(tval, 1e10)
+    # Estimate the p-values using the Cumulative Distribution Function (CDF)
+    cdfval = sps.t.cdf(tval, 1e10)
+    # Set a minimum threshold for p-values to avoid infinite z-scores
+    pval = np.array(np.minimum(np.maximum(pval, 1.e-300), 1. - 1.e-16))
+    cdfval = np.array(np.minimum(np.maximum(cdfval, 1.e-300), 1. - 1.e-16))
+    # Compute z-score from the p-value estimated with the SF
+    zval_sf = norm.isf(pval)
+    # Compute z-score from the p-value estimated with the CDF
+    zval_cdf = norm.ppf(cdfval)
+    # Create the final array of z-scores, ...
+    zval = np.zeros(pval.size)
+    # ... in which z-scores < 0 estimated w/ SF are replaced by z-scores < 0
+    # estimated w/ CDF
+    zval[np.atleast_1d(zval_sf < 0)] = zval_cdf[zval_sf < 0]
+    # ... and z-scores >=0 estimated from SF are kept.
+    zval[np.atleast_1d(zval_sf >= 0)] = zval_sf[zval_sf >= 0]
+    # Test 'z_score' function in 'nilearn/glm/contrasts.py'
+    assert_array_almost_equal(z_score(pval, one_minus_pvalue=cdfval), zval)
+    # Test 'z_score' function in 'nilearn/glm/contrasts.py',
+    # when one_minus_pvalue is None
+    assert_array_almost_equal(norm.sf(z_score(pval)), pval)
+    # ################# Check z-scores computed from F-values #################
+    # Randomly draw samples from the F distribution
+    fval = np.random.RandomState(42).f(1, 48, size=10)
+    # Estimate the p-values using the Survival Function (SF)
+    p_val = sps.f.sf(fval, 42, 1e10)
+    # Estimate the p-values using the Cumulative Distribution Function (CDF)
+    cdf_val = sps.f.cdf(fval, 42, 1e10)
+    # Set a minimum threshold for p-values to avoid infinite z-scores
+    p_val = np.array(np.minimum(np.maximum(p_val, 1.e-300), 1. - 1.e-16))
+    cdf_val = np.array(np.minimum(np.maximum(cdf_val, 1.e-300), 1. - 1.e-16))
+    # Compute z-score from the p-value estimated with the SF
+    z_val_sf = norm.isf(p_val)
+    # Compute z-score from the p-value estimated with the CDF
+    z_val_cdf = norm.ppf(cdf_val)
+    # Create the final array of z-scores, ...
+    z_val = np.zeros(p_val.size)
+    # ... in which z-scores < 0 estimated w/ SF are replaced by z-scores < 0
+    # estimated w/ CDF
+    z_val[np.atleast_1d(z_val_sf < 0)] = z_val_cdf[z_val_sf < 0]
+    # ... and z-scores >=0 estimated from SF are kept.
+    z_val[np.atleast_1d(z_val_sf >= 0)] = z_val_sf[z_val_sf >= 0]
+    # Test 'z_score' function in 'nilearn/glm/contrasts.py'
+    assert_array_almost_equal(z_score(p_val, one_minus_pvalue=cdf_val), z_val)
+    # Test 'z_score' function in 'nilearn/glm/contrasts.py',
+    # when one_minus_pvalue is None
+    assert_array_almost_equal(norm.sf(z_score(p_val)), p_val)
+    # ##################### Check the numerical precision #####################
+    for t in [33.75, -8.3]:
+        p = sps.t.sf(t, 1e10)
+        cdf = sps.t.cdf(t, 1e10)
+        z_sf = norm.isf(p)
+        z_cdf = norm.ppf(cdf)
+        if p <= .5:
+            z = z_sf
+        else:
+            z = z_cdf
+        assert_array_almost_equal(z_score(p, one_minus_pvalue=cdf), z)
+
+
+def test_z_score_opposite_contrast():
+    fmri, mask = generate_fake_fmri(shape=(50, 20, 50), length=96,
+                                    rand_gen=np.random.RandomState(42))
+
+    nifti_masker = NiftiMasker(mask_img=mask)
+    data = nifti_masker.fit_transform(fmri)
+
+    frametimes = np.linspace(0, (96 - 1) * 2, 96)
+
+    for i in [0, 20]:
+        design_matrix = make_first_level_design_matrix(
+            frametimes, hrf_model='spm',
+            add_regs=np.array(data[:, i]).reshape(-1, 1))
+        c1 = np.array([1] + [0] * (design_matrix.shape[1] - 1))
+        c2 = np.array([0] + [1] + [0] * (design_matrix.shape[1] - 2))
+        contrasts = {'seed1 - seed2': c1 - c2, 'seed2 - seed1': c2 - c1}
+        fmri_glm = FirstLevelModel(t_r=2., 
+                                   noise_model='ar1', 
+                                   standardize=False, 
+                                   hrf_model='spm', 
+                                   drift_model='cosine')
+        fmri_glm.fit(fmri, design_matrices=design_matrix)
+        z_map_seed1_vs_seed2 = fmri_glm.compute_contrast(
+            contrasts['seed1 - seed2'], output_type='z_score')
+        z_map_seed2_vs_seed1 = fmri_glm.compute_contrast(
+            contrasts['seed2 - seed1'], output_type='z_score')
+        assert_almost_equal(z_map_seed1_vs_seed2.get_data().min(),
+                            -z_map_seed2_vs_seed1.get_data().max(),
+                            decimal=10)
+        assert_almost_equal(z_map_seed1_vs_seed2.get_data().max(),
+                            -z_map_seed2_vs_seed1.get_data().min(),
+                            decimal=10)
 
 
 def test_mahalanobis():