Repository: nilearn/nilearn

Issue:
confounds removal
I am unsure about what the signal.clean is supposed to output. Isn't the cleaned signal the residual from the  least squares regression ?
I have been comparing the residuals obtained using scipy.linalg.lstsq and the output of signal.clean. With confounds with full rank, the results match. On the other hand, I have a systhematic mismatch between  the two outputs when the confounds do not have full rank (adding for instance a constant array to confounds or using the same confound twice).



Discussion:
Thanks, this is very kind of you to crawl the code and highlight such problems! Could you provide a small script that reproduce the problem?

It is with great pleasure ! plus I saw it spontaneously working on ADHD since it has a constant confound in the .csv confounds file. This is the code showing regression works well only when full rank confounds

``` Python
import numpy as np
from scipy.linalg import lstsq
from nilearn.signal import clean, _standardize
from nilearn.tests.test_signal import generate_signals

# Generate a normalized signal
length = 30
signal, noise, confounds = generate_signals(n_features=1, n_confounds=5,
                                            length=length)
to_clean = _standardize(signal + noise, detrend=False, normalize=True)

# Generate full rank confounds, with a confound non present in the signal
confounds_full = np.hstack((confounds, np.random.rand(length, 1)))

# Generate non-full rank confounds
confounds_twice = np.hstack((confounds,
                             confounds[:, -1][:, np.newaxis]))
confounds_cst = np.hstack((confounds, np.ones((length, 1))))


# Compare with scipy residuals
for confounds in [confounds_full, confounds_twice, confounds_cst]:
    # Standardize the confounds, as done in signal.clean
    confounds = _standardize(confounds, detrend=False, normalize=True)
    cleaned_signal = clean(to_clean, confounds=confounds, detrend=False,
                           standardize=False)
    beta = lstsq(confounds, to_clean)[0]
    residual = to_clean - confounds.dot(beta)
    print('{0} confounds, rank is {1}, mismatch is {2}'.format(
        confounds.shape[-1],
        np.linalg.matrix_rank(confounds),
        np.max(np.abs(cleaned_signal - residual))))
```


Resolving patch:
diff --git a/nilearn/signal.py b/nilearn/signal.py
--- a/nilearn/signal.py
+++ b/nilearn/signal.py
@@ -41,6 +41,7 @@ def _standardize(signals, detrend=False, normalize=True):
     std_signals: numpy.ndarray
         copy of signals, normalized.
     """
+
     if detrend:
         signals = _detrend(signals, inplace=False)
     else:
@@ -51,9 +52,9 @@ def _standardize(signals, detrend=False, normalize=True):
         return signals
 
     if normalize:
-        # remove mean if not already detrended
         if not detrend:
-            signals -= signals.mean(axis=0)
+            # remove mean if not already detrended
+            signals = signals - signals.mean(axis=0)
 
         std = np.sqrt((signals ** 2).sum(axis=0))
         std[std < np.finfo(np.float).eps] = 1.  # avoid numerical problems
@@ -452,8 +453,22 @@ def clean(signals, detrend=True, standardize=True, confounds=None,
 
         confounds = _ensure_float(confounds)
         confounds = _standardize(confounds, normalize=True, detrend=detrend)
-        Q = linalg.qr(confounds, mode='economic')[0]
-        signals -= np.dot(Q, np.dot(Q.T, signals))
+
+        if (LooseVersion(scipy.__version__) > LooseVersion('0.9.0')):
+            # Pivoting in qr decomposition was added in scipy 0.10
+            Q, R, _ = linalg.qr(confounds, mode='economic', pivoting=True)
+            Q = Q[:, np.abs(np.diag(R)) > np.finfo(np.float).eps * 100.]
+            signals -= Q.dot(Q.T).dot(signals)
+        else:
+            Q, R = linalg.qr(confounds, mode='economic')
+            non_null_diag = np.abs(np.diag(R)) > np.finfo(np.float).eps * 100.
+            if np.all(non_null_diag):
+                signals -= Q.dot(Q.T).dot(signals)
+            elif np.any(non_null_diag):
+                R = R[:, non_null_diag]
+                confounds = confounds[:, non_null_diag]
+                inv = scipy.linalg.inv(np.dot(R.T, R))
+                signals -= confounds.dot(inv).dot(confounds.T).dot(signals)
 
     if low_pass is not None or high_pass is not None:
         signals = butterworth(signals, sampling_rate=1. / t_r,


Test patch:
diff --git a/nilearn/tests/test_signal.py b/nilearn/tests/test_signal.py
--- a/nilearn/tests/test_signal.py
+++ b/nilearn/tests/test_signal.py
@@ -278,9 +278,15 @@ def test_clean_confounds():
 
     # With signal: output must be orthogonal to confounds
     cleaned_signals = nisignal.clean(signals + noises, confounds=confounds,
-                                      detrend=True, standardize=False)
+                                      detrend=False, standardize=True)
     assert_true(abs(np.dot(confounds.T, cleaned_signals)).max() < 1000. * eps)
 
+    # Same output when a constant confound is added
+    confounds1 = np.hstack((np.ones((45, 1)), confounds))
+    cleaned_signals1 = nisignal.clean(signals + noises, confounds=confounds1,
+                                      detrend=False, standardize=True)
+    np.testing.assert_almost_equal(cleaned_signals1, cleaned_signals)
+
     # Test detrending. No trend should exist in the output.
     # Use confounds with a trend.
     temp = confounds.T