Repository: statsmodels/statsmodels

Issue:
streamline linear algebra for linear model
df_model uses a separate svd to get rank of exog

we can use svd/pinv or qr directly to find the rank of exog, but it would need to be moved to fit instead of initialize.

do we need to check jacobian during WLS iteration in GLM and RLM ?

Related: For nonlinear models we know the rank of the Jacobian only in fit, not during `__init__`



Discussion:

Resolving patch:
diff --git a/statsmodels/regression/linear_model.py b/statsmodels/regression/linear_model.py
--- a/statsmodels/regression/linear_model.py
+++ b/statsmodels/regression/linear_model.py
@@ -35,7 +35,7 @@
 from scipy import stats
 from scipy.stats.stats import ss
 from statsmodels.tools.tools import (add_constant, rank,
-                                             recipr, chain_dot)
+                                             recipr, chain_dot, pinv_extended)
 from statsmodels.tools.decorators import (resettable_cache,
         cache_readonly, cache_writable)
 import statsmodels.base.model as base
@@ -69,6 +69,7 @@ def _get_sigma(sigma, nobs):
 
     return sigma, cholsigmainv
 
+
 class RegressionModel(base.LikelihoodModel):
     """
     Base class for linear regression models not used by users.
@@ -80,15 +81,38 @@ def __init__(self, endog, exog, **kwargs):
         self._data_attr.extend(['pinv_wexog', 'wendog', 'wexog', 'weights'])
 
     def initialize(self):
-        #print "calling initialize, now whitening"  #for debugging
         self.wexog = self.whiten(self.exog)
         self.wendog = self.whiten(self.endog)
         # overwrite nobs from class Model:
         self.nobs = float(self.wexog.shape[0])
-        self.rank = rank(self.exog)
-        self.df_model = float(self.rank - self.k_constant)
-        self.df_resid = self.nobs - self.rank
-        self.df_model = float(rank(self.exog) - self.k_constant)
+
+        self._df_model = None
+        self._df_resid = None
+        self.rank = None
+
+    @property
+    def df_model(self):
+        if self._df_model is None:
+            if self.rank is None:
+                self.rank = rank(self.exog)
+            self._df_model = float(self.rank - self.k_constant)
+        return self._df_model
+
+    @df_model.setter
+    def df_model(self, value):
+        self._df_model = value
+
+    @property
+    def df_resid(self):
+        if self._df_resid is None:
+            if self.rank is None:
+                self.rank = rank(self.exog)
+            self._df_resid = self.nobs - self.rank
+        return self._df_resid
+
+    @df_resid.setter
+    def df_resid(self, value):
+        self._df_resid = value
 
     def fit(self, method="pinv", **kwargs):
         """
@@ -117,32 +141,44 @@ def fit(self, method="pinv", **kwargs):
         The fit method uses the pseudoinverse of the design/exogenous variables
         to solve the least squares minimization.
         """
-        exog = self.wexog
-        endog = self.wendog
-
         if method == "pinv":
             if ((not hasattr(self, 'pinv_wexog')) or
-                (not hasattr(self, 'normalized_cov_params'))):
-                #print "recalculating pinv"   #for debugging
-                self.pinv_wexog = pinv_wexog = np.linalg.pinv(self.wexog)
-                self.normalized_cov_params = np.dot(pinv_wexog,
-                                                 np.transpose(pinv_wexog))
-            beta = np.dot(self.pinv_wexog, endog)
+                (not hasattr(self, 'normalized_cov_params')) or
+                (not hasattr(self, 'rank'))):
+
+                self.pinv_wexog, singular_values = pinv_extended(self.wexog)
+                self.normalized_cov_params = np.dot(self.pinv_wexog,
+                                        np.transpose(self.pinv_wexog))
+
+                # Cache these singular values for use later.
+                self.wexog_singular_values = singular_values
+                self.rank = rank(np.diag(singular_values))
+
+            beta = np.dot(self.pinv_wexog, self.wendog)
 
         elif method == "qr":
             if ((not hasattr(self, 'exog_Q')) or
-                (not hasattr(self, 'normalized_cov_params'))):
-                Q, R = np.linalg.qr(exog)
+                (not hasattr(self, 'normalized_cov_params')) or
+                (not hasattr(self, 'rank'))):
+                Q, R = np.linalg.qr(self.wexog)
                 self.exog_Q, self.exog_R = Q, R
                 self.normalized_cov_params = np.linalg.inv(np.dot(R.T, R))
+
+                # Cache singular values from R.
+                self.wexog_singular_values = np.linalg.svd(R, 0, 0)
+                self.rank = rank(R)
             else:
                 Q, R = self.exog_Q, self.exog_R
 
             # used in ANOVA
-            self.effects = effects = np.dot(Q.T, endog)
+            self.effects = effects = np.dot(Q.T, self.wendog)
             beta = np.linalg.solve(R, effects)
 
-            # no upper triangular solve routine in numpy/scipy?
+        if self._df_model is None:
+            self._df_model = float(self.rank - self.k_constant)
+        if self._df_resid is None:
+            self.df_resid = self.nobs - self.rank
+
         if isinstance(self, OLS):
             lfit = OLSResults(self, beta,
                        normalized_cov_params=self.normalized_cov_params)
@@ -168,7 +204,7 @@ def predict(self, params, exog=None):
 
         Notes
         -----
-        If the model as not yet been fit, params is not optional.
+        If the model has not yet been fit, params is not optional.
         """
         #JP: this doesn't look correct for GLMAR
         #SS: it needs its own predict method
@@ -709,6 +745,7 @@ def yule_walker(X, order=1, method="unbiased", df=None, inv=False, demean=True):
     else:
         return rho, np.sqrt(sigmasq)
 
+
 class RegressionResults(base.LikelihoodModelResults):
     """
     This class summarizes the fit of a linear regression model.
@@ -855,6 +892,10 @@ def __init__(self, model, params, normalized_cov_params=None, scale=1.):
                                                  normalized_cov_params,
                                                  scale)
         self._cache = resettable_cache()
+        if hasattr(model, 'wexog_singular_values'):
+            self._wexog_singular_values = model.wexog_singular_values
+        else:
+            self._wexog_singular_values = None
 
     def __str__(self):
         self.summary()
@@ -973,7 +1014,7 @@ def mse_total(self):
         if self.k_constant:
             return self.centered_tss / (self.df_resid + self.df_model)
         else:
-            return self.uncentered_tss/ (self.df_resid + self.df_model)
+            return self.uncentered_tss / (self.df_resid + self.df_model)
 
     @cache_readonly
     def fvalue(self):
@@ -1000,6 +1041,25 @@ def bic(self):
         return (-2 * self.llf + np.log(self.nobs) * (self.df_model +
                                                      self.k_constant))
 
+    @cache_readonly
+    def eigenvals(self):
+        """
+        Return eigenvalues sorted in decreasing order.
+        """
+        if self._wexog_singular_values is not None:
+            eigvals = self._wexog_singular_values ** 2
+        else:
+            eigvals = np.linalg.linalg.eigvalsh(np.dot(self.model.wexog.T, self.model.wexog))
+        return np.sort(eigvals)[::-1]
+
+    @cache_readonly
+    def condition_number(self):
+        """
+        Return condition number of exogenous matrix, calculated as ratio of largest to smallest eigenvalue.
+        """
+        eigvals = self.eigenvals
+        return np.sqrt(eigvals[0]/eigvals[-1])
+
     #TODO: make these properties reset bse
     def _HCCM(self, scale):
         H = np.dot(self.model.pinv_wexog,
@@ -1194,16 +1254,12 @@ def summary(self, yname=None, xname=None, title=None, alpha=.05):
         jb, jbpv, skew, kurtosis = jarque_bera(self.wresid)
         omni, omnipv = omni_normtest(self.wresid)
 
-        #TODO: reuse condno from somewhere else ?
-        #condno = np.linalg.cond(np.dot(self.wexog.T, self.wexog))
-        wexog = self.model.wexog
-        eigvals = np.linalg.linalg.eigvalsh(np.dot(wexog.T, wexog))
-        eigvals = np.sort(eigvals) #in increasing order
-        condno = np.sqrt(eigvals[-1]/eigvals[0])
+        eigvals = self.eigenvals
+        condno = self.condition_number
 
         self.diagn = dict(jb=jb, jbpv=jbpv, skew=skew, kurtosis=kurtosis,
                           omni=omni, omnipv=omnipv, condno=condno,
-                          mineigval=eigvals[0])
+                          mineigval=eigvals[-1])
 
         #TODO not used yet
         #diagn_left_header = ['Models stats']
@@ -1265,12 +1321,12 @@ def summary(self, yname=None, xname=None, title=None, alpha=.05):
         if self.model.exog.shape[0] < self.model.exog.shape[1]:
             wstr = "The input rank is higher than the number of observations."
             etext.append(wstr)
-        if eigvals[0] < 1e-10:
+        if eigvals[-1] < 1e-10:
             wstr = "The smallest eigenvalue is %6.3g. This might indicate "
             wstr += "that there are\n"
             wstr += "strong multicollinearity problems or that the design "
             wstr += "matrix is singular."
-            wstr = wstr % eigvals[0]
+            wstr = wstr % eigvals[-1]
             etext.append(wstr)
         elif condno > 1000:  #TODO: what is recommended
             wstr = "The condition number is large, %6.3g. This might "
@@ -1334,13 +1390,13 @@ def summary2(self, yname=None, xname=None, title=None, alpha=.05,
         from statsmodels.stats.stattools import (jarque_bera,
                                                  omni_normtest,
                                                  durbin_watson)
-        from numpy.linalg import (cond, eigvalsh)
+
         from statsmodels.compatnp.collections import OrderedDict
         jb, jbpv, skew, kurtosis = jarque_bera(self.wresid)
         omni, omnipv = omni_normtest(self.wresid)
         dw = durbin_watson(self.wresid)
-        condno = cond(self.model.wexog)
-        eigvals = eigvalsh(np.dot(self.model.wexog.T, self.model.wexog))
+        eigvals = self.eigenvals
+        condno = self.condition_number
         eigvals = np.sort(eigvals) #in increasing order
         diagnostic = OrderedDict([
                      ('Omnibus:',  "%.3f" % omni),
@@ -1361,10 +1417,10 @@ def summary2(self, yname=None, xname=None, title=None, alpha=.05,
         smry.add_dict(diagnostic)
 
         # Warnings
-        if eigvals[0] < 1e-10:
+        if eigvals[-1] < 1e-10:
             warn = "The smallest eigenvalue is %6.3g. This might indicate that\
             there are strong multicollinearity problems or that the design\
-            matrix is singular." % eigvals[0]
+            matrix is singular." % eigvals[-1]
             smry.add_text(warn)
         if condno > 1000:
             warn = "* The condition number is large (%.g). This might indicate \
diff --git a/statsmodels/regression/quantile_regression.py b/statsmodels/regression/quantile_regression.py
--- a/statsmodels/regression/quantile_regression.py
+++ b/statsmodels/regression/quantile_regression.py
@@ -22,7 +22,7 @@
 import scipy.stats as stats
 from scipy.linalg import pinv
 from scipy.stats import norm
-from statsmodels.tools.tools import chain_dot
+from statsmodels.tools.tools import chain_dot, rank
 from statsmodels.tools.decorators import cache_readonly
 from statsmodels.regression.linear_model import (RegressionModel,
                                                  RegressionResults,
@@ -136,11 +136,14 @@ def fit(self, q=.5, vcov='robust', kernel='epa', bandwidth='hsheather',
         endog = self.endog
         exog = self.exog
         nobs = self.nobs
-        rank = self.rank
+        exog_rank = rank(self.exog)
+        self.rank = exog_rank
+        self.df_model = float(self.rank - self.k_constant)
+        self.df_resid = self.nobs - self.rank
         n_iter = 0
         xstar = exog
 
-        beta = np.ones(rank)
+        beta = np.ones(exog_rank)
         # TODO: better start, initial beta is used only for convergence check
 
         # Note the following doesn't work yet,
@@ -365,12 +368,8 @@ def summary(self, yname=None, xname=None, title=None, alpha=.05):
         jb, jbpv, skew, kurtosis = jarque_bera(self.wresid)
         omni, omnipv = omni_normtest(self.wresid)
 
-        #TODO: reuse condno from somewhere else ?
-        #condno = np.linalg.cond(np.dot(self.wexog.T, self.wexog))
-        wexog = self.model.wexog
-        eigvals = np.linalg.linalg.eigvalsh(np.dot(wexog.T, wexog))
-        eigvals = np.sort(eigvals) #in increasing order
-        condno = np.sqrt(eigvals[-1]/eigvals[0])
+        eigvals = self.eigenvals
+        condno = self.condition_number
 
         self.diagn = dict(jb=jb, jbpv=jbpv, skew=skew, kurtosis=kurtosis,
                           omni=omni, omnipv=omnipv, condno=condno,
@@ -421,12 +420,12 @@ def summary(self, yname=None, xname=None, title=None, alpha=.05):
 
         #add warnings/notes, added to text format only
         etext =[]
-        if eigvals[0] < 1e-10:
+        if eigvals[-1] < 1e-10:
             wstr = "The smallest eigenvalue is %6.3g. This might indicate "
             wstr += "that there are\n"
-            wstr = "strong multicollinearity problems or that the design "
+            wstr += "strong multicollinearity problems or that the design "
             wstr += "matrix is singular."
-            wstr = wstr % eigvals[0]
+            wstr = wstr % eigvals[-1]
             etext.append(wstr)
         elif condno > 1000:  #TODO: what is recommended
             wstr = "The condition number is large, %6.3g. This might "
diff --git a/statsmodels/tools/tools.py b/statsmodels/tools/tools.py
--- a/statsmodels/tools/tools.py
+++ b/statsmodels/tools/tools.py
@@ -345,6 +345,29 @@ def isestimable(C, D):
     return True
 
 
+def pinv_extended(X, rcond=1e-15):
+    """
+    Return the pinv of an array X as well as the singular values
+    used in computation.
+
+    Code adapted from numpy.
+    """
+    X = np.asarray(X)
+    X = X.conjugate()
+    u, s, vt = np.linalg.svd(X, 0)
+    s_orig = np.copy(s)
+    m = u.shape[0]
+    n = vt.shape[1]
+    cutoff = rcond * np.maximum.reduce(s)
+    for i in range(min(n, m)):
+        if s[i] > cutoff:
+            s[i] = 1./s[i]
+        else:
+            s[i] = 0.
+    res = np.dot(np.transpose(vt), np.multiply(s[:, np.core.newaxis], np.transpose(u)))
+    return res, s_orig
+
+
 def recipr(X):
     """
     Return the reciprocal of an array, setting all entries less than or


Test patch:
diff --git a/statsmodels/regression/tests/test_regression.py b/statsmodels/regression/tests/test_regression.py
--- a/statsmodels/regression/tests/test_regression.py
+++ b/statsmodels/regression/tests/test_regression.py
@@ -99,7 +99,7 @@ def test_mse_model(self):
     decimal_mse_total = DECIMAL_4
     def test_mse_total(self):
         assert_almost_equal(self.res1.mse_total, self.res2.mse_total,
-                    self.decimal_mse_total)
+                    self.decimal_mse_total, err_msg="Test class %s" % self)
 
     decimal_fvalue = DECIMAL_4
     def test_fvalue(self):
@@ -596,7 +596,6 @@ def setupClass(cls):
         cls.degen_exog = cls.exog_n_one[:-1]
         cls.mod1 = OLS(cls.endog_n_one, cls.exog_n_one)
         cls.mod1.df_model += 1
-        #cls.mod1.df_resid -= 1
         cls.res1 = cls.mod1.fit()
         # Note that these are created for every subclass..
         # A little extra overhead probably
diff --git a/statsmodels/tools/tests/test_tools.py b/statsmodels/tools/tests/test_tools.py
--- a/statsmodels/tools/tests/test_tools.py
+++ b/statsmodels/tools/tests/test_tools.py
@@ -10,6 +10,8 @@
 
 from statsmodels.datasets import longley
 from statsmodels.tools import tools
+from statsmodels.tools.tools import pinv_extended
+
 
 class TestTools(TestCase):
 
@@ -52,6 +54,23 @@ def test_rank(self):
         X[:,0] = X[:,1] + X[:,2]
         self.assertEquals(tools.rank(X), 9)
 
+    def test_extendedpinv(self):
+        X = standard_normal((40, 10))
+        np_inv = np.linalg.pinv(X)
+        np_sing_vals = np.linalg.svd(X, 0, 0)
+        sm_inv, sing_vals = pinv_extended(X)
+        assert_almost_equal(np_inv, sm_inv)
+        assert_almost_equal(np_sing_vals, sing_vals)
+
+    def test_extendedpinv_singular(self):
+        X = standard_normal((40, 10))
+        X[:, 5] = X[:, 1] + X[:, 3]
+        np_inv = np.linalg.pinv(X)
+        np_sing_vals = np.linalg.svd(X, 0, 0)
+        sm_inv, sing_vals = pinv_extended(X)
+        assert_almost_equal(np_inv, sm_inv)
+        assert_almost_equal(np_sing_vals, sing_vals)
+
     def test_fullrank(self):
         X = standard_normal((40,10))
         X[:,0] = X[:,1] + X[:,2]