Repository: statsmodels/statsmodels

Issue:
GLM.fit does not allow start_params
GLM fit signature should allow starting parameters to be specified. This came up as I was reading about how IRLS is not a robust fitting algorithm.

http://www.r-bloggers.com/how-robust-is-logistic-regression/



Discussion:
reading a bit how the initial estimate works, I think
`mu = self.family.starting_mu(self.endog)`
should depend on data_weights. It uses y.mean() instead of weighted mean in Binomial.

an example that breaks with the default starting values is at a comment on the original post
http://www.win-vector.com/blog/2012/08/how-robust-is-logistic-regression/

discrete Logit fails with singular matrix exception when using the default newton method.

method='ncg' converges very fast (10 iterations) to the correct solution (result of R glm with adjusted starting values)

example script for the case in the comment

```
# -*- coding: utf-8 -*-
"""

Created on Sat Aug 25 10:14:55 2012
Author: Josef Perktold

convergence failure of GLM Binomial and Logit
example based on comment in 
http://www.win-vector.com/blog/2012/08/how-robust-is-logistic-regression/

Note: example has one value with predicted probability zero
>>> (reslogit.predict() < 1e-10).sum()
1
>>> reslogit.predict().min()
9.6285423080907814e-233

method='ncg' looks good

"""

import numpy as np
from numpy.testing import assert_almost_equal, assert_equal

import statsmodels.api as sm

y2 = np.array('0 1 0 0 0 1'.split(), int)
wt = np.array([50,1,50,1,5,10])
y2 = np.repeat(y2, wt)
x2 = np.repeat([0,0,0.001,100,-1,-1], wt)
assert_equal(y2.sum(), 11)
assert_equal(len(y2), 117)
#y2.mean() is 11./117 = 0.094017094017094016

resglm2 = sm.GLM(y2, sm.add_constant(x2, prepend=True), family=sm.families.Binomial()).fit()
print resglm2.params #nan

try:
    reslogit = sm.Logit(y2, sm.add_constant(x2, prepend=True)).fit()
except np.linalg.linalg.LinAlgError:
    print 'numpy.linalg.linalg.LinAlgError'
#numpy.linalg.linalg.LinAlgError: Singular matrix

reslogit = sm.Logit(y2, sm.add_constant(x2, prepend=True)).fit(method='nm')
#Warning: Maximum number of iterations has been exceeded
reslogit = sm.Logit(y2, sm.add_constant(x2, prepend=True)).fit(method='ncg')
#from R glm with starting values (-4, -5)
#prints:
rglm_params = [-4.603050220003291, -5.296345452725506]
rglm_aic = 34.31049560846446
assert_almost_equal(reslogit.params, rglm_params, decimal=8)
assert_almost_equal(reslogit.aic, rglm_aic, decimal=12)

#with starting values
reslogit_s = sm.Logit(y2, sm.add_constant(x2, prepend=True)).fit(
                                        start_params=[-4,-5])

assert_almost_equal(reslogit_s.params, rglm_params, decimal=8)
assert_almost_equal(reslogit_s.aic, rglm_aic, decimal=12)
```


Resolving patch:
diff --git a/statsmodels/genmod/generalized_linear_model.py b/statsmodels/genmod/generalized_linear_model.py
--- a/statsmodels/genmod/generalized_linear_model.py
+++ b/statsmodels/genmod/generalized_linear_model.py
@@ -20,8 +20,7 @@
 
 import numpy as np
 from . import families
-from statsmodels.tools.decorators import (cache_readonly,
-        resettable_cache)
+from statsmodels.tools.decorators import cache_readonly, resettable_cache
 
 import statsmodels.base.model as base
 import statsmodels.regression.linear_model as lm
@@ -32,12 +31,14 @@
 
 __all__ = ['GLM']
 
+
 def _check_convergence(criterion, iteration, tol, maxiter):
     return not ((np.fabs(criterion[iteration] - criterion[iteration-1]) > tol)
-            and iteration <= maxiter)
+                and iteration <= maxiter)
+
 
 class GLM(base.LikelihoodModel):
-    __doc__ = '''
+    __doc__ = """
     Generalized Linear Models class
 
     GLM inherits from statsmodels.LikelihoodModel
@@ -182,10 +183,10 @@ class GLM(base.LikelihoodModel):
         The value of the weights after the last iteration of fit.  Only
         available after fit is called.  See statsmodels.families.family for
         the specific distribution weighting functions.
-    ''' % {'extra_params' : base._missing_param_doc}
+    """ % {'extra_params' : base._missing_param_doc}
 
     def __init__(self, endog, exog, family=None, offset=None, exposure=None,
-                        missing='none'):
+                 missing='none'):
         self._check_inputs(family, offset, exposure, endog)
         super(GLM, self).__init__(endog, exog, missing=missing,
                                   offset=self.offset, exposure=self.exposure)
@@ -208,7 +209,7 @@ def initialize(self):
 
         self.pinv_wexog = np.linalg.pinv(self.exog)
         self.normalized_cov_params = np.dot(self.pinv_wexog,
-                                        np.transpose(self.pinv_wexog))
+                                            np.transpose(self.pinv_wexog))
 
         self.df_model = np_matrix_rank(self.exog)-1
         self.df_resid = self.exog.shape[0] - np_matrix_rank(self.exog)
@@ -288,8 +289,8 @@ def estimate_scale(self, mu):
                 return 1.
             else:
                 resid = self.endog - mu
-                return ((np.power(resid, 2) / self.family.variance(mu)).sum() \
-                    / self.df_resid)
+                return ((np.power(resid, 2) / self.family.variance(mu)).sum()
+                        / self.df_resid)
 
         if isinstance(self.scaletype, float):
             return np.array(self.scaletype)
@@ -297,17 +298,17 @@ def estimate_scale(self, mu):
         if isinstance(self.scaletype, str):
             if self.scaletype.lower() == 'x2':
                 resid = self.endog - mu
-                return ((np.power(resid, 2) / self.family.variance(mu)).sum() \
-                    / self.df_resid)
+                return ((np.power(resid, 2) / self.family.variance(mu)).sum()
+                        / self.df_resid)
             elif self.scaletype.lower() == 'dev':
                 return self.family.deviance(self.endog, mu)/self.df_resid
             else:
-                raise ValueError("Scale %s with type %s not understood" %\
-                    (self.scaletype,type(self.scaletype)))
+                raise ValueError("Scale %s with type %s not understood" %
+                                 (self.scaletype, type(self.scaletype)))
 
         else:
-            raise ValueError("Scale %s with type %s not understood" %\
-                (self.scaletype, type(self.scaletype)))
+            raise ValueError("Scale %s with type %s not understood" %
+                             (self.scaletype, type(self.scaletype)))
 
     def predict(self, params, exog=None, linear=False):
         """
@@ -335,11 +336,12 @@ def predict(self, params, exog=None, linear=False):
         if linear:
             return np.dot(exog, params) + offset + exposure
         else:
-            return self.family.fitted(np.dot(exog, params) + exposure + \
-                                                             offset)
+            return self.family.fitted(np.dot(exog, params) + exposure +
+                                      offset)
 
-    def fit(self, maxiter=100, method='IRLS', tol=1e-8, scale=None):
-        '''
+    def fit(self, start_params=None, maxiter=100, method='IRLS', tol=1e-8,
+            scale=None):
+        """
         Fits a generalized linear model for a given family.
 
         parameters
@@ -358,16 +360,20 @@ def fit(self, maxiter=100, method='IRLS', tol=1e-8, scale=None):
             `dev` is the deviance divided by df_resid
         tol : float
             Convergence tolerance.  Default is 1e-8.
-        '''
+        start_params : array-like, optional
+            Initial guess of the solution for the loglikelihood maximization.
+            The default is family-specific and is given by the
+            ``family.starting_mu(endog)``. If start_params is given then the
+            initial mean will be calculated as ``np.dot(exog, start_params)``.
+        """
         endog = self.endog
         if endog.ndim > 1 and endog.shape[1] == 2:
-            data_weights = endog.sum(1) # weights are total trials
+            data_weights = endog.sum(1)  # weights are total trials
         else:
             data_weights = np.ones((endog.shape[0]))
         self.data_weights = data_weights
-        if np.shape(self.data_weights) == () and self.data_weights>1:
-            self.data_weights = self.data_weights *\
-                    np.ones((endog.shape[0]))
+        if np.shape(self.data_weights) == () and self.data_weights > 1:
+            self.data_weights = self.data_weights * np.ones((endog.shape[0]))
         self.scaletype = scale
         if isinstance(self.family, families.Binomial):
         # this checks what kind of data is given for Binomial.
@@ -383,8 +389,11 @@ def fit(self, maxiter=100, method='IRLS', tol=1e-8, scale=None):
             offset = 0
         #TODO: would there ever be both and exposure and an offset?
 
-        mu = self.family.starting_mu(self.endog)
         wlsexog = self.exog
+        if start_params is None:
+            mu = self.family.starting_mu(self.endog)
+        else:
+            mu = self.family.fitted(np.dot(wlsexog, start_params))
         eta = self.family.predict(mu)
         dev = self.family.deviance(self.endog, mu)
         if np.isnan(dev):
@@ -392,17 +401,16 @@ def fit(self, maxiter=100, method='IRLS', tol=1e-8, scale=None):
                              "returned a nan.  This could be a boundary "
                              " problem and should be reported.")
 
-
         # first guess on the deviance is assumed to be scaled by 1.
         # params are none to start, so they line up with the deviance
-        history = dict(params = [None, None], deviance=[np.inf,dev])
+        history = dict(params=[None, start_params], deviance=[np.inf, dev])
         iteration = 0
         converged = 0
         criterion = history['deviance']
         while not converged:
             self.weights = data_weights*self.family.weights(mu)
-            wlsendog = eta + self.family.link.deriv(mu) * (self.endog-mu) \
-                - offset
+            wlsendog = (eta + self.family.link.deriv(mu) * (self.endog-mu)
+                        - offset)
             wls_results = lm.WLS(wlsendog, wlsexog, self.weights).fit()
             eta = np.dot(self.exog, wls_results.params) + offset
             mu = self.family.fitted(eta)
@@ -412,8 +420,7 @@ def fit(self, maxiter=100, method='IRLS', tol=1e-8, scale=None):
             if endog.squeeze().ndim == 1 and np.allclose(mu - endog, 0):
                 msg = "Perfect separation detected, results not available"
                 raise PerfectSeparationError(msg)
-            converged = _check_convergence(criterion, iteration, tol,
-                                            maxiter)
+            converged = _check_convergence(criterion, iteration, tol, maxiter)
         self.mu = mu
         glm_results = GLMResults(self, wls_results.params,
                                  wls_results.normalized_cov_params,
@@ -422,8 +429,9 @@ def fit(self, maxiter=100, method='IRLS', tol=1e-8, scale=None):
         glm_results.fit_history = history
         return GLMResultsWrapper(glm_results)
 
+
 class GLMResults(base.LikelihoodModelResults):
-    '''
+    """
     Class to contain GLM results.
 
     GLMResults inherits from statsmodels.LikelihoodModelResults
@@ -457,7 +465,8 @@ class GLMResults(base.LikelihoodModelResults):
         dot(exog, params)
     llf : float
         Value of the loglikelihood function evalued at params.
-        See statsmodels.families.family for distribution-specific loglikelihoods.
+        See statsmodels.families.family for distribution-specific
+        loglikelihoods.
     model : class instance
         Pointer to GLM model instance that called fit.
     mu : array
@@ -507,11 +516,12 @@ class GLMResults(base.LikelihoodModelResults):
     See Also
     --------
     statsmodels.LikelihoodModelResults
-    '''
+    """
 
     def __init__(self, model, params, normalized_cov_params, scale):
         super(GLMResults, self).__init__(model, params,
-                normalized_cov_params=normalized_cov_params, scale=scale)
+                                         normalized_cov_params=
+                                         normalized_cov_params, scale=scale)
         self.family = model.family
         self._endog = model.endog
         self.nobs = model.endog.shape[0]
@@ -530,8 +540,8 @@ def resid_response(self):
 
     @cache_readonly
     def resid_pearson(self):
-        return np.sqrt(self._data_weights) * (self._endog-self.mu)/\
-                        np.sqrt(self.family.variance(self.mu))
+        return (np.sqrt(self._data_weights) * (self._endog-self.mu) /
+                np.sqrt(self.family.variance(self.mu)))
 
     @cache_readonly
     def resid_working(self):
@@ -547,10 +557,9 @@ def resid_anscombe(self):
     def resid_deviance(self):
         return self.family.resid_dev(self._endog, self.mu)
 
-
     @cache_readonly
     def pearson_chi2(self):
-        chisq =  (self._endog- self.mu)**2 / self.family.variance(self.mu)
+        chisq = (self._endog - self.mu)**2 / self.family.variance(self.mu)
         chisq *= self._data_weights
         chisqsum = np.sum(chisq)
         return chisqsum
@@ -569,7 +578,7 @@ def null(self):
                        family=self.family).fit().mu
         elif hasattr(model, 'exposure'):
             return GLM(endog, exog, exposure=model.exposure,
-                    family=self.family).fit().mu
+                       family=self.family).fit().mu
         else:
             wls_model = lm.WLS(endog, exog, weights=self._data_weights)
             return wls_model.fit().fittedvalues
@@ -586,11 +595,10 @@ def null_deviance(self):
     def llf(self):
         _modelfamily = self.family
         if isinstance(_modelfamily, families.NegativeBinomial):
-            val = _modelfamily.loglike(self.model.endog,
-                        fittedvalues = np.dot(self.model.exog,self.params))
+            XB = np.dot(self.model.exog, self.params)
+            val = _modelfamily.loglike(self.model.endog, fittedvalues=XB)
         else:
-            val = _modelfamily.loglike(self._endog, self.mu,
-                                    scale=self.scale)
+            val = _modelfamily.loglike(self._endog, self.mu, scale=self.scale)
         return val
 
     @cache_readonly
@@ -604,7 +612,7 @@ def bic(self):
     def remove_data(self):
         #GLM has alias/reference in result instance
         self._data_attr.extend([i for i in self.model._data_attr
-                                           if not '_data.' in i])
+                                if not '_data.' in i])
         super(self.__class__, self).remove_data()
 
         #TODO: what are these in results?
@@ -613,9 +621,9 @@ def remove_data(self):
 
     remove_data.__doc__ = base.LikelihoodModelResults.remove_data.__doc__
 
-
     def summary(self, yname=None, xname=None, title=None, alpha=.05):
-        """Summarize the Regression Results
+        """
+        Summarize the Regression Results
 
         Parameters
         -----------
@@ -643,14 +651,15 @@ def summary(self, yname=None, xname=None, title=None, alpha=.05):
         """
 
         top_left = [('Dep. Variable:', None),
-                  ('Model:', None),
-                  ('Model Family:', [self.family.__class__.__name__]),
-                  ('Link Function:', [self.family.link.__class__.__name__]),
-                  ('Method:', ['IRLS']),
-                  ('Date:', None),
-                  ('Time:', None),
-                  ('No. Iterations:', ["%d" % self.fit_history['iteration']]),
-                  ]
+                    ('Model:', None),
+                    ('Model Family:', [self.family.__class__.__name__]),
+                    ('Link Function:', [self.family.link.__class__.__name__]),
+                    ('Method:', ['IRLS']),
+                    ('Date:', None),
+                    ('Time:', None),
+                    ('No. Iterations:',
+                     ["%d" % self.fit_history['iteration']]),
+                    ]
 
         top_right = [('No. Observations:', None),
                      ('Df Residuals:', None),
@@ -667,10 +676,10 @@ def summary(self, yname=None, xname=None, title=None, alpha=.05):
         #create summary tables
         from statsmodels.iolib.summary import Summary
         smry = Summary()
-        smry.add_table_2cols(self, gleft=top_left, gright=top_right, #[],
-                          yname=yname, xname=xname, title=title)
+        smry.add_table_2cols(self, gleft=top_left, gright=top_right,  # [],
+                             yname=yname, xname=xname, title=title)
         smry.add_table_params(self, yname=yname, xname=xname, alpha=alpha,
-                             use_t=True)
+                              use_t=True)
 
         #diagnostic table is not used yet:
         #smry.add_table_2cols(self, gleft=diagn_left, gright=diagn_right,
@@ -679,9 +688,8 @@ def summary(self, yname=None, xname=None, title=None, alpha=.05):
 
         return smry
 
-
     def summary2(self, yname=None, xname=None, title=None, alpha=.05,
-                float_format="%.4f"):
+                 float_format="%.4f"):
 
         """Experimental summary for regression Results
 
@@ -715,7 +723,7 @@ def summary2(self, yname=None, xname=None, title=None, alpha=.05,
         from statsmodels.iolib import summary2
         smry = summary2.Summary()
         smry.add_base(results=self, alpha=alpha, float_format=float_format,
-                xname=xname, yname=yname, title=title)
+                      xname=xname, yname=yname, title=title)
 
         return smry
 
@@ -743,7 +751,6 @@ class GLMResultsWrapper(lm.RegressionResultsWrapper):
 ##    print(GLMT[2])
     GLMTp = GLMmod.summary(title='Test GLM')
 
-
     """
 From Stata
 . webuse beetle


Test patch:
diff --git a/statsmodels/genmod/tests/test_glm.py b/statsmodels/genmod/tests/test_glm.py
--- a/statsmodels/genmod/tests/test_glm.py
+++ b/statsmodels/genmod/tests/test_glm.py
@@ -519,6 +519,38 @@ def test_attribute_writable_resettable():
     glm_model2 = sm.GLM(endog, exog)
     assert_equal(glm_model2.family.link.power, 1.0)
 
+
+class Test_start_params(CheckModelResultsMixin):
+    def __init__(self):
+        '''
+        Test Gaussian family with canonical identity link
+        '''
+        # Test Precisions
+        self.decimal_resids = DECIMAL_3
+        self.decimal_params = DECIMAL_2
+        self.decimal_bic = DECIMAL_0
+        self.decimal_bse = DECIMAL_3
+
+        from statsmodels.datasets.longley import load
+        self.data = load()
+        self.data.exog = add_constant(self.data.exog, prepend=False)
+        params = sm.OLS(self.data.endog, self.data.exog).fit().params
+        self.res1 = GLM(self.data.endog, self.data.exog,
+                        family=sm.families.Gaussian()).fit(start_params=params)
+        from .results.results_glm import Longley
+        self.res2 = Longley()
+
+
+def test_glm_start_params():
+    # see 1604
+    y2 = np.array('0 1 0 0 0 1'.split(), int)
+    wt = np.array([50,1,50,1,5,10])
+    y2 = np.repeat(y2, wt)
+    x2 = np.repeat([0,0,0.001,100,-1,-1], wt)
+    mod = sm.GLM(y2, sm.add_constant(x2), family=sm.families.Binomial())
+    res = mod.fit(start_params=[-4, -5])
+    np.testing.assert_almost_equal(res.params, [-4.60305022, -5.29634545], 6)
+
 if __name__=="__main__":
     #run_module_suite()
     #taken from Fernando Perez: