Repository: statsmodels/statsmodels

Issue:
BUG/Design: integer overflow when endog is int
In countmodels the endog is int32 in a simple example. We don't currently promote to float.

In GeneralizedPoisson, the hessian produces wrong results in some cases for dldada because of integer overflow.

Is there a problem if we always promote endog in count models to float64? or at least promote if int?

The problem shows up if there are some computation that only involve endog. e.g.
`y**2 * (y - 1) ` overflows, while `y**2. * (y - 1.)` looks fine

(also never trust an integer power)

(several hours of debugging to see what causes the "sometimes" bug)


Discussion:
For now I promote dtype is CountModels. With the exp anything less than float64 is too fragile and prone to overflowing. (i.e. no lower memory dtypes are allowed)

```
       # promote dtype to float64 if needed
        dt = np.promote_types(self.endog.dtype, np.float64)
        self.endog = np.asarray(self.endog, dt)
        dt = np.promote_types(self.exog.dtype, np.float64)
        self.exog = np.asarray(self.exog, dt)
```

changes will be included in upcoming `llnull` PR

Resolving patch:
diff --git a/statsmodels/discrete/discrete_model.py b/statsmodels/discrete/discrete_model.py
--- a/statsmodels/discrete/discrete_model.py
+++ b/statsmodels/discrete/discrete_model.py
@@ -114,6 +114,17 @@
     trimmed : Boolean array
         trimmed[i] == True if the ith parameter was trimmed from the model."""
 
+_get_start_params_null_docs = """
+Compute one-step moment estimator for null (constant-only) model
+
+This is a preliminary estimator used as start_params.
+
+Returns
+-------
+params : ndarray
+    parameter estimate based one one-step moment matching
+
+"""
 
 # helper for MNLogit (will be generally useful later)
 
@@ -725,6 +736,13 @@ def __init__(self, endog, exog, offset=None, exposure=None, missing='none',
         if exposure is None:
             delattr(self, 'exposure')
 
+        # promote dtype to float64 if needed
+        dt = np.promote_types(self.endog.dtype, np.float64)
+        self.endog = np.asarray(self.endog, dt)
+        dt = np.promote_types(self.exog.dtype, np.float64)
+        self.exog = np.asarray(self.exog, dt)
+
+
     def _check_inputs(self, offset, exposure, endog):
         if offset is not None and offset.shape[0] != endog.shape[0]:
             raise ValueError("offset is not the same length as endog")
@@ -993,6 +1011,15 @@ def loglikeobs(self, params):
         #np.sum(stats.poisson.logpmf(endog, np.exp(XB)))
         return -np.exp(XB) +  endog*XB - gammaln(endog+1)
 
+    def _get_start_params_null(self):
+        offset = getattr(self, "offset", 0)
+        exposure = getattr(self, "exposure", 0)
+        const = (self.endog / np.exp(offset + exposure)).mean()
+        params = [np.log(const)]
+        return params
+
+    _get_start_params_null.__doc__ = _get_start_params_null_docs
+
     def fit(self, start_params=None, method='newton', maxiter=35,
             full_output=1, disp=1, callback=None, **kwargs):
         cntfit = super(CountModel, self).fit(start_params=start_params,
@@ -1287,6 +1314,22 @@ def loglikeobs(self, params):
         return (np.log(mu) + (endog - 1) * np.log(a2) - endog *
                 np.log(a1) - gammaln(endog + 1) - a2 / a1)
 
+    def _get_start_params_null(self):
+        offset = getattr(self, "offset", 0)
+        exposure = getattr(self, "exposure", 0)
+        q = self.parameterization
+
+        const = (self.endog / np.exp(offset + exposure)).mean()
+        params = [np.log(const)]
+        mu = const * np.exp(offset + exposure)
+        resid = self.endog - mu
+        a = ((np.abs(resid) / np.sqrt(mu) - 1) * mu**(-q)).mean()
+        params.append(a)
+
+        return np.array(params)
+
+    _get_start_params_null.__doc__ = _get_start_params_null_docs
+
     def fit(self, start_params=None, method='bfgs', maxiter=35,
             full_output=1, disp=1, callback=None, use_transparams = False,
             cov_type='nonrobust', cov_kwds=None, use_t=None, **kwargs):
@@ -1508,7 +1551,7 @@ def hessian(self, params):
         hess_arr[:-1,-1] = dldpda
 
         # for dl/dalpha dalpha
-        dldada = mu_p**2 * (3 * y / a1**2 - y**2 * (y - 1) / a2**2 - 2 * a2 /
+        dldada = mu_p**2 * (3 * y / a1**2 - (y / a2)**2. * (y - 1) - 2 * a2 /
                             a1**3)
 
         hess_arr[-1,-1] = dldada.sum()
@@ -2621,13 +2664,30 @@ def score_obs(self, params):
         sc = approx_fprime_cs(params, self.loglikeobs)
         return sc
 
+    def _get_start_params_null(self):
+        offset = getattr(self, "offset", 0)
+        exposure = getattr(self, "exposure", 0)
+        const = (self.endog / np.exp(offset + exposure)).mean()
+        params = [np.log(const)]
+        mu = const * np.exp(offset + exposure)
+        resid = self.endog - mu
+        if self.loglike_method == 'nb2':
+            #params.append(np.linalg.pinv(mu[:,None]).dot(resid**2 / mu - 1))
+            a = ((resid**2 / mu - 1) / mu).mean()
+            params.append(a)
+        else: #self.loglike_method == 'nb1':
+            params.append((resid**2 / mu - 1).mean())
+        return np.array(params)
+
+    _get_start_params_null.__doc__ = _get_start_params_null_docs
+
     def fit(self, start_params=None, method='bfgs', maxiter=35,
             full_output=1, disp=1, callback=None,
             cov_type='nonrobust', cov_kwds=None, use_t=None, **kwargs):
 
         # Note: don't let super handle robust covariance because it has
         # transformed params
-
+        self._transparams = False # always define attribute
         if self.loglike_method.startswith('nb') and method not in ['newton',
                                                                    'ncg']:
             self._transparams = True # in case same Model instance is refit
@@ -2644,6 +2704,11 @@ def fit(self, start_params=None, method='bfgs', maxiter=35,
             start_params = mod_poi.fit(disp=0).params
             if self.loglike_method.startswith('nb'):
                 start_params = np.append(start_params, 0.1)
+        else:
+            if self._transparams is True:
+                # transform user provided start_params dispersion, see #3918
+                start_params = np.array(start_params, copy=True)
+                start_params[-1] = np.log(start_params[-1])
 
         if callback is None:
             # work around perfect separation callback #3895
@@ -2946,6 +3011,22 @@ def hessian(self, params):
 
         return hess_arr
 
+    def _get_start_params_null(self):
+        offset = getattr(self, "offset", 0)
+        exposure = getattr(self, "exposure", 0)
+        q = self.parameterization - 1
+
+        const = (self.endog / np.exp(offset + exposure)).mean()
+        params = [np.log(const)]
+        mu = const * np.exp(offset + exposure)
+        resid = self.endog - mu
+        a = ((resid**2 / mu - 1) * mu**(-q)).mean()
+        params.append(a)
+
+        return np.array(params)
+
+    _get_start_params_null.__doc__ = _get_start_params_null_docs
+
     def fit(self, start_params=None, method='bfgs', maxiter=35,
             full_output=1, disp=1, callback=None, use_transparams = False,
             cov_type='nonrobust', cov_kwds=None, use_t=None, **kwargs):
@@ -3166,6 +3247,47 @@ def llr(self):
     def llr_pvalue(self):
         return stats.chisqprob(self.llr, self.df_model)
 
+    def set_null_options(self, llnull=None, attach_results=True, **kwds):
+        """set fit options for Null (constant-only) model
+
+        This resets the cache for related attributes which is potentially
+        fragile. This only sets the option, the null model is estimated
+        when llnull is accessed, if llnull is not yet in cache.
+
+        Parameters
+        ----------
+        llnull : None or float
+            If llnull is not None, then the value will be directly assigned to
+            the cached attribute "llnull".
+        attach_results : bool
+            Sets an internal flag whether the results instance of the null
+            model should be attached. By default without calling this method,
+            thenull model results are not attached and only the loglikelihood
+            value llnull is stored.
+        kwds : keyword arguments
+            `kwds` are directly used as fit keyword arguments for the null
+            model, overriding any provided defaults.
+
+        Returns
+        -------
+        no returns, modifies attributes of this instance
+
+        """
+        # reset cache, note we need to add here anything that depends on
+        # llnullor the null model. If something is missing, then the attribute
+        # might be incorrect.
+        self._cache.pop('llnull', None)
+        self._cache.pop('llr', None)
+        self._cache.pop('llr_pvalue', None)
+        self._cache.pop('prsquared', None)
+        if hasattr(self, 'res_null'):
+            del self.res_null
+
+        if llnull is not None:
+            self._cache['llnull'] = llnull
+        self._attach_nullmodel = attach_results
+        self._optim_kwds_null = kwds
+
     @cache_readonly
     def llnull(self):
 
@@ -3178,8 +3300,33 @@ def llnull(self):
         # TODO: consider catching and warning on convergence failure?
         # in the meantime, try hard to converge. see
         # TestPoissonConstrained1a.test_smoke
-        res_null = mod_null.fit(disp=0, warn_convergence=False,
-                                maxiter=10000)
+
+        optim_kwds = getattr(self, '_optim_kwds_null', {}).copy()
+
+        if 'start_params' in optim_kwds:
+            # user provided
+            sp_null = optim_kwds.pop('start_params')
+        elif hasattr(model, '_get_start_params_null'):
+            # get moment estimates if available
+            sp_null = model._get_start_params_null()
+        else:
+            sp_null = None
+
+        opt_kwds = dict(method='bfgs', warn_convergence=False, maxiter=10000,
+                        disp=0)
+        opt_kwds.update(optim_kwds)
+
+        if optim_kwds:
+            res_null = mod_null.fit(start_params=sp_null, **opt_kwds)
+        else:
+            # this should be a reasonably method case across versions
+            res_null = mod_null.fit(start_params=sp_null, method='nm',
+                                    warn_convergence=False,
+                                    maxiter=10000, disp=0)
+            res_null = mod_null.fit(start_params=res_null.params, method='bfgs',
+                                    warn_convergence=False,
+                                    maxiter=10000, disp=0)
+
         if getattr(self, '_attach_nullmodel', False) is not False:
             self.res_null = res_null
 


Test patch:
diff --git a/statsmodels/discrete/tests/test_discrete.py b/statsmodels/discrete/tests/test_discrete.py
--- a/statsmodels/discrete/tests/test_discrete.py
+++ b/statsmodels/discrete/tests/test_discrete.py
@@ -1779,7 +1779,7 @@ def test_mean_var(self):
     def test_predict_prob(self):
         res = self.res
         endog = res.model.endog
-        freq = np.bincount(endog)
+        freq = np.bincount(endog.astype(int))
 
         pr = res.predict(which='prob')
         pr2 = sm.distributions.genpoisson_p.pmf(np.arange(6)[:, None],
@@ -2077,6 +2077,150 @@ def test_predict_prob_p2(self):
             nbinom.pmf(np.arange(8)[:,None], size, prob).T,
             atol=1e-2, rtol=1e-2)
 
+class CheckNull(object):
+
+    @classmethod
+    def _get_data(cls):
+        x = np.array([ 20.,  25.,  30.,  35.,  40.,  45.,  50.])
+        nobs = len(x)
+        exog = np.column_stack((np.ones(nobs), x))
+        endog = np.array([ 469, 5516, 6854, 6837, 5952, 4066, 3242])
+        return endog, exog
+
+    def test_llnull(self):
+        res = self.model.fit(start_params=self.start_params)
+        res._results._attach_nullmodel = True
+        llf0 = res.llnull
+        res_null0 = res.res_null
+        assert_allclose(llf0, res_null0.llf, rtol=1e-6)
+
+        res_null1 = self.res_null
+        assert_allclose(llf0, res_null1.llf, rtol=1e-6)
+        # Note default convergence tolerance doesn't get lower rtol
+        # from different starting values (using bfgs)
+        assert_allclose(res_null0.params, res_null1.params, rtol=5e-5)
+
+
+class TestPoissonNull(CheckNull):
+
+    @classmethod
+    def setup_class(cls):
+        endog, exog = cls._get_data()
+        cls.model = Poisson(endog, exog)
+        cls.res_null = Poisson(endog, exog[:, 0]).fit(start_params=[8.5])
+        # use start params to avoid warnings
+        cls.start_params = [8.5, 0]
+
+
+class TestNegativeBinomialNB1Null(CheckNull):
+
+    @classmethod
+    def setup_class(cls):
+        endog, exog = cls._get_data()
+        cls.model = NegativeBinomial(endog, exog, loglike_method='nb1')
+        cls.model_null = NegativeBinomial(endog, exog[:, 0], loglike_method='nb1')
+        cls.res_null = cls.model_null.fit(start_params=[8, 1000],
+                                          method='bfgs', gtol=1e-08, maxiter=300)
+        # for convergence with bfgs, I needed to round down alpha start_params
+        cls.start_params = np.array([7.730452, 2.01633068e-02, 1763.0])
+
+
+class TestNegativeBinomialNB2Null(CheckNull):
+
+    @classmethod
+    def setup_class(cls):
+        endog, exog = cls._get_data()
+        cls.model = NegativeBinomial(endog, exog, loglike_method='nb2')
+        cls.model_null = NegativeBinomial(endog, exog[:, 0], loglike_method='nb2')
+        cls.res_null = cls.model_null.fit(start_params=[8, 0.5],
+                                          method='bfgs', gtol=1e-06, maxiter=300)
+        cls.start_params = np.array([ 8.07216448,  0.01087238,  0.44024134])
+
+
+class TestNegativeBinomialNBP2Null(CheckNull):
+
+    @classmethod
+    def setup_class(cls):
+        endog, exog = cls._get_data()
+        cls.model = NegativeBinomialP(endog, exog, p=2)
+        cls.model_null = NegativeBinomialP(endog, exog[:, 0], p=2)
+        cls.res_null = cls.model_null.fit(start_params=[8, 1],
+                                          method='bfgs', gtol=1e-06, maxiter=300)
+        cls.start_params = np.array([ 8.07216448,  0.01087238,  0.44024134])
+
+    def test_start_null(self):
+        endog, exog = self.model.endog, self.model.exog
+        model_nb2 = NegativeBinomial(endog, exog, loglike_method='nb2')
+        sp1 = model_nb2._get_start_params_null()
+        sp0 = self.model._get_start_params_null()
+        assert_allclose(sp0, sp1, rtol=1e-12)
+
+
+class TestNegativeBinomialNBP1Null(CheckNull):
+
+    @classmethod
+    def setup_class(cls):
+        endog, exog = cls._get_data()
+        cls.model = NegativeBinomialP(endog, exog, p=1.)
+        cls.model_null = NegativeBinomialP(endog, exog[:, 0], p=1)
+        cls.res_null = cls.model_null.fit(start_params=[8, 1],
+                                          method='bfgs', gtol=1e-06, maxiter=300)
+        cls.start_params = np.array([7.730452, 2.01633068e-02, 1763.0])
+
+    def test_start_null(self):
+        endog, exog = self.model.endog, self.model.exog
+        model_nb2 = NegativeBinomial(endog, exog, loglike_method='nb1')
+        sp1 = model_nb2._get_start_params_null()
+        sp0 = self.model._get_start_params_null()
+        assert_allclose(sp0, sp1, rtol=1e-12)
+
+
+class TestGeneralizedPoissonNull(CheckNull):
+
+    @classmethod
+    def setup_class(cls):
+        endog, exog = cls._get_data()
+        cls.model = GeneralizedPoisson(endog, exog, p=1.5)
+        cls.model_null = GeneralizedPoisson(endog, exog[:, 0], p=1.5)
+        cls.res_null = cls.model_null.fit(start_params=[8.4, 1],
+                                          method='bfgs', gtol=1e-08, maxiter=300)
+        cls.start_params = np.array([6.91127148, 0.04501334, 0.88393736])
+
+def test_null_options():
+    # this is a "nice" case because we only check that options are used
+    # correctly
+    nobs = 10
+    exog = np.ones((20, 2))
+    exog[:nobs // 2, 1] = 0
+    mu = np.exp(exog.sum(1))
+    endog = np.random.poisson(mu)  # Note no size=nobs in np.random
+    res = Poisson(endog, exog).fit(start_params=np.log([1, 1]))
+    llnull0 = res.llnull
+    assert_(hasattr(res, 'res_llnull') is False)
+    res.set_null_options(attach_results=True)
+    # default optimization
+    lln = res.llnull  # access to trigger computation
+    assert_allclose(res.res_null.mle_settings['start_params'], np.log(endog.mean()), rtol=1e-10)
+    assert_equal(res.res_null.mle_settings['optimizer'], 'bfgs')
+    assert_allclose(lln, llnull0)
+
+    res.set_null_options(attach_results=True, start_params=[0.5], method='nm')
+    lln = res.llnull  # access to trigger computation
+    assert_allclose(res.res_null.mle_settings['start_params'], [0.5], rtol=1e-10)
+    assert_equal(res.res_null.mle_settings['optimizer'], 'nm')
+
+    res.summary()  # call to fill cache
+    assert_('prsquared' in res._cache)
+    assert_equal(res._cache['llnull'],  lln)
+
+    assert_('prsquared' in res._cache)
+    assert_equal(res._cache['llnull'],  lln)
+
+    # check setting cache
+    res.set_null_options(llnull=999)
+    assert_('prsquared' not in res._cache)
+    assert_equal(res._cache['llnull'],  999)
+
 
 if __name__ == "__main__":
     import pytest