Repository: statsmodels/statsmodels

Issue:
refactor exposure in count models
Problem with exposure: we don't store "exposure" we store log(exposure)

I think that's not a good design, storing a different variable under the original name

found while working on #1221,  #1093 and #1608



Discussion:

Resolving patch:
diff --git a/statsmodels/base/model.py b/statsmodels/base/model.py
--- a/statsmodels/base/model.py
+++ b/statsmodels/base/model.py
@@ -63,6 +63,21 @@ def __init__(self, endog, exog=None, **kwargs):
         self._data_attr = []
         self._data_attr.extend(['exog', 'endog', 'data.exog', 'data.endog',
                                 'data.orig_endog', 'data.orig_exog'])
+        # store keys for extras if we need to recreate model instance
+        # we don't need 'missing', maybe we need 'hasconst'
+        self._init_keys = list(kwargs.keys())
+        if hasconst is not None:
+            self._init_keys.append('hasconst')
+
+
+    def _get_init_kwds(self):
+        """return dictionary with extra keys used in model.__init__
+        """
+        kwds = dict(((key, getattr(self, key, None))
+                         for key in self._init_keys))
+
+        return kwds
+
 
     def _handle_data(self, endog, exog, missing, hasconst, **kwargs):
         data = handle_data(endog, exog, missing, hasconst, **kwargs)
diff --git a/statsmodels/discrete/discrete_model.py b/statsmodels/discrete/discrete_model.py
--- a/statsmodels/discrete/discrete_model.py
+++ b/statsmodels/discrete/discrete_model.py
@@ -657,6 +657,16 @@ def _check_inputs(self, offset, exposure, endog):
                 raise ValueError("exposure is not the same length as endog")
         self.exposure = exposure
 
+
+    def _get_init_kwds(self):
+        # this is a temporary fixup because exposure has been transformed
+        # see #1609
+        kwds = super(CountModel, self)._get_init_kwds()
+        if 'exposure' in kwds and kwds['exposure'] is not None:
+            kwds['exposure'] = np.exp(kwds['exposure'])
+        return kwds
+
+
     #TODO: are these two methods only for Poisson? or also Negative Binomial?
     def predict(self, params, exog=None, exposure=None, offset=None,
                 linear=False):
@@ -1787,6 +1797,9 @@ def __init__(self, endog, exog, loglike_method='nb2', offset=None,
         self._initialize()
         if loglike_method in ['nb2', 'nb1']:
             self.exog_names.append('alpha')
+        # store keys for extras if we need to recreate model instance
+        # we need to append keys that don't go to super
+        self._init_keys.append('loglike_method')
 
     def _initialize(self):
         if self.loglike_method == 'nb2':
@@ -2137,10 +2150,13 @@ def llr_pvalue(self):
 
     @cache_readonly
     def llnull(self):
+
         model = self.model
+        kwds = model._get_init_kwds()
         #TODO: what parameters to pass to fit?
-        null = model.__class__(model.endog, np.ones(self.nobs)).fit(disp=0)
-        return null.llf
+        mod_null = model.__class__(model.endog, np.ones(self.nobs), **kwds)
+        res_null = mod_null.fit(disp=0)
+        return res_null.llf
 
     @cache_readonly
     def fittedvalues(self):


Test patch:
diff --git a/statsmodels/discrete/tests/results/results_discrete.py b/statsmodels/discrete/tests/results/results_discrete.py
--- a/statsmodels/discrete/tests/results/results_discrete.py
+++ b/statsmodels/discrete/tests/results/results_discrete.py
@@ -942,9 +942,9 @@ def negativebinomial_geometric_bfgs(self):
         self.df_model = 9.0
         self.df_resid = 20180.0
         self.llf = -43540.58000589021
-        self.llnull = -44199.27443567125
-        self.llr = 1317.3888595620811
-        self.llr_pvalue = 5.4288002863296022e-278
+        self.llnull = -44586.650971362695 # old value -44199.27443567125
+        self.llr = 2092.1425097129977 # old value 1317.3888595620811
+        self.llr_pvalue = 0 # old value 5.4288002863296022e-278
         self.fittedvalues = [ 0.89348994,  0.89348994,  0.89348994,
                 0.89348994,  0.89348994, 0.9365745 ,  0.9365745 ,  0.9365745 ,
                 0.9365745 ,  0.9365745 ]
diff --git a/statsmodels/discrete/tests/test_discrete.py b/statsmodels/discrete/tests/test_discrete.py
--- a/statsmodels/discrete/tests/test_discrete.py
+++ b/statsmodels/discrete/tests/test_discrete.py
@@ -12,7 +12,7 @@
 import os
 import numpy as np
 from numpy.testing import (assert_, assert_raises, assert_almost_equal,
-                           assert_equal, assert_array_equal)
+                           assert_equal, assert_array_equal, assert_allclose)
 
 from statsmodels.discrete.discrete_model import (Logit, Probit, MNLogit,
                                                  Poisson, NegativeBinomial)
@@ -53,7 +53,7 @@ def test_params(self):
         assert_almost_equal(self.res1.params, self.res2.params, DECIMAL_4)
 
     def test_conf_int(self):
-        assert_almost_equal(self.res1.conf_int(), self.res2.conf_int, DECIMAL_4)
+        assert_allclose(self.res1.conf_int(), self.res2.conf_int, rtol=8e-5)
 
     def test_zstat(self):
         assert_almost_equal(self.res1.tvalues, self.res2.z, DECIMAL_4)
diff --git a/statsmodels/discrete/tests/test_sandwich_cov.py b/statsmodels/discrete/tests/test_sandwich_cov.py
--- a/statsmodels/discrete/tests/test_sandwich_cov.py
+++ b/statsmodels/discrete/tests/test_sandwich_cov.py
@@ -31,6 +31,7 @@
 exog_data = data['yr_con op_75_79'.split()]
 exog = add_constant(exog_data, prepend=False)
 group = np.asarray(data['ship'], int)
+exposure = np.asarray(data['service'])
 
 
 # TODO get the test methods from regression/tests
@@ -69,6 +70,13 @@ def get_robust_clu(cls):
         cls.corr_fact = np.sqrt(corr_fact)
 
 
+    def test_oth(self):
+        res1 = self.res1
+        res2 = self.res2
+        assert_allclose(res1._results.llf, res2.ll, 1e-4)
+        assert_allclose(res1._results.llnull, res2.ll_0, 1e-4)
+
+
 class TestPoissonClu(CheckCountRobustMixin):
 
     @classmethod
@@ -79,6 +87,16 @@ def setup_class(cls):
         cls.get_robust_clu()
 
 
+class TestPoissonCluExposure(CheckCountRobustMixin):
+
+    @classmethod
+    def setup_class(cls):
+        cls.res2 = results_st.results_poisson_exposure_clu #nonrobust
+        mod = smd.Poisson(endog, exog, exposure=exposure)
+        cls.res1 = mod.fit(disp=False)
+        cls.get_robust_clu()
+
+
 class TestNegbinClu(CheckCountRobustMixin):
 
     @classmethod
@@ -89,6 +107,17 @@ def setup_class(cls):
         cls.get_robust_clu()
 
 
+# TODO: check what's going on with the params
+class T_estNegbinCluExposure(CheckCountRobustMixin):
+
+    @classmethod
+    def setup_class(cls):
+        cls.res2 = results_st.results_negbin_exposure_clu #nonrobust
+        mod = smd.NegativeBinomial(endog, exog, exposure=exposure)
+        cls.res1 = mod.fit(disp=False)
+        cls.get_robust_clu()
+
+
 #        mod_nbe = smd.NegativeBinomial(endog, exog, exposure=data['service'])
 #        res_nbe = mod_nbe.fit()
 #        mod_nb = smd.NegativeBinomial(endog, exog)