Repository: statsmodels/statsmodels

Issue:
BUG GLM gamma get_distribution assumes scale=1
parameterization is incorrect if scale is not equal to 1

see
https://github.com/statsmodels/statsmodels/issues/106#issuecomment-586072934

(there is now a 3rd question on stackoverflow for this.)


Discussion:

Resolving patch:
diff --git a/statsmodels/genmod/families/family.py b/statsmodels/genmod/families/family.py
--- a/statsmodels/genmod/families/family.py
+++ b/statsmodels/genmod/families/family.py
@@ -10,7 +10,7 @@
 import warnings
 import inspect
 import numpy as np
-from scipy import special
+from scipy import special, stats
 from . import links as L
 from . import varfuncs as V
 
@@ -493,6 +493,28 @@ def resid_anscombe(self, endog, mu, var_weights=1., scale=1.):
         resid *= np.sqrt(var_weights)
         return resid
 
+    def get_distribution(self, mu, scale=1., var_weights=1.):
+        r"""
+        Frozen Poisson distribution instance for given parameters
+
+        Parameters
+        ----------
+        mu : ndarray
+            Usually but not always the fitted mean response variable.
+        scale : float
+            The scale parameter is ignored.
+        var_weights : array_like
+            1d array of variance (analytic) weights. The default is 1.
+            var_weights are ignored for Poisson.
+
+        Returns
+        -------
+        distribution instance
+
+        """
+
+        return stats.poisson(mu)
+
 
 class Gaussian(Family):
     """
@@ -637,6 +659,28 @@ def resid_anscombe(self, endog, mu, var_weights=1., scale=1.):
         resid *= np.sqrt(var_weights)
         return resid
 
+    def get_distribution(self, mu, scale, var_weights=1.):
+        r"""
+        Frozen Gaussian distribution instance for given parameters
+
+        Parameters
+        ----------
+        mu : ndarray
+            Usually but not always the fitted mean response variable.
+        scale : float
+            The scale parameter is required argument for get_distribution.
+        var_weights : array_like
+            1d array of variance (analytic) weights. The default is 1.
+
+        Returns
+        -------
+        distribution instance
+
+        """
+
+        scale_n = scale / var_weights
+        return stats.norm(loc=mu, scale=np.sqrt(scale_n))
+
 
 class Gamma(Family):
     """
@@ -771,6 +815,30 @@ def resid_anscombe(self, endog, mu, var_weights=1., scale=1.):
         resid *= np.sqrt(var_weights)
         return resid
 
+    def get_distribution(self, mu, scale, var_weights=1.):
+        r"""
+        Frozen Gamma distribution instance for given parameters
+
+        Parameters
+        ----------
+        mu : ndarray
+            Usually but not always the fitted mean response variable.
+        scale : float
+            The scale parameter is required argument for get_distribution.
+        var_weights : array_like
+            1d array of variance (analytic) weights. The default is 1.
+
+        Returns
+        -------
+        distribution instance
+
+        """
+        # combine var_weights with scale
+        scale_ = scale / var_weights
+        shape = 1 / scale_
+        scale_g = mu * scale_
+        return stats.gamma(shape, scale=scale_g)
+
 
 class Binomial(Family):
     """
@@ -1010,6 +1078,31 @@ def cox_snell(x):
         resid *= np.sqrt(var_weights)
         return resid
 
+    def get_distribution(self, mu, scale=1., var_weights=1., n_trials=1):
+        r"""
+        Frozen Binomial distribution instance for given parameters
+
+        Parameters
+        ----------
+        mu : ndarray
+            Usually but not always the fitted mean response variable.
+        scale : float
+            The scale parameter is ignored.
+        var_weights : array_like
+            1d array of variance (analytic) weights. The default is 1.
+            var_weights are ignored for Poisson.
+        n_trials : int
+            Number of trials for the binomial distribution. The default is 1
+            which corresponds to a Bernoulli random variable.
+
+        Returns
+        -------
+        distribution instance
+
+        """
+
+        return stats.binom(n=n_trials, p=mu)
+
 
 class InverseGaussian(Family):
     """
@@ -1144,10 +1237,33 @@ def resid_anscombe(self, endog, mu, var_weights=1., scale=1.):
         resid *= np.sqrt(var_weights)
         return resid
 
+    def get_distribution(self, mu, scale, var_weights=1.):
+        r"""
+        Frozen Inverse Gaussian distribution instance for given parameters
+
+        Parameters
+        ----------
+        mu : ndarray
+            Usually but not always the fitted mean response variable.
+        scale : float
+            The scale parameter is required argument for get_distribution.
+        var_weights : array_like
+            1d array of variance (analytic) weights. The default is 1.
+
+        Returns
+        -------
+        distribution instance
+
+        """
+        # combine var_weights with scale
+        scale_ = scale / var_weights
+        mu_ig = mu * scale_
+        return stats.invgauss(mu_ig, scale=1 / scale_)
+
 
 class NegativeBinomial(Family):
     r"""
-    Negative Binomial exponential family.
+    Negative Binomial exponential family (corresponds to NB2).
 
     Parameters
     ----------
@@ -1330,6 +1446,29 @@ def hyp2f1(x):
         resid *= np.sqrt(var_weights)
         return resid
 
+    def get_distribution(self, mu, scale=1., var_weights=1.):
+        r"""
+        Frozen NegativeBinomial distribution instance for given parameters
+
+        Parameters
+        ----------
+        mu : ndarray
+            Usually but not always the fitted mean response variable.
+        scale : float
+            The scale parameter is ignored.
+        var_weights : array_like
+            1d array of variance (analytic) weights. The default is 1.
+            var_weights are ignored for NegativeBinomial.
+
+        Returns
+        -------
+        distribution instance
+
+        """
+        size = 1. / self.alpha
+        prob = size / (size + mu)
+        return stats.nbinom(size, prob)
+
 
 class Tweedie(Family):
     """
@@ -1356,7 +1495,7 @@ class Tweedie(Family):
         ``variance`` is an instance of
         statsmodels.genmod.families.varfuncs.Power
     Tweedie.var_power : float
-        The power of the variance function.
+        The power parameter of the variance function.
 
     See Also
     --------
diff --git a/statsmodels/genmod/generalized_linear_model.py b/statsmodels/genmod/generalized_linear_model.py
--- a/statsmodels/genmod/generalized_linear_model.py
+++ b/statsmodels/genmod/generalized_linear_model.py
@@ -899,10 +899,10 @@ def predict(self, params, exog=None, exposure=None, offset=None,
         else:
             return self.family.fitted(linpred)
 
-    def get_distribution(self, params, scale=1., exog=None, exposure=None,
-                         offset=None):
+    def get_distribution(self, params, scale=None, exog=None, exposure=None,
+                         offset=None, var_weights=1., n_trials=1.):
         """
-        Return a random number generator for the predictive distribution.
+        Return a instance of the predictive distribution.
 
         Parameters
         ----------
@@ -912,12 +912,22 @@ def get_distribution(self, params, scale=1., exog=None, exposure=None,
             The scale parameter.
         exog : array_like
             The predictor variable matrix.
+        offset : array_like or None
+            Offset variable for predicted mean.
+        exposure : array_like or None
+            Log(exposure) will be added to the linear prediction.
+        var_weights : array_like
+            1d array of variance (analytic) weights. The default is None.
+        n_trials : int
+            Number of trials for the binomial distribution. The default is 1
+            which corresponds to a Bernoulli random variable.
 
         Returns
         -------
         gen
-            Frozen random number generator object.  Use the ``rvs`` method to
-            generate random values.
+            Instance of a scipy frozen distribution based on estimated
+            parameters.
+            Use the ``rvs`` method to generate random values.
 
         Notes
         -----
@@ -928,26 +938,22 @@ def get_distribution(self, params, scale=1., exog=None, exposure=None,
         results will be produced.
         """
         scale = float_like(scale, "scale", optional=True)
-        fit = self.predict(params, exog, exposure, offset, linear=False)
+        # use scale=1, independent of QMLE scale for discrete
+        if isinstance(self.family, (families.Binomial, families.Poisson,
+                                    families.NegativeBinomial)):
+            scale = 1.
 
-        import scipy.stats.distributions as dist
+        mu = self.predict(params, exog, exposure, offset, linear=False)
 
-        if isinstance(self.family, families.Gaussian):
-            return dist.norm(loc=fit, scale=np.sqrt(scale))
+        kwds = {}
+        if (np.any(n_trials != 1) and
+                isinstance(self.family, families.Binomial)):
 
-        elif isinstance(self.family, families.Binomial):
-            return dist.binom(n=1, p=fit)
+            kwds["n_trials"] = n_trials
 
-        elif isinstance(self.family, families.Poisson):
-            return dist.poisson(mu=fit)
-
-        elif isinstance(self.family, families.Gamma):
-            alpha = fit / float(scale)
-            return dist.gamma(alpha, scale=scale)
-
-        else:
-            raise ValueError("get_distribution not implemented for %s" %
-                             self.family.name)
+        distr = self.family.get_distribution(mu, scale,
+                                             var_weights=var_weights, **kwds)
+        return distr
 
     def _setup_binomial(self):
         # this checks what kind of data is given for Binomial.


Test patch:
diff --git a/statsmodels/genmod/tests/test_glm.py b/statsmodels/genmod/tests/test_glm.py
--- a/statsmodels/genmod/tests/test_glm.py
+++ b/statsmodels/genmod/tests/test_glm.py
@@ -205,6 +205,28 @@ def test_summary2(self):
             warnings.simplefilter("ignore", DomainWarning)
             self.res1.summary2()
 
+    def test_get_distribution(self):
+        res1 = self.res1
+        if not hasattr(res1.model.family, "get_distribution"):
+            # only Tweedie has not get_distribution
+            pytest.skip("get_distribution not available")
+
+        if isinstance(res1.model.family, sm.families.NegativeBinomial):
+            res_scale = 1  # QMLE scale can differ from 1
+        else:
+            res_scale = res1.scale
+
+        distr = res1.model.family.get_distribution(res1.fittedvalues,
+                                                   res_scale)
+        var_endog = res1.model.family.variance(res1.fittedvalues) * res_scale
+        m, v = distr.stats()
+        assert_allclose(res1.fittedvalues, m, rtol=1e-13)
+        assert_allclose(var_endog, v, rtol=1e-13)
+        # check model method
+        distr2 = res1.model.get_distribution(res1.params, res_scale)
+        for k in distr2.kwds:
+            assert_allclose(distr.kwds[k], distr2.kwds[k], rtol=1e-13)
+
 
 class CheckComparisonMixin(object):
 
@@ -438,6 +460,26 @@ def test_invalid_endog_formula(self, reset_randomstate):
             sm.GLM.from_formula("y ~ x1 + x2", data,
                                 family=sm.families.Binomial())
 
+    def test_get_distribution_binom_count(self):
+        # test for binomial counts with n_trials > 1
+        res1 = self.res1
+        res_scale = 1  # QMLE scale can differ from 1
+
+        mu_prob = res1.fittedvalues
+        n = res1.model.n_trials
+        distr = res1.model.family.get_distribution(mu_prob, res_scale,
+                                                   n_trials=n)
+        var_endog = res1.model.family.variance(mu_prob) * res_scale
+        m, v = distr.stats()
+        assert_allclose(mu_prob * n, m, rtol=1e-13)
+        assert_allclose(var_endog * n, v, rtol=1e-13)
+
+        # check model method
+        distr2 = res1.model.get_distribution(res1.params, res_scale,
+                                             n_trials=n)
+        for k in distr2.kwds:
+            assert_allclose(distr.kwds[k], distr2.kwds[k], rtol=1e-13)
+
 
 # FIXME: enable/xfail/skip or delete
 # TODO:
@@ -658,11 +700,20 @@ def setup_class(cls):
 
         from .results.results_glm import InvGauss
         res2 = InvGauss()
-        res1 = GLM(res2.endog, res2.exog, \
-                family=sm.families.InverseGaussian()).fit()
+        res1 = GLM(res2.endog, res2.exog,
+                   family=sm.families.InverseGaussian()).fit()
         cls.res1 = res1
         cls.res2 = res2
 
+    def test_get_distribution(self):
+        res1 = self.res1
+        distr = res1.model.family.get_distribution(res1.fittedvalues,
+                                                   res1.scale)
+        var_endog = res1.model.family.variance(res1.fittedvalues) * res1.scale
+        m, v = distr.stats()
+        assert_allclose(res1.fittedvalues, m, rtol=1e-13)
+        assert_allclose(var_endog, v, rtol=1e-13)
+
 
 class TestGlmInvgaussLog(CheckModelResultsMixin):
     @classmethod