Repository: statsmodels/statsmodels

Issue:
BUG: GEEResults fittedvalues ignores offset and exposure
GEEResults.fittedvalues uses explicit computation instead of going through predict

```
    @cache_readonly
    def fittedvalues(self):
        """
        Returns the fitted values from the model.
        """
        return self.model.family.link.inverse(np.dot(self.model.exog,
                                                     self.params))
```

all the resids are defined in terms of fittedvalues and will have the same behavior as consequence

this is incorrect in case of offset and inconsistent definition for exposure
GLM uses the correctly defined mu=predict() including user provided offset and exposure.

found when I tried to get an overview about how fittedvalues are defined
https://github.com/statsmodels/statsmodels/issues/37#issuecomment-424019626


Discussion:
I haven't checked what is in the unit tests for GEE with offset.

**update**
test_gee.py has an offset example for predict but doesn't check fittedvalues nor resid
test_gee_glm.py compares resid between gee and glm but does not have an offset example

@kshedden Is there a reason why fittedvalues is defined this way that I missed?
This _looks_ like it has an easy fix:
```
--- statsmodels-0.11.0.orig/statsmodels/genmod/generalized_estimating_equations.py
+++ statsmodels-0.11.0/statsmodels/genmod/generalized_estimating_equations.py
@@ -2000,8 +2000,7 @@ class GEEResults(base.LikelihoodModelRes
         """
         Returns the fitted values from the model.
         """
-        return self.model.family.link.inverse(np.dot(self.model.exog,
-                                                     self.params))
+        return self.model.predict(self.params)
 
     @Appender(_plot_added_variable_doc % {'extra_params_doc': ''})
     def plot_added_variable(self, focus_exog, resid_type=None,
```
...but given that it's a wrong-results bug over a year old, does having not done that already imply it isn't that easy?  (I haven't tested it yet.)

Sort of related: **GLM**Results.fittedvalues' docstring says it's the linear predictor; since it isn't, we should probably stop saying that.
```
--- statsmodels-0.11.0.orig/statsmodels/genmod/generalized_linear_model.py
+++ statsmodels-0.11.0/statsmodels/genmod/generalized_linear_model.py
@@ -1564,8 +1564,7 @@ class GLMResults(base.LikelihoodModelRes
     @cached_data
     def fittedvalues(self):
         """
-        Linear predicted values for the fitted model.
-        dot(exog, params)
+        Predicted values for the fitted model.
         """
         return self.mu
 
```
I don't think there is a good reason for this.  We should fix it.

Resolving patch:
diff --git a/statsmodels/genmod/generalized_estimating_equations.py b/statsmodels/genmod/generalized_estimating_equations.py
--- a/statsmodels/genmod/generalized_estimating_equations.py
+++ b/statsmodels/genmod/generalized_estimating_equations.py
@@ -37,7 +37,7 @@
 import statsmodels.base.wrapper as wrap
 
 from statsmodels.genmod import families
-from statsmodels.genmod.generalized_linear_model import GLM
+from statsmodels.genmod.generalized_linear_model import GLM, GLMResults
 from statsmodels.genmod import cov_struct as cov_structs
 
 import statsmodels.genmod.families.varfuncs as varfuncs
@@ -475,7 +475,7 @@ def _check_args(endog, exog, groups, time, offset, exposure):
         raise ValueError("'exposure' and 'endog' should have the same size")
 
 
-class GEE(base.Model):
+class GEE(GLM):
 
     __doc__ = (
         "    Marginal Regression Model using Generalized Estimating "
@@ -534,7 +534,7 @@ def __init__(self, endog, exog, groups, time=None, family=None,
                                   time=time, offset=offset,
                                   exposure=exposure, weights=weights,
                                   dep_data=dep_data, missing=missing,
-                                  **kwargs)
+                                  family=family, **kwargs)
 
         self._init_keys.extend(["update_dep", "constraint", "family",
                                 "cov_struct"])
@@ -558,21 +558,6 @@ def __init__(self, endog, exog, groups, time=None, family=None,
 
         self.cov_struct = cov_struct
 
-        # Handle the offset and exposure
-        self._offset_exposure = None
-        if offset is not None:
-            self._offset_exposure = self.offset.copy()
-            self.offset = offset
-        if exposure is not None:
-            if not isinstance(self.family.link, families.links.Log):
-                raise ValueError(
-                    "exposure can only be used with the log link function")
-            if self._offset_exposure is not None:
-                self._offset_exposure += np.log(exposure)
-            else:
-                self._offset_exposure = np.log(exposure)
-            self.exposure = exposure
-
         # Handle the constraint
         self.constraint = None
         if constraint is not None:
@@ -625,10 +610,12 @@ def __init__(self, endog, exog, groups, time=None, family=None,
                  for y in self.endog_li]
             self.time = np.concatenate(self.time_li)
 
-        if self._offset_exposure is not None:
-            self.offset_li = self.cluster_list(self._offset_exposure)
-        else:
+        if (self._offset_exposure is None or
+            (np.isscalar(self._offset_exposure) and
+             self._offset_exposure == 0.)):
             self.offset_li = None
+        else:
+            self.offset_li = self.cluster_list(self._offset_exposure)
         if constraint is not None:
             self.constraint.exog_fulltrans_li = \
                 self.cluster_list(self.constraint.exog_fulltrans)
@@ -748,10 +735,16 @@ def from_formula(cls, formula, groups, data, subset=None,
                 dep_data = data[dep_data]
             kwargs["dep_data"] = np.asarray(dep_data)
 
+        family = None
+        if "family" in kwargs:
+            family = kwargs["family"]
+            del kwargs["family"]
+
         model = super(GEE, cls).from_formula(formula, data=data, subset=subset,
                                              groups=groups, time=time,
                                              offset=offset,
                                              exposure=exposure,
+                                             family=family,
                                              *args, **kwargs)
 
         if dep_data_names is not None:
@@ -1164,98 +1157,15 @@ def _bc_covmat(self, cov_naive):
 
         return cov_robust_bc
 
-    def predict(self, params, exog=None, offset=None,
-                exposure=None, linear=False):
-        """
-        Return predicted values for a marginal regression model fit
-        using GEE.
-
-        Parameters
-        ----------
-        params : array_like
-            Parameters / coefficients of a marginal regression model.
-        exog : array_like, optional
-            Design / exogenous data. If exog is None, model exog is
-            used.
-        offset : array_like, optional
-            Offset for exog if provided.  If offset is None, model
-            offset is used.
-        exposure : array_like, optional
-            Exposure for exog, if exposure is None, model exposure is
-            used.  Only allowed if link function is the logarithm.
-        linear : bool
-            If True, returns the linear predicted values.  If False,
-            returns the value of the inverse of the model's link
-            function at the linear predicted values.
-
-        Returns
-        -------
-        An array of fitted values
-
-        Notes
-        -----
-        Using log(V) as the offset is equivalent to using V as the
-        exposure.  If exposure U and offset V are both provided, then
-        log(U) + V is added to the linear predictor.
-        """
-
-        # TODO: many paths through this, not well covered in tests
-
-        if exposure is not None:
-            if not isinstance(self.family.link, families.links.Log):
-                raise ValueError(
-                    "exposure can only be used with the log link function")
-
-        # This is the combined offset and exposure
-        _offset = 0.
-
-        # Using model exog
-        if exog is None:
-            exog = self.exog
-
-            if not isinstance(self.family.link, families.links.Log):
-                # Do not need to worry about exposure
-                if offset is None:
-                    if self._offset_exposure is not None:
-                        _offset = self._offset_exposure.copy()
-                else:
-                    _offset = offset
+    def _starting_params(self):
 
-            else:
-                if offset is None and exposure is None:
-                    if self._offset_exposure is not None:
-                        _offset = self._offset_exposure
-                elif offset is None and exposure is not None:
-                    _offset = np.log(exposure)
-                    if hasattr(self, "offset"):
-                        _offset = _offset + self.offset
-                elif offset is not None and exposure is None:
-                    _offset = offset
-                    if hasattr(self, "exposure"):
-                        _offset = offset + np.log(self.exposure)
-                else:
-                    _offset = offset + np.log(exposure)
-
-        # exog is provided: this is simpler than above because we
-        # never use model exog or exposure if exog is provided.
+        if np.isscalar(self._offset_exposure):
+            offset = None
         else:
-            if offset is not None:
-                _offset = _offset + offset
-            if exposure is not None:
-                _offset += np.log(exposure)
-
-        lin_pred = _offset + np.dot(exog, params)
-
-        if not linear:
-            return self.family.link.inverse(lin_pred)
-
-        return lin_pred
-
-    def _starting_params(self):
+            offset = self._offset_exposure
 
         model = GLM(self.endog, self.exog, family=self.family,
-                    offset=self._offset_exposure,
-                    freq_weights=self.weights)
+                    offset=offset, freq_weights=self.weights)
         result = model.fit()
         return result.params
 
@@ -1778,7 +1688,7 @@ def qic(self, params, scale, cov_params):
         return ql, qic, qicu
 
 
-class GEEResults(base.LikelihoodModelResults):
+class GEEResults(GLMResults):
 
     __doc__ = (
         "This class summarizes the fit of a marginal regression model "
@@ -1825,6 +1735,13 @@ def __init__(self, model, params, cov_params, scale,
                 raise ValueError('cov_type in argument is different from '
                                  'already attached cov_type')
 
+    @cache_readonly
+    def resid(self):
+        """
+        The response residuals.
+        """
+        return self.resid_response
+
     def standard_errors(self, cov_type="robust"):
         """
         This is a convenience function that returns the standard
@@ -1863,14 +1780,6 @@ def standard_errors(self, cov_type="robust"):
     def bse(self):
         return self.standard_errors(self.cov_type)
 
-    @cache_readonly
-    def resid(self):
-        """
-        Returns the residuals, the endogeneous data minus the fitted
-        values from the model.
-        """
-        return self.model.endog - self.fittedvalues
-
     def score_test(self):
         """
         Return the results of a score test for a linear constraint.
@@ -1971,38 +1880,6 @@ def qic(self, scale=None):
     centered_resid = resid_centered
     split_centered_resid = resid_centered_split
 
-    @cache_readonly
-    def resid_response(self):
-        return self.model.endog - self.fittedvalues
-
-    @cache_readonly
-    def resid_pearson(self):
-        val = self.model.endog - self.fittedvalues
-        val = val / np.sqrt(self.family.variance(self.fittedvalues))
-        return val
-
-    @cache_readonly
-    def resid_working(self):
-        val = self.resid_response
-        val = val * self.family.link.deriv(self.fittedvalues)
-        return val
-
-    @cache_readonly
-    def resid_anscombe(self):
-        return self.family.resid_anscombe(self.model.endog, self.fittedvalues)
-
-    @cache_readonly
-    def resid_deviance(self):
-        return self.family.resid_dev(self.model.endog, self.fittedvalues)
-
-    @cache_readonly
-    def fittedvalues(self):
-        """
-        Returns the fitted values from the model.
-        """
-        return self.model.family.link.inverse(np.dot(self.model.exog,
-                                                     self.params))
-
     @Appender(_plot_added_variable_doc % {'extra_params_doc': ''})
     def plot_added_variable(self, focus_exog, resid_type=None,
                             use_glm_weights=True, fit_kwargs=None,
@@ -2472,9 +2349,10 @@ def setup_ordinal(self, endog, exog, groups, time, offset):
         return endog_out, exog_out, groups_out, time_out, offset_out
 
     def _starting_params(self):
+        exposure = getattr(self, "exposure", None)
         model = GEE(self.endog, self.exog, self.groups,
                     time=self.time, family=families.Binomial(),
-                    offset=self.offset, exposure=self.exposure)
+                    offset=self.offset, exposure=exposure)
         result = model.fit()
         return result.params
 
@@ -2677,9 +2555,10 @@ def __init__(self, endog, exog, groups, time=None, family=None,
             offset, dep_data, constraint)
 
     def _starting_params(self):
+        exposure = getattr(self, "exposure", None)
         model = GEE(self.endog, self.exog, self.groups,
                     time=self.time, family=families.Binomial(),
-                    offset=self.offset, exposure=self.exposure)
+                    offset=self.offset, exposure=exposure)
         result = model.fit()
         return result.params
 
diff --git a/statsmodels/genmod/generalized_linear_model.py b/statsmodels/genmod/generalized_linear_model.py
--- a/statsmodels/genmod/generalized_linear_model.py
+++ b/statsmodels/genmod/generalized_linear_model.py
@@ -1476,7 +1476,7 @@ def __init__(self, model, params, normalized_cov_params, scale,
     @cached_data
     def resid_response(self):
         """
-        Respnose residuals.  The response residuals are defined as
+        Response residuals.  The response residuals are defined as
         `endog` - `fittedvalues`
         """
         return self._n_trials * (self._endog-self.mu)
@@ -1564,8 +1564,12 @@ def pearson_chi2(self):
     @cached_data
     def fittedvalues(self):
         """
-        Linear predicted values for the fitted model.
-        dot(exog, params)
+        The estimated mean response.
+
+        This is the value of the inverse of the link function at
+        lin_pred, where lin_pred is the linear predicted value
+        obtained by multiplying the design matrix by the coefficient
+        vector.
         """
         return self.mu
 


Test patch: