Repository: statsmodels/statsmodels

Issue:
ENH hazard regression prediction with formula
When hazard_regression was fit with a formula, the predict method doesn't accept a new exog data frame to be processed through the formula.  Needs refactoring.



Discussion:

Resolving patch:
diff --git a/statsmodels/duration/hazard_regression.py b/statsmodels/duration/hazard_regression.py
--- a/statsmodels/duration/hazard_regression.py
+++ b/statsmodels/duration/hazard_regression.py
@@ -22,6 +22,60 @@
 """
 
 
+_predict_docstring = """
+    Returns predicted values from the proportional hazards
+    regression model.
+
+    Parameters
+    ----------
+    params : array-like
+        The proportional hazards model parameters.
+    exog : array-like
+        Data to use as `exog` in forming predictions.  If not
+        provided, the `exog` values from the model used to fit the
+        data are used.%(cov_params_doc)s
+    endog : array-like
+        Duration (time) values at which the predictions are made.
+        Only used if pred_type is either 'cumhaz' or 'surv'.  If
+        using model `exog`, defaults to model `endog` (time), but
+        may be provided explicitly to make predictions at
+        alternative times.
+    strata : array-like
+        A vector of stratum values used to form the predictions.
+        Not used (may be 'None') if pred_type is 'lhr' or 'hr'.
+        If `exog` is None, the model stratum values are used.  If
+        `exog` is not None and pred_type is 'surv' or 'cumhaz',
+        stratum values must be provided (unless there is only one
+        stratum).
+    offset : array-like
+        Offset values used to create the predicted values.
+    pred_type : string
+        If 'lhr', returns log hazard ratios, if 'hr' returns
+        hazard ratios, if 'surv' returns the survival function, if
+        'cumhaz' returns the cumulative hazard function.
+
+    Returns
+    -------
+    A bunch containing two fields: `predicted_values` and
+    `standard_errors`.
+
+    Notes
+    -----
+    Standard errors are only returned when predicting the log
+    hazard ratio (pred_type is 'lhr').
+
+    Types `surv` and `cumhaz` require estimation of the cumulative
+    hazard function.
+"""
+
+_predict_cov_params_docstring = """
+    cov_params : array-like
+        The covariance matrix of the estimated `params` vector,
+        used to obtain prediction errors if pred_type='lhr',
+        otherwise optional."""
+
+
+
 class PHSurvivalTime(object):
 
     def __init__(self, time, status, exog, strata=None, entry=None,
@@ -1204,57 +1258,9 @@ def baseline_cumulative_hazard_function(self, params):
 
         return cumhaz_f
 
-    def predict(self, params, cov_params=None, endog=None, exog=None,
+    def predict(self, params, exog=None, cov_params=None, endog=None,
                 strata=None, offset=None, pred_type="lhr"):
-        """
-        Returns predicted values from the proportional hazards
-        regression model.
-
-        Parameters
-        ----------
-        params : array-like
-            The proportional hazards model parameters.
-        cov_params : array-like
-            The covariance matrix of the estimated `params` vector,
-            used to obtain prediction errors if pred_type='lhr',
-            otherwise optional.
-        endog : array-like
-            Duration (time) values at which the predictions are made.
-            Only used if pred_type is either 'cumhaz' or 'surv'.  If
-            using model `exog`, defaults to model `endog` (time), but
-            may be provided explicitly to make predictions at
-            alternative times.
-        exog : array-like
-            Data to use as `exog` in forming predictions.  If not
-            provided, the `exog` values from the model used to fit the
-            data are used.
-        strata : array-like
-            A vector of stratum values used to form the predictions.
-            Not used (may be 'None') if pred_type is 'lhr' or 'hr'.
-            If `exog` is None, the model stratum values are used.  If
-            `exog` is not None and pred_type is 'surv' or 'cumhaz',
-            stratum values must be provided (unless there is only one
-            stratum).
-        offset : array-like
-            Offset values used to create the predicted values.
-        pred_type : string
-            If 'lhr', returns log hazard ratios, if 'hr' returns
-            hazard ratios, if 'surv' returns the survival function, if
-            'cumhaz' returns the cumulative hazard function.
-
-        Returns
-        -------
-        A bunch containing two fields: `predicted_values` and
-        `standard_errors`.
-
-        Notes
-        -----
-        Standard errors are only returned when predicting the log
-        hazard ratio (pred_type is 'lhr').
-
-        Types `surv` and `cumhaz` require estimation of the cumulative
-        hazard function.
-        """
+        # docstring attached below
 
         pred_type = pred_type.lower()
         if pred_type not in ["lhr", "hr", "surv", "cumhaz"]:
@@ -1286,9 +1292,10 @@ class bunch:
 
         if pred_type == "lhr":
             ret_val.predicted_values = lhr
-            mat = np.dot(exog, cov_params)
-            va = (mat * exog).sum(1)
-            ret_val.standard_errors = np.sqrt(va)
+            if cov_params is not None:
+                mat = np.dot(exog, cov_params)
+                va = (mat * exog).sum(1)
+                ret_val.standard_errors = np.sqrt(va)
             return ret_val
 
         hr = np.exp(lhr)
@@ -1330,6 +1337,8 @@ class bunch:
 
         return ret_val
 
+    predict.__doc__ = _predict_docstring % {'cov_params_doc': _predict_cov_params_docstring}
+
     def get_distribution(self, params):
         """
         Returns a scipy distribution object corresponding to the
@@ -1489,56 +1498,18 @@ def get_distribution(self):
 
 
     def predict(self, endog=None, exog=None, strata=None,
-                offset=None, pred_type="lhr"):
-        """
-        Returns predicted values from the fitted proportional hazards
-        regression model.
-
-        Parameters
-        ----------
-        params : array-;like
-            The proportional hazards model parameters.
-        endog : array-like
-            Duration (time) values at which the predictions are made.
-            Only used if pred_type is either 'cumhaz' or 'surv'.  If
-            using model `exog`, defaults to model `endog` (time), but
-            may be provided explicitly to make predictions at
-            alternative times.
-        exog : array-like
-            Data to use as `exog` in forming predictions.  If not
-            provided, the `exog` values from the model used to fit the
-            data are used.
-        strata : array-like
-            A vector of stratum values used to form the predictions.
-            Not used (may be 'None') if pred_type is 'lhr' or 'hr'.
-            If `exog` is None, the model stratum values are used.  If
-            `exog` is not None and pred_type is 'surv' or 'cumhaz',
-            stratum values must be provided (unless there is only one
-            stratum).
-        offset : array-like
-            Offset values used to create the predicted values.
-        pred_type : string
-            If 'lhr', returns log hazard ratios, if 'hr' returns
-            hazard ratios, if 'surv' returns the survival function, if
-            'cumhaz' returns the cumulative hazard function.
-
-        Returns
-        -------
-        A bunch containing two fields: `predicted_values` and
-        `standard_errors`.
+                offset=None, transform=True, pred_type="lhr"):
+        # docstring attached below
 
-        Notes
-        -----
-        Standard errors are only returned when predicting the log
-        hazard ratio (pred_type is 'lhr').
-
-        Types `surv` and `cumhaz` require estimation of the cumulative
-        hazard function.
-        """
+        return super(PHRegResults, self).predict(exog=exog,
+                                                 transform=transform,
+                                                 cov_params=self.cov_params(),
+                                                 endog=endog,
+                                                 strata=strata,
+                                                 offset=offset,
+                                                 pred_type=pred_type)
 
-        return self.model.predict(self.params, self.cov_params(),
-                                  endog, exog, strata, offset,
-                                  pred_type)
+    predict.__doc__ = _predict_docstring % {'cov_params_doc': ''}
 
     def _group_stats(self, groups):
         """
@@ -1864,5 +1835,3 @@ def _opt_1d(funcs, start, L1_wt, tol):
         return brent(funcs[0], brack=(x-0.2, x+0.2), tol=tol)
 
     return x
-
-


Test patch:
diff --git a/statsmodels/duration/tests/test_phreg.py b/statsmodels/duration/tests/test_phreg.py
--- a/statsmodels/duration/tests/test_phreg.py
+++ b/statsmodels/duration/tests/test_phreg.py
@@ -167,6 +167,39 @@ def test_formula(self):
         assert_allclose(rslt1.bse, rslt2.bse)
         assert_allclose(rslt1.bse, rslt3.bse)
 
+    def test_predict_formula(self):
+
+        n = 100
+        np.random.seed(34234)
+        time = 50 * np.random.uniform(size=n)
+        status = np.random.randint(0, 2, n).astype(np.float64)
+        exog = np.random.uniform(1, 2, size=(n, 2))
+
+        df = pd.DataFrame({"time": time, "status": status,
+                           "exog1": exog[:, 0], "exog2": exog[:, 1]})
+
+        fml = "time ~ 0 + exog1 + np.log(exog2) + exog1*exog2"
+        model1 = PHReg.from_formula(fml, df, status=status)
+        result1 = model1.fit()
+
+        from patsy import dmatrix
+        dfp = dmatrix(model1.data.orig_exog.design_info.builder, df)
+
+        pr1 = result1.predict()
+        pr2 = result1.predict(exog=df)
+        pr3 = model1.predict(result1.params, exog=dfp) # No standard errors
+        pr4 = model1.predict(result1.params, cov_params=result1.cov_params(), exog=dfp)
+
+        prl = (pr1, pr2, pr3, pr4)
+        for i in range(4):
+            for j in range(i):
+                assert_allclose(prl[i].predicted_values, prl[j].predicted_values)
+
+        prl = (pr1, pr2, pr4)
+        for i in range(3):
+            for j in range(i):
+                assert_allclose(prl[i].standard_errors, prl[j].standard_errors)
+
     def test_offset(self):
 
         np.random.seed(34234)