Repository: statsmodels/statsmodels

Issue:
Re-enable LazyDicts in predict
Hello,

I wrote  a quick PR here to close #3609.

I think this keeps all the current functionality, but allows `predict` to use the "lazy" dict concept 

Happy to hear comments. 

I've also removed a few lines from the `predict` method that, as far as I could tell, was unreachable. So that explains why the `warning` bit goes away. 


Discussion:

Resolving patch:
diff --git a/statsmodels/base/model.py b/statsmodels/base/model.py
--- a/statsmodels/base/model.py
+++ b/statsmodels/base/model.py
@@ -165,7 +165,7 @@ def from_formula(cls, formula, data, subset=None, drop_cols=None,
                         cols.remove(col)
                     except ValueError:
                         pass  # OK if not present
-                design_info = design_info.builder.subset(cols).design_info
+                design_info = design_info.subset(cols).design_info
 
         kwargs.update({'missing_idx': missing_idx,
                        'missing': missing,
@@ -777,7 +777,7 @@ def predict(self, exog=None, transform=True, *args, **kwargs):
         Parameters
         ----------
         exog : array-like, optional
-            The values for which you want to predict.
+            The values for which you want to predict. see Notes below.
         transform : bool, optional
             If the model was fit via a formula, do you want to pass
             exog through the formula. Default is True. E.g., if you fit
@@ -794,25 +794,54 @@ def predict(self, exog=None, transform=True, *args, **kwargs):
         prediction : ndarray, pandas.Series or pandas.DataFrame
             See self.model.predict
 
+        Notes
+        -----
+        The types of exog that are supported depends on whether a formula
+        was used in the specification of the model.
+
+        If a formula was used, then exog is processed in the same way as
+        the original data. This transformation needs to have key access to the
+        same variable names, and can be a pandas DataFrame or a dict like
+        object.
+
+        If no formula was used, then the provided exog needs to have the
+        same number of columns as the original exog in the model. No
+        transformation of the data is performed except converting it to
+        a numpy array.
+
+        Row indices as in pandas data frames are supported, and added to the
+        returned prediction.
+
         """
         import pandas as pd
 
-        exog_index = exog.index if _is_using_pandas(exog, None) else None
+        is_pandas = _is_using_pandas(exog, None)
+
+        exog_index = exog.index if is_pandas else None
 
-        if transform and hasattr(self.model, 'formula') and exog is not None:
+        if transform and hasattr(self.model, 'formula') and (exog is not None):
+            design_info = self.model.data.design_info
             from patsy import dmatrix
-            exog = pd.DataFrame(exog)  # user may pass series, if one predictor
-            if exog_index is None:  # user passed in a dictionary
-                exog_index = exog.index
-            exog = dmatrix(self.model.data.design_info.builder,
-                           exog, return_type="dataframe")
-            if len(exog) < len(exog_index):
-                # missing values, rows have been dropped
-                if exog_index is not None:
-                    exog = exog.reindex(exog_index)
+            if isinstance(exog, pd.Series):
+                # we are guessing whether it should be column or row
+                if (hasattr(exog, 'name') and
+                    isinstance(exog.name, str) and
+                    exog.name in design_info.describe()):
+                    # assume we need one column
+                    exog = pd.DataFrame(exog)
+                else:
+                    # assume we need a row
+                    exog = pd.DataFrame(exog).T
+            orig_exog_len = len(exog)
+            is_dict = isinstance(exog, dict)
+            exog = dmatrix(design_info, exog, return_type="dataframe")
+            if orig_exog_len > len(exog) and not is_dict:
+                import warnings
+                if exog_index is None:
+                    warnings.warn('nan values have been dropped', ValueWarning)
                 else:
-                    import warnings
-                    warnings.warn("nan rows have been dropped", ValueWarning)
+                    exog = exog.reindex(exog_index)
+            exog_index = exog.index
 
         if exog is not None:
             exog = np.asarray(exog)
@@ -821,17 +850,16 @@ def predict(self, exog=None, transform=True, *args, **kwargs):
                 exog = exog[:, None]
             exog = np.atleast_2d(exog)  # needed in count model shape[1]
 
-        predict_results = self.model.predict(self.params, exog, *args, **kwargs)
-
-        if exog_index is not None and not hasattr(predict_results, 'predicted_values'):
+        predict_results = self.model.predict(self.params, exog, *args,
+                                             **kwargs)
 
+        if exog_index is not None and not hasattr(predict_results,
+                                                  'predicted_values'):
             if predict_results.ndim == 1:
                 return pd.Series(predict_results, index=exog_index)
             else:
                 return pd.DataFrame(predict_results, index=exog_index)
-
         else:
-
             return predict_results
 
     def summary(self):
@@ -1695,7 +1723,7 @@ def t_test_pairwise(self, term_name, method='hs', alpha=0.05,
         >>> res = ols("np.log(Days+1) ~ C(Weight) + C(Duration)", data).fit()
         >>> pw = res.t_test_pairwise("C(Weight)")
         >>> pw.result_frame
-                 coef   std err         t         P>|t|  Conf. Int. Low  
+                 coef   std err         t         P>|t|  Conf. Int. Low
         2-1  0.632315  0.230003  2.749157  8.028083e-03        0.171563
         3-1  1.302555  0.230003  5.663201  5.331513e-07        0.841803
         3-2  0.670240  0.230003  2.914044  5.119126e-03        0.209488


Test patch:
diff --git a/statsmodels/base/tests/test_predict.py b/statsmodels/base/tests/test_predict.py
new file mode 100644
--- /dev/null
+++ b/statsmodels/base/tests/test_predict.py
@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for Results.predict
+"""
+
+import numpy as np
+import pandas as pd
+
+from numpy.testing import assert_allclose, assert_equal
+import pandas.util.testing as pdt
+
+from statsmodels.regression.linear_model import OLS
+from statsmodels.genmod.generalized_linear_model import GLM
+
+
+class CheckPredictReturns(object):
+
+    def test_2d(self):
+        res = self.res
+        data = self.data
+
+        fitted = res.fittedvalues.iloc[1:10:2]
+
+        pred = res.predict(data.iloc[1:10:2])
+        pdt.assert_index_equal(pred.index, fitted.index)
+        assert_allclose(pred.values, fitted.values, rtol=1e-13)
+
+        # plain dict
+        xd = dict(zip(data.columns, data.iloc[1:10:2].values.T))
+        pred = res.predict(xd)
+        assert_equal(pred.index, np.arange(len(pred)))
+        assert_allclose(pred.values, fitted.values, rtol=1e-13)
+
+
+    def test_1d(self):
+        # one observation
+        res = self.res
+        data = self.data
+
+        pred = res.predict(data.iloc[:1])
+        pdt.assert_index_equal(pred.index, data.iloc[:1].index)
+        assert_allclose(pred.values, res.fittedvalues[0], rtol=1e-13)
+
+        fittedm = res.fittedvalues.mean()
+        xmean = data.mean()
+        pred = res.predict(xmean.to_frame().T)
+        assert_equal(pred.index, np.arange(1))
+        assert_allclose(pred, fittedm, rtol=1e-13)
+
+
+        # Series
+        pred = res.predict(data.mean())
+        assert_equal(pred.index, np.arange(1))
+        assert_allclose(pred.values, fittedm, rtol=1e-13)
+
+        # dict with scalar value (is plain dict)
+        # Note: this warns about dropped nan, even though there are None -FIXED
+        pred = res.predict(data.mean().to_dict())
+        assert_equal(pred.index, np.arange(1))
+        assert_allclose(pred.values, fittedm, rtol=1e-13)
+
+    def test_nopatsy(self):
+        res = self.res
+        data = self.data
+        fitted = res.fittedvalues.iloc[1:10:2]
+
+        # plain numpy array
+        pred = res.predict(res.model.exog[1:10:2], transform=False)
+        assert_allclose(pred, fitted.values, rtol=1e-13)
+
+        # pandas DataFrame
+        x = pd.DataFrame(res.model.exog[1:10:2],
+                         index = data.index[1:10:2],
+                         columns=res.model.exog_names)
+        pred = res.predict(x)
+        pdt.assert_index_equal(pred.index, fitted.index)
+        assert_allclose(pred.values, fitted.values, rtol=1e-13)
+
+        # one observation - 1-D
+        pred = res.predict(res.model.exog[1], transform=False)
+        assert_allclose(pred, fitted.values[0], rtol=1e-13)
+
+        # one observation - pd.Series
+        pred = res.predict(x.iloc[0])
+        pdt.assert_index_equal(pred.index, fitted.index[:1])
+        assert_allclose(pred.values[0], fitted.values[0], rtol=1e-13)
+
+
+class TestPredictOLS(CheckPredictReturns):
+
+    @classmethod
+    def setup_class(cls):
+        nobs = 30
+        np.random.seed(987128)
+        x = np.random.randn(nobs, 3)
+        y = x.sum(1) + np.random.randn(nobs)
+        index = ['obs%02d' % i for i in range(nobs)]
+        # add one extra column to check that it doesn't matter
+        cls.data = pd.DataFrame(np.round(np.column_stack((y, x)), 4),
+                                columns='y var1 var2 var3'.split(),
+                                index=index)
+
+        cls.res = OLS.from_formula('y ~ var1 + var2', data=cls.data).fit()
+
+
+class TestPredictGLM(CheckPredictReturns):
+
+    @classmethod
+    def setup_class(cls):
+        nobs = 30
+        np.random.seed(987128)
+        x = np.random.randn(nobs, 3)
+        y = x.sum(1) + np.random.randn(nobs)
+        index = ['obs%02d' % i for i in range(nobs)]
+        # add one extra column to check that it doesn't matter
+        cls.data = pd.DataFrame(np.round(np.column_stack((y, x)), 4),
+                                columns='y var1 var2 var3'.split(),
+                                index=index)
+
+        cls.res = GLM.from_formula('y ~ var1 + var2', data=cls.data).fit()
+
+    def test_predict_offset(self):
+        res = self.res
+        data = self.data
+
+        fitted = res.fittedvalues.iloc[1:10:2]
+        offset = np.arange(len(fitted))
+        fitted = fitted + offset
+
+        pred = res.predict(data.iloc[1:10:2], offset=offset)
+        pdt.assert_index_equal(pred.index, fitted.index)
+        assert_allclose(pred.values, fitted.values, rtol=1e-13)
+
+        # plain dict
+        xd = dict(zip(data.columns, data.iloc[1:10:2].values.T))
+        pred = res.predict(xd, offset=offset)
+        assert_equal(pred.index, np.arange(len(pred)))
+        assert_allclose(pred.values, fitted.values, rtol=1e-13)
+
+        # offset as pandas.Series
+        data2 = data.iloc[1:10:2].copy()
+        data2['offset'] = offset
+        pred = res.predict(data2, offset=data2['offset'])
+        pdt.assert_index_equal(pred.index, fitted.index)
+        assert_allclose(pred.values, fitted.values, rtol=1e-13)
+
+        # check nan in exog is ok, preserves index matching offset length
+        data2 = data.iloc[1:10:2].copy()
+        data2['offset'] = offset
+        data2.iloc[0, 1] = np.nan
+        pred = res.predict(data2, offset=data2['offset'])
+        pdt.assert_index_equal(pred.index, fitted.index)
+        fitted_nan = fitted.copy()
+        fitted_nan[0] = np.nan
+        assert_allclose(pred.values, fitted_nan.values, rtol=1e-13)
diff --git a/statsmodels/formula/tests/test_formula.py b/statsmodels/formula/tests/test_formula.py
--- a/statsmodels/formula/tests/test_formula.py
+++ b/statsmodels/formula/tests/test_formula.py
@@ -5,9 +5,11 @@
 from statsmodels.formula.formulatools import make_hypotheses_matrices
 from statsmodels.tools import add_constant
 from statsmodels.datasets.longley import load, load_pandas
+from statsmodels.datasets import cpunish
 
 import numpy.testing as npt
 from statsmodels.tools.testing import assert_equal
+import numpy as np
 
 
 longley_formula = 'TOTEMP ~ GNPDEFL + GNP + UNEMP + ARMED + POP + YEAR'
@@ -118,3 +120,52 @@ def test_formula_predict_series():
     result = results.predict({"x": [1, 2, 3]})
     expected = pd.Series([1., 2., 3.], index=[0, 1, 2])
     tm.assert_series_equal(result, expected)
+
+
+def test_patsy_lazy_dict():
+    class LazyDict(dict):
+        def __init__(self, data):
+            self.data = data
+
+        def __missing__(self, key):
+            return np.array(self.data[key])
+
+    data = cpunish.load_pandas().data
+    data = LazyDict(data)
+    res = ols('EXECUTIONS ~ SOUTH + INCOME', data=data).fit()
+
+    res2 = res.predict(data)
+    npt.assert_allclose(res.fittedvalues, res2)
+
+    data = cpunish.load_pandas().data
+    data['INCOME'].loc[0] = None
+
+    data = LazyDict(data)
+    data.index = cpunish.load_pandas().data.index
+    res = ols('EXECUTIONS ~ SOUTH + INCOME', data=data).fit()
+
+    res2 = res.predict(data)
+    assert_equal(res.fittedvalues, res2)  # Should lose a record
+    assert_equal(len(res2) + 1, len(cpunish.load_pandas().data))
+
+
+def test_patsy_missing_data():
+    # Test pandas-style first
+    data = cpunish.load_pandas().data
+    data['INCOME'].loc[0] = None
+    res = ols('EXECUTIONS ~ SOUTH + INCOME', data=data).fit()
+    res2 = res.predict(data)
+    # First record will be dropped during fit, but not during predict
+    assert_equal(res.fittedvalues, res2[1:])
+
+    # Non-pandas version
+    data = cpunish.load_pandas().data
+    data['INCOME'].loc[0] = None
+    data = data.to_records(index=False)
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        res2 = res.predict(data)
+        assert_equal(repr(w[-1].message),
+                     "ValueWarning('nan values have been dropped',)")
+    # Frist record will be dropped in both cases
+    assert_equal(res.fittedvalues, res2)