Repository: statsmodels/statsmodels

Issue:
BUG: rlm errors on missing values
Hello,

I just upgraded to statsmodels v. 0.6.0 and found my code was not running as expected compared to v.0.5.0. After somme digging, I narrowed the error to the following problem of rlm with the formula api. Since the missing kwarg is set to 'drop' by default, I'm guessing this is a bug.

``` python
import statsmodels.formula.api as smf
import pandas as pd

d = {'Foo': [1, 2, 10, 149], 'Bar': [1, 2, 3, np.nan]}
df = pd.DataFrame(d)
mod = smf.rlm('Foo ~ Bar', data=df)
```

which raises the following Exception

``` python
  File "statsmodels\base\model.py", line 150, in from_formula
    mod = cls(endog, exog, *args, **kwargs)

  File "statsmodels\robust\robust_linear_model.py", line 117, in __init__
    missing=missing, **kwargs)

  File "statsmodels\base\model.py", line 60, in __init__
    **kwargs)

  File "statsmodels\base\model.py", line 84, in _handle_data
    data = handle_data(endog, exog, missing, hasconst, **kwargs)

  File "statsmodels\base\data.py", line 539, in handle_data
    **kwargs)

  File "statsmodels\base\data.py", line 61, in __init__
    **kwargs)

  File "statsmodels\base\data.py", line 198, in handle_missing
    nan_mask = missing_idx | _nan_rows(*combined)

  File "statsmodels\base\data.py", line 47, in _nan_rows
    return reduce(_nan_row_maybe_two_inputs, arrs).squeeze()

TypeError: reduce() of empty sequence with no initial value
```



Discussion:
Thanks for the report (unfortunately 0.6 is out)

it looks like there is no protection for a empty `combined`.
However, I don't understand (yet) why this works for `ols` but not for `rlm`. It should use exactly the same code in the generic data handling.

using arrays also works correctly

```
endog = d['Foo']
exog = np.column_stack((np.ones(len(endog)), d['Bar']))
mod_np = sm.RLM(endog, exog, missing='drop')
```

```
>>> patsy.__version__
'0.3.0'
```

The fix looks like it should specifically check for empty `combined`

```
>>> smd._nan_rows(*[])
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "e:\josef\eclipsegworkspace\statsmodels-git\statsmodels-all-new2_py27\statsmodels\statsmodels\base\data.py", line 47, in _nan_rows
    return reduce(_nan_row_maybe_two_inputs, arrs).squeeze()
TypeError: reduce() of empty sequence with no initial value
```

aside: an empty list doesn't raise an exception in reduce

```
>>> smd._nan_rows([])
array([], dtype=bool)
```

Looks like OLS always add `weights`:

(adding a raise to get to the right spot with pdb)

```

  File "e:\josef\eclipsegworkspace\statsmodels-git\statsmodels-all-new2_py27\statsmodels\statsmodels\base\data.py", line 197, in handle_missing
    raise(ValueError)
ValueError
locals().keys()
['value_array', 'combined_names', 'missing', 'endog', 'combined_2d', 'combined', 'key', 'kwargs', 'none_array_names', 'combined_2d_names', 'missing_idx', 'exog', 'cls']
(Pdb) missing_idx
array([False, False, False,  True], dtype=bool)
(Pdb) combined
(array([ 1.,  1.,  1.,  1.]),)
(Pdb) combined_names
['weights']
(Pdb) kwargs
{'weights': array([ 1.,  1.,  1.,  1.])}
```


Resolving patch:
diff --git a/statsmodels/base/data.py b/statsmodels/base/data.py
--- a/statsmodels/base/data.py
+++ b/statsmodels/base/data.py
@@ -180,27 +180,46 @@ def handle_missing(cls, endog, exog, missing, **kwargs):
                     continue
                 # grab 1d arrays
                 if value_array.ndim == 1:
-                    combined += (value_array,)
+                    combined += (np.asarray(value_array),)
                     combined_names += [key]
                 elif value_array.squeeze().ndim == 1:
-                    combined += (value_array,)
+                    combined += (np.asarray(value_array),)
                     combined_names += [key]
 
                 # grab 2d arrays that are _assumed_ to be symmetric
                 elif value_array.ndim == 2:
-                    combined_2d += (value_array,)
+                    combined_2d += (np.asarray(value_array),)
                     combined_2d_names += [key]
                 else:
                     raise ValueError("Arrays with more than 2 dimensions "
                                      "aren't yet handled")
 
         if missing_idx is not None:
-            nan_mask = missing_idx | _nan_rows(*combined)
+            nan_mask = missing_idx
+            updated_row_mask = None
+            if combined:  # there were extra arrays not handled by patsy
+                combined_nans = _nan_rows(*combined)
+                if combined_nans.shape[0] != nan_mask.shape[0]:
+                    raise ValueError("Shape mismatch between endog/exog "
+                                     "and extra arrays given to model.")
+                # for going back and updated endog/exog
+                updated_row_mask = combined_nans[~nan_mask]
+                nan_mask |= combined_nans  # for updating extra arrays only
+            if combined_2d:
+                combined_2d_nans = _nan_rows(combined_2d)
+                if combined_2d_nans.shape[0] != nan_mask.shape[0]:
+                    raise ValueError("Shape mismatch between endog/exog "
+                                     "and extra 2d arrays given to model.")
+                if updated_row_mask is not None:
+                    updated_row_mask |= combined_2d_nans[~nan_mask]
+                else:
+                    updated_row_mask = combined_2d_nans[~nan_mask]
+                nan_mask |= combined_2d_nans
+
         else:
             nan_mask = _nan_rows(*combined)
-
-        if combined_2d:
-            nan_mask = _nan_rows(*(nan_mask[:, None],) + combined_2d)
+            if combined_2d:
+                nan_mask = _nan_rows(*(nan_mask[:, None],) + combined_2d)
 
         if not np.any(nan_mask):  # no missing don't do anything
             combined = dict(zip(combined_names, combined))
@@ -225,6 +244,19 @@ def handle_missing(cls, endog, exog, missing, **kwargs):
             drop_nans = lambda x: cls._drop_nans(x, nan_mask)
             drop_nans_2d = lambda x: cls._drop_nans_2d(x, nan_mask)
             combined = dict(zip(combined_names, lmap(drop_nans, combined)))
+
+            if missing_idx is not None:
+                if updated_row_mask is not None:
+                    updated_row_mask = ~updated_row_mask
+                    # update endog/exog with this new information
+                    endog = cls._drop_nans(endog, updated_row_mask)
+                    if exog is not None:
+                        exog = cls._drop_nans(exog, updated_row_mask)
+
+                combined.update({'endog': endog})
+                if exog is not None:
+                    combined.update({'exog': exog})
+
             if combined_2d:
                 combined.update(dict(zip(combined_2d_names,
                                          lmap(drop_nans_2d, combined_2d))))
@@ -232,11 +264,6 @@ def handle_missing(cls, endog, exog, missing, **kwargs):
                 combined.update(dict(zip(none_array_names,
                                          [None] * len(none_array_names))))
 
-            if missing_idx is not None:
-                combined.update({'endog': endog})
-                if exog is not None:
-                    combined.update({'exog': exog})
-
             return combined, np.where(~nan_mask)[0].tolist()
         else:
             raise ValueError("missing option %s not understood" % missing)
diff --git a/statsmodels/discrete/discrete_model.py b/statsmodels/discrete/discrete_model.py
--- a/statsmodels/discrete/discrete_model.py
+++ b/statsmodels/discrete/discrete_model.py
@@ -707,27 +707,23 @@ def _derivative_exog(self, params, exog=None, transform='dydx',
 class CountModel(DiscreteModel):
     def __init__(self, endog, exog, offset=None, exposure=None, missing='none',
                  **kwargs):
-        self._check_inputs(offset, exposure, endog) # attaches if needed
         super(CountModel, self).__init__(endog, exog, missing=missing,
-                offset=self.offset, exposure=self.exposure, **kwargs)
+                                         offset=offset,
+                                         exposure=exposure, **kwargs)
+        if exposure is not None:
+            self.exposure = np.log(self.exposure)
+        self._check_inputs(self.offset, self.exposure, self.endog)
         if offset is None:
             delattr(self, 'offset')
         if exposure is None:
             delattr(self, 'exposure')
 
     def _check_inputs(self, offset, exposure, endog):
-        if offset is not None:
-            offset = np.asarray(offset)
-            if offset.shape[0] != endog.shape[0]:
-                raise ValueError("offset is not the same length as endog")
-        self.offset = offset
-
-        if exposure is not None:
-            exposure = np.log(exposure)
-            if exposure.shape[0] != endog.shape[0]:
-                raise ValueError("exposure is not the same length as endog")
-        self.exposure = exposure
+        if offset is not None and offset.shape[0] != endog.shape[0]:
+            raise ValueError("offset is not the same length as endog")
 
+        if exposure is not None and exposure.shape[0] != endog.shape[0]:
+            raise ValueError("exposure is not the same length as endog")
 
     def _get_init_kwds(self):
         # this is a temporary fixup because exposure has been transformed
@@ -737,7 +733,6 @@ def _get_init_kwds(self):
             kwds['exposure'] = np.exp(kwds['exposure'])
         return kwds
 
-
     def predict(self, params, exog=None, exposure=None, offset=None,
                 linear=False):
         """
diff --git a/statsmodels/genmod/generalized_linear_model.py b/statsmodels/genmod/generalized_linear_model.py
--- a/statsmodels/genmod/generalized_linear_model.py
+++ b/statsmodels/genmod/generalized_linear_model.py
@@ -190,10 +190,14 @@ class GLM(base.LikelihoodModel):
 
     def __init__(self, endog, exog, family=None, offset=None, exposure=None,
                  missing='none', **kwargs):
-        self._check_inputs(family, offset, exposure, endog)
+        if exposure is not None:
+            exposure = np.log(exposure)
+        if offset is not None:  # this should probably be done upstream
+            offset = np.asarray(offset)
         super(GLM, self).__init__(endog, exog, missing=missing,
-                                  offset=self.offset, exposure=self.exposure,
+                                  offset=offset, exposure=exposure,
                                   **kwargs)
+        self._check_inputs(family, self.offset, self.exposure, self.endog)
         if offset is None:
             delattr(self, 'offset')
         if exposure is None:
@@ -227,22 +231,16 @@ def _check_inputs(self, family, offset, exposure, endog):
             family = families.Gaussian()
         self.family = family
 
-        if exposure is not None and not isinstance(self.family.link, families.links.Log):
-            raise ValueError("exposure can only be used with the log link function")
+        if exposure is not None:
+            if not isinstance(self.family.link, families.links.Log):
+                raise ValueError("exposure can only be used with the log "
+                                 "link function")
+            elif exposure.shape[0] != endog.shape[0]:
+                raise ValueError("exposure is not the same length as endog")
 
         if offset is not None:
-            offset = np.asarray(offset)
             if offset.shape[0] != endog.shape[0]:
                 raise ValueError("offset is not the same length as endog")
-        self.offset = offset
-
-        if exposure is not None:
-            exposure = np.asarray(exposure)
-            if exposure.shape[0] != endog.shape[0]:
-                raise ValueError("exposure is not the same length as endog")
-            exposure = np.log(exposure)
-        self.exposure = exposure
-
 
     def loglike(self, *args):
         """


Test patch:
diff --git a/statsmodels/base/tests/test_data.py b/statsmodels/base/tests/test_data.py
--- a/statsmodels/base/tests/test_data.py
+++ b/statsmodels/base/tests/test_data.py
@@ -1,9 +1,10 @@
 import numpy as np
-from numpy.testing import assert_equal, assert_
+from numpy.testing import assert_equal, assert_, assert_raises
 import pandas
 import pandas.util.testing as ptesting
 
 from statsmodels.base import data as sm_data
+from statsmodels.formula import handle_formula_data
 
 #class TestDates(object):
 #    @classmethod
@@ -811,6 +812,80 @@ def test_dtype_object():
 
     np.testing.assert_raises(ValueError, sm_data.handle_data, y, df)
 
+
+def test_formula_missing_extra_arrays():
+    np.random.seed(1)
+    # because patsy can't turn off missing data-handling as of 0.3.0, we need
+    # separate tests to make sure that missing values are handled correctly
+    # when going through formulas
+
+    # there is a handle_formula_data step
+    # then there is the regular handle_data step
+    # see 2083
+
+    # the untested cases are endog/exog have missing. extra has missing.
+    # endog/exog are fine. extra has missing.
+    # endog/exog do or do not have missing and extra has wrong dimension
+    y = np.random.randn(10)
+    y_missing = y.copy()
+    y_missing[[2, 5]] = np.nan
+    X = np.random.randn(10)
+    X_missing = X.copy()
+    X_missing[[1, 3]] = np.nan
+
+    weights = np.random.uniform(size=10)
+    weights_missing = weights.copy()
+    weights_missing[[6]] = np.nan
+
+    weights_wrong_size = np.random.randn(12)
+
+    data = {'y': y,
+            'X': X,
+            'y_missing': y_missing,
+            'X_missing': X_missing,
+            'weights': weights,
+            'weights_missing': weights_missing}
+    data = pandas.DataFrame.from_dict(data)
+    data['constant'] = 1
+
+    formula = 'y_missing ~ X_missing'
+
+    (endog, exog), missing_idx = handle_formula_data(data, None, formula,
+                                                     depth=2,
+                                                     missing='drop')
+
+    kwargs = {'missing_idx': missing_idx, 'missing': 'drop',
+              'weights': data['weights_missing']}
+
+    model_data = sm_data.handle_data(endog, exog, **kwargs)
+    data_nona = data.dropna()
+    assert_equal(data_nona['y'].values, model_data.endog)
+    assert_equal(data_nona[['constant', 'X']].values, model_data.exog)
+    assert_equal(data_nona['weights'].values, model_data.weights)
+
+    (endog, exog), missing_idx = handle_formula_data(data, None, formula,
+                                                     depth=2,
+                                                     missing='drop')
+    weights_2d = np.random.randn(10, 10)
+    weights_2d[[8, 7], [7, 8]] = np.nan  #symmetric missing values
+    kwargs.update({'weights': weights_2d,
+                   'missing_idx': missing_idx})
+
+    model_data2 = sm_data.handle_data(endog, exog, **kwargs)
+
+    good_idx = [0, 4, 6, 9]
+    assert_equal(data.ix[good_idx, 'y'], model_data2.endog)
+    assert_equal(data.ix[good_idx, ['constant', 'X']], model_data2.exog)
+    assert_equal(weights_2d[good_idx][:, good_idx], model_data2.weights)
+
+    (endog, exog), missing_idx = handle_formula_data(data, None, formula,
+                                                     depth=2,
+                                                     missing='drop')
+    kwargs.update({'weights': weights_wrong_size,
+                   'missing_idx': missing_idx})
+    assert_raises(ValueError, sm_data.handle_data, endog, exog, **kwargs)
+
+
 if __name__ == "__main__":
     import nose
     #nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
diff --git a/statsmodels/discrete/tests/test_discrete.py b/statsmodels/discrete/tests/test_discrete.py
--- a/statsmodels/discrete/tests/test_discrete.py
+++ b/statsmodels/discrete/tests/test_discrete.py
@@ -1388,6 +1388,25 @@ def test_mnlogit_factor():
     summary = res2.summary()
 
 
+def test_formula_missing_exposure():
+    # see 2083
+    import statsmodels.formula.api as smf
+    import pandas as pd
+
+    d = {'Foo': [1, 2, 10, 149], 'Bar': [1, 2, 3, np.nan],
+         'constant': [1] * 4, 'exposure' : np.random.uniform(size=4),
+         'x': [1, 3, 2, 1.5]}
+    df = pd.DataFrame(d)
+
+    # should work
+    mod1 = smf.poisson('Foo ~ Bar', data=df, exposure=df['exposure'])
+    assert_(type(mod1.exposure) is np.ndarray, msg='Exposure is not ndarray')
+
+    # make sure this raises
+    exposure = pd.Series(np.random.randn(5))
+    assert_raises(ValueError, sm.Poisson, df.Foo, df[['constant', 'Bar']],
+                  exposure=exposure)
+
 if __name__ == "__main__":
     import nose
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb'],
diff --git a/statsmodels/genmod/tests/test_glm.py b/statsmodels/genmod/tests/test_glm.py
--- a/statsmodels/genmod/tests/test_glm.py
+++ b/statsmodels/genmod/tests/test_glm.py
@@ -762,6 +762,30 @@ def llf(params):
         like = llf(params)
         assert_almost_equal(like, res.llf)
 
+
+def test_formula_missing_exposure():
+    # see 2083
+    import statsmodels.formula.api as smf
+    import pandas as pd
+
+    d = {'Foo': [1, 2, 10, 149], 'Bar': [1, 2, 3, np.nan],
+         'constant': [1] * 4, 'exposure' : np.random.uniform(size=4),
+         'x': [1, 3, 2, 1.5]}
+    df = pd.DataFrame(d)
+
+    family = sm.families.Gaussian(link=sm.families.links.log)
+
+    mod = smf.glm("Foo ~ Bar", data=df, exposure=df.exposure,
+                  family=family)
+    assert_(type(mod.exposure) is np.ndarray, msg='Exposure is not ndarray')
+
+    exposure = pd.Series(np.random.uniform(size=5))
+    assert_raises(ValueError, smf.glm, "Foo ~ Bar", data=df,
+                  exposure=exposure, family=family)
+    assert_raises(ValueError, GLM, df.Foo, df[['constant', 'Bar']],
+                  exposure=exposure, family=family)
+
+
 if __name__=="__main__":
     #run_module_suite()
     #taken from Fernando Perez:
diff --git a/statsmodels/robust/tests/test_rlm.py b/statsmodels/robust/tests/test_rlm.py
--- a/statsmodels/robust/tests/test_rlm.py
+++ b/statsmodels/robust/tests/test_rlm.py
@@ -305,3 +305,11 @@ def setup(self):
 #                        r.rlm, psi="psi.huber")
         from .results.results_rlm import Huber
         self.res2 = Huber()
+
+
+def test_missing():
+    # see 2083
+    import statsmodels.formula.api as smf
+
+    d = {'Foo': [1, 2, 10, 149], 'Bar': [1, 2, 3, np.nan]}
+    mod = smf.rlm('Foo ~ Bar', data=d)