Repository: statsmodels/statsmodels

Issue:
statsmodels.formula.api.mixedlm does to know how to handle missing data
Hi there,
The formula interface for MixedLM does not know how to handle missing values. It also does not handle categorical data very well, too.

Minimal Example

```
import numpy as np
import pandas as pd
from pandas import Series, DataFrame
from patsy import dmatrix
from patsy import dmatrices
import statsmodels.api as sm
import statsmodels.formula.api as smf

x = Series(random.randint(0,2,50), dtype='category')
x.cat.categories = ['foo', 'bar']

y = Series(random.randint(0,3,50), dtype='category')
y.cat.categories = ['low', 'medium', 'high']

z = Series(random.randint(0,2,50), dtype='category')
z.cat.categories = ['male', 'female']

data = DataFrame({
    'response': random.rand(50),
    'assistant': x,
    'risk' : y,
    'sex': z,
    'age': random.randint(20,60,50)})
```

Which gives us:

```
In [59]: data.head()
Out[59]: 
   age assistant  response    risk     sex
0   38       foo  0.524707  medium  female
1   38       foo  0.902656    high  female
2   39       foo  0.934783     low    male
3   53       foo  0.136764    high    male
4   34       foo  0.013332     low    male
```

The grouping in the data frame is a categorical value, which mixedlm complains about.

```
In [65]: fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()
/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py:160: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  self._params = np.zeros(self.k_tot)
/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py:1717: ConvergenceWarning: The MLE may be on the boundary of the parameter space.
  warnings.warn(msg, ConvergenceWarning)
```

If we set some covariates to missing, we get an error.

```
In [66]: data.risk[10] = np.nan

In [67]: fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-67-decde0c23284> in <module>()
----> 1 fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()

/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in from_formula(cls, formula, data, re_formula, subset, *args, **kwargs)
    651                                                subset=None,
    652                                                exog_re=exog_re,
--> 653                                                *args, **kwargs)
    654 
    655         # expand re names to account for pairs of RE

/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/base/model.py in from_formula(cls, formula, data, subset, *args, **kwargs)
    148         kwargs.update({'missing_idx': missing_idx,
    149                        'missing': missing})
--> 150         mod = cls(endog, exog, *args, **kwargs)
    151         mod.formula = formula
    152 

/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in __init__(self, endog, exog, groups, exog_re, use_sqrt, missing, **kwargs)
    537 
    538         # Split the data by groups
--> 539         self.endog_li = self.group_list(self.endog)
    540         self.exog_li = self.group_list(self.exog)
    541         self.exog_re_li = self.group_list(self.exog_re)

/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in group_list(self, array)
    671         if array.ndim == 1:
    672             return [np.array(array[self.row_indices[k]])
--> 673                     for k in self.group_labels]
    674         else:
    675             return [np.array(array[self.row_indices[k], :])

/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in <listcomp>(.0)
    671         if array.ndim == 1:
    672             return [np.array(array[self.row_indices[k]])
--> 673                     for k in self.group_labels]
    674         else:
    675             return [np.array(array[self.row_indices[k], :])

IndexError: index 49 is out of bounds for axis 1 with size 49
```

The problem is obviously that `group=data['assistant']` is now too long and does not fit the design matrix build by patsy any more.

Q: How does the formula api make sure that the order in data['assistant'] and the design matrix produced by patsy always match? I don't know what patsy does under the hood, but is it allowed to change the order in which samples are given to it?

If there are missing values in data['assistant'] it also breaks.

```
In [69]: data.dropna(inplace=True)
In [70]: data.assistant[21] = np.nan
In [71]: fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-71-decde0c23284> in <module>()
----> 1 fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()

/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in from_formula(cls, formula, data, re_formula, subset, *args, **kwargs)
    651                                                subset=None,
    652                                                exog_re=exog_re,
--> 653                                                *args, **kwargs)
    654 
    655         # expand re names to account for pairs of RE

/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/base/model.py in from_formula(cls, formula, data, subset, *args, **kwargs)
    148         kwargs.update({'missing_idx': missing_idx,
    149                        'missing': missing})
--> 150         mod = cls(endog, exog, *args, **kwargs)
    151         mod.formula = formula
    152 

/home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in __init__(self, endog, exog, groups, exog_re, use_sqrt, missing, **kwargs)
    528         # list of arrays, corresponding to the groups.
    529         group_labels = list(set(groups))
--> 530         group_labels.sort()
    531         row_indices = dict((s, []) for s in group_labels)
    532         for i,g in enumerate(groups):

TypeError: unorderable types: str() < float()
```

A work around is to subset the data frame to only contain the variables needed in the model fit and drop all missings. It is just not very practical during model building and selection when a lot of variables are added and removed on the fly.

Maybe it would be more intuitive to define the grouping as a formula, too, say `groups='~assistant'` and that mixedlm works out the details by itself? I don't know. You are the experts.

Thank you for your help, your efforts, and a great package!


Discussion:
Thanks for your comments.  I will look into this again.  Last time I
checked it was difficult to get missing value handling to work correctly
with models that have auxiliary variables.

I believe that currently the groups argument can be a string containing the
grouping variable name, or the grouping variable itself.  It might also be
possible to extend this to allow the group variable to be constructed on
the fly from a formula.  I will keep that in mind.

Kerby

On Tue, Nov 1, 2016 at 11:28 AM, Thomas W. D. Möbius <
notifications@github.com> wrote:

> Hi there,
> The formula interface for MixedLM does not know how to handle missing
> values. It also does not handle categorical data very well, too.
> 
> Minimal Example
> 
> import numpy as np
> import pandas as pd
> from pandas import Series, DataFrame
> from patsy import dmatrix
> from patsy import dmatrices
> import statsmodels.api as sm
> import statsmodels.formula.api as smf
> 
> x = Series(random.randint(0,2,50), dtype='category')
> x.cat.categories = ['foo', 'bar']
> 
> y = Series(random.randint(0,3,50), dtype='category')
> y.cat.categories = ['low', 'medium', 'high']
> 
> z = Series(random.randint(0,2,50), dtype='category')
> z.cat.categories = ['male', 'female']
> 
> data = DataFrame({
>     'response': random.rand(50),
>     'assistant': x,
>     'risk' : y,
>     'sex': z,
>     'age': random.randint(20,60,50)})
> 
> Which gives us:
> 
> In [59]: data.head()
> Out[59]:
>    age assistant  response    risk     sex
> 0   38       foo  0.524707  medium  female
> 1   38       foo  0.902656    high  female
> 2   39       foo  0.934783     low    male
> 3   53       foo  0.136764    high    male
> 4   34       foo  0.013332     low    male
> 
> The grouping in the data frame is a categorical value, which mixedlm
> complains about.
> 
> In [65]: fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py:160: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
>   self._params = np.zeros(self.k_tot)
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py:1717: ConvergenceWarning: The MLE may be on the boundary of the parameter space.
>   warnings.warn(msg, ConvergenceWarning)
> 
> If we set some covariates to missing, we get an error.
> 
> In [66]: data.risk[10] = np.nan
> 
> ## In [67]: fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()
> 
> IndexError                                Traceback (most recent call last)
> <ipython-input-67-decde0c23284> in <module>()
> ----> 1 fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()
> 
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in from_formula(cls, formula, data, re_formula, subset, _args, *_kwargs)
>     651                                                subset=None,
>     652                                                exog_re=exog_re,
> --> 653                                                _args, *_kwargs)
>     654
>     655         # expand re names to account for pairs of RE
> 
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/base/model.py in from_formula(cls, formula, data, subset, _args, *_kwargs)
>     148         kwargs.update({'missing_idx': missing_idx,
>     149                        'missing': missing})
> --> 150         mod = cls(endog, exog, _args, *_kwargs)
>     151         mod.formula = formula
>     152
> 
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in **init**(self, endog, exog, groups, exog_re, use_sqrt, missing, **kwargs)
>     537
>     538         # Split the data by groups
> --> 539         self.endog_li = self.group_list(self.endog)
>     540         self.exog_li = self.group_list(self.exog)
>     541         self.exog_re_li = self.group_list(self.exog_re)
> 
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in group_list(self, array)
>     671         if array.ndim == 1:
>     672             return [np.array(array[self.row_indices[k]])
> --> 673                     for k in self.group_labels]
>     674         else:
>     675             return [np.array(array[self.row_indices[k], :])
> 
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in <listcomp>(.0)
>     671         if array.ndim == 1:
>     672             return [np.array(array[self.row_indices[k]])
> --> 673                     for k in self.group_labels]
>     674         else:
>     675             return [np.array(array[self.row_indices[k], :])
> 
> IndexError: index 49 is out of bounds for axis 1 with size 49
> 
> The problem is obviously that group=data['assistant'] is now too long and
> does not fit the design matrix build by patsy any more.
> 
> Q: How does the formula api make sure that the order in data['assistant']
> and the design matrix produced by patsy always match? I don't know what
> patsy does under the hood, but is it allowed to change the order in which
> samples are given to it?
> 
> If there are missing values in data['assistant'] it also breaks.
> 
> In [69]: data.dropna(inplace=True)
> In [70]: data.assistant[21] = np.nan
> 
> ## In [71]: fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()
> 
> TypeError                                 Traceback (most recent call last)
> <ipython-input-71-decde0c23284> in <module>()
> ----> 1 fit = smf.mixedlm('response ~ sex + age + risk', groups=data['assistant'], data=data).fit()
> 
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in from_formula(cls, formula, data, re_formula, subset, _args, *_kwargs)
>     651                                                subset=None,
>     652                                                exog_re=exog_re,
> --> 653                                                _args, *_kwargs)
>     654
>     655         # expand re names to account for pairs of RE
> 
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/base/model.py in from_formula(cls, formula, data, subset, _args, *_kwargs)
>     148         kwargs.update({'missing_idx': missing_idx,
>     149                        'missing': missing})
> --> 150         mod = cls(endog, exog, _args, *_kwargs)
>     151         mod.formula = formula
>     152
> 
> /home/moebius/software/python-environments/for/lib/python3.5/site-packages/statsmodels/regression/mixed_linear_model.py in **init**(self, endog, exog, groups, exog_re, use_sqrt, missing, **kwargs)
>     528         # list of arrays, corresponding to the groups.
>     529         group_labels = list(set(groups))
> --> 530         group_labels.sort()
>     531         row_indices = dict((s, []) for s in group_labels)
>     532         for i,g in enumerate(groups):
> 
> TypeError: unorderable types: str() < float()
> 
> A work around is to subset the data frame to only contain the variables
> needed in the model fit and drop all missings. It is just not very
> practical during model building and selection when a lot of variables are
> added and removed on the fly.
> 
> Maybe it would be more intuitive to define the grouping as a formula, too,
> say groups='~assistant' and that mixedlm works out the details by itself?
> I don't know. You are the experts.
> 
> Thank you for your help, your efforts, and a great package!
> 
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> https://github.com/statsmodels/statsmodels/issues/3248, or mute the
> thread
> https://github.com/notifications/unsubscribe-auth/ACiwwzNtahjP_jc5Gv2zsQ9_MWqU_KIsks5q51q2gaJpZM4KmN5B
> .


Resolving patch:
diff --git a/statsmodels/regression/mixed_linear_model.py b/statsmodels/regression/mixed_linear_model.py
--- a/statsmodels/regression/mixed_linear_model.py
+++ b/statsmodels/regression/mixed_linear_model.py
@@ -153,6 +153,7 @@
 import pandas as pd
 import patsy
 from statsmodels.compat.collections import OrderedDict
+from statsmodels.compat.python import string_types
 from statsmodels.compat import range
 import warnings
 from statsmodels.tools.sm_exceptions import ConvergenceWarning
@@ -639,7 +640,7 @@ def __init__(self, endog, exog, groups, exog_re=None,
             self.k_re2 = 1
             self.exog_re = np.ones((len(endog), 1), dtype=np.float64)
             self.data.exog_re = self.exog_re
-            names = ['Group RE']
+            names = ['Group Var']
             self.data.param_names = self.exog_names + names
             self.data.exog_re_names = names
             self.data.exog_re_names_full = names
@@ -741,19 +742,21 @@ def _make_param_names(self, exog_re):
         for i in range(len(exog_re_names)):
             for j in range(i + 1):
                 if i == j:
-                    param_names.append(exog_re_names[i] + " RE")
+                    param_names.append(exog_re_names[i] + " Var")
                 else:
-                    param_names.append(exog_re_names[j] + " RE x " +
-                                       exog_re_names[i] + " RE")
+                    param_names.append(exog_re_names[j] + " x " +
+                                       exog_re_names[i] + " Cov")
                 jj += 1
 
-        vc_names = [x + " RE" for x in self._vc_names]
+        vc_names = [x + " Var" for x in self._vc_names]
 
         return exog_names + param_names + vc_names, exog_re_names, param_names
 
+
     @classmethod
     def from_formula(cls, formula, data, re_formula=None, vc_formula=None,
-                     subset=None, use_sparse=False, *args, **kwargs):
+                     subset=None, use_sparse=False, missing='none', *args,
+                     **kwargs):
         """
         Create a Model from a formula and dataframe.
 
@@ -777,6 +780,8 @@ def from_formula(cls, formula, data, re_formula=None, vc_formula=None,
             An array-like object of booleans, integers, or index
             values that indicate the subset of df to use in the
             model. Assumes df is a `pandas.DataFrame`
+        missing : string
+            Either 'none' or 'drop'
         args : extra arguments
             These are passed to the model
         kwargs : extra keyword arguments
@@ -812,10 +817,6 @@ def from_formula(cls, formula, data, re_formula=None, vc_formula=None,
         be affected by whether the group labels are distinct or
         re-used over the top-level groups.
 
-        This method currently does not correctly handle missing
-        values, so missing values should be explicitly dropped from
-        the DataFrame before calling this method.
-
         Examples
         --------
         Suppose we have an educational data set with students nested
@@ -853,19 +854,26 @@ def from_formula(cls, formula, data, re_formula=None, vc_formula=None,
 
         if "groups" not in kwargs.keys():
             raise AttributeError("'groups' is a required keyword argument in MixedLM.from_formula")
+        groups = kwargs["groups"]
 
         # If `groups` is a variable name, retrieve the data for the
         # groups variable.
         group_name = "Group"
-        if type(kwargs["groups"]) == str:
-            group_name = kwargs["groups"]
-            kwargs["groups"] = np.asarray(data[kwargs["groups"]])
+        if isinstance(groups, string_types):
+            group_name = groups
+            groups = np.asarray(data[groups])
+        del kwargs["groups"]
+
+        # Bypass all upstream missing data handling to properly handle variance components
+        if missing == 'drop':
+            data, groups = _handle_missing(data, groups, formula, re_formula, vc_formula)
+            missing = 'none'
 
         if re_formula is not None:
             if re_formula.strip() == "1":
                 # Work around Patsy bug, fixed by 0.3.
                 exog_re = np.ones((data.shape[0], 1))
-                exog_re_names = ["Group"]
+                exog_re_names = [group_name]
             else:
                 eval_env = kwargs.get('eval_env', None)
                 if eval_env is None:
@@ -875,13 +883,14 @@ def from_formula(cls, formula, data, re_formula=None, vc_formula=None,
                     eval_env = EvalEnvironment({})
                 exog_re = patsy.dmatrix(re_formula, data, eval_env=eval_env)
                 exog_re_names = exog_re.design_info.column_names
+                exog_re_names = [x.replace("Intercept", group_name) for x in exog_re_names]
                 exog_re = np.asarray(exog_re)
             if exog_re.ndim == 1:
                 exog_re = exog_re[:, None]
         else:
             exog_re = None
             if vc_formula is None:
-                exog_re_names = ["groups"]
+                exog_re_names = [group_name]
             else:
                 exog_re_names = []
 
@@ -894,8 +903,7 @@ def from_formula(cls, formula, data, re_formula=None, vc_formula=None,
                 eval_env = EvalEnvironment({})
 
             exog_vc = {}
-            data["_group"] = kwargs["groups"]
-            gb = data.groupby("_group")
+            gb = data.groupby(groups)
             kylist = list(gb.groups.keys())
             kylist.sort()
             for vc_name in vc_formula.keys():
@@ -917,6 +925,7 @@ def from_formula(cls, formula, data, re_formula=None, vc_formula=None,
                                                subset=None,
                                                exog_re=exog_re,
                                                exog_vc=exog_vc,
+                                               groups=groups,
                                                *args, **kwargs)
 
         # expand re names to account for pairs of RE
@@ -1036,7 +1045,7 @@ def fit_regularized(self, start_params=None, method='l1', alpha=0,
         http://statweb.stanford.edu/~tibs/stat315a/Supplements/fuse.pdf
         """
 
-        if type(method) == str and (method.lower() != 'l1'):
+        if isinstance(method, string_types) and (method.lower() != 'l1'):
             raise ValueError("Invalid regularization method")
 
         # If method is a smooth penalty just optimize directly.
@@ -2580,3 +2589,42 @@ class MixedLMResultsWrapper(base.LikelihoodResultsWrapper):
     _methods = {}
     _upstream_methods = base.LikelihoodResultsWrapper._wrap_methods
     _wrap_methods = base.wrap.union_dicts(_methods, _upstream_methods)
+
+
+def _handle_missing(data, groups, formula, re_formula, vc_formula):
+
+    tokens = set([])
+
+    forms = [formula]
+    if re_formula is not None:
+        forms.append(re_formula)
+    if vc_formula is not None:
+        forms.extend(vc_formula.values())
+
+    import tokenize
+    from statsmodels.compat import PY3
+    from statsmodels.compat.python import StringIO, asunicode
+    skiptoks = {"(", ")", "*", ":", "+", "-", "**", "/"}
+
+    for fml in forms:
+        # Unicode conversion is for Py2 compatability
+        rl = StringIO(fml)
+        def rlu():
+            line = rl.readline()
+            return asunicode(line, 'ascii')
+        g = tokenize.generate_tokens(rlu)
+        for tok in g:
+            if tok not in skiptoks:
+                if PY3:
+                    tokens.add(tok.string)
+                else:
+                    tokens.add(tok[1])
+    tokens = list(tokens & set(data.columns))
+    tokens.sort()
+
+    data = data[tokens]
+    ii = pd.notnull(data).all(1)
+    if type(groups) != "str":
+        ii &= pd.notnull(groups)
+
+    return data.loc[ii, :], groups[np.asarray(ii)]


Test patch:
diff --git a/statsmodels/regression/tests/test_lme.py b/statsmodels/regression/tests/test_lme.py
--- a/statsmodels/regression/tests/test_lme.py
+++ b/statsmodels/regression/tests/test_lme.py
@@ -472,7 +472,7 @@ def test_formulas(self):
         # test the names
         assert_(mod1.data.xnames == ["x1", "x2", "x3", "x4"])
         assert_(mod1.data.exog_re_names == ["x_re1"])
-        assert_(mod1.data.exog_re_names_full == ["x_re1 RE"])
+        assert_(mod1.data.exog_re_names_full == ["x_re1 Var"])
         rslt1 = mod1.fit()
 
         # Fit with a formula, passing groups as the actual values.
@@ -487,7 +487,7 @@ def test_formulas(self):
 
         assert_(mod2.data.xnames == ["exog0", "exog1", "exog2", "exog3"])
         assert_(mod2.data.exog_re_names == ["exog_re"])
-        assert_(mod2.data.exog_re_names_full == ["exog_re RE"])
+        assert_(mod2.data.exog_re_names_full == ["exog_re Var"])
 
         rslt2 = mod2.fit()
         assert_almost_equal(rslt1.params, rslt2.params)
@@ -498,7 +498,7 @@ def test_formulas(self):
                                     groups="groups")
         assert_(mod3.data.xnames == ["exog0", "exog1", "exog2", "exog3"])
         assert_(mod3.data.exog_re_names == ["exog_re"])
-        assert_(mod3.data.exog_re_names_full == ["exog_re RE"])
+        assert_(mod3.data.exog_re_names_full == ["exog_re Var"])
 
         rslt3 = mod3.fit(start_params=rslt2.params)
         assert_allclose(rslt1.params, rslt3.params, rtol=1e-4)
@@ -514,7 +514,7 @@ def test_formulas(self):
         from statsmodels.formula.api import mixedlm
         mod5 = mixedlm(fml, df, groups="groups")
         assert_(mod5.data.exog_re_names == ["groups"])
-        assert_(mod5.data.exog_re_names_full == ["groups RE"])
+        assert_(mod5.data.exog_re_names_full == ["groups Var"])
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
             rslt5 = mod5.fit()
@@ -652,9 +652,9 @@ def test_mixed_lm_wrapper():
     result.summary()
 
     xnames = ["exog0", "exog1", "exog2", "exog3"]
-    re_names = ["Intercept", "exog_re"]
-    re_names_full = ["Intercept RE", "Intercept RE x exog_re RE",
-                     "exog_re RE"]
+    re_names = ["Group", "exog_re"]
+    re_names_full = ["Group Var", "Group x exog_re Cov",
+                     "exog_re Var"]
 
     assert_(mod2.data.xnames == xnames)
     assert_(mod2.data.exog_re_names == re_names)
@@ -726,6 +726,63 @@ def test_random_effects():
     assert_(isinstance(re[0], pd.Series))
     assert_(len(re[0]) == 2)
 
+
+def test_handle_missing():
+
+    np.random.seed(23423)
+    df = np.random.normal(size=(100, 6))
+    df = pd.DataFrame(df)
+    df.columns = ["y", "g", "x1", "z1", "c1", "c2"]
+    df["g"] = np.kron(np.arange(50), np.ones(2))
+    re = np.random.normal(size=(50, 4))
+    re = np.kron(re, np.ones((2, 1)))
+    df["y"] = re[:, 0] + re[:, 1] * df.z1 + re[:, 2] * df.c1
+    df["y"] += re[:, 3] * df.c2 + np.random.normal(size=100)
+    df.loc[1, "y"] = np.NaN
+    df.loc[2, "g"] = np.NaN
+    df.loc[3, "x1"] = np.NaN
+    df.loc[4, "z1"] = np.NaN
+    df.loc[5, "c1"] = np.NaN
+    df.loc[6, "c2"] = np.NaN
+
+    fml = "y ~ x1"
+    re_formula = "1 + z1"
+    vc_formula = {"a": "0 + c1", "b": "0 + c2"}
+    for include_re in False, True:
+        for include_vc in False, True:
+            kwargs = {}
+            dx = df.copy()
+            va = ["y", "g", "x1"]
+            if include_re:
+                kwargs["re_formula"] = re_formula
+                va.append("z1")
+            if include_vc:
+                kwargs["vc_formula"] = vc_formula
+                va.extend(["c1", "c2"])
+
+            dx = dx[va].dropna()
+
+            # Some of these models are severely misspecified with
+            # small n, so produce convergence warnings.  Not relevant
+            # to what we are checking here.
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+
+                # Drop missing externally
+                model1 = MixedLM.from_formula(
+                    fml, groups="g", data=dx, **kwargs)
+                result1 = model1.fit()
+
+                # MixeLM handles missing
+                model2 = MixedLM.from_formula(
+                    fml, groups="g", data=df, missing='drop', **kwargs)
+                result2 = model2.fit()
+
+                assert_allclose(result1.params, result2.params)
+                assert_allclose(result1.bse, result2.bse)
+                assert_equal(len(result1.fittedvalues), result1.nobs)
+
+
 if __name__ == "__main__":
 
     import nose