Repository: statsmodels/statsmodels

Issue:
Error in accessing mixedlm results with random slopes
Per the email thread on the [support page](https://groups.google.com/forum/?hl=en#!topic/pystatsmodels/lQmsdMa6LeU), I'm opening an issue describing an error accessing MixedLM results.

import pandas as pd
    import statsmodels.api as sm
    import statsmodels.formula.api as smf

```
data = sm.datasets.get_rdataset('dietox', 'geepack').data
mod = smf.mixedlm('Weight ~ Time', data, groups = 'Pig')
modf = mod.fit()
print modf.summary()
```

When I try to include a random slope, the model runs and gets fitted:

```
mod  = smf.mixedlm('Weight ~ Time', data, groups = 'Pig', re_formula = '~Time')
modf = mod.fit()
```

but I get the following error:

```
print modf.summary()

Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 1, in <module>
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/statsmodels/regression/mixed_linear_model.py", line 1954, in summary
sdf = pd.DataFrame(index=self.model.data.param_names, data=sdf)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/pandas-0.15.0_14_g8e0155c-py2.7-macosx-10.9x86_64.egg/pandas/core/frame.py", line 238, in __init__
copy=copy)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/pandas-0.15.0_14_g8e0155c-py2.7-macosx-10.9-x86_64.egg/pandas/core/frame.py", line 371, in _init_ndarray
return create_block_manager_from_blocks([values.T], [columns, index])
```

  File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/pandas-0.15.0_14_g8e0155c-py2.7-macosx-10.9-x86_64.egg/pandas/core/internals.py", line 3489, in create_block_manager_from_blocks
    construction_error(tot_items, blocks[0].shape[1:], axes, e)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/pandas-0.15.0_14_g8e0155c-py2.7-macosx-10.9-x86_64.egg/pandas/core/internals.py", line 3470, in construction_error
    passed,implied))
ValueError: Shape of passed values is (6, 5), indices imply (6, 4)



Discussion:

Resolving patch:
diff --git a/statsmodels/regression/mixed_linear_model.py b/statsmodels/regression/mixed_linear_model.py
--- a/statsmodels/regression/mixed_linear_model.py
+++ b/statsmodels/regression/mixed_linear_model.py
@@ -100,6 +100,7 @@
 import statsmodels.base.model as base
 from scipy.optimize import fmin_ncg, fmin_cg, fmin_bfgs, fmin
 from statsmodels.tools.decorators import cache_readonly
+from statsmodels.tools import data as data_tools
 from scipy.stats.distributions import norm
 import pandas as pd
 import patsy
@@ -114,8 +115,16 @@
 
 
 def _get_exog_re_names(exog_re):
-    if isinstance(exog_re, (pd.Series, pd.DataFrame)):
+    """
+    Passes through if given a list of names. Otherwise, gets pandas names
+    or creates some generic variable names as needed.
+    """
+    if isinstance(exog_re, pd.DataFrame):
         return exog_re.columns.tolist()
+    elif isinstance(exog_re, pd.Series) and exog_re.name is not None:
+        return [exog_re.name]
+    elif isinstance(exog_re, list):
+        return exog_re
     return ["Z{0}".format(k + 1) for k in range(exog_re.shape[1])]
 
 
@@ -152,8 +161,7 @@ def __init__(self, k_fe, k_re, use_sqrt=True):
 
     def from_packed(params, k_fe, use_sqrt):
         """
-        Factory method to create a MixedLMParams object based on the
-        given packed parameter vector.
+        Create a MixedLMParams object from packed parameter vector.
 
         Parameters
         ----------
@@ -186,8 +194,7 @@ def from_packed(params, k_fe, use_sqrt):
     def from_components(fe_params, cov_re=None, cov_re_sqrt=None,
                         use_sqrt=True):
         """
-        Factory method to create a MixedLMParams object from given
-        values for each parameter component.
+        Create a MixedLMParams object from each parameter component.
 
         Parameters
         ----------
@@ -466,10 +473,14 @@ def __init__(self, endog, exog, groups, exog_re=None,
         # vector, convert these to 2d arrays.
         # TODO: Can this be moved up in the class hierarchy?
         #       yes, it should be done up the hierarchy
-        if exog is not None and exog.ndim == 1:
-            exog = exog[:,None]
-        if exog_re is not None and exog_re.ndim == 1:
-            exog_re = exog_re[:,None]
+        if (exog is not None and
+                data_tools._is_using_ndarray_type(exog, None) and
+                exog.ndim == 1):
+            exog = exog[:, None]
+        if (exog_re is not None and
+                data_tools._is_using_ndarray_type(exog_re, None) and
+                exog_re.ndim == 1):
+            exog_re = exog_re[:, None]
 
         # Calling super creates self.endog, etc. as ndarrays and the
         # original exog, endog, etc. are self.data.endog, etc.
@@ -485,21 +496,27 @@ def __init__(self, endog, exog, groups, exog_re=None,
             self.k_re2 = 1
             self.exog_re = np.ones((len(endog), 1), dtype=np.float64)
             self.data.exog_re = self.exog_re
-            self.data.param_names = self.exog_names + ['Intercept']
+            self.data.param_names = self.exog_names + ['Intercept RE']
         else:
             # Process exog_re the same way that exog is handled
             # upstream
             # TODO: this is wrong and should be handled upstream wholly
             self.data.exog_re = exog_re
             self.exog_re = np.asarray(exog_re)
+            if self.exog_re.ndim == 1:
+                self.exog_re = self.exog_re[:, None]
             if not self.data._param_names:
                 # HACK: could've been set in from_formula already
                 # needs refactor
-                (self.data.param_names,
-                 self.data.exog_re_names) = self._make_param_names(exog_re)
+                (param_names,
+                 exog_re_names,
+                 exog_re_names_full) = self._make_param_names(exog_re)
+                self.data.param_names = param_names
+                self.data.exog_re_names = exog_re_names
+                self.data.exog_re_names_full = exog_re_names_full
             # Model dimensions
             # Number of random effect covariates
-            self.k_re = exog_re.shape[1]
+            self.k_re = self.exog_re.shape[1]
             # Number of covariance parameters
             self.k_re2 = self.k_re * (self.k_re + 1) // 2
 
@@ -535,21 +552,26 @@ def __init__(self, endog, exog, groups, exog_re=None,
                                range(self.exog.shape[1])]
 
     def _make_param_names(self, exog_re):
+        """
+        Returns the full parameter names list, just the exogenous random
+        effects variables, and the exogenous random effects variables with
+        the interaction terms.
+        """
         exog_names = list(self.exog_names)
         exog_re_names = _get_exog_re_names(exog_re)
         param_names = []
 
         jj = self.k_fe
-        for i in range(exog_re.shape[1]):
+        for i in range(len(exog_re_names)):
             for j in range(i + 1):
                 if i == j:
                     param_names.append(exog_re_names[i] + " RE")
                 else:
-                    param_names.append(exog_re_names[j] + " x " +
+                    param_names.append(exog_re_names[j] + " RE x " +
                                        exog_re_names[i] + " RE")
                 jj += 1
 
-        return exog_names + exog_re_names, exog_re_names
+        return exog_names + param_names, exog_re_names, param_names
 
     @classmethod
     def from_formula(cls, formula, data, re_formula=None, subset=None,
@@ -621,15 +643,20 @@ def from_formula(cls, formula, data, re_formula=None, subset=None,
         else:
             exog_re = np.ones((data.shape[0], 1),
                               dtype=np.float64)
-            exog_re_names = ["Intercept RE"]
+            exog_re_names = ["Intercept"]
 
         mod = super(MixedLM, cls).from_formula(formula, data,
                                                subset=None,
                                                exog_re=exog_re,
                                                *args, **kwargs)
 
-        mod.data.param_names = mod.exog_names + exog_re_names
+        # expand re names to account for pairs of RE
+        (param_names,
+         exog_re_names,
+         exog_re_names_full) = mod._make_param_names(exog_re_names)
+        mod.data.param_names = param_names
         mod.data.exog_re_names = exog_re_names
+        mod.data.exog_re_names_full = exog_re_names_full
 
         return mod
 
@@ -2016,6 +2043,9 @@ def profile_re(self, re_ix, num_low=5, dist_low=1., num_high=5,
         # Define the sequence of values to which the parameter of
         # interest will be constrained.
         ru0 = cov_re[0, 0]
+        if dist_low > ru0:
+            raise ValueError("dist_low is too large and would result in a "
+                             "negative number. Try a smaller value.")
         left = np.linspace(ru0 - dist_low, ru0, num_low + 1)
         right = np.linspace(ru0, ru0 + dist_high, num_high+1)[1:]
         rvalues = np.concatenate((left, right))
@@ -2041,7 +2071,7 @@ def profile_re(self, re_ix, num_low=5, dist_low=1., num_high=5,
             cov_re[0, 0] = x
             params.set_cov_re(cov_re)
             rslt = model.fit(start_params=params, free=free,
-                             reml=self.reml, cov_pen=self.cov_pen)
+                             reml=self.reml, cov_pen=self.cov_pen)._results
             likev.append([rslt.cov_re[0, 0], rslt.llf])
         likev = np.asarray(likev)
 
@@ -2052,9 +2082,11 @@ def profile_re(self, re_ix, num_low=5, dist_low=1., num_high=5,
 
 
 class MixedLMResultsWrapper(base.LikelihoodResultsWrapper):
-    _attrs = {'bse_re': ('generic_columns', 'exog_re_names'),
+    _attrs = {'bse_re': ('generic_columns', 'exog_re_names_full'),
+              'fe_params': ('generic_columns', 'xnames'),
               'bse_fe': ('generic_columns', 'xnames'),
               'cov_re': ('generic_columns_2d', 'exog_re_names'),
+              'cov_re_unscaled': ('generic_columns_2d', 'exog_re_names'),
               }
     _upstream_attrs = base.LikelihoodResultsWrapper._wrap_attrs
     _wrap_attrs = base.wrap.union_dicts(_attrs, _upstream_attrs)
diff --git a/tools/nbgenerate.py b/tools/nbgenerate.py
--- a/tools/nbgenerate.py
+++ b/tools/nbgenerate.py
@@ -290,17 +290,23 @@ def nb2rst(nb, files_dir):
                 new_html = os.path.join(rst_target_dir, fname_only+".rst")
                 # get the title out of the notebook because sphinx needs it
                 title_cell = nb['worksheets'][0]['cells'].pop(0)
-                try:
-                    assert title_cell['cell_type'] == 'heading'
-                except:
+                if title_cell['cell_type'] == 'heading':
+                    pass
+                elif (title_cell['cell_type'] == 'markdown'
+                      and title_cell['source'].strip().startswith('#')):
+                    # IPython 3.x got rid of header cells
+                    pass
+                else:
                     print "Title not in first cell for ", fname_only
                     print "Not generating rST"
+                    continue
 
                 html_out = nb2html(nb)
                 # indent for insertion into raw html block in rST
                 html_out = "\n".join(["   "+i for i in html_out.split("\n")])
                 with io.open(new_html, "w", encoding="utf-8") as f:
-                    f.write(title_cell["source"]+u"\n")
+                    f.write(title_cell["source"].replace("#",
+                                                         "").strip() + u"\n")
                     f.write(u"="*len(title_cell["source"])+u"\n\n")
                     f.write(notebook_template.substitute(name=fname_only,
                                                          body=html_out))


Test patch:
diff --git a/statsmodels/regression/tests/test_lme.py b/statsmodels/regression/tests/test_lme.py
--- a/statsmodels/regression/tests/test_lme.py
+++ b/statsmodels/regression/tests/test_lme.py
@@ -3,7 +3,7 @@
 import pandas as pd
 from statsmodels.regression.mixed_linear_model import MixedLM, MixedLMParams
 from numpy.testing import (assert_almost_equal, assert_equal, assert_allclose,
-                           dec)
+                           dec, assert_)
 from . import lme_r_results
 from statsmodels.base import _penalties as penalties
 import os
@@ -187,7 +187,6 @@ def test_profile(self):
                                num_high=1)
 
     def test_formulas(self):
-
         np.random.seed(2410)
         exog = np.random.normal(size=(300,4))
         exog_re = np.random.normal(size=300)
@@ -197,6 +196,11 @@ def test_formulas(self):
         endog = exog.sum(1) + g_errors + np.random.normal(size=300)
 
         mod1 = MixedLM(endog, exog, groups, exog_re)
+        # test the names
+        assert_(mod1.data.xnames == ["x1", "x2", "x3", "x4"])
+        assert_(mod1.data.exog_re_names == ["Z1"])
+        assert_(mod1.data.exog_re_names_full == ["Z1 RE"])
+
         rslt1 = mod1.fit()
 
         # Fit with a formula, passing groups as the actual values.
@@ -208,6 +212,11 @@ def test_formulas(self):
         re_fml = "0 + exog_re"
         mod2 = MixedLM.from_formula(fml, df, re_formula=re_fml,
                                     groups=groups)
+
+        assert_(mod2.data.xnames == ["exog0", "exog1", "exog2", "exog3"])
+        assert_(mod2.data.exog_re_names == ["exog_re"])
+        assert_(mod2.data.exog_re_names_full == ["exog_re RE"])
+
         rslt2 = mod2.fit()
         assert_almost_equal(rslt1.params, rslt2.params)
 
@@ -215,6 +224,10 @@ def test_formulas(self):
         df["groups"] = groups
         mod3 = MixedLM.from_formula(fml, df, re_formula=re_fml,
                                     groups="groups")
+        assert_(mod3.data.xnames == ["exog0", "exog1", "exog2", "exog3"])
+        assert_(mod3.data.exog_re_names == ["exog_re"])
+        assert_(mod3.data.exog_re_names_full == ["exog_re RE"])
+
         rslt3 = mod3.fit(start_params=rslt2.params)
         assert_allclose(rslt1.params, rslt3.params, rtol=1e-4)
 
@@ -227,6 +240,8 @@ def test_formulas(self):
             rslt4 = mod4.fit(start_params=rslt2.params)
         from statsmodels.formula.api import mixedlm
         mod5 = mixedlm(fml, df, groups="groups")
+        assert_(mod5.data.exog_re_names == ["Intercept"])
+        assert_(mod5.data.exog_re_names_full == ["Intercept RE"])
         rslt5 = mod5.fit(start_params=rslt2.params)
         assert_almost_equal(rslt4.params, rslt5.params)
 
@@ -326,6 +341,61 @@ def test_r(self):
                     yield self.do1, reml, irf, ds_ix
 
 
+def test_mixed_lm_wrapper():
+    # a bit more complicated model to test
+    np.random.seed(2410)
+    exog = np.random.normal(size=(300, 4))
+    exog_re = np.random.normal(size=300)
+    groups = np.kron(np.arange(100), [1, 1, 1])
+    g_errors = exog_re * np.kron(np.random.normal(size=100),
+                                 [1, 1, 1])
+    endog = exog.sum(1) + g_errors + np.random.normal(size=300)
+
+    # Fit with a formula, passing groups as the actual values.
+    df = pd.DataFrame({"endog": endog})
+    for k in range(exog.shape[1]):
+        df["exog%d" % k] = exog[:, k]
+    df["exog_re"] = exog_re
+    fml = "endog ~ 0 + exog0 + exog1 + exog2 + exog3"
+    re_fml = "~ exog_re"
+    mod2 = MixedLM.from_formula(fml, df, re_formula=re_fml,
+                                groups=groups)
+    result = mod2.fit()
+    smoke = result.summary()
+
+    xnames = ["exog0", "exog1", "exog2", "exog3"]
+    re_names = ["Intercept", "exog_re"]
+    re_names_full = ["Intercept RE", "Intercept RE x exog_re RE",
+                     "exog_re RE"]
+
+    assert_(mod2.data.xnames == xnames)
+    assert_(mod2.data.exog_re_names == re_names)
+    assert_(mod2.data.exog_re_names_full == re_names_full)
+
+    params = result.params
+    assert_(params.index.tolist() == xnames + re_names_full)
+    bse = result.bse
+    assert_(bse.index.tolist() == xnames + re_names_full)
+    tvalues = result.tvalues
+    assert_(tvalues.index.tolist() == xnames + re_names_full)
+    cov_params = result.cov_params()
+    assert_(cov_params.index.tolist() == xnames + re_names_full)
+    assert_(cov_params.columns.tolist() == xnames + re_names_full)
+    fe = result.fe_params
+    assert_(fe.index.tolist() == xnames)
+    bse_fe = result.bse_fe
+    assert_(bse_fe.index.tolist() == xnames)
+    cov_re = result.cov_re
+    assert_(cov_re.index.tolist() == re_names)
+    assert_(cov_re.columns.tolist() == re_names)
+    cov_re_u = result.cov_re_unscaled
+    assert_(cov_re_u.index.tolist() == re_names)
+    assert_(cov_re_u.columns.tolist() == re_names)
+    bse_re = result.bse_re
+    assert_(bse_re.index.tolist() == re_names_full)
+
+
+
 
 if  __name__=="__main__":