Repository: statsmodels/statsmodels

Issue:
BUG: Flatten pandas MultiIndexes
Flatten pandas MultiIndexes when extracting column names in the
data module. Prior to this commit, the MultiIndexes would be
exposed as a list of tuples, which would then result in the
DataFrame being treated as having more than 2 dimensions.

closes #5414
Exception in data module when using DataFrames with MultiIndex columns
The data module appears to fail when working with DataFrames that use a MultiIndex for columns. This can be easily demonstrated in `statsmodels.base.tests.test_data.TestDataFrames` by changing the creation of a DataFrame from
```python
exog =  pandas.DataFrame(np.random.random((10,2)),
                                 columns=['x_1','x_2'])
```
to
```python
exog = pandas.DataFrame(np.random.random((10,2)),
                                 columns=pandas.MultiIndex.from_product([['x'], ['1', '2']]))
```

Using MultiIndexes for DataFrame columns then propagates itself to other places because using the DataFrame.columns value as a simple list doesn't work (in a MultiIndex, DataFrame.columns is exposed as an iterable of tuples). The specific issue I ran into was when using a MultiIndex DataFrame with exponential smoothing, resulting in the following stack trace:
```
Traceback (most recent call last):
  File "xxx/__init__.py", line 451, in do_algorithm_usecase
    if af.compute():
  File "xxx/algorithms/es.py", line 153, in compute
    preds = self.fitted_model.predict(predict_from, predict_until)
  File "xxx/myenv/lib/python3.7/site-packages/statsmodels/base/wrapper.py", line 95, in wrapper
    obj = data.wrap_output(func(results, *args, **kwargs), how)
  File "xxx/myenv/lib/python3.7/site-packages/statsmodels/base/data.py", line 416, in wrap_output
    return self.attach_dates(obj)
  File "xxx/myenv/lib/python3.7/site-packages/statsmodels/base/data.py", line 563, in attach_dates
    columns=self.ynames)
  File "xxx/myenv/lib/python3.7/site-packages/pandas/core/frame.py", line 379, in __init__
    copy=copy)
  File "xxx/myenv/lib/python3.7/site-packages/pandas/core/frame.py", line 536, in _init_ndarray
    return create_block_manager_from_blocks([values], [columns, index])
  File "xxx/myenv/lib/python3.7/site-packages/pandas/core/internals.py", line 4866, in create_block_manager_from_blocks
    construction_error(tot_items, blocks[0].shape[1:], axes, e)
  File "xxx/myenv/lib/python3.7/site-packages/pandas/core/internals.py", line 4843, in construction_error
    passed, implied))
ValueError: Shape of passed values is (1, 3), indices imply (3, 3)
```


Discussion:

Resolving patch:
diff --git a/statsmodels/base/data.py b/statsmodels/base/data.py
--- a/statsmodels/base/data.py
+++ b/statsmodels/base/data.py
@@ -3,10 +3,12 @@
 results, and doing data cleaning
 """
 from statsmodels.compat.python import reduce, iteritems, lmap, zip, range
+
 import numpy as np
-from pandas import DataFrame, Series, isnull
-from statsmodels.tools.decorators import cache_readonly, cache_writable
+from pandas import DataFrame, Series, isnull, MultiIndex
+
 import statsmodels.tools.data as data_util
+from statsmodels.tools.decorators import cache_readonly, cache_writable
 from statsmodels.tools.sm_exceptions import MissingDataError
 
 
@@ -365,7 +367,12 @@ def _get_row_labels(self, arr):
 
     def _get_names(self, arr):
         if isinstance(arr, DataFrame):
-            return list(arr.columns)
+            if isinstance(arr.columns, MultiIndex):
+                # Flatten MultiIndexes into "simple" column names
+                return ['_'.join((level for level in c if level))
+                        for c in arr.columns]
+            else:
+                return list(arr.columns)
         elif isinstance(arr, Series):
             if arr.name:
                 return [arr.name]


Test patch:
diff --git a/statsmodels/base/tests/test_data.py b/statsmodels/base/tests/test_data.py
--- a/statsmodels/base/tests/test_data.py
+++ b/statsmodels/base/tests/test_data.py
@@ -136,12 +136,37 @@ def test_attach(self):
         # this makes sure what the wrappers need work but not the wrapped
         # results themselves
         ptesting.assert_series_equal(data.wrap_output(self.col_input,
-                                                  'columns'),
-                                self.col_result)
+                                                      'columns'),
+                                     self.col_result)
         ptesting.assert_series_equal(data.wrap_output(self.row_input, 'rows'),
-                                self.row_result)
+                                     self.row_result)
         ptesting.assert_frame_equal(data.wrap_output(self.cov_input, 'cov'),
-                                self.cov_result)
+                                    self.cov_result)
+
+
+class TestDataFramesWithMultiIndex(TestDataFrames):
+    @classmethod
+    def setup_class(cls):
+        cls.endog = pandas.DataFrame(np.random.random(10), columns=['y_1'])
+        mi = pandas.MultiIndex.from_product([['x'], ['1', '2']])
+        exog = pandas.DataFrame(np.random.random((10, 2)), columns=mi)
+        exog_flattened_idx = pandas.Index(['const', 'x_1', 'x_2'])
+        exog.insert(0, 'const', 1)
+        cls.exog = exog
+        cls.data = sm_data.handle_data(cls.endog, cls.exog)
+        nrows = 10
+        nvars = 3
+        cls.col_input = np.random.random(nvars)
+        cls.col_result = pandas.Series(cls.col_input, index=exog_flattened_idx)
+        cls.row_input = np.random.random(nrows)
+        cls.row_result = pandas.Series(cls.row_input, index=exog.index)
+        cls.cov_input = np.random.random((nvars, nvars))
+        cls.cov_result = pandas.DataFrame(cls.cov_input,
+                                          index=exog_flattened_idx,
+                                          columns=exog_flattened_idx)
+        cls.xnames = ['const', 'x_1', 'x_2']
+        cls.ynames = 'y_1'
+        cls.row_labels = cls.exog.index
 
 
 class TestLists(TestArrays):
diff --git a/statsmodels/iolib/tests/test_table.py b/statsmodels/iolib/tests/test_table.py
--- a/statsmodels/iolib/tests/test_table.py
+++ b/statsmodels/iolib/tests/test_table.py
@@ -9,9 +9,10 @@
 ltx_fmt1 = default_latex_fmt.copy()
 html_fmt1 = default_html_fmt.copy()
 
+
 class TestSimpleTable(object):
-    def test_SimpleTable_1(self):
-        # Basic test, test_SimpleTable_1
+    def test_simple_table_1(self):
+        # Basic test, test_simple_table_1
         desired = '''
 =====================
       header1 header2
@@ -28,7 +29,7 @@ def test_SimpleTable_1(self):
         actual = '\n%s\n' % actual.as_text()
         assert_equal(desired, str(actual))
 
-    def test_SimpleTable_2(self):
+    def test_simple_table_2(self):
         #  Test SimpleTable.extend_right()
         desired = '''
 =============================================================
@@ -50,7 +51,7 @@ def test_SimpleTable_2(self):
         actual = '\n%s\n' % actual1.as_text()
         assert_equal(desired, str(actual))
 
-    def test_SimpleTable_3(self):
+    def test_simple_table_3(self):
         # Test SimpleTable.extend() as in extend down
         desired = '''
 ==============================
@@ -76,8 +77,8 @@ def test_SimpleTable_3(self):
         actual = '\n%s\n' % actual1.as_text()
         assert_equal(desired, str(actual))
 
-    def test_SimpleTable_4(self):
-        # Basic test, test_SimpleTable_4 test uses custom txt_fmt
+    def test_simple_table_4(self):
+        # Basic test, test_simple_table_4 test uses custom txt_fmt
         txt_fmt1 = dict(data_fmts = ['%3.2f', '%d'],
                         empty_cell = ' ',
                         colwidths = 1,
@@ -121,6 +122,7 @@ def test_txt_fmt1(self):
             #print(actual)
             #print(desired)
             assert_equal(actual, desired)
+
         def test_ltx_fmt1(self):
             # Limited test of custom ltx_fmt
             desired = r"""
@@ -145,6 +147,7 @@ def test_ltx_fmt1(self):
 """ % desired[1:-1]
             actual_centered = '\n%s\n' % tbl.as_latex_tabular()
             assert_equal(actual_centered, desired_centered)
+
         def test_html_fmt1(self):
             # Limited test of custom html_fmt
             desired = """
@@ -165,8 +168,9 @@ def test_html_fmt1(self):
         test_txt_fmt1(self)
         test_ltx_fmt1(self)
         test_html_fmt1(self)
-    def test_SimpleTable_special_chars(self):
-    # Simple table with characters: (%, >, |, _, $, &, #)
+
+    def test_simple_table_special_chars(self):
+        # Simple table with characters: (%, >, |, _, $, &, #)
         cell0c_data = 22
         cell1c_data = 1053
         row0c_data = [cell0c_data, cell1c_data]
@@ -190,31 +194,32 @@ def test_ltx_special_chars(self):
             actual = '\n%s\n' % tbl_c.as_latex_tabular(center=False)
             assert_equal(actual, desired)
         test_ltx_special_chars(self)
+
     def test_regression_with_tuples(self):
-        i = pandas.Series( [1,2,3,4]*10 , name="i")
-        y = pandas.Series( [1,2,3,4,5]*8, name="y")
-        x = pandas.Series( [1,2,3,4,5,6,7,8]*5, name="x")
+        i = pandas.Series([1, 2, 3, 4] * 10, name="i")
+        y = pandas.Series([1, 2, 3, 4, 5] * 8, name="y")
+        x = pandas.Series([1, 2, 3, 4, 5, 6, 7, 8] * 5, name="x")
 
-        df = pandas.DataFrame( index=i.index )
-        df = df.join( i )
-        endo = df.join( y )
-        exo = df.join( x )
+        df = pandas.DataFrame(index=i.index)
+        df = df.join(i)
+        endo = df.join(y)
+        exo = df.join(x)
         endo_groups = endo.groupby("i")
         exo_groups = exo.groupby("i")
-        exo_Df = exo_groups.agg( [np.sum, np.max] )
-        endo_Df = endo_groups.agg( [np.sum, np.max] )
-        reg = OLS(exo_Df[[("x", "sum")]],endo_Df).fit()
+        exo_df = exo_groups.agg([np.sum, np.max])
+        endo_df = endo_groups.agg([np.sum, np.max])
+        reg = OLS(exo_df[[("x", "sum")]], endo_df).fit()
         interesting_lines = []
         import warnings
         with warnings.catch_warnings():
             # Catch ominormal warning, not interesting here
             warnings.simplefilter("ignore")
-            for line in str( reg.summary() ).splitlines():
-                if "('" in line:
-                    interesting_lines.append( line[:38] )
+            for line in str(reg.summary()).splitlines():
+                if "_" in line:
+                    interesting_lines.append(line[:38])
 
-        desired = ["Dep. Variable:           ('x', 'sum') ",
-                   "('y', 'sum')      1.4595      0.209   ",
-                   "('y', 'amax')     0.2432      0.035   "]
+        desired = ["Dep. Variable:                  x_sum ",
+                   "y_sum          1.4595      0.209      ",
+                   "y_amax         0.2432      0.035      "]
 
-        assert_equal(sorted(desired), sorted(interesting_lines)  )
+        assert_equal(sorted(desired), sorted(interesting_lines))