Repository: statsmodels/statsmodels

Issue:
BUG: get_robustcov_results('cluster') fails if `groups` is 2D vector
``` python
import pandas as pd
import numpy as np
from numpy.random import randn, randint

import statsmodels.api as sm

N = 1000000
Ng = 100000
df = pd.DataFrame({'y': randn(N), 'a': randn(N), 'b': randn(N),
                   'group': randint(Ng, size=N)})
reg = sm.OLS.from_formula('y ~ a + b', df).fit()
if 1 == 1:
    # Variation 1: An (N, 1) array
    clust = reg.get_robustcov_results('cluster', groups=df[['group']].values)
else:
    # Variation 2: An (N, 1) DataFrame
    clust = reg.get_robustcov_results('cluster', groups=df[['group']])
```

As far as I can tell from the documentation, passing these kind of arrays should be supported. It also seems like it should be very easy to fix both. I would have just gone straight to a pull request, but I haven't been using statsmodels very long and I'm still getting a feel for the code, philosophy, etc etc. Wanted to check first before spending the time.

Versions: win7, python 2.7.7 (Anaconda), pandas 0.15.2, numpy 1.9.1, statsmodels 0.7.0 (pulled about 10 minutes ago)

BUG: get_robustcov_results('cluster') fails if `groups` is 2D vector
``` python
import pandas as pd
import numpy as np
from numpy.random import randn, randint

import statsmodels.api as sm

N = 1000000
Ng = 100000
df = pd.DataFrame({'y': randn(N), 'a': randn(N), 'b': randn(N),
                   'group': randint(Ng, size=N)})
reg = sm.OLS.from_formula('y ~ a + b', df).fit()
if 1 == 1:
    # Variation 1: An (N, 1) array
    clust = reg.get_robustcov_results('cluster', groups=df[['group']].values)
else:
    # Variation 2: An (N, 1) DataFrame
    clust = reg.get_robustcov_results('cluster', groups=df[['group']])
```

As far as I can tell from the documentation, passing these kind of arrays should be supported. It also seems like it should be very easy to fix both. I would have just gone straight to a pull request, but I haven't been using statsmodels very long and I'm still getting a feel for the code, philosophy, etc etc. Wanted to check first before spending the time.

Versions: win7, python 2.7.7 (Anaconda), pandas 0.15.2, numpy 1.9.1, statsmodels 0.7.0 (pulled about 10 minutes ago)



Discussion:
related to #2169
most likely insufficient input checking and handling.

In this case a `squeeze` is missing.  groups can be a Series, or it can be 2d with 2columns (IIRC).
groups cannot be a one column array or DataFrame, i.e. shape (nobs, 1)

`clust = reg.get_robustcov_results('cluster', groups=df[['group']].values.squeeze())`
works

A PR would be welcome, if you find where the array conversion and `squeeze` needs to go, + unit tests.
Otherwise, I will fix this before 0.7, i.e. soon.

Thanks for the report.

Happy to, once I get familiar with the unit tests.

related to #2169
most likely insufficient input checking and handling.

In this case a `squeeze` is missing.  groups can be a Series, or it can be 2d with 2columns (IIRC).
groups cannot be a one column array or DataFrame, i.e. shape (nobs, 1)

`clust = reg.get_robustcov_results('cluster', groups=df[['group']].values.squeeze())`
works

A PR would be welcome, if you find where the array conversion and `squeeze` needs to go, + unit tests.
Otherwise, I will fix this before 0.7, i.e. soon.

Thanks for the report.

Happy to, once I get familiar with the unit tests.


Resolving patch:
diff --git a/statsmodels/base/covtype.py b/statsmodels/base/covtype.py
--- a/statsmodels/base/covtype.py
+++ b/statsmodels/base/covtype.py
@@ -183,6 +183,10 @@ def get_robustcov_results(self, cov_type='HC1', use_t=None, **kwds):
         groups = kwds['groups']
         if not hasattr(groups, 'shape'):
             groups = np.asarray(groups).T
+
+        if groups.ndim >= 2:
+            groups = groups.squeeze()
+
         res.cov_kwds['groups'] = groups
         use_correction = kwds.get('use_correction', True)
         res.cov_kwds['use_correction'] = use_correction
@@ -195,6 +199,9 @@ def get_robustcov_results(self, cov_type='HC1', use_t=None, **kwds):
                                              use_correction=use_correction)
 
         elif groups.ndim == 2:
+            if hasattr(groups, 'values'):
+                groups = groups.values
+
             if adjust_df:
                 # need to find number of groups
                 # duplicate work
diff --git a/statsmodels/regression/linear_model.py b/statsmodels/regression/linear_model.py
--- a/statsmodels/regression/linear_model.py
+++ b/statsmodels/regression/linear_model.py
@@ -1813,6 +1813,10 @@ def get_robustcov_results(self, cov_type='HC1', use_t=None, **kwds):
             groups = kwds['groups']
             if not hasattr(groups, 'shape'):
                 groups = np.asarray(groups).T
+
+            if groups.ndim >= 2:
+                groups = groups.squeeze()
+
             res.cov_kwds['groups'] = groups
             use_correction = kwds.get('use_correction', True)
             res.cov_kwds['use_correction'] = use_correction
@@ -1825,6 +1829,9 @@ def get_robustcov_results(self, cov_type='HC1', use_t=None, **kwds):
                                                  use_correction=use_correction)
 
             elif groups.ndim == 2:
+                if hasattr(groups, 'values'):
+                    groups = groups.values
+
                 if adjust_df:
                     # need to find number of groups
                     # duplicate work


Test patch:
diff --git a/statsmodels/regression/tests/test_robustcov.py b/statsmodels/regression/tests/test_robustcov.py
--- a/statsmodels/regression/tests/test_robustcov.py
+++ b/statsmodels/regression/tests/test_robustcov.py
@@ -9,7 +9,9 @@
 import numpy as np
 from scipy import stats
 
-from numpy.testing import assert_allclose, assert_equal, assert_warns
+from numpy.testing import (assert_allclose, assert_equal, assert_warns,
+                           assert_raises)
+
 
 from statsmodels.regression.linear_model import OLS, WLS
 import statsmodels.stats.sandwich_covariance as sw
@@ -380,6 +382,45 @@ def setup(self):
         self.rtolh = 1e-10
 
 
+class TestOLSRobustCluster2Input(CheckOLSRobustCluster, CheckOLSRobustNewMixin):
+    # compare with `reg cluster`
+
+    def setup(self):
+        import pandas as pd
+        fat_array = self.groups.reshape(-1, 1)
+        fat_groups = pd.DataFrame(fat_array)
+
+        res_ols = self.res1.get_robustcov_results('cluster',
+                                                  groups=fat_groups,
+                                                  use_correction=True,
+                                                  use_t=True)
+        self.res3 = self.res1
+        self.res1 = res_ols
+        self.bse_robust = res_ols.bse
+        self.cov_robust = res_ols.cov_params()
+        cov1 = sw.cov_cluster(self.res1, self.groups, use_correction=True)
+        se1 =  sw.se_cov(cov1)
+        self.bse_robust2 = se1
+        self.cov_robust2 = cov1
+        self.small = True
+        self.res2 = res2.results_cluster
+
+        self.rtol = 1e-6
+        self.rtolh = 1e-10
+
+    def test_too_many_groups(self):
+        long_groups = self.groups.reshape(-1, 1)
+        groups3 = np.hstack((long_groups, long_groups, long_groups))
+        assert_raises(ValueError, self.res1.get_robustcov_results,'cluster',
+                      groups=groups3, use_correction=True, use_t=True)
+
+    def test_2way_dataframe(self):
+        import pandas as pd
+        long_groups = self.groups.reshape(-1, 1)
+        groups2 = pd.DataFrame(np.hstack((long_groups, long_groups)))
+        res = self.res1.get_robustcov_results(
+            'cluster', groups=groups2, use_correction=True, use_t=True)
+
 class TestOLSRobustCluster2Fit(CheckOLSRobustCluster, CheckOLSRobustNewMixin):
     # copy, past uses fit method
     # compare with `reg cluster`