Repository: statsmodels/statsmodels

Issue:
NameErrors in grouputils
"g" is not defined at this point: https://github.com/statsmodels/statsmodels/blob/master/statsmodels/tools/grouputils.py#L114

One of the arguments is "group".  Is that supposed to be `x[group==cat]` ?

Same thing here: https://github.com/statsmodels/statsmodels/blob/master/statsmodels/tools/grouputils.py#L180







Discussion:
Yes, The first one should be group instead of g

I'm not sure about the second one. I never added the sparse version of the sandwiches to statsmodels, where I think I used this. I have a new version of the sparse dummy matrix function in one of the sparse/absorb panel data PR. 

given that groups is supposed to be consecutive integer similar to arange, I think g should be `groups` in the second case.

NameError no big deal.  Closing.

Resolving patch:
diff --git a/statsmodels/sandbox/panel/panel_short.py b/statsmodels/sandbox/panel/panel_short.py
--- a/statsmodels/sandbox/panel/panel_short.py
+++ b/statsmodels/sandbox/panel/panel_short.py
@@ -24,7 +24,7 @@
 
 import numpy as np
 from statsmodels.regression.linear_model import OLS, GLS
-from statsmodels.tools.grouputils import Group, GroupSorted
+from statsmodels.tools.grouputils import GroupSorted
 
 
 def sum_outer_product_loop(x, group_iter):
diff --git a/statsmodels/stats/sandwich_covariance.py b/statsmodels/stats/sandwich_covariance.py
--- a/statsmodels/stats/sandwich_covariance.py
+++ b/statsmodels/stats/sandwich_covariance.py
@@ -105,7 +105,7 @@
 import pandas as pd
 import numpy as np
 
-from statsmodels.tools.grouputils import Group, group_sums
+from statsmodels.tools.grouputils import combine_indices, group_sums
 from statsmodels.stats.moment_helpers import se_cov
 
 __all__ = ['cov_cluster', 'cov_cluster_2groups', 'cov_hac', 'cov_nw_panel',
@@ -589,10 +589,9 @@ def cov_cluster_2groups(results, group, group2=None, use_correction=True):
     #[0] because we get still also returns bse
     cov1 = cov_cluster(results, group1, use_correction=use_correction)
 
-    group_intersection = Group(group)
-    #cov of cluster formed by intersection of two groups
+    # cov of cluster formed by intersection of two groups
     cov01 = cov_cluster(results,
-                        group_intersection.group_int,
+                        combine_indices(group)[0],
                         use_correction=use_correction)
 
     #robust cov matrix for union of groups
diff --git a/statsmodels/tools/grouputils.py b/statsmodels/tools/grouputils.py
--- a/statsmodels/tools/grouputils.py
+++ b/statsmodels/tools/grouputils.py
@@ -110,7 +110,7 @@ def group_sums(x, group, use_bincount=True):
         uniques = np.unique(group)
         result = np.zeros([len(uniques)] + list(x.shape[1:]))
         for ii, cat in enumerate(uniques):
-            result[ii] = x[g == cat].sum(0)
+            result[ii] = x[group == cat].sum(0)
         return result
 
 
@@ -551,102 +551,3 @@ def dummy_sparse(self, level=0):
         indptr = np.arange(len(groups)+1)
         data = np.ones(len(groups), dtype=np.int8)
         self._dummies = sparse.csr_matrix((data, groups, indptr))
-
-
-if __name__ == '__main__':
-
-    # ---------- examples combine_indices
-    from numpy.testing import assert_equal
-
-    np.random.seed(985367)
-    groups = np.random.randint(0, 2, size=(10, 2))
-    uv, ux, u, label = combine_indices(groups, return_labels=True)
-    uv, ux, u, label = combine_indices(groups, prefix='g1,g2=', sep=',',
-                                       return_labels=True)
-
-    group0 = np.array(['sector0', 'sector1'])[groups[:, 0]]
-    group1 = np.array(['region0', 'region1'])[groups[:, 1]]
-    uv, ux, u, label = combine_indices((group0, group1),
-                                       prefix='sector,region=',
-                                       sep=',',
-                                       return_labels=True)
-    uv, ux, u, label = combine_indices((group0, group1), prefix='', sep='.',
-                                       return_labels=True)
-    group_joint = np.array(label)[uv]
-    group_joint_expected = np.array(['sector1.region0', 'sector0.region1',
-                                     'sector0.region0', 'sector0.region1',
-                                     'sector1.region1', 'sector0.region0',
-                                     'sector1.region0', 'sector1.region0',
-                                     'sector0.region1', 'sector0.region0'],
-                                    dtype='|S15')
-    assert_equal(group_joint, group_joint_expected)
-
-    """
-    >>> uv
-    array([2, 1, 0, 0, 1, 0, 2, 0, 1, 0])
-    >>> label
-    ['sector0.region0', 'sector1.region0', 'sector1.region1']
-    >>> np.array(label)[uv]
-    array(['sector1.region1', 'sector1.region0', 'sector0.region0',
-           'sector0.region0', 'sector1.region0', 'sector0.region0',
-           'sector1.region1', 'sector0.region0', 'sector1.region0',
-           'sector0.region0'],
-          dtype='|S15')
-    >>> np.column_stack((group0, group1))
-    array([['sector1', 'region1'],
-           ['sector1', 'region0'],
-           ['sector0', 'region0'],
-           ['sector0', 'region0'],
-           ['sector1', 'region0'],
-           ['sector0', 'region0'],
-           ['sector1', 'region1'],
-           ['sector0', 'region0'],
-           ['sector1', 'region0'],
-           ['sector0', 'region0']],
-          dtype='|S7')
-      """
-
-    # ------------- examples sparse_dummies
-    from scipy import sparse
-
-    g = np.array([0, 0, 1, 2, 1, 1, 2, 0])
-    u = lrange(3)
-    indptr = np.arange(len(g)+1)
-    data = np.ones(len(g), dtype=np.int8)
-    a = sparse.csr_matrix((data, g, indptr))
-    print(a.todense())
-    print(np.all(a.todense() == (g[:, None] == np.arange(3)).astype(int)))
-
-    x = np.arange(len(g)*3).reshape(len(g), 3, order='F')
-
-    print('group means')
-    print(x.T * a)
-    print(np.dot(x.T, g[:, None] == np.arange(3)))
-    print(np.array([np.bincount(g, weights=x[:, col]) for col in range(3)]))
-    for cat in u:
-        print(x[g == cat].sum(0))
-    for cat in u:
-        x[g == cat].sum(0)
-
-    cc = sparse.csr_matrix([[0, 1, 0, 1, 0, 0, 0, 0, 0],
-                            [1, 0, 1, 0, 1, 0, 0, 0, 0],
-                            [0, 1, 0, 0, 0, 1, 0, 0, 0],
-                            [1, 0, 0, 0, 1, 0, 1, 0, 0],
-                            [0, 1, 0, 1, 0, 1, 0, 1, 0],
-                            [0, 0, 1, 0, 1, 0, 0, 0, 1],
-                            [0, 0, 0, 1, 0, 0, 0, 1, 0],
-                            [0, 0, 0, 0, 1, 0, 1, 0, 1],
-                            [0, 0, 0, 0, 0, 1, 0, 1, 0]])
-
-    # ------------- groupsums
-    print(group_sums(np.arange(len(g)*3*2).reshape(len(g), 3, 2), g,
-                     use_bincount=False).T)
-    print(group_sums(np.arange(len(g)*3*2).reshape(len(g), 3, 2)[:, :, 0], g))
-    print(group_sums(np.arange(len(g)*3*2).reshape(len(g), 3, 2)[:, :, 1], g))
-
-    # ------------- examples class
-    x = np.arange(len(g)*3).reshape(len(g), 3, order='F')
-    mygroup = Group(g)
-    print(mygroup.group_int)
-    print(mygroup.group_sums(x))
-    print(mygroup.labels())


Test patch:
diff --git a/statsmodels/stats/tests/test_groups_sw.py b/statsmodels/stats/tests/test_groups_sw.py
--- a/statsmodels/stats/tests/test_groups_sw.py
+++ b/statsmodels/stats/tests/test_groups_sw.py
@@ -10,7 +10,8 @@
 import numpy as np
 from numpy.testing import assert_equal, assert_raises
 import statsmodels.stats.sandwich_covariance as sw
-from statsmodels.tools.grouputils import Group, GroupSorted
+from statsmodels.tools.grouputils import GroupSorted
+
 
 class CheckPanelLagMixin(object):
 
diff --git a/statsmodels/tools/tests/test_grouputils.py b/statsmodels/tools/tests/test_grouputils.py
--- a/statsmodels/tools/tests/test_grouputils.py
+++ b/statsmodels/tools/tests/test_grouputils.py
@@ -1,13 +1,17 @@
-from statsmodels.compat.python import PY37
+from statsmodels.compat.python import PY37, lrange
 
 import numpy as np
+from numpy.testing import assert_equal
 import pandas as pd
-from statsmodels.tools.grouputils import Grouping
+from pandas.util import testing as tm
+
+from statsmodels.tools.grouputils import (
+    Grouping, Group, combine_indices, group_sums)
 from statsmodels.tools.tools import categorical
 from statsmodels.datasets import grunfeld, anes96
-from pandas.util import testing as ptesting
 import pytest
 
+
 class CheckGrouping(object):
 
     def test_reindex(self):
@@ -23,7 +27,7 @@ def test_sort(self):
         sorted_data, index = self.grouping.sort(self.data)
         expected_sorted_data = self.data.sort_index()
 
-        ptesting.assert_frame_equal(sorted_data, expected_sorted_data)
+        tm.assert_frame_equal(sorted_data, expected_sorted_data)
         np.testing.assert_(isinstance(sorted_data, pd.DataFrame))
         np.testing.assert_(not index.equals(self.grouping.index))
 
@@ -42,7 +46,7 @@ def test_sort(self):
         sorted_data, index = self.grouping.sort(series)
 
         expected_sorted_data = series.sort_index()
-        ptesting.assert_series_equal(sorted_data, expected_sorted_data)
+        tm.assert_series_equal(sorted_data, expected_sorted_data)
         np.testing.assert_(isinstance(sorted_data, pd.Series))
         if hasattr(sorted_data, 'equals'):
             np.testing.assert_(not sorted_data.equals(series))
@@ -254,3 +258,53 @@ def test_init_api():
     grouping = Grouping(list_groups)
     np.testing.assert_array_equal(grouping.group_names,
                                   ['group0', 'group1', 'group2'])
+
+
+def test_combine_indices():
+    # Moved from grouputils __main__ section
+    np.random.seed(985367)
+    groups = np.random.randint(0, 2, size=(10, 2))
+    uv, ux, u, label = combine_indices(groups, return_labels=True)
+    uv, ux, u, label = combine_indices(groups, prefix='g1,g2=', sep=',',
+                                       return_labels=True)
+
+    group0 = np.array(['sector0', 'sector1'])[groups[:, 0]]
+    group1 = np.array(['region0', 'region1'])[groups[:, 1]]
+    uv, ux, u, label = combine_indices((group0, group1),
+                                       prefix='sector,region=',
+                                       sep=',',
+                                       return_labels=True)
+    uv, ux, u, label = combine_indices((group0, group1), prefix='', sep='.',
+                                       return_labels=True)
+    group_joint = np.array(label)[uv]
+    group_joint_expected = np.array(['sector1.region0', 'sector0.region1',
+                                     'sector0.region0', 'sector0.region1',
+                                     'sector1.region1', 'sector0.region0',
+                                     'sector1.region0', 'sector1.region0',
+                                     'sector0.region1', 'sector0.region0'],
+                                    dtype='|U15')
+    assert_equal(group_joint, group_joint_expected)
+
+
+@pytest.mark.smoke
+def test_group_sums():
+    # Moved from grouputils __main__ section
+    g = np.array([0, 0, 1, 2, 1, 1, 2, 0])
+
+    group_sums(np.arange(len(g)*3*2).reshape(len(g), 3, 2), g,
+               use_bincount=False).T
+    group_sums(np.arange(len(g)*3*2).reshape(len(g), 3, 2)[:, :, 0], g)
+    group_sums(np.arange(len(g)*3*2).reshape(len(g), 3, 2)[:, :, 1], g)
+
+
+@pytest.mark.smoke
+def test_group_class():
+    # Moved from grouputils __main__ section
+    g = np.array([0, 0, 1, 2, 1, 1, 2, 0])
+
+    x = np.arange(len(g)*3).reshape(len(g), 3, order='F')
+    mygroup = Group(g)
+
+    mygroup.group_int
+    mygroup.group_sums(x)
+    mygroup.labels()