Repository: pandas-dev/pandas

Issue:
AssertionError while boolean indexing with non-unique columns
Hit this `AssertionError: cannot create BlockManager._ref_locs ...` while processing a dataframe with multiple empty column names. Appears to fail with 0.12/master and pass with 0.11.

```
import numpy as np
import pandas as pd
df = pd.DataFrame(np.random.rand(3,4), columns=['', '', 'C', 'D'])
df[df.C > .5]
```

&nbsp;

```
Traceback (most recent call last):
  File "test_fail.py", line 5, in <module>
    df[df.C > .5]
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/frame.py", line 1830, in __getitem__
    return self._getitem_frame(key)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/frame.py", line 1898, in _getitem_frame
    return self.where(key)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/generic.py", line 2372, in where
    cond = cond.reindex(**self._construct_axes_dict())
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/generic.py", line 1118, in reindex
    return self._reindex_axes(axes, level, limit, method, fill_value, copy, takeable=takeable)._propogate_attributes(self)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/frame.py", line 2413, in _reindex_axes
    fill_value, limit, takeable=takeable)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/frame.py", line 2436, in _reindex_columns
    copy=copy, fill_value=fill_value, allow_dups=takeable)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/generic.py", line 1218, in _reindex_with_indexers
    fill_value=fill_value, allow_dups=allow_dups)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/internals.py", line 2840, in reindex_indexer
    return self._reindex_indexer_items(new_axis, indexer, fill_value)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/internals.py", line 2885, in _reindex_indexer_items
    return self.__class__(new_blocks, new_axes)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/internals.py", line 1735, in __init__
    self._set_ref_locs(do_refs=True)
  File "/home/gmd/.virtualenvs/pandas-master/local/lib/python2.7/site-packages/pandas-0.12.0_476_gb889fda-py2.7-linux-i686.egg/pandas/core/internals.py", line 1863, in _set_ref_locs
    "does not have _ref_locs set" % (block, labels))
AssertionError: cannot create BlockManager._ref_locs because block [BoolBlock: [C], 1 x 3, dtype: bool] with duplicate items [Index([u'', u'', u'C', u'D'], dtype=object)] does not have _ref_locs set
```



Discussion:

Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -1840,8 +1840,12 @@ def _getitem_column(self, key):
         if self.columns.is_unique:
             return self._get_item_cache(key)
 
-        # duplicate columns
-        return self._constructor(self._data.get(key))
+        # duplicate columns & possible reduce dimensionaility
+        result = self._constructor(self._data.get(key))
+        if result.columns.is_unique:
+            result = result[key]
+
+        return result
 
     def _getitem_slice(self, key):
         return self._slice(key, axis=0)
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -401,7 +401,7 @@ def _astype(self, dtype, copy=False, raise_on_error=True, values=None,
             if values is None:
                 values = com._astype_nansafe(self.values, dtype, copy=True)
             newb = make_block(
-                values, self.items, self.ref_items, ndim=self.ndim,
+                values, self.items, self.ref_items, ndim=self.ndim, placement=self._ref_locs,
                 fastpath=True, dtype=dtype, klass=klass)
         except:
             if raise_on_error is True:
@@ -716,7 +716,7 @@ def create_block(v, m, n, item, reshape=True):
         if inplace:
             return [self]
 
-        return [make_block(new_values, self.items, self.ref_items, fastpath=True)]
+        return [make_block(new_values, self.items, self.ref_items, placement=self._ref_locs, fastpath=True)]
 
     def interpolate(self, method='pad', axis=0, inplace=False,
                     limit=None, fill_value=None, coerce=False,
@@ -2853,12 +2853,13 @@ def _reindex_indexer_items(self, new_items, indexer, fill_value):
         # TODO: less efficient than I'd like
 
         item_order = com.take_1d(self.items.values, indexer)
+        new_axes = [new_items] + self.axes[1:]
+        new_blocks = []
+        is_unique = new_items.is_unique
 
         # keep track of what items aren't found anywhere
+        l = np.arange(len(item_order))
         mask = np.zeros(len(item_order), dtype=bool)
-        new_axes = [new_items] + self.axes[1:]
-
-        new_blocks = []
         for blk in self.blocks:
             blk_indexer = blk.items.get_indexer(item_order)
             selector = blk_indexer != -1
@@ -2872,12 +2873,19 @@ def _reindex_indexer_items(self, new_items, indexer, fill_value):
             new_block_items = new_items.take(selector.nonzero()[0])
             new_values = com.take_nd(blk.values, blk_indexer[selector], axis=0,
                                      allow_fill=False)
-            new_blocks.append(make_block(new_values, new_block_items,
-                                         new_items, fastpath=True))
+            placement = l[selector] if not is_unique else None
+            new_blocks.append(make_block(new_values,
+                                         new_block_items,
+                                         new_items,
+                                         placement=placement,
+                                         fastpath=True))
 
         if not mask.all():
             na_items = new_items[-mask]
-            na_block = self._make_na_block(na_items, new_items,
+            placement = l[-mask] if not is_unique else None
+            na_block = self._make_na_block(na_items,
+                                           new_items,
+                                           placement=placement,
                                            fill_value=fill_value)
             new_blocks.append(na_block)
             new_blocks = _consolidate(new_blocks, new_items)
@@ -2943,7 +2951,7 @@ def reindex_items(self, new_items, indexer=None, copy=True, fill_value=None):
 
         return self.__class__(new_blocks, new_axes)
 
-    def _make_na_block(self, items, ref_items, fill_value=None):
+    def _make_na_block(self, items, ref_items, placement=None, fill_value=None):
         # TODO: infer dtypes other than float64 from fill_value
 
         if fill_value is None:
@@ -2954,8 +2962,7 @@ def _make_na_block(self, items, ref_items, fill_value=None):
         dtype, fill_value = com._infer_dtype_from_scalar(fill_value)
         block_values = np.empty(block_shape, dtype=dtype)
         block_values.fill(fill_value)
-        na_block = make_block(block_values, items, ref_items)
-        return na_block
+        return make_block(block_values, items, ref_items, placement=placement)
 
     def take(self, indexer, new_index=None, axis=1, verify=True):
         if axis < 1:


Test patch:
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -3150,6 +3150,36 @@ def check(result, expected=None):
         expected = DataFrame([[1],[1],[1]],columns=['bar'])
         check(df,expected)
 
+    def test_column_dups_indexing(self):
+
+        def check(result, expected=None):
+            if expected is not None:
+                assert_frame_equal(result,expected)
+            result.dtypes
+            str(result)
+
+        # boolean indexing
+        # GH 4879
+        dups = ['A', 'A', 'C', 'D']
+        df = DataFrame(np.arange(12).reshape(3,4), columns=['A', 'B', 'C', 'D'],dtype='float64')
+        expected = df[df.C > 6]
+        expected.columns = dups
+        df = DataFrame(np.arange(12).reshape(3,4), columns=dups,dtype='float64')
+        result = df[df.C > 6]
+        check(result,expected)
+
+        # where
+        df = DataFrame(np.arange(12).reshape(3,4), columns=['A', 'B', 'C', 'D'],dtype='float64')
+        expected = df[df > 6]
+        expected.columns = dups
+        df = DataFrame(np.arange(12).reshape(3,4), columns=dups,dtype='float64')
+        result = df[df > 6]
+        check(result,expected)
+
+        # boolean with the duplicate raises
+        df = DataFrame(np.arange(12).reshape(3,4), columns=dups,dtype='float64')
+        self.assertRaises(ValueError, lambda : df[df.A > 6])
+
     def test_insert_benchmark(self):
         # from the vb_suite/frame_methods/frame_insert_columns
         N = 10
diff --git a/pandas/tests/test_indexing.py b/pandas/tests/test_indexing.py
--- a/pandas/tests/test_indexing.py
+++ b/pandas/tests/test_indexing.py
@@ -1233,9 +1233,10 @@ def test_mi_access(self):
 
         # GH 4146, not returning a block manager when selecting a unique index
         # from a duplicate index
-        expected = DataFrame([['a',1,1]],index=['A1'],columns=['h1','h3','h5'],).T
+        # as of 4879, this returns a Series (which is similar to what happens with a non-unique)
+        expected = Series(['a',1,1],index=['h1','h3','h5'])
         result = df2['A']['A1']
-        assert_frame_equal(result,expected)
+        assert_series_equal(result,expected)
 
         # selecting a non_unique from the 2nd level
         expected = DataFrame([['d',4,4],['e',5,5]],index=Index(['B2','B2'],name='sub'),columns=['h1','h3','h5'],).T