Repository: pandas-dev/pandas

Issue:
Slice by column then by index fails if columns/rows are repeated.
We've found a problem where repeating a row and a column in a DataFrame fails with a "Cannot create BlockManager._ref_locs" assertion error.

The dataframe is very simple:

``` python
df = pd.DataFrame(np.arange(25.).reshape(5,5),
                            index=['a', 'b', 'c', 'd', 'e'],
                            columns=['a', 'b', 'c', 'd', 'e'])
```

And we pull the data out like this:

``` python
z = df[['a', 'c', 'a']]
z.ix[['a', 'c', 'a']]
Traceback (most recent call last):
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/ipython-1.1.0_1_ahl1-py2.7.egg/IPython/core/interactiveshell.py", line 2830, in run_code
    exec code_obj in self.user_global_ns, self.user_ns
  File "<ipython-input-87-3bdc0aacc4b5>", line 1, in <module>
    z.ix[['a', 'c', 'a']]
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_292_g4dcecb0-py2.7-linux-x86_64.egg/pandas/core/indexing.py", line 56, in __getitem__
    return self._getitem_axis(key, axis=0)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_292_g4dcecb0-py2.7-linux-x86_64.egg/pandas/core/indexing.py", line 744, in _getitem_axis
    return self._getitem_iterable(key, axis=axis)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_292_g4dcecb0-py2.7-linux-x86_64.egg/pandas/core/indexing.py", line 816, in _getitem_iterable
    convert=False)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_292_g4dcecb0-py2.7-linux-x86_64.egg/pandas/core/generic.py", line 1164, in take
    new_data = self._data.take(indices, axis=baxis)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_292_g4dcecb0-py2.7-linux-x86_64.egg/pandas/core/internals.py", line 3366, in take
    ref_items=new_axes[0], axis=axis)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_292_g4dcecb0-py2.7-linux-x86_64.egg/pandas/core/internals.py", line 2337, in apply
    do_integrity_check=do_integrity_check)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_292_g4dcecb0-py2.7-linux-x86_64.egg/pandas/core/internals.py", line 1990, in __init__
    self._set_ref_locs(do_refs=True)
  File "/users/is/dbew/pyenvs/timeseries/lib/python2.7/site-packages/pandas-0.13.0_292_g4dcecb0-py2.7-linux-x86_64.egg/pandas/core/internals.py", line 2130, in _set_ref_locs
    'have _ref_locs set' % (block, labels))
AssertionError: Cannot create BlockManager._ref_locs because block [FloatBlock: [a], 1 x 3, dtype: float64] with duplicate items [Index([u'a', u'c', u'a'], dtype='object')] does not have _ref_locs set
```

If instead we take a copy of the intermediate step, then it works:

``` python
z = df[['a', 'c', 'a']].copy()
z.ix[['a', 'c', 'a']]
Out[89]: 
    a   c   a
a   0   2   0
c  10  12  10
a   0   2   0

[3 rows x 3 columns]
```

This means that if you several functions which each do a part of the data processing, you need to know the history of an object to know whether what you're doing works. I think .ix should _always_ succeed on a DataFrame or Series, regardless of how it was constructed.

(I've read the discussion at https://github.com/pydata/pandas/issues/6056 about chained operations - but it's not something you can avoid if you have a pipeline of small steps instead of one big step).

This wasn't an issue in 0.11.0 but is failing in 0.13.0 and the latest master. Here's the output of installed versions when running on the master:

commit: None
python: 2.7.3.final.0
python-bits: 64
OS: Linux
OS-release: 2.6.18-308.el5
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_GB

pandas: 0.13.0-292-g4dcecb0
Cython: 0.16
numpy: 1.7.1
scipy: 0.9.0
statsmodels: None
patsy: None
scikits.timeseries: None
dateutil: 1.5
pytz: None
bottleneck: 0.6.0
tables: 2.3.1-1
numexpr: 2.0.1
matplotlib: 1.1.1
openpyxl: None
xlrd: 0.8.0
xlwt: None
xlsxwriter: None
sqlalchemy: None
lxml: 2.3.6
bs4: None
html5lib: None
bq: None
apiclient: None



Discussion:

Resolving patch:
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -116,6 +116,25 @@ def ref_locs(self):
             self._ref_locs = indexer
         return self._ref_locs
 
+    def take_ref_locs(self, indexer):
+        """
+        need to preserve the ref_locs and just shift them
+        return None if ref_locs is None
+
+        see GH6509
+        """
+
+        ref_locs = self._ref_locs
+        if ref_locs is None:
+            return None
+
+        tindexer = np.ones(len(ref_locs),dtype=bool)
+        tindexer[indexer] = False
+        tindexer = tindexer.astype(int).cumsum()[indexer]
+        ref_locs = ref_locs[indexer]
+        ref_locs -= tindexer
+        return ref_locs
+
     def reset_ref_locs(self):
         """ reset the block ref_locs """
         self._ref_locs = np.empty(len(self.items), dtype='int64')
@@ -866,13 +885,20 @@ def func(x):
                   ndim=self.ndim, klass=self.__class__, fastpath=True)]
         return self._maybe_downcast(blocks, downcast)
 
-    def take(self, indexer, ref_items, axis=1):
+    def take(self, indexer, ref_items, new_axis, axis=1):
         if axis < 1:
             raise AssertionError('axis must be at least 1, got %d' % axis)
         new_values = com.take_nd(self.values, indexer, axis=axis,
                                  allow_fill=False)
+
+        # need to preserve the ref_locs and just shift them
+        # GH6121
+        ref_locs = None
+        if not new_axis.is_unique:
+            ref_locs = self._ref_locs
+
         return [make_block(new_values, self.items, ref_items, ndim=self.ndim,
-                           klass=self.__class__, fastpath=True)]
+                           klass=self.__class__, placement=ref_locs, fastpath=True)]
 
     def get_values(self, dtype=None):
         return self.values
@@ -1820,7 +1846,7 @@ def shift(self, indexer, periods, axis=0):
             new_values[periods:] = fill_value
         return [self.make_block(new_values)]
 
-    def take(self, indexer, ref_items, axis=1):
+    def take(self, indexer, ref_items, new_axis, axis=1):
         """ going to take our items
             along the long dimension"""
         if axis < 1:
@@ -2601,18 +2627,7 @@ def get_slice(self, slobj, axis=0, raise_on_error=False):
             if len(self.blocks) == 1:
 
                 blk = self.blocks[0]
-
-                # see GH 6059
-                ref_locs = blk._ref_locs
-                if ref_locs is not None:
-
-                    # need to preserve the ref_locs and just shift them
-                    indexer = np.ones(len(ref_locs),dtype=bool)
-                    indexer[slobj] = False
-                    indexer = indexer.astype(int).cumsum()[slobj]
-                    ref_locs = ref_locs[slobj]
-                    ref_locs -= indexer
-
+                ref_locs = blk.take_ref_locs(slobj)
                 newb = make_block(blk._slice(slobj), new_items, new_items,
                                   klass=blk.__class__, fastpath=True,
                                   placement=ref_locs)
@@ -3371,6 +3386,7 @@ def take(self, indexer, new_index=None, axis=1, verify=True):
         if axis < 1:
             raise AssertionError('axis must be at least 1, got %d' % axis)
 
+        self._consolidate_inplace()
         if isinstance(indexer, list):
             indexer = np.array(indexer)
 
@@ -3388,8 +3404,12 @@ def take(self, indexer, new_index=None, axis=1, verify=True):
             new_index = self.axes[axis].take(indexer)
 
         new_axes[axis] = new_index
-        return self.apply('take', axes=new_axes, indexer=indexer,
-                          ref_items=new_axes[0], axis=axis)
+        return self.apply('take',
+                          axes=new_axes,
+                          indexer=indexer,
+                          ref_items=new_axes[0],
+                          new_axis=new_axes[axis],
+                          axis=axis)
 
     def merge(self, other, lsuffix=None, rsuffix=None):
         if not self._is_indexed_like(other):


Test patch:
diff --git a/pandas/tests/test_frame.py b/pandas/tests/test_frame.py
--- a/pandas/tests/test_frame.py
+++ b/pandas/tests/test_frame.py
@@ -3303,6 +3303,21 @@ def check(result, expected=None):
         result = dfbool[['one', 'three', 'one']]
         check(result,expected)
 
+        # multi-axis dups
+        # GH 6121
+        df = DataFrame(np.arange(25.).reshape(5,5),
+                       index=['a', 'b', 'c', 'd', 'e'],
+                       columns=['A', 'B', 'C', 'D', 'E'])
+        z = df[['A', 'C', 'A']].copy()
+        expected = z.ix[['a', 'c', 'a']]
+
+        df = DataFrame(np.arange(25.).reshape(5,5),
+                       index=['a', 'b', 'c', 'd', 'e'],
+                       columns=['A', 'B', 'C', 'D', 'E'])
+        z = df[['A', 'C', 'A']]
+        result = z.ix[['a', 'c', 'a']]
+        check(result,expected)
+
     def test_insert_benchmark(self):
         # from the vb_suite/frame_methods/frame_insert_columns
         N = 10