Repository: pandas-dev/pandas

Issue:
multilevel df.ix fails with IndexError on Multindex with duplicate entries
The following example works:

df = DataFrame(np.arange(20).reshape(5,4))
df2 = df.set_index(keys=[0,1,2])
df2.ix[(4,5,6)]

for using a MultiIndex

So I created a file sample_data.csv that looks like this:

col1,col2,year,amount 
111111,3.5,2012,700 
111112,3.5,2011,600 
222221,4.0,2012,222 
... 

I then ran the following:

import numpy as np 
import pandas as pd 
sd=pd.read_csv('sample_data.csv') 
sd2=sd.set_index(keys=['col2','year']) 
sd2.ix[(4.0,2012)] 

But this produces the following error: IndexError: index out of bounds
Can anyone explain why the former example works, but the latter doesn't ?



Discussion:
Can't reproduce on 0.10.1.

``` ipython
In [54]: import numpy as np
    ...: import pandas as pd
    ...: sd=pd.read_csv('/tmp/1b.csv')
    ...: sd2=sd.set_index(keys=['col2','year'])
    ...: sd2.ix[(4.0,2012)]
Out[54]: 
col1      222221
amount       222
Name: (4.0, 2012)
```

I realize now that the problem occurs if the MultiIndex isn't unique, hence the following entries cause problems:
col1,col2,year,amount
111111,3.5,2012,700
111112,3.5,2011,600
222221,4.0,2012,222 
222234,4.0,2012,333
222256,4.0,2012,444

marked as bug.

Test case:

``` python
In [67]: df
Out[67]: 
          amount
col year        
3.5 2012     700
    2011     600
4.0 2012     222
    2012     333
    2012     444
```

``` python
import pandas as pd
data={'amount': {0: 700, 1: 600, 2: 222, 3: 333, 4: 444},
 'col': {0: 3.5, 1: 3.5, 2: 4.0, 3: 4.0, 4: 4.0},
 'year': {0: 2012, 1: 2011, 2: 2012, 3: 2012, 4: 2012}}
df=pd.DataFrame(data).set_index(keys=['col','year'])
df.ix[(4.0,2012)]
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-66-e55f14b02b89> in <module>()
      4  'year': {0: 2012, 1: 2011, 2: 2012, 3: 2012, 4: 2012}}
      5 df=pd.DataFrame(data).set_index(keys=['col','year'])
----> 6 df.ix[(4.0,2012)]

/home/user1/src/pandas/pandas/core/indexing.pyc in __getitem__(self, key)
     32                 pass
     33 
---> 34             return self._getitem_tuple(key)
     35         else:
     36             return self._getitem_axis(key, axis=0)

/home/user1/src/pandas/pandas/core/indexing.pyc in _getitem_tuple(self, tup)
    205     def _getitem_tuple(self, tup):
    206         try:
--> 207             return self._getitem_lowerdim(tup)
    208         except IndexingError:
    209             pass

/home/user1/src/pandas/pandas/core/indexing.pyc in _getitem_lowerdim(self, tup)
    326                         new_key, = new_key
    327 
--> 328                 return section.ix[new_key]
    329 
    330         raise IndexingError('not applicable')

/home/user1/src/pandas/pandas/core/indexing.pyc in __getitem__(self, key)
     32                 pass
     33 
---> 34             return self._getitem_tuple(key)
     35         else:
     36             return self._getitem_axis(key, axis=0)

/home/user1/src/pandas/pandas/core/indexing.pyc in _getitem_tuple(self, tup)
    205     def _getitem_tuple(self, tup):
    206         try:
--> 207             return self._getitem_lowerdim(tup)
    208         except IndexingError:
    209             pass

/home/user1/src/pandas/pandas/core/indexing.pyc in _getitem_lowerdim(self, tup)
    307         for i, key in enumerate(tup):
    308             if _is_label_like(key) or isinstance(key, tuple):
--> 309                 section = self._getitem_axis(key, axis=i)
    310 
    311                 # might have been a MultiIndex

/home/user1/src/pandas/pandas/core/indexing.pyc in _getitem_axis(self, key, axis)
    361             lab = key
    362             if com.is_integer(key) and not _is_integer_index(labels):
--> 363                 return self._get_loc(key, axis=axis)
    364             return self._get_label(lab, axis=axis)
    365 

/home/user1/src/pandas/pandas/core/indexing.pyc in _get_loc(self, key, axis)
     49 
     50     def _get_loc(self, key, axis=0):
---> 51         return self.obj._ixs(key, axis=axis)
     52 
     53     def _slice(self, obj, axis=0):

/home/user1/src/pandas/pandas/core/frame.pyc in _ixs(self, i, axis)
   1918             return self.irow(i)
   1919         else:
-> 1920             return self.icol(i)
   1921 
   1922     def iget_value(self, i, j):

/home/user1/src/pandas/pandas/core/frame.pyc in icol(self, i)
   1900         column : Series (int) or DataFrame (slice, sequence)
   1901         """
-> 1902         label = self.columns[i]
   1903         if isinstance(i, slice):
   1904             # need to return view

/home/user1/src/pandas/pandas/core/index.pyc in __getitem__(self, key)
    356         arr_idx = self.view(np.ndarray)
    357         if np.isscalar(key):
--> 358             return arr_idx[key]
    359         else:
    360             if com._is_bool_indexer(key):

IndexError: index out of bounds
```

```
```

@y-p this is blowing up because of the float indexer
We should outlaw float indexes in multiindex, too complicated

you mean it's numerical issues? is there type conversion going on
which makes the initial 4.0 different from the latter 4.0?

no it just makes `ix` really complicated, better to use `iloc/loc` in this case
and floating indices don't make sense IMHO anyhow, just select on the values (and don't use them as an index)

I agree, but if not numerical issues why then is it failing? just a bug in ix for floats?

This is actually because the index is not sorted,
see below

however, I think its a bug because this error is not raised (instead you get index out of bounds),
so going to leave open and push to 0.12 for that reason

@femibyte see below

```
(Pdb) df
          amount
col year        
3.5 2012     700
    2011     600
4.0 2012     222
    2012     333
    2012     444
(Pdb) df.sortlevel(inplace=True)
          amount
col year        
3.5 2011     600
    2012     700
4.0 2012     222
    2012     333
    2012     444

(Pdb) df.ix[(4.0,2012)]
          amount
col year        
4   2012     222
    2012     333
    2012     444
```


Resolving patch:
diff --git a/pandas/core/index.py b/pandas/core/index.py
--- a/pandas/core/index.py
+++ b/pandas/core/index.py
@@ -274,6 +274,9 @@ def values(self):
     def is_monotonic(self):
         return self._engine.is_monotonic
 
+    def is_lexsorted_for_tuple(self, tup):
+        return True
+
     @cache_readonly
     def is_unique(self):
         return self._engine.is_unique
@@ -1692,6 +1695,12 @@ def is_lexsorted(self):
         """
         return self.lexsort_depth == self.nlevels
 
+    def is_lexsorted_for_tuple(self, tup):
+        """
+        Return True if we are correctly lexsorted given the passed tuple
+        """
+        return len(tup) <= self.lexsort_depth
+
     @cache_readonly
     def lexsort_depth(self):
         if self.sortorder is not None:
diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py
--- a/pandas/core/indexing.py
+++ b/pandas/core/indexing.py
@@ -311,6 +311,10 @@ def _getitem_lowerdim(self, tup):
             except Exception, e1:
                 if isinstance(tup[0], (slice, Index)):
                     raise IndexingError
+
+                # raise the error if we are not sorted
+                if not ax0.is_lexsorted_for_tuple(tup):
+                    raise e1
                 try:
                     loc = ax0.get_loc(tup[0])
                 except KeyError:


Test patch:
diff --git a/pandas/tests/test_indexing.py b/pandas/tests/test_indexing.py
--- a/pandas/tests/test_indexing.py
+++ b/pandas/tests/test_indexing.py
@@ -697,6 +697,23 @@ def test_loc_multiindex(self):
         xp = mi_int.ix[4]
         assert_frame_equal(rs,xp)
 
+    def test_ix_general(self):
+
+        # ix general issues
+
+        # GH 2817
+        data={'amount': {0: 700, 1: 600, 2: 222, 3: 333, 4: 444},
+              'col': {0: 3.5, 1: 3.5, 2: 4.0, 3: 4.0, 4: 4.0},
+              'year': {0: 2012, 1: 2011, 2: 2012, 3: 2012, 4: 2012}}
+        df = DataFrame(data).set_index(keys=['col','year'])
+
+        # this should raise correct error
+        self.assertRaises(KeyError, df.ix.__getitem__, tuple([4.0,2012]))
+
+        # this is ok
+        df.sortlevel(inplace=True)
+        df.ix[(4.0,2012)]
+
 if __name__ == '__main__':
     import nose
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],