Repository: pandas-dev/pandas

Issue:
Slicing non-unique index fails when slice endpoint is not in index
This looks like a regression from 0.13.1. If you have a simple dataframe with a non-unique but sorted index like:

``` python
df = pd.DataFrame(np.arange(5.), index=[datetime(2001, 1, i, 10, 00)
                                        for i in [1,2,2,3,4]])
df
Out[23]: 
                     0
2001-01-01 10:00:00  0
2001-01-02 10:00:00  1
2001-01-02 10:00:00  2
2001-01-03 10:00:00  3
2001-01-04 10:00:00  4

[5 rows x 1 columns]
```

Then in pandas 0.13.1 you could slice a range out of that dataframe with a date not in the index like so:

``` python
# pandas 0.13.1
df.ix[datetime(2001, 1, 2, 11, 0):]
Out[22]: 
                     0
2001-01-03 10:00:00  3
2001-01-04 10:00:00  4

[2 rows x 1 columns]
```

In the HEAD of pandas/master now, you can't. Instead you get a TypeError with long long traceback.

``` python
# pandas 0.14.1
In [58]: df.ix[datetime(2001, 1, 2, 11, 0):]
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-58-c4fe729e46fe> in <module>()
----> 1 df.ix[datetime(2001, 1, 2, 11, 0):]

/users/is/dbew/src/pandas/pandas/core/indexing.py in __getitem__(self, key)
     69             return self._getitem_tuple(key)
     70         else:
---> 71             return self._getitem_axis(key, axis=0)
     72 
     73     def _get_label(self, label, axis=0):

/users/is/dbew/src/pandas/pandas/core/indexing.py in _getitem_axis(self, key, axis, validate_iterable)
    843         labels = self.obj._get_axis(axis)
    844         if isinstance(key, slice):
--> 845             return self._get_slice_axis(key, axis=axis)
    846         elif _is_list_like(key) and not (isinstance(key, tuple) and
    847                                          isinstance(labels, MultiIndex)):

/users/is/dbew/src/pandas/pandas/core/indexing.py in _get_slice_axis(self, slice_obj, axis)
   1090         if not _need_slice(slice_obj):
   1091             return obj
-> 1092         indexer = self._convert_slice_indexer(slice_obj, axis)
   1093 
   1094         if isinstance(indexer, slice):

/users/is/dbew/src/pandas/pandas/core/indexing.py in _convert_slice_indexer(self, key, axis)
    161         # if we are accessing via lowered dim, use the last dim
    162         ax = self.obj._get_axis(min(axis, self.ndim - 1))
--> 163         return ax._convert_slice_indexer(key, typ=self.name)
    164 
    165     def _has_valid_setitem_indexer(self, indexer):

/users/is/dbew/src/pandas/pandas/core/index.py in _convert_slice_indexer(self, key, typ)
    589         else:
    590             try:
--> 591                 indexer = self.slice_indexer(start, stop, step)
    592             except Exception:
    593                 if is_index_slice:

/users/is/dbew/src/pandas/pandas/tseries/index.py in slice_indexer(self, start, end, step)
   1350             raise TypeError('Cannot index datetime64 with float keys')
   1351 
-> 1352         return Index.slice_indexer(self, start, end, step)
   1353 
   1354     def slice_locs(self, start=None, end=None):

/users/is/dbew/src/pandas/pandas/core/index.py in slice_indexer(self, start, end, step)
   1712 
   1713         # loc indexers
-> 1714         return Index(start_slice) & Index(end_slice)
   1715 
   1716     def slice_locs(self, start=None, end=None):

/users/is/dbew/src/pandas/pandas/core/index.py in __new__(cls, data, dtype, copy, name, fastpath, tupleize_cols, **kwargs)
    170                         pass  # python2 - MultiIndex fails on mixed types
    171             # other iterable of some kind
--> 172             subarr = com._asarray_tuplesafe(data, dtype=object)
    173 
    174         if dtype is None:

/users/is/dbew/src/pandas/pandas/core/common.py in _asarray_tuplesafe(values, dtype)
   2106     if not (isinstance(values, (list, tuple))
   2107             or hasattr(values, '__array__')):
-> 2108         values = list(values)
   2109     elif isinstance(values, Index):
   2110         return values.values

TypeError: 'datetime.datetime' object is not iterable
```



Discussion:
Looks like this is specific to DatetimeIndex - I can't see the same behaviour with a simple int index.


Resolving patch:
diff --git a/pandas/core/index.py b/pandas/core/index.py
--- a/pandas/core/index.py
+++ b/pandas/core/index.py
@@ -1758,7 +1758,11 @@ def _get_slice(starting_value, offset, search_side, slice_property,
 
             except KeyError:
                 if self.is_monotonic:
-                    if not is_unique:
+
+                    # we are duplicated but non-unique
+                    # so if we have an indexer then we are done
+                    # else search for it (GH 7523)
+                    if not is_unique and is_integer(search_value):
                         slc = search_value
                     else:
                         slc = self.searchsorted(search_value,


Test patch:
diff --git a/pandas/tseries/tests/test_timeseries.py b/pandas/tseries/tests/test_timeseries.py
--- a/pandas/tseries/tests/test_timeseries.py
+++ b/pandas/tseries/tests/test_timeseries.py
@@ -2966,6 +2966,42 @@ def test_slice_locs_indexerror(self):
         s = Series(lrange(100000), times)
         s.ix[datetime(1900, 1, 1):datetime(2100, 1, 1)]
 
+    def test_slicing_datetimes(self):
+
+        # GH 7523
+
+        # unique
+        df = DataFrame(np.arange(4.,dtype='float64'),
+                       index=[datetime(2001, 1, i, 10, 00) for i in [1,2,3,4]])
+        result = df.ix[datetime(2001,1,1,10):]
+        assert_frame_equal(result,df)
+        result = df.ix[:datetime(2001,1,4,10)]
+        assert_frame_equal(result,df)
+        result = df.ix[datetime(2001,1,1,10):datetime(2001,1,4,10)]
+        assert_frame_equal(result,df)
+
+        result = df.ix[datetime(2001,1,1,11):]
+        expected = df.iloc[1:]
+        assert_frame_equal(result,expected)
+        result = df.ix['20010101 11':]
+        assert_frame_equal(result,expected)
+
+        # duplicates
+        df = pd.DataFrame(np.arange(5.,dtype='float64'),
+                          index=[datetime(2001, 1, i, 10, 00) for i in [1,2,2,3,4]])
+
+        result = df.ix[datetime(2001,1,1,10):]
+        assert_frame_equal(result,df)
+        result = df.ix[:datetime(2001,1,4,10)]
+        assert_frame_equal(result,df)
+        result = df.ix[datetime(2001,1,1,10):datetime(2001,1,4,10)]
+        assert_frame_equal(result,df)
+
+        result = df.ix[datetime(2001,1,1,11):]
+        expected = df.iloc[1:]
+        assert_frame_equal(result,expected)
+        result = df.ix['20010101 11':]
+        assert_frame_equal(result,expected)
 
 class TestSeriesDatetime64(tm.TestCase):
 
@@ -3054,7 +3090,7 @@ def test_intersection(self):
         for tz in [None, 'Asia/Tokyo']:
             rng = date_range('6/1/2000', '6/30/2000', freq='D', name='idx')
 
-            # if target has the same name, it is preserved 
+            # if target has the same name, it is preserved
             rng2 = date_range('5/15/2000', '6/20/2000', freq='D', name='idx')
             expected2 = date_range('6/1/2000', '6/20/2000', freq='D', name='idx')