Repository: pandas-dev/pandas

Issue:
PERF: Float64Index to use Float hashtable backend
https://groups.google.com/forum/m/#!topic/pydata/zUxl7rOHVNY

Currently `Float64Index` uses the `Object` back-end for ease of implementation. 
See Here: https://github.com/pydata/pandas/blob/master/pandas/core/index.py#L1881
- change the backend to use `Float64Engine` - the cythob code needs fixin in order to make it work.
- needs vbench for perf indications
- needs pickle validation that it is backward compat (it should be w/o changes)



Discussion:
@jreback can I take this or do u already have something worked up?

go for it!

its very close....I just didn't have time to debug it....most of the code was already there from @wesm originally

Cool Ill take a look

@cpcloud any progress on this?

Yep ... should be able to put up a WIP PR this weekend

gr8 thanks

`nan` isn't really a singleton, probably because of its internal representation as some large integer. Small integers are singletons in Python, whereas larger ones are new instances of `int` every time and so their id changes. The first set membership test checks the object `nan` whose reference I put in to the array whereas `flt` has a new instance of nan. What's weird is that `hash(nan) == 0` and yet the membership check fails. I thought the order of lookup for sets was `hash` -> `eq` (if no `__eq__` method then compare `id`), but since `nan` is an instance of float it obviously has that method. @jreback Am I missing something here?

nope that's why you can lookup object dtyped floats/nan pretty easiky
but actual Nan's that are floats need special logic 
in essence you have to segregate Nan's and keep a reference to their locations and not hash them 
so keep the float hash table for non-nan then getitem checks for nan before you test the hash

@cpcloud coming along?

Yep coming along. Couple of errors left to go. Mostly just renaming and keeping those refs to nan locs around. 

gr8


Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -1843,7 +1843,6 @@ def eval(self, expr, **kwargs):
         kwargs['resolvers'] = kwargs.get('resolvers', ()) + resolvers
         return _eval(expr, **kwargs)
 
-
     def _box_item_values(self, key, values):
         items = self.columns[self.columns.get_loc(key)]
         if values.ndim == 2:
@@ -2566,7 +2565,7 @@ def sort_index(self, axis=0, by=None, ascending=True, inplace=False,
         -------
         sorted : DataFrame
         """
-        
+
         from pandas.core.groupby import _lexsort_indexer, _nargsort
         axis = self._get_axis_number(axis)
         if axis not in [0, 1]:  # pragma: no cover
@@ -2622,7 +2621,7 @@ def trans(v):
         else:
             indexer = _nargsort(labels, kind=kind, ascending=ascending,
                                 na_position=na_position)
-            
+
         if inplace:
             if axis == 1:
                 new_data = self._data.reindex_items(
@@ -3285,7 +3284,7 @@ def apply(self, func, axis=0, broadcast=False, raw=False, reduce=None,
         code path. This can lead to unexpected behavior if func has
         side-effects, as they will take effect twice for the first
         column/row.
-        
+
         Examples
         --------
         >>> df.apply(numpy.sqrt) # returns DataFrame
diff --git a/pandas/core/index.py b/pandas/core/index.py
--- a/pandas/core/index.py
+++ b/pandas/core/index.py
@@ -16,7 +16,8 @@
 from pandas.util.decorators import cache_readonly, deprecate
 from pandas.core.common import isnull, array_equivalent
 import pandas.core.common as com
-from pandas.core.common import _values_from_object, is_float, is_integer, ABCSeries
+from pandas.core.common import (_values_from_object, is_float, is_integer,
+                                ABCSeries)
 from pandas.core.config import get_option
 
 # simplify
@@ -27,6 +28,13 @@
 __all__ = ['Index']
 
 
+def _try_get_item(x):
+    try:
+        return x.item()
+    except AttributeError:
+        return x
+
+
 def _indexOp(opname):
     """
     Wrapper function for index comparison operations, to avoid
@@ -1911,11 +1919,17 @@ class Float64Index(Index):
 
     Notes
     -----
-    An Index instance can **only** contain hashable objects
+    An Float64Index instance can **only** contain hashable objects
     """
 
     # when this is not longer object dtype this can be changed
-    #_engine_type = _index.Float64Engine
+    _engine_type = _index.Float64Engine
+    _groupby = _algos.groupby_float64
+    _arrmap = _algos.arrmap_float64
+    _left_indexer_unique = _algos.left_join_indexer_unique_float64
+    _left_indexer = _algos.left_join_indexer_float64
+    _inner_indexer = _algos.inner_join_indexer_float64
+    _outer_indexer = _algos.outer_join_indexer_float64
 
     def __new__(cls, data, dtype=None, copy=False, name=None, fastpath=False):
 
@@ -1938,9 +1952,9 @@ def __new__(cls, data, dtype=None, copy=False, name=None, fastpath=False):
             raise TypeError('Unsafe NumPy casting, you must '
                             'explicitly cast')
 
-        # coerce to object for storage
-        if not subarr.dtype == np.object_:
-            subarr = subarr.astype(object)
+        # coerce to float64 for storage
+        if subarr.dtype != np.float64:
+            subarr = subarr.astype(np.float64)
 
         subarr = subarr.view(cls)
         subarr.name = name
@@ -1951,13 +1965,12 @@ def inferred_type(self):
         return 'floating'
 
     def astype(self, dtype):
-        if np.dtype(dtype) != np.object_:
-            raise TypeError('Setting %s dtype to anything other than object '
-                            'is not supported' % self.__class__)
-        return Index(self.values, name=self.name, dtype=object)
+        if np.dtype(dtype) not in (np.object, np.float64):
+            raise TypeError('Setting %s dtype to anything other than '
+                            'float64 or object is not supported' % self.__class__)
+        return Index(self.values, name=self.name, dtype=dtype)
 
     def _convert_scalar_indexer(self, key, typ=None):
-
         if typ == 'iloc':
             return super(Float64Index, self)._convert_scalar_indexer(key,
                                                                      typ=typ)
@@ -1968,8 +1981,6 @@ def _convert_slice_indexer(self, key, typ=None):
             unless we are iloc """
         if typ == 'iloc':
             return self._convert_slice_indexer_iloc(key)
-        elif typ == 'getitem':
-            pass
 
         # allow floats here
         self._validate_slicer(
@@ -2008,13 +2019,75 @@ def equals(self, other):
         try:
             if not isinstance(other, Float64Index):
                 other = self._constructor(other)
-            if self.dtype != other.dtype or self.shape != other.shape: return False
+            if self.dtype != other.dtype or self.shape != other.shape:
+                return False
             left, right = self.values, other.values
-            return ((left == right) | (isnull(left) & isnull(right))).all()
+            return ((left == right) | (self._isnan & other._isnan)).all()
         except TypeError:
             # e.g. fails in numpy 1.6 with DatetimeIndex #1681
             return False
 
+    def __contains__(self, other):
+        if super(Float64Index, self).__contains__(other):
+            return True
+
+        try:
+            # if other is a sequence this throws a ValueError
+            return np.isnan(other) and self._hasnans
+        except ValueError:
+            try:
+                return len(other) <= 1 and _try_get_item(other) in self
+            except TypeError:
+                return False
+
+    def get_loc(self, key):
+        if np.isnan(key):
+            try:
+                return self._nan_idxs.item()
+            except ValueError:
+                return self._nan_idxs
+        return super(Float64Index, self).get_loc(key)
+
+    @property
+    def is_all_dates(self):
+        """
+        Checks that all the labels are datetime objects
+        """
+        return False
+
+    @cache_readonly
+    def _nan_idxs(self):
+        w, = self._isnan.nonzero()
+        return w
+
+    @cache_readonly
+    def _isnan(self):
+        return np.isnan(self.values)
+
+    @cache_readonly
+    def _hasnans(self):
+        return self._isnan.any()
+
+    @cache_readonly
+    def is_unique(self):
+        return super(Float64Index, self).is_unique and self._nan_idxs.size < 2
+
+    def isin(self, values):
+        """
+        Compute boolean array of whether each index value is found in the
+        passed set of values
+
+        Parameters
+        ----------
+        values : set or sequence of values
+
+        Returns
+        -------
+        is_contained : ndarray (boolean dtype)
+        """
+        value_set = set(values)
+        return lib.ismember_nans(self._array_values(), value_set,
+                                 self._hasnans)
 
 class MultiIndex(Index):
 
diff --git a/vb_suite/index_object.py b/vb_suite/index_object.py
--- a/vb_suite/index_object.py
+++ b/vb_suite/index_object.py
@@ -59,3 +59,49 @@
 index_str_slice_indexer_even = Benchmark('idx[::2]', setup)
 index_str_boolean_indexer = Benchmark('idx[mask]', setup)
 index_str_boolean_series_indexer = Benchmark('idx[series_mask]', setup)
+
+#----------------------------------------------------------------------
+# float64 index
+#----------------------------------------------------------------------
+# construction
+setup = common_setup + """
+baseidx = np.arange(1e6)
+"""
+
+index_float64_construct = Benchmark('Index(baseidx)', setup,
+                                    name='index_float64_construct',
+                                    start_date=datetime(2014, 4, 13))
+
+setup = common_setup + """
+idx = tm.makeFloatIndex(1000000)
+
+mask = np.arange(idx.size) % 3 == 0
+series_mask = Series(mask)
+"""
+#----------------------------------------------------------------------
+# getting
+index_float64_get = Benchmark('idx[1]', setup, name='index_float64_get',
+                              start_date=datetime(2014, 4, 13))
+
+
+#----------------------------------------------------------------------
+# slicing
+index_float64_slice_indexer_basic = Benchmark('idx[:-1]', setup,
+                                              name='index_float64_slice_indexer_basic',
+                                              start_date=datetime(2014, 4, 13))
+index_float64_slice_indexer_even = Benchmark('idx[::2]', setup,
+                                             name='index_float64_slice_indexer_even',
+                                             start_date=datetime(2014, 4, 13))
+index_float64_boolean_indexer = Benchmark('idx[mask]', setup,
+                                          name='index_float64_boolean_indexer',
+                                          start_date=datetime(2014, 4, 13))
+index_float64_boolean_series_indexer = Benchmark('idx[series_mask]', setup,
+                                                 name='index_float64_boolean_series_indexer',
+                                                 start_date=datetime(2014, 4, 13))
+
+#----------------------------------------------------------------------
+# arith ops
+index_float64_mul = Benchmark('idx * 2', setup, name='index_float64_mul',
+                              start_date=datetime(2014, 4, 13))
+index_float64_div = Benchmark('idx / 2', setup, name='index_float64_div',
+                              start_date=datetime(2014, 4, 13))


Test patch:
diff --git a/pandas/tests/test_index.py b/pandas/tests/test_index.py
--- a/pandas/tests/test_index.py
+++ b/pandas/tests/test_index.py
@@ -37,6 +37,7 @@ def _skip_if_need_numpy_1_7():
     if _np_version_under1p7:
         raise nose.SkipTest('numpy >= 1.7 required')
 
+
 class TestIndex(tm.TestCase):
     _multiprocess_can_split_ = True
 
@@ -835,15 +836,15 @@ def test_constructor(self):
         self.assertIsInstance(index, Float64Index)
         index = Float64Index(np.array([1.,2,3,4,5]))
         self.assertIsInstance(index, Float64Index)
-        self.assertEqual(index.dtype, object)
+        self.assertEqual(index.dtype, float)
 
         index = Float64Index(np.array([1.,2,3,4,5]),dtype=np.float32)
         self.assertIsInstance(index, Float64Index)
-        self.assertEqual(index.dtype, object)
+        self.assertEqual(index.dtype, np.float64)
 
         index = Float64Index(np.array([1,2,3,4,5]),dtype=np.float32)
         self.assertIsInstance(index, Float64Index)
-        self.assertEqual(index.dtype, object)
+        self.assertEqual(index.dtype, np.float64)
 
         # nan handling
         result = Float64Index([np.nan, np.nan])
@@ -904,6 +905,15 @@ def test_equals(self):
         i2 = Float64Index([1.0,np.nan])
         self.assertTrue(i.equals(i2))
 
+    def test_contains_nans(self):
+        i = Float64Index([1.0, 2.0, np.nan])
+        self.assertTrue(np.nan in i)
+
+    def test_contains_not_nans(self):
+        i = Float64Index([1.0, 2.0, np.nan])
+        self.assertTrue(1.0 in i)
+
+
 class TestInt64Index(tm.TestCase):
     _multiprocess_can_split_ = True
 
diff --git a/pandas/tests/test_indexing.py b/pandas/tests/test_indexing.py
--- a/pandas/tests/test_indexing.py
+++ b/pandas/tests/test_indexing.py
@@ -4,23 +4,20 @@
 import warnings
 
 from pandas.compat import range, lrange, lzip, StringIO, lmap, map
-from numpy import random, nan
+from numpy import nan
 from numpy.random import randn
 import numpy as np
-from numpy.testing import assert_array_equal
 
 import pandas as pd
 import pandas.core.common as com
-from pandas.core.api import (DataFrame, Index, Series, Panel, notnull, isnull,
-                             MultiIndex, DatetimeIndex, Float64Index, Timestamp)
+from pandas.core.api import (DataFrame, Index, Series, Panel, isnull,
+                             MultiIndex, Float64Index, Timestamp)
 from pandas.util.testing import (assert_almost_equal, assert_series_equal,
                                  assert_frame_equal, assert_panel_equal)
-from pandas import compat, concat
+from pandas import concat
 
 import pandas.util.testing as tm
-import pandas.lib as lib
 from pandas import date_range
-from numpy.testing.decorators import slow
 
 _verbose = False
 
@@ -1201,17 +1198,23 @@ def test_ix_general(self):
         # ix general issues
 
         # GH 2817
-        data={'amount': {0: 700, 1: 600, 2: 222, 3: 333, 4: 444},
-              'col': {0: 3.5, 1: 3.5, 2: 4.0, 3: 4.0, 4: 4.0},
-              'year': {0: 2012, 1: 2011, 2: 2012, 3: 2012, 4: 2012}}
-        df = DataFrame(data).set_index(keys=['col','year'])
+        data = {'amount': {0: 700, 1: 600, 2: 222, 3: 333, 4: 444},
+                'col': {0: 3.5, 1: 3.5, 2: 4.0, 3: 4.0, 4: 4.0},
+                'year': {0: 2012, 1: 2011, 2: 2012, 3: 2012, 4: 2012}}
+        df = DataFrame(data).set_index(keys=['col', 'year'])
+        key = 4.0, 2012
 
         # this should raise correct error
-        self.assertRaises(KeyError, df.ix.__getitem__, tuple([4.0,2012]))
+        with tm.assertRaises(KeyError):
+            df.ix[key]
 
         # this is ok
         df.sortlevel(inplace=True)
-        df.ix[(4.0,2012)]
+        res = df.ix[key]
+        index = MultiIndex.from_arrays([[4] * 3, [2012] * 3],
+                                       names=['col', 'year'])
+        expected = DataFrame({'amount': [222, 333, 444]}, index=index)
+        tm.assert_frame_equal(res, expected)
 
     def test_ix_weird_slicing(self):
         ## http://stackoverflow.com/q/17056560/1240268