Repository: pandas-dev/pandas

Issue:
Speed up DatetimeConverter for plotting
I've recently started using pandas (impressed so far!) and found that plotting large data (from around 100k) samples is quite slow. I traced the bottleneck to the _dt_to_float_ordinal helper function called by DatetimeConverter.(https://github.com/pydata/pandas/blob/master/pandas/tseries/converter.py#L144).

More specifically, this function uses matplotlib's date2num, which converts arrays and iterables using a slow list comprehension. Since pandas seem to natively store datetimes as epoch+nanoseconds in an int64 array, it would be much faster to use matplotlib's vectorized epoch2num instead. In a testcase with 1 million points, using epoch2num is about 100 times faster than date2num:

``` python
from pandas import date_range, DataFrame
from numpy import int64, arange
from matplotlib import pyplot, dates
import time

n = 1e6

df = DataFrame(arange(n), index = date_range('20130101', periods=n, freq='S'))

start = time.time()
pyplot.plot(df.index, df)
print('date2num took {0:g}s'.format(time.time() - start))
pyplot.show()

# monkey patch
import pandas.tseries.converter
def _my_dt_to_float_ordinal(dt):
    try:
        base = dates.epoch2num(dt.astype(int64) / 1.0E9)
    except AttributeError:
        base = dates.date2num(dt)
    return base
pandas.tseries.converter._dt_to_float_ordinal = _my_dt_to_float_ordinal

start = time.time()
pyplot.plot(df.index, df)
print('epoch2num took {0:g}s'.format(time.time() - start))
pyplot.show()
```

Unfortunately, I am not familiar enough with pandas to know whether date2num is  used intentionally or to implement a proper patch myself that works in all cases.



Discussion:
I think they are the same effect, so this should be good. @TomAugspurger 

@agijsberts pls do a pull-request and can get this in.

I think may need to manually validate that the graphs are correct as we don't do comparison graphs per se (more of a validation that they plot and and the returned objects are 'ok').

pls add a vbench for this as well.

good catch

https://github.com/pydata/pandas/wiki section on how-to do the PR

Applying your changed didn't seem to break any tests so this should be good. @jreback know if there will be any problems on 32-bit systems?

@agijsberts a pull request would be great for this. Let me know if you have any trouble.

@agijsberts yep let's give a try on this

Sorry, things are moving a bit slow since it's my first time preparing a PR (setting up git, virtualenv etc.). I'm manually checking the correctness of the plots at the moment (at least w.r.t. the current implementation). Expect a PR later today.

Note by the way that pandas' plotting functions (e.g., DataFrame.plot) do not benefit from this patch, as they do other trickery with time axes. The patch is therefore unfortunately only helpful in use-cases where matplotlib's functions are used directly with the time index.


Resolving patch:
diff --git a/pandas/tseries/converter.py b/pandas/tseries/converter.py
--- a/pandas/tseries/converter.py
+++ b/pandas/tseries/converter.py
@@ -16,6 +16,7 @@
 import pandas.core.common as com
 from pandas.core.index import Index
 
+from pandas.core.series import Series
 from pandas.tseries.index import date_range
 import pandas.tseries.tools as tools
 import pandas.tseries.frequencies as frequencies
@@ -144,7 +145,10 @@ def _dt_to_float_ordinal(dt):
     preserving hours, minutes, seconds and microseconds.  Return value
     is a :func:`float`.
     """
-    base = dates.date2num(dt)
+    if isinstance(dt, (np.ndarray, Series)) and com.is_datetime64_ns_dtype(dt):
+        base = dates.epoch2num(dt.asi8 / 1.0E9)
+    else:
+        base = dates.date2num(dt)
     return base
 
 
diff --git a/vb_suite/timeseries.py b/vb_suite/timeseries.py
--- a/vb_suite/timeseries.py
+++ b/vb_suite/timeseries.py
@@ -269,3 +269,15 @@ def date_range(start=None, end=None, periods=None, freq=None):
 dataframe_resample_max_numpy = \
     Benchmark("df.resample('1s', how=np.max)", setup)
 
+
+#----------------------------------------------------------------------
+# DatetimeConverter
+
+setup = common_setup + """
+from pandas.tseries.converter import DatetimeConverter
+"""
+
+datetimeindex_converter = \
+    Benchmark('DatetimeConverter.convert(rng, None, None)', 
+              setup, start_date=datetime(2013, 1, 1))
+


Test patch:
diff --git a/pandas/tseries/tests/test_converter.py b/pandas/tseries/tests/test_converter.py
--- a/pandas/tseries/tests/test_converter.py
+++ b/pandas/tseries/tests/test_converter.py
@@ -5,8 +5,11 @@
 import nose
 
 import numpy as np
+from numpy.testing import assert_almost_equal as np_assert_almost_equal
+from pandas import Timestamp
 from pandas.compat import u
 import pandas.util.testing as tm
+from pandas.tseries.offsets import Second, Milli, Micro
 
 try:
     import pandas.tseries.converter as converter
@@ -46,9 +49,48 @@ def test_conversion(self):
         rs = self.dtc.convert('2012-1-1', None, None)
         self.assertEqual(rs, xp)
 
+        rs = self.dtc.convert(Timestamp('2012-1-1'), None, None)
+        self.assertEqual(rs, xp)
+
+    def test_conversion_float(self):
+        decimals = 9
+
+        rs = self.dtc.convert(Timestamp('2012-1-1 01:02:03', tz='UTC'), None, None)
+        xp = converter.dates.date2num(Timestamp('2012-1-1 01:02:03', tz='UTC'))
+        np_assert_almost_equal(rs, xp, decimals)
+
+        rs = self.dtc.convert(Timestamp('2012-1-1 09:02:03', tz='Asia/Hong_Kong'), None, None)
+        np_assert_almost_equal(rs, xp, decimals)
+
+        rs = self.dtc.convert(datetime(2012, 1, 1, 1, 2, 3), None, None)
+        np_assert_almost_equal(rs, xp, decimals)
+
     def test_time_formatter(self):
         self.tc(90000)
 
+    def test_dateindex_conversion(self):
+        decimals = 9
+
+        for freq in ('B', 'L', 'S'):
+            dateindex = tm.makeDateIndex(k = 10, freq = freq)
+            rs = self.dtc.convert(dateindex, None, None)
+            xp = converter.dates.date2num(dateindex)
+            np_assert_almost_equal(rs, xp, decimals)
+
+    def test_resolution(self):
+        def _assert_less(ts1, ts2):
+            val1 = self.dtc.convert(ts1, None, None)
+            val2 = self.dtc.convert(ts2, None, None)
+            if not val1 < val2:
+                raise AssertionError('{0} is not less than {1}.'.format(val1, val2))
+
+        # Matplotlib's time representation using floats cannot distinguish intervals smaller 
+        # than ~10 microsecond in the common range of years. 
+        ts = Timestamp('2012-1-1')
+        _assert_less(ts, ts + Second())
+        _assert_less(ts, ts + Milli())
+        _assert_less(ts, ts + Micro(50))
+
 
 if __name__ == '__main__':
     import nose