Repository: pandas-dev/pandas

Issue:
Serious performance regression in DataFrame construction with monthly DatetimeIndex
Hi,

After upgrading from v0.12.0 to v0.13.1, I noticed about a 100% slowdown on a pandas-heavy project. I've just started looking, but I've come up with a test case that shows a time-complexity change from O(1) to O(n) (~240x slowdown for my inputs).

Here's the comparison for v0.12.0 (y-axis is milliseconds):

![perf_12](https://f.cloud.github.com/assets/440095/2263404/746ae6ce-9e65-11e3-92c2-5d7d3d2c6cf9.png)

And the comparison for v0.13.1:

![perf_131](https://f.cloud.github.com/assets/440095/2263407/791db6a6-9e65-11e3-940e-cb97ffc53809.png)

The test code (I'll convert this to vbench later):

```
rows = 1000
columns = 10
data = DataFrame(np.random.random((rows, columns)), index=DatetimeIndex(start='1/1/1900', periods=rows, freq='M'))

d = {}

for col in data:
    d[col] = data[col]

%timeit DataFrame(d)
```

Daily indices don't appear to be affected, though I suspect other frequencies are impacted. I'm seeing similar regressions in v0.13.0.



Discussion:
I've identified the cause:

Using 'D' frequency:

```
Function: _fast_union at line 1033
Total time: 0.000723 s

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
  1033                                               def _fast_union(self, other):
  1034         9           19      2.1      2.6          if len(other) == 0:
  1035                                                       return self.view(type(self))
  1036                                           
  1037         9           11      1.2      1.5          if len(self) == 0:
  1038                                                       return other.view(type(self))
  1039                                           
  1040                                                   # to make our life easier, "sort" the two ranges
  1041         9          261     29.0     36.1          if self[0] <= other[0]:
  1042         9           11      1.2      1.5              left, right = self, other
  1043                                                   else:
  1044                                                       left, right = other, self
  1045                                           
  1046         9          245     27.2     33.9          left_start, left_end = left[0], left[-1]
  1047         9          121     13.4     16.7          right_end = right[-1]
  1048                                           
  1049         9           34      3.8      4.7          if not self.offset._should_cache():
  1050                                                       # concatenate dates
  1051         9           12      1.3      1.7              if left_end < right_end:
  1052                                                           loc = right.searchsorted(left_end, side='right')
  1053                                                           right_chunk = right.values[loc:]
  1054                                                           dates = com._concat_compat((left.values, right_chunk))
  1055                                                           return self._view_like(dates)
  1056                                                       else:
  1057         9            9      1.0      1.2                  return left
  1058                                                   else:
  1059                                                       return type(self)(start=left_start,
  1060                                                                         end=max(left_end, right_end),
  1061                                                                         freq=left.offset)
```

And with 'M':

```
Function: _fast_union at line 1033
Total time: 0.851928 s

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
  1033                                               def _fast_union(self, other):
  1034         9           33      3.7      0.0          if len(other) == 0:
  1035                                                       return self.view(type(self))
  1036                                           
  1037         9           11      1.2      0.0          if len(self) == 0:
  1038                                                       return other.view(type(self))
  1039                                           
  1040                                                   # to make our life easier, "sort" the two ranges
  1041         9          323     35.9      0.0          if self[0] <= other[0]:
  1042         9           12      1.3      0.0              left, right = self, other
  1043                                                   else:
  1044                                                       left, right = other, self
  1045                                           
  1046         9          253     28.1      0.0          left_start, left_end = left[0], left[-1]
  1047         9          139     15.4      0.0          right_end = right[-1]
  1048                                           
  1049         9           46      5.1      0.0          if not self.offset._should_cache():
  1050                                                       # concatenate dates
  1051                                                       if left_end < right_end:
  1052                                                           loc = right.searchsorted(left_end, side='right')
  1053                                                           right_chunk = right.values[loc:]
  1054                                                           dates = com._concat_compat((left.values, right_chunk))
  1055                                                           return self._view_like(dates)
  1056                                                       else:
  1057                                                           return left
  1058                                                   else:
  1059         9           13      1.4      0.0              return type(self)(start=left_start,
  1060         9           38      4.2      0.0                                end=max(left_end, right_end),
  1061         9       851060  94562.2     99.9                                freq=left.offset)
```

Which suggests this is due to a change in either `Offset.isAnchored()` or `Offset._cacheable`.

yep it seems the offsets had the cache able class reversed with the offset
easy fix
I'll put up a pr but would appreciate of you could add some more vbenches so that this can be caught if things change again

Sure, I'll take a look tonight.


Resolving patch:
diff --git a/pandas/tseries/offsets.py b/pandas/tseries/offsets.py
--- a/pandas/tseries/offsets.py
+++ b/pandas/tseries/offsets.py
@@ -291,7 +291,7 @@ def _from_name(cls, suffix=None):
         return cls()
 
 
-class BusinessDay(CacheableOffset, SingleConstructorOffset):
+class BusinessDay(SingleConstructorOffset):
     """
     DateOffset subclass representing possibly n business days
     """
@@ -399,7 +399,7 @@ def apply(self, other):
                 n -= 5 * k
                 if n == 0 and result.weekday() > 4:
                     n -= 1
-            
+
             while n != 0:
                 k = n // abs(n)
                 result = result + timedelta(k)
@@ -548,7 +548,7 @@ def name(self):
             return "%s-%s" % (self.rule_code, _int_to_month[self.n])
 
 
-class MonthEnd(CacheableOffset, MonthOffset):
+class MonthEnd(MonthOffset):
     """DateOffset of one month end"""
 
     def apply(self, other):
@@ -572,7 +572,7 @@ def onOffset(cls, dt):
     _prefix = 'M'
 
 
-class MonthBegin(CacheableOffset, MonthOffset):
+class MonthBegin(MonthOffset):
     """DateOffset of one month at beginning"""
 
     def apply(self, other):
@@ -591,7 +591,7 @@ def onOffset(cls, dt):
     _prefix = 'MS'
 
 
-class BusinessMonthEnd(CacheableOffset, MonthOffset):
+class BusinessMonthEnd(MonthOffset):
     """DateOffset increments between business EOM dates"""
 
     def isAnchored(self):
@@ -619,7 +619,7 @@ def apply(self, other):
     _prefix = 'BM'
 
 
-class BusinessMonthBegin(CacheableOffset, MonthOffset):
+class BusinessMonthBegin(MonthOffset):
     """DateOffset of one business month at beginning"""
 
     def apply(self, other):
@@ -654,7 +654,7 @@ def onOffset(cls, dt):
     _prefix = 'BMS'
 
 
-class Week(CacheableOffset, DateOffset):
+class Week(DateOffset):
     """
     Weekly offset
 
@@ -744,7 +744,7 @@ class WeekDay(object):
 _weekday_to_int = dict((v, k) for k, v in _int_to_weekday.items())
 
 
-class WeekOfMonth(CacheableOffset, DateOffset):
+class WeekOfMonth(DateOffset):
     """
     Describes monthly dates like "the Tuesday of the 2nd week of each month"
 
@@ -830,7 +830,7 @@ def _from_name(cls, suffix=None):
         weekday = _weekday_to_int[suffix[1:]]
         return cls(week=week, weekday=weekday)
 
-class LastWeekOfMonth(CacheableOffset, DateOffset):
+class LastWeekOfMonth(DateOffset):
     """
     Describes monthly dates in last week of month like "the last Tuesday of each month"
 
@@ -940,7 +940,7 @@ def rule_code(self):
         return '%s-%s' % (self._prefix, _int_to_month[self.startingMonth])
 
 
-class BQuarterEnd(CacheableOffset, QuarterOffset):
+class BQuarterEnd(QuarterOffset):
     """DateOffset increments between business Quarter dates
     startingMonth = 1 corresponds to dates like 1/31/2007, 4/30/2007, ...
     startingMonth = 2 corresponds to dates like 2/28/2007, 5/31/2007, ...
@@ -999,7 +999,7 @@ def onOffset(self, dt):
 
 
 # TODO: This is basically the same as BQuarterEnd
-class BQuarterBegin(CacheableOffset, QuarterOffset):
+class BQuarterBegin(QuarterOffset):
     _outputName = "BusinessQuarterBegin"
     # I suspect this is wrong for *all* of them.
     _default_startingMonth = 3
@@ -1036,7 +1036,7 @@ def apply(self, other):
         return as_timestamp(result)
 
 
-class QuarterEnd(CacheableOffset, QuarterOffset):
+class QuarterEnd(QuarterOffset):
     """DateOffset increments between business Quarter dates
     startingMonth = 1 corresponds to dates like 1/31/2007, 4/30/2007, ...
     startingMonth = 2 corresponds to dates like 2/28/2007, 5/31/2007, ...
@@ -1077,7 +1077,7 @@ def onOffset(self, dt):
         return MonthEnd().onOffset(dt) and modMonth == 0
 
 
-class QuarterBegin(CacheableOffset, QuarterOffset):
+class QuarterBegin(QuarterOffset):
     _outputName = 'QuarterBegin'
     _default_startingMonth = 3
     _from_name_startingMonth = 1
@@ -1129,7 +1129,7 @@ def rule_code(self):
         return '%s-%s' % (self._prefix, _int_to_month[self.month])
 
 
-class BYearEnd(CacheableOffset, YearOffset):
+class BYearEnd(YearOffset):
     """DateOffset increments between business EOM dates"""
     _outputName = 'BusinessYearEnd'
     _default_month = 12
@@ -1166,7 +1166,7 @@ def apply(self, other):
         return result
 
 
-class BYearBegin(CacheableOffset, YearOffset):
+class BYearBegin(YearOffset):
     """DateOffset increments between business year begin dates"""
     _outputName = 'BusinessYearBegin'
     _default_month = 1
@@ -1198,7 +1198,7 @@ def apply(self, other):
         return as_timestamp(datetime(other.year, self.month, first))
 
 
-class YearEnd(CacheableOffset, YearOffset):
+class YearEnd(YearOffset):
     """DateOffset increments between calendar year ends"""
     _default_month = 12
     _prefix = 'A'
@@ -1254,7 +1254,7 @@ def onOffset(self, dt):
         return self.month == dt.month and dt.day == days_in_month
 
 
-class YearBegin(CacheableOffset, YearOffset):
+class YearBegin(YearOffset):
     """DateOffset increments between calendar year begin dates"""
     _default_month = 1
     _prefix = 'AS'
@@ -1300,7 +1300,7 @@ def onOffset(self, dt):
         return dt.month == self.month and dt.day == 1
 
 
-class FY5253(CacheableOffset, DateOffset):
+class FY5253(DateOffset):
     """
     Describes 52-53 week fiscal year. This is also known as a 4-4-5 calendar.
 
@@ -1501,7 +1501,7 @@ def _from_name(cls, *args):
         return cls(**cls._parse_suffix(*args))
 
 
-class FY5253Quarter(CacheableOffset, DateOffset):
+class FY5253Quarter(DateOffset):
     """
     DateOffset increments between business quarter dates
     for 52-53 week fiscal year (also known as a 4-4-5 calendar).
@@ -1772,7 +1772,7 @@ def _delta_to_nanoseconds(delta):
             + delta.microseconds) * 1000
 
 
-class Day(CacheableOffset, Tick):
+class Day(Tick):
     _inc = timedelta(1)
     _prefix = 'D'
 
diff --git a/vb_suite/frame_ctor.py b/vb_suite/frame_ctor.py
--- a/vb_suite/frame_ctor.py
+++ b/vb_suite/frame_ctor.py
@@ -1,7 +1,15 @@
 from vbench.benchmark import Benchmark
 from datetime import datetime
+try:
+    import pandas.tseries.offsets as offsets
+except:
+    import pandas.core.datetools as offsets
 
 common_setup = """from pandas_vb_common import *
+try:
+    from pandas.tseries.offsets import *
+except:
+    from pandas.core.datetools import *
 """
 
 #----------------------------------------------------------------------
@@ -36,6 +44,21 @@
 """
 frame_ctor_nested_dict_int64 = Benchmark("DataFrame(data)", setup)
 
+# dynamically generate benchmarks for every offset
+dynamic_benchmarks = {}
+n_steps = [1, 2]
+for offset in offsets.__all__:
+    for n in n_steps:
+        setup = common_setup + """
+df = DataFrame(np.random.randn(1000,10),index=date_range('1/1/1900',periods=1000,freq={}({})))
+d = dict([ (col,df[col]) for col in df.columns ])
+""".format(offset, n)
+        key = 'frame_ctor_dtindex_{}({})'.format(offset, n)
+        dynamic_benchmarks[key] = Benchmark("DataFrame(d)", setup, name=key)
+
+# Have to stuff them in globals() so vbench detects them
+globals().update(dynamic_benchmarks)
+
 # from a mi-series
 setup = common_setup + """
 mi = MultiIndex.from_tuples([(x,y) for x in range(100) for y in range(100)])


Test patch:
diff --git a/pandas/tseries/tests/test_offsets.py b/pandas/tseries/tests/test_offsets.py
--- a/pandas/tseries/tests/test_offsets.py
+++ b/pandas/tseries/tests/test_offsets.py
@@ -316,7 +316,7 @@ def test_apply_large_n(self):
         rs = st + off
         xp = datetime(2011, 12, 26)
         self.assertEqual(rs, xp)
-        
+
         off = BDay() * 10
         rs = datetime(2014, 1, 5) + off # see #5890
         xp = datetime(2014, 1, 17)
@@ -2427,25 +2427,9 @@ def get_all_subclasses(cls):
     return ret
 
 class TestCaching(tm.TestCase):
-    no_simple_ctr = [WeekOfMonth, FY5253,
-                     FY5253Quarter,
-                     LastWeekOfMonth]
-
-    def test_should_cache_month_end(self):
-        self.assertTrue(MonthEnd()._should_cache())
-
-    def test_should_cache_bmonth_end(self):
-        self.assertTrue(BusinessMonthEnd()._should_cache())
-
-    def test_should_cache_week_month(self):
-        self.assertTrue(WeekOfMonth(weekday=1, week=2)._should_cache())
 
-    def test_all_cacheableoffsets(self):
-        for subclass in get_all_subclasses(CacheableOffset):
-            if subclass.__name__[0] == "_" \
-                or subclass in TestCaching.no_simple_ctr:
-                continue
-            self.run_X_index_creation(subclass)
+    # as of GH 6479 (in 0.14.0), offset caching is turned off
+    # as of v0.12.0 only BusinessMonth/Quarter were actually caching
 
     def setUp(self):
         _daterange_cache.clear()
@@ -2462,20 +2446,35 @@ def run_X_index_creation(self, cls):
         DatetimeIndex(start=datetime(2013,1,31), end=datetime(2013,3,31), freq=inst1, normalize=True)
         self.assertTrue(cls() in _daterange_cache, cls)
 
+    def test_should_cache_month_end(self):
+        self.assertFalse(MonthEnd()._should_cache())
+
+    def test_should_cache_bmonth_end(self):
+        self.assertFalse(BusinessMonthEnd()._should_cache())
+
+    def test_should_cache_week_month(self):
+        self.assertFalse(WeekOfMonth(weekday=1, week=2)._should_cache())
+
+    def test_all_cacheableoffsets(self):
+        for subclass in get_all_subclasses(CacheableOffset):
+            if subclass.__name__[0] == "_" \
+                or subclass in TestCaching.no_simple_ctr:
+                continue
+            self.run_X_index_creation(subclass)
+
     def test_month_end_index_creation(self):
         DatetimeIndex(start=datetime(2013,1,31), end=datetime(2013,3,31), freq=MonthEnd(), normalize=True)
-        self.assertTrue(MonthEnd() in _daterange_cache)
+        self.assertFalse(MonthEnd() in _daterange_cache)
 
     def test_bmonth_end_index_creation(self):
         DatetimeIndex(start=datetime(2013,1,31), end=datetime(2013,3,29), freq=BusinessMonthEnd(), normalize=True)
-        self.assertTrue(BusinessMonthEnd() in _daterange_cache)
+        self.assertFalse(BusinessMonthEnd() in _daterange_cache)
 
     def test_week_of_month_index_creation(self):
         inst1 = WeekOfMonth(weekday=1, week=2)
         DatetimeIndex(start=datetime(2013,1,31), end=datetime(2013,3,29), freq=inst1, normalize=True)
         inst2 = WeekOfMonth(weekday=1, week=2)
-        self.assertTrue(inst2 in _daterange_cache)
-
+        self.assertFalse(inst2 in _daterange_cache)
 
 class TestReprNames(tm.TestCase):
     def test_str_for_named_is_name(self):