Repository: pandas-dev/pandas

Issue:
Fix PyDateTime_CheckExact checks as proxy for Timestamp checks in cython files
This is a followup to the discussion in #25746.

PyDateTime_CheckExact is being used as a proxy for isinstance(obj, Timestamp) checks for performance reasons.  However, if a subclass of the std datetime lib is being used, these checks are not sufficient to determine if an object is a Timestamp or not.  Thus a performant replacement must be found.

I have a working fix, but need to do some more performance testing and add proper test cases.  If that goes well, I will submit a PR for this.


Discussion:

Resolving patch:
diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py
--- a/pandas/core/arrays/datetimelike.py
+++ b/pandas/core/arrays/datetimelike.py
@@ -7,11 +7,11 @@
 import numpy as np
 
 from pandas._libs import NaT, NaTType, Timestamp, algos, iNaT, lib
+from pandas._libs.tslibs.c_timestamp import maybe_integer_op_deprecated
 from pandas._libs.tslibs.period import (
     DIFFERENT_FREQ, IncompatibleFrequency, Period)
 from pandas._libs.tslibs.timedeltas import Timedelta, delta_to_nanoseconds
-from pandas._libs.tslibs.timestamps import (
-    RoundTo, maybe_integer_op_deprecated, round_nsint64)
+from pandas._libs.tslibs.timestamps import RoundTo, round_nsint64
 from pandas.compat.numpy import function as nv
 from pandas.errors import (
     AbstractMethodError, NullFrequencyError, PerformanceWarning)
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -312,6 +312,7 @@ class CheckSDist(sdist_class):
                  'pandas/_libs/sparse.pyx',
                  'pandas/_libs/ops.pyx',
                  'pandas/_libs/parsers.pyx',
+                 'pandas/_libs/tslibs/c_timestamp.pyx',
                  'pandas/_libs/tslibs/ccalendar.pyx',
                  'pandas/_libs/tslibs/period.pyx',
                  'pandas/_libs/tslibs/strptime.pyx',
@@ -592,6 +593,11 @@ def srcpath(name=None, suffix='.pyx', subdir='src'):
         'include': ts_include,
         'depends': tseries_depends,
         'sources': np_datetime_sources},
+    '_libs.tslibs.c_timestamp': {
+        'pyxfile': '_libs/tslibs/c_timestamp',
+        'include': ts_include,
+        'depends': tseries_depends,
+        'sources': np_datetime_sources},
     '_libs.tslibs.ccalendar': {
         'pyxfile': '_libs/tslibs/ccalendar',
         'include': []},


Test patch:
diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py
--- a/pandas/tests/arithmetic/test_datetime64.py
+++ b/pandas/tests/arithmetic/test_datetime64.py
@@ -2351,3 +2351,22 @@ def test_shift_months(years, months):
            for x in dti]
     expected = DatetimeIndex(raw)
     tm.assert_index_equal(actual, expected)
+
+
+class SubDatetime(datetime):
+    pass
+
+
+@pytest.mark.parametrize("lh,rh", [
+    (SubDatetime(2000, 1, 1),
+     Timedelta(hours=1)),
+    (Timedelta(hours=1),
+     SubDatetime(2000, 1, 1))
+])
+def test_dt_subclass_add_timedelta(lh, rh):
+    # GH 25851
+    # ensure that subclassed datetime works for
+    # Timedelta operations
+    result = lh + rh
+    expected = SubDatetime(2000, 1, 1, 1)
+    assert result == expected
diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py
--- a/pandas/tests/scalar/timestamp/test_timestamp.py
+++ b/pandas/tests/scalar/timestamp/test_timestamp.py
@@ -603,6 +603,18 @@ def test_dont_convert_dateutil_utc_to_pytz_utc(self):
         expected = Timestamp(datetime(2018, 1, 1)).tz_localize(tzutc())
         assert result == expected
 
+    def test_constructor_subclassed_datetime(self):
+        # GH 25851
+        # ensure that subclassed datetime works for
+        # Timestamp creation
+        class SubDatetime(datetime):
+            pass
+
+        data = SubDatetime(2000, 1, 1)
+        result = Timestamp(data)
+        expected = Timestamp(2000, 1, 1)
+        assert result == expected
+
 
 class TestTimestamp(object):
 
diff --git a/pandas/tests/tslibs/test_api.py b/pandas/tests/tslibs/test_api.py
--- a/pandas/tests/tslibs/test_api.py
+++ b/pandas/tests/tslibs/test_api.py
@@ -6,7 +6,8 @@
 
 def test_namespace():
 
-    submodules = ['ccalendar',
+    submodules = ['c_timestamp',
+                  'ccalendar',
                   'conversion',
                   'fields',
                   'frequencies',
diff --git a/pandas/tests/tslibs/test_array_to_datetime.py b/pandas/tests/tslibs/test_array_to_datetime.py
--- a/pandas/tests/tslibs/test_array_to_datetime.py
+++ b/pandas/tests/tslibs/test_array_to_datetime.py
@@ -9,6 +9,7 @@
 from pandas._libs import iNaT, tslib
 from pandas.compat.numpy import np_array_datetime64_compat
 
+from pandas import Timestamp
 import pandas.util.testing as tm
 
 
@@ -154,3 +155,27 @@ def test_to_datetime_barely_out_of_bounds():
 
     with pytest.raises(tslib.OutOfBoundsDatetime, match=msg):
         tslib.array_to_datetime(arr)
+
+
+class SubDatetime(datetime):
+    pass
+
+
+@pytest.mark.parametrize("data,expected", [
+    ([SubDatetime(2000, 1, 1)],
+     ["2000-01-01T00:00:00.000000000-0000"]),
+    ([datetime(2000, 1, 1)],
+     ["2000-01-01T00:00:00.000000000-0000"]),
+    ([Timestamp(2000, 1, 1)],
+     ["2000-01-01T00:00:00.000000000-0000"])
+])
+def test_datetime_subclass(data, expected):
+    # GH 25851
+    # ensure that subclassed datetime works with
+    # array_to_datetime
+
+    arr = np.array(data, dtype=object)
+    result, _ = tslib.array_to_datetime(arr)
+
+    expected = np_array_datetime64_compat(expected, dtype="M8[ns]")
+    tm.assert_numpy_array_equal(result, expected)
diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py
--- a/pandas/tests/tslibs/test_conversion.py
+++ b/pandas/tests/tslibs/test_conversion.py
@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+from datetime import datetime
+
 import numpy as np
 import pytest
 from pytz import UTC
@@ -7,7 +9,7 @@
 from pandas._libs.tslib import iNaT
 from pandas._libs.tslibs import conversion, timezones
 
-from pandas import date_range
+from pandas import Timestamp, date_range
 import pandas.util.testing as tm
 
 
@@ -66,3 +68,24 @@ def test_length_zero_copy(dtype, copy):
     arr = np.array([], dtype=dtype)
     result = conversion.ensure_datetime64ns(arr, copy=copy)
     assert result.base is (None if copy else arr)
+
+
+class SubDatetime(datetime):
+    pass
+
+
+@pytest.mark.parametrize("dt, expected", [
+    pytest.param(Timestamp("2000-01-01"),
+                 Timestamp("2000-01-01", tz=UTC), id="timestamp"),
+    pytest.param(datetime(2000, 1, 1),
+                 datetime(2000, 1, 1, tzinfo=UTC),
+                 id="datetime"),
+    pytest.param(SubDatetime(2000, 1, 1),
+                 SubDatetime(2000, 1, 1, tzinfo=UTC),
+                 id="subclassed_datetime")])
+def test_localize_pydatetime_dt_types(dt, expected):
+    # GH 25851
+    # ensure that subclassed datetime works with
+    # localize_pydatetime
+    result = conversion.localize_pydatetime(dt, UTC)
+    assert result == expected
diff --git a/pandas/tests/tslibs/test_normalize_date.py b/pandas/tests/tslibs/test_normalize_date.py
--- a/pandas/tests/tslibs/test_normalize_date.py
+++ b/pandas/tests/tslibs/test_normalize_date.py
@@ -6,6 +6,7 @@
 import pytest
 
 from pandas._libs import tslibs
+from pandas._libs.tslibs.timestamps import Timestamp
 
 
 @pytest.mark.parametrize("value,expected", [
@@ -16,3 +17,19 @@
 def test_normalize_date(value, expected):
     result = tslibs.normalize_date(value)
     assert result == expected
+
+
+class SubDatetime(datetime):
+    pass
+
+
+@pytest.mark.parametrize("dt, expected", [
+    (Timestamp(2000, 1, 1, 1), Timestamp(2000, 1, 1, 0)),
+    (datetime(2000, 1, 1, 1), datetime(2000, 1, 1, 0)),
+    (SubDatetime(2000, 1, 1, 1), SubDatetime(2000, 1, 1, 0))])
+def test_normalize_date_sub_types(dt, expected):
+    # GH 25851
+    # ensure that subclassed datetime works with
+    # normalize_date
+    result = tslibs.normalize_date(dt)
+    assert result == expected