Repository: ibis-project/ibis

Issue:
bug: pandas and dask interpret cast of integer to timestamps as nanos since epoch
Every other backend interprets these as seconds since epoch. This is a breaking change.


Discussion:

Resolving patch:
diff --git a/ibis/backends/dask/execution/generic.py b/ibis/backends/dask/execution/generic.py
--- a/ibis/backends/dask/execution/generic.py
+++ b/ibis/backends/dask/execution/generic.py
@@ -255,6 +255,29 @@ def execute_cast_series_group_by(op, data, type, **kwargs):
     return result.groupby(data.index)
 
 
+def cast_scalar_to_timestamp(data, tz):
+    if isinstance(data, str):
+        return pd.Timestamp(data, tz=tz)
+    return pd.Timestamp(data, unit="s", tz=tz)
+
+
+@execute_node.register(ops.Cast, dd.core.Scalar, dt.Timestamp)
+def execute_cast_scalar_timestamp(op, data, type, **kwargs):
+    return dd.map_partitions(
+        cast_scalar_to_timestamp, data, tz=type.timezone, meta="datetime64[ns]"
+    )
+
+
+def cast_series_to_timestamp(data, tz):
+    if pd.api.types.is_string_dtype(data):
+        timestamps = to_datetime(data, infer_datetime_format=True)
+    else:
+        timestamps = to_datetime(data, unit="s")
+    if getattr(timestamps.dtype, "tz", None) is not None:
+        return timestamps.dt.tz_convert(tz)
+    return timestamps.dt.tz_localize(tz)
+
+
 @execute_node.register(ops.Cast, dd.Series, dt.Timestamp)
 def execute_cast_series_timestamp(op, data, type, **kwargs):
     arg = op.arg
@@ -264,22 +287,17 @@ def execute_cast_series_timestamp(op, data, type, **kwargs):
         return data
 
     tz = type.timezone
+    dtype = 'M8[ns]' if tz is None else DatetimeTZDtype('ns', tz)
 
     if from_type.is_timestamp() or from_type.is_date():
-        return data.astype('M8[ns]' if tz is None else DatetimeTZDtype('ns', tz))
+        return data.astype(dtype)
 
     if from_type.is_string() or from_type.is_integer():
-        timestamps = data.map_partitions(
-            to_datetime,
-            infer_datetime_format=True,
-            meta=(data.name, 'datetime64[ns]'),
+        return data.map_partitions(
+            cast_series_to_timestamp,
+            tz,
+            meta=(data.name, dtype),
         )
-        # TODO - is there a better way to do this
-        timestamps = timestamps.astype(timestamps.head(1).dtype)
-        if getattr(timestamps.dtype, "tz", None) is not None:
-            return timestamps.dt.tz_convert(tz)
-        else:
-            return timestamps.dt.tz_localize(tz)
 
     raise TypeError(f"Don't know how to cast {from_type} to {type}")
 
diff --git a/ibis/backends/pandas/execution/generic.py b/ibis/backends/pandas/execution/generic.py
--- a/ibis/backends/pandas/execution/generic.py
+++ b/ibis/backends/pandas/execution/generic.py
@@ -103,7 +103,13 @@ def execute_cast_series_group_by(op, data, type, **kwargs):
 
 @execute_node.register(ops.Cast, pd.Series, dt.DataType)
 def execute_cast_series_generic(op, data, type, **kwargs):
-    return data.astype(constants.IBIS_TYPE_TO_PANDAS_TYPE[type])
+    out = data.astype(constants.IBIS_TYPE_TO_PANDAS_TYPE[type])
+    if type.is_integer():
+        if op.arg.output_dtype.is_timestamp():
+            return out.floordiv(int(1e9))
+        elif op.arg.output_dtype.is_date():
+            return out.floordiv(int(24 * 60 * 60 * 1e9))
+    return out
 
 
 @execute_node.register(ops.Cast, pd.Series, dt.Array)
@@ -132,7 +138,10 @@ def execute_cast_series_timestamp(op, data, type, **kwargs):
         return data.astype('M8[ns]' if tz is None else DatetimeTZDtype('ns', tz))
 
     if from_type.is_string() or from_type.is_integer():
-        timestamps = pd.to_datetime(data.values, infer_datetime_format=True)
+        if from_type.is_integer():
+            timestamps = pd.to_datetime(data.values, unit="s")
+        else:
+            timestamps = pd.to_datetime(data.values, infer_datetime_format=True)
         if getattr(timestamps.dtype, "tz", None) is not None:
             method_name = "tz_convert"
         else:
@@ -342,15 +351,10 @@ def execute_cast_datetime_or_timestamp_to_string(op, data, type, **kwargs):
 
 
 @execute_node.register(ops.Cast, datetime.datetime, dt.Int64)
-def execute_cast_datetime_to_integer(op, data, type, **kwargs):
-    """Cast datetimes to integers."""
-    return pd.Timestamp(data).value
-
-
-@execute_node.register(ops.Cast, pd.Timestamp, dt.Int64)
 def execute_cast_timestamp_to_integer(op, data, type, **kwargs):
     """Cast timestamps to integers."""
-    return data.value
+    t = pd.Timestamp(data)
+    return pd.NA if pd.isna(t) else int(t.timestamp())
 
 
 @execute_node.register(ops.Cast, (np.bool_, bool), dt.Timestamp)
@@ -373,18 +377,26 @@ def execute_cast_bool_to_interval(op, data, type, **kwargs):
     )
 
 
-@execute_node.register(ops.Cast, integer_types + (str,), dt.Timestamp)
-def execute_cast_simple_literal_to_timestamp(op, data, type, **kwargs):
-    """Cast integer and strings to timestamps."""
+@execute_node.register(ops.Cast, integer_types, dt.Timestamp)
+def execute_cast_integer_to_timestamp(op, data, type, **kwargs):
+    """Cast integer to timestamp."""
+    return pd.Timestamp(data, unit="s", tz=type.timezone)
+
+
+@execute_node.register(ops.Cast, str, dt.Timestamp)
+def execute_cast_string_to_timestamp(op, data, type, **kwargs):
+    """Cast string to timestamp."""
     return pd.Timestamp(data, tz=type.timezone)
 
 
-@execute_node.register(ops.Cast, pd.Timestamp, dt.Timestamp)
+@execute_node.register(ops.Cast, datetime.datetime, dt.Timestamp)
 def execute_cast_timestamp_to_timestamp(op, data, type, **kwargs):
     """Cast timestamps to other timestamps including timezone if necessary."""
     input_timezone = data.tz
     target_timezone = type.timezone
 
+    data = pd.Timestamp(data)
+
     if input_timezone == target_timezone:
         return data
 
@@ -394,11 +406,6 @@ def execute_cast_timestamp_to_timestamp(op, data, type, **kwargs):
     return data.tz_convert(target_timezone)
 
 
-@execute_node.register(ops.Cast, datetime.datetime, dt.Timestamp)
-def execute_cast_datetime_to_datetime(op, data, type, **kwargs):
-    return execute_cast_timestamp_to_timestamp(op, data, type, **kwargs).to_pydatetime()
-
-
 @execute_node.register(ops.Cast, fixed_width_types + (str,), dt.DataType)
 def execute_cast_string_literal(op, data, type, **kwargs):
     try:


Test patch:
diff --git a/ibis/backends/dask/tests/execution/test_cast.py b/ibis/backends/dask/tests/execution/test_cast.py
--- a/ibis/backends/dask/tests/execution/test_cast.py
+++ b/ibis/backends/dask/tests/execution/test_cast.py
@@ -77,7 +77,7 @@ def test_cast_timestamp_column(t, df, column, to, expected):
     ('to', 'expected'),
     [
         ('string', str),
-        ('int64', lambda x: x.value),
+        ('int64', lambda x: x.value // int(1e9)),
         param(
             'double',
             float,
@@ -101,7 +101,7 @@ def test_cast_timestamp_scalar_naive(to, expected):
     ('to', 'expected'),
     [
         ('string', str),
-        ('int64', lambda x: x.value),
+        ('int64', lambda x: x.value // int(1e9)),
         param('double', float, marks=pytest.mark.notimpl(["dask"])),
         (
             dt.Timestamp('America/Los_Angeles'),
diff --git a/ibis/backends/dask/tests/execution/test_temporal.py b/ibis/backends/dask/tests/execution/test_temporal.py
--- a/ibis/backends/dask/tests/execution/test_temporal.py
+++ b/ibis/backends/dask/tests/execution/test_temporal.py
@@ -105,7 +105,7 @@ def test_cast_integer_to_temporal_type(t, df, column):
     df_computed = df.compute()
     expected = dd.from_pandas(
         pd.Series(
-            pd.to_datetime(df_computed.plain_int64.values, unit='ns').values,
+            pd.to_datetime(df_computed.plain_int64.values, unit='s').values,
             index=df_computed.index,
             name='plain_int64',
         ).dt.tz_localize(column_type.timezone),
diff --git a/ibis/backends/pandas/tests/execution/test_cast.py b/ibis/backends/pandas/tests/execution/test_cast.py
--- a/ibis/backends/pandas/tests/execution/test_cast.py
+++ b/ibis/backends/pandas/tests/execution/test_cast.py
@@ -89,7 +89,7 @@ def test_cast_timestamp_column(t, df, column, to, expected):
     ('to', 'expected'),
     [
         ('string', str),
-        ('int64', lambda x: x.value),
+        ('int64', lambda x: x.value // int(1e9)),
         param(
             'double',
             float,
@@ -113,7 +113,7 @@ def test_cast_timestamp_scalar_naive(to, expected):
     ('to', 'expected'),
     [
         ('string', str),
-        ('int64', lambda x: x.value),
+        ('int64', lambda x: x.value // int(1e9)),
         param(
             'double',
             float,
diff --git a/ibis/backends/pandas/tests/execution/test_temporal.py b/ibis/backends/pandas/tests/execution/test_temporal.py
--- a/ibis/backends/pandas/tests/execution/test_temporal.py
+++ b/ibis/backends/pandas/tests/execution/test_temporal.py
@@ -93,7 +93,7 @@ def test_cast_integer_to_temporal_type(t, df, column):
     expr = t.plain_int64.cast(column_type)
     result = expr.execute()
     expected = pd.Series(
-        pd.to_datetime(df.plain_int64.values, unit='ns').values,
+        pd.to_datetime(df.plain_int64.values, unit='s').values,
         index=df.index,
         name='plain_int64',
     ).dt.tz_localize(column_type.timezone)
diff --git a/ibis/backends/tests/test_temporal.py b/ibis/backends/tests/test_temporal.py
--- a/ibis/backends/tests/test_temporal.py
+++ b/ibis/backends/tests/test_temporal.py
@@ -922,17 +922,21 @@ def test_timestamp_extract_milliseconds_with_big_value(con):
 
 
 @pytest.mark.notimpl(["bigquery", "datafusion", "mssql"])
-@pytest.mark.broken(
-    ["dask", "pandas"],
-    reason="Pandas and Dask interpret integers as nanoseconds since epoch",
-)
-def test_integer_cast_to_timestamp(backend, alltypes, df):
+def test_integer_cast_to_timestamp_column(backend, alltypes, df):
     expr = alltypes.int_col.cast("timestamp")
     expected = pd.to_datetime(df.int_col, unit="s").rename(expr.get_name())
     result = expr.execute()
     backend.assert_series_equal(result, expected)
 
 
+@pytest.mark.notimpl(["datafusion", "pyspark", "mssql"])
+def test_integer_cast_to_timestamp_scalar(backend, alltypes, df):
+    expr = alltypes.int_col.min().cast("timestamp")
+    result = expr.execute()
+    expected = pd.to_datetime(df.int_col.min(), unit="s")
+    assert result == expected
+
+
 @pytest.mark.broken(
     ["clickhouse", "impala"],
     reason=(