Repository: ibis-project/ibis

Issue:
Milliseconds extraction problem in duckdb (and possibly, postgres)
Consider the following code snippet:
```
import ibis
from sqlalchemy.dialects import postgresql

table = ibis.table(name='A', schema=[('a', 'timestamp')])
print(str(
    ibis.duckdb.compile(
        table['a'].millisecond()
    ).compile(
        dialect=postgresql.dialect(paramstyle="named"),
        compile_kwargs={"literal_binds": True}
    )
))
```
The result is the following:
`'SELECT CAST(floor(EXTRACT(millisecond FROM t0.a)) AS SMALLINT) % 1000 AS millisecond \nFROM "A" AS t0'`
Here typecast to SMALLINT cannot be done in general case. For example:
```
import duckdb
import pyarrow as pa
from datetime import datetime

dtm = datetime.fromisoformat

arrow_table = pa.table([
    pa.array([
        dtm('2020-01-12T23:34:45.567891'),
        dtm('2022-02-24T04:34:13.067123'),
        None
    ])],
    names=['a']
)
conn = duckdb.connect(database=':memory:')
conn.register('A', arrow_table)
conn.execute('SELECT EXTRACT(millisecond FROM a) AS a FROM A').arrow()
```
Here the result is
```
pyarrow.Table
a: int64
----
a: [[45567,13067,null]]
```
This it is ok to do '% 1000' to eventually get milliseconds, since we see here whole seconds as well, but SMALLINT is not able to store 45567 because it has a range [-32768 to 32767](https://duckdb.org/docs/sql/data_types/numeric), while we need 0 to 60000. So, USMALLINT should be good for milliseconds.
From the example above if we try to run a query on duckdb connection:
`conn.execute('SELECT CAST(floor(EXTRACT(millisecond FROM t0.a)) AS SMALLINT) % 1000 AS millisecond \nFROM "A" AS t0').fetchall()`
it produces an error:
`'RuntimeError: Conversion Error: Type DOUBLE with value 45567.0 can't be cast because the value is out of range for the destination type INT16'`
which makes sense.

PROPOSAL: USMALLINT looks like a better choice then SMALLINT


Discussion:
Also, i tried to do the same thing with duckdb-engine (`pip install duckdb-engine`). The result is the same:
```
import ibis
from sqlalchemy import create_engine

eng = create_engine("duckdb:///:memory:")
table = ibis.table(name='A', schema=[('a', 'timestamp')])

str(
    ibis.duckdb.compile(
        table['a'].millisecond()
    ).compile(eng, compile_kwargs={"literal_binds": True})
)
```
the output:
`'SELECT CAST(floor(EXTRACT(millisecond FROM t0.a)) AS SMALLINT) % 1000 AS millisecond \nFROM "A" AS t0'`
@evgenygr Thanks for the bug report. Indeed this looks like a bug. Ibis is assuming that the result of `extract(milliseconds from thing)` which is `seconds * 1000 + milliseconds` is less than the max value of a signed 16-bit integer (`32767`) which is incorrect.

Resolving patch:
diff --git a/ibis/backends/base/sql/registry/literal.py b/ibis/backends/base/sql/registry/literal.py
--- a/ibis/backends/base/sql/registry/literal.py
+++ b/ibis/backends/base/sql/registry/literal.py
@@ -60,7 +60,7 @@ def _date_literal_format(translator, expr):
 def _timestamp_literal_format(translator, expr):
     value = expr.op().value
     if isinstance(value, datetime.datetime):
-        value = value.strftime('%Y-%m-%d %H:%M:%S')
+        value = value.isoformat()
 
     return repr(value)
 
diff --git a/ibis/backends/postgres/registry.py b/ibis/backends/postgres/registry.py
--- a/ibis/backends/postgres/registry.py
+++ b/ibis/backends/postgres/registry.py
@@ -51,9 +51,9 @@ def _millisecond(t, expr):
     # we get total number of milliseconds including seconds with extract so we
     # mod 1000
     (sa_arg,) = map(t.translate, expr.op().args)
-    return (
-        sa.cast(sa.func.floor(sa.extract('millisecond', sa_arg)), sa.SMALLINT)
-        % 1000
+    return sa.cast(
+        sa.func.floor(sa.extract('millisecond', sa_arg)) % 1000,
+        sa.SMALLINT,
     )
 
 
diff --git a/ibis/backends/sqlite/__init__.py b/ibis/backends/sqlite/__init__.py
--- a/ibis/backends/sqlite/__init__.py
+++ b/ibis/backends/sqlite/__init__.py
@@ -14,9 +14,11 @@
 
 from __future__ import annotations
 
+import sqlite3
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import pandas as pd
 import sqlalchemy as sa
 
 if TYPE_CHECKING:
@@ -70,6 +72,8 @@ def do_connect(self, path: str | Path | None = None) -> None:
             f"sqlite:///{path if path is not None else ':memory:'}"
         )
 
+        sqlite3.register_adapter(pd.Timestamp, lambda value: value.isoformat())
+
         @sa.event.listens_for(engine, "connect")
         def connect(dbapi_connection, connection_record):
             """Register UDFs on connection."""


Test patch:
diff --git a/ibis/backends/impala/tests/test_value_exprs.py b/ibis/backends/impala/tests/test_value_exprs.py
--- a/ibis/backends/impala/tests/test_value_exprs.py
+++ b/ibis/backends/impala/tests/test_value_exprs.py
@@ -329,17 +329,17 @@ def test_timestamp_deltas(table, unit, compiled_unit):
     [
         pytest.param(
             lambda v: L(pd.Timestamp(v)),
-            "'2015-01-01 12:34:56'",
+            "'2015-01-01T12:34:56'",
             id="literal_pd_timestamp",
         ),
         pytest.param(
             lambda v: L(pd.Timestamp(v).to_pydatetime()),
-            "'2015-01-01 12:34:56'",
+            "'2015-01-01T12:34:56'",
             id="literal_pydatetime",
         ),
         pytest.param(
             lambda v: ibis.timestamp(v),
-            "'2015-01-01 12:34:56'",
+            "'2015-01-01T12:34:56'",
             id="ibis_timestamp_function",
         ),
     ],
@@ -355,18 +355,18 @@ def test_timestamp_literals(expr_fn, expected):
     [
         pytest.param(
             lambda v: v.day_of_week.index(),
-            "pmod(dayofweek('2015-09-01 01:00:23') - 2, 7)",
+            "pmod(dayofweek('2015-09-01T01:00:23') - 2, 7)",
             id="index",
         ),
         pytest.param(
             lambda v: v.day_of_week.full_name(),
-            "dayname('2015-09-01 01:00:23')",
+            "dayname('2015-09-01T01:00:23')",
             id="full_name",
         ),
     ],
 )
 def test_timestamp_day_of_week(expr_fn, expected):
-    expr = expr_fn(ibis.timestamp('2015-09-01 01:00:23'))
+    expr = expr_fn(ibis.timestamp('2015-09-01T01:00:23'))
     result = translate(expr)
     assert result == expected
 
diff --git a/ibis/backends/tests/test_temporal.py b/ibis/backends/tests/test_temporal.py
--- a/ibis/backends/tests/test_temporal.py
+++ b/ibis/backends/tests/test_temporal.py
@@ -718,3 +718,13 @@ def test_date_column_from_iso(con, alltypes, df):
     )
     actual = result.dt.strftime('%Y-%m-%d')
     tm.assert_series_equal(golden.rename('tmp'), actual.rename('tmp'))
+
+
+@pytest.mark.notimpl(["datafusion"])
+@pytest.mark.notyet(["clickhouse", "pyspark"])
+@pytest.mark.broken(["mysql"])
+def test_timestamp_extract_milliseconds_with_big_value(con):
+    timestamp = ibis.timestamp("2021-01-01 01:30:59.333")
+    millis = timestamp.millisecond()
+    result = con.execute(millis)
+    assert result == 333
diff --git a/ibis/tests/sql/test_compiler.py b/ibis/tests/sql/test_compiler.py
--- a/ibis/tests/sql/test_compiler.py
+++ b/ibis/tests/sql/test_compiler.py
@@ -294,14 +294,14 @@ def test_table_drop_with_filter():
 FROM (
   SELECT t2.`a`
   FROM (
-    SELECT `a`, `b`, '2018-01-01 00:00:00' AS `the_date`
+    SELECT `a`, `b`, '2018-01-01T00:00:00' AS `the_date`
     FROM (
       SELECT *
       FROM (
         SELECT `a`, `b`, `c` AS `C`
         FROM t
       ) t5
-      WHERE `C` = '2018-01-01 00:00:00'
+      WHERE `C` = '2018-01-01T00:00:00'
     ) t4
   ) t2
     INNER JOIN s t1
diff --git a/ibis/tests/sql/test_select_sql.py b/ibis/tests/sql/test_select_sql.py
--- a/ibis/tests/sql/test_select_sql.py
+++ b/ibis/tests/sql/test_select_sql.py
@@ -251,9 +251,9 @@ def test_where_analyze_scalar_op(functional_alltypes):
     expected = """\
 SELECT count(*) AS `count`
 FROM functional_alltypes
-WHERE (`timestamp_col` < date_add(cast({} as timestamp), INTERVAL 3 MONTH)) AND
+WHERE (`timestamp_col` < date_add(cast({!r} as timestamp), INTERVAL 3 MONTH)) AND
       (`timestamp_col` < date_add(cast(now() as timestamp), INTERVAL 10 DAY))"""  # noqa: E501
-    assert result == expected.format("'2010-01-01 00:00:00'")
+    assert result == expected.format("2010-01-01T00:00:00")
 
 
 def test_bug_duplicated_where(airlines):