Repository: ibis-project/ibis

Issue:
bug(duckdb): backend doesn't support time-based range windows for aggregation
### What happened?

```python
>>> import pandas as pd
>>> quotes = pd.DataFrame(
...     {
...         "time": [
...             pd.Timestamp("2016-05-25 13:30:00.023"),
...             pd.Timestamp("2016-05-25 13:30:00.023"),
...             pd.Timestamp("2016-05-25 13:30:00.030"),
...             pd.Timestamp("2016-05-25 13:30:00.041"),
...             pd.Timestamp("2016-05-25 13:30:00.048"),
...             pd.Timestamp("2016-05-25 13:30:00.049"),
...             pd.Timestamp("2016-05-25 13:30:00.072"),
...             pd.Timestamp("2016-05-25 13:30:00.075")
...         ],
...         "ticker": [
...                "GOOG",
...                "MSFT",
...                "MSFT",
...                "MSFT",
...                "GOOG",
...                "AAPL",
...                "GOOG",
...                "MSFT"
...            ],
...            "bid": [720.50, 51.95, 51.97, 51.99, 720.50, 97.99, 720.50, 52.01],
...            "ask": [720.93, 51.96, 51.98, 52.00, 720.93, 98.01, 720.88, 52.03]
...     }
... )
>>> con = ibis.duckdb.connect()
>>> t = con.create_table("quotes", quotes)
>>> agged = t[
...     "ticker",
...     t.bid.sum()
...     .over(range=(-ibis.interval(seconds=10), 0), order_by=t.time)
...     .name("total_bid_amount"),
... ]
>>> agged.to_pandas()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/deepyaman/github/claypotai/ibis/ibis/expr/types/relations.py", line 3232, in to_pandas
    return self.execute(**kwargs)
  File "/Users/deepyaman/github/claypotai/ibis/ibis/expr/types/core.py", line 322, in execute
    return self._find_backend(use_default=True).execute(
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/__init__.py", line 335, in execute
    sql = query_ast.compile()
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/compiler/base.py", line 37, in compile
    compiled_queries = [q.compile() for q in self.queries]
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/compiler/base.py", line 37, in <listcomp>
    compiled_queries = [q.compile() for q in self.queries]
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/alchemy/query_builder.py", line 216, in compile
    frag = step(frag)
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/alchemy/query_builder.py", line 248, in _add_select
    arg = self._translate(op, named=True)
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/compiler/query_builder.py", line 239, in _translate
    return translator.get_result()
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/compiler/translator.py", line 224, in get_result
    translated = self.translate(self.node)
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/compiler/translator.py", line 256, in translate
    return formatter(self, op)
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/alchemy/registry.py", line 227, in _alias
    return t.translate(op.arg)
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/compiler/translator.py", line 256, in translate
    return formatter(self, op)
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/alchemy/registry.py", line 355, in _window_function
    end = _translate_window_boundary(window.frame.end)
  File "/Users/deepyaman/github/claypotai/ibis/ibis/backends/base/sql/alchemy/registry.py", line 321, in _translate_window_boundary
    raise com.TranslationError("Window boundaries must be literal values")
ibis.common.exceptions.TranslationError: Window boundaries must be literal values
>>> 
```

### What did you expect to happen?

Same thing, but with `pandas` backend, yields:

```python
>>> agged.to_pandas()
  ticker  total_bid_amount
0   GOOG            720.50
1   MSFT            772.45
2   MSFT            824.42
3   MSFT            876.41
4   GOOG           1596.91
5   AAPL           1694.90
6   GOOG           2415.40
7   MSFT           2467.41
>>> 
```

### What version of ibis are you using?

7.1.0 (c572eabaef7e02ec36eff146e33b693e74d8a6f0)

### What backend(s) are you using, if any?

DuckDB

### Relevant log output

_No response_

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
It looks like there are an annoying number of missing pieces here:

1. Ibis doesn't allow non-literals through
2. Even if it did, sqlalchemy `sa.over` implicitly doesn't allow expressions there because it's calling `int` on the range bounds.
3. Even if we could workaround the previous point, the sqlalchemy compiler assumes the range bounds can be compared to other objects with `<` 😞

I'll look into a workaround. Worst case we have to compile `sa.Over` (capital-`O`) ourselves.
Ok, I was able to hack around this ... it's not pretty.

I'll PR it though, in case it's an acceptable hack :)

Resolving patch:
diff --git a/ibis/backends/base/sql/alchemy/registry.py b/ibis/backends/base/sql/alchemy/registry.py
--- a/ibis/backends/base/sql/alchemy/registry.py
+++ b/ibis/backends/base/sql/alchemy/registry.py
@@ -7,6 +7,7 @@
 
 import sqlalchemy as sa
 from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.sql.elements import RANGE_CURRENT, RANGE_UNBOUNDED
 from sqlalchemy.sql.functions import FunctionElement, GenericFunction
 
 import ibis.common.exceptions as com
@@ -308,17 +309,84 @@ def _endswith(t, op):
     return t.translate(op.arg).endswith(t.translate(op.end))
 
 
-def _translate_window_boundary(boundary):
-    if boundary is None:
-        return None
+def _reinterpret_range_bound(bound):
+    if bound is None:
+        return RANGE_UNBOUNDED
 
-    if isinstance(boundary.value, ops.Literal):
-        if boundary.preceding:
-            return -boundary.value.value
-        else:
-            return boundary.value.value
+    try:
+        lower = int(bound)
+    except ValueError as err:
+        sa.util.raise_(
+            sa.exc.ArgumentError(
+                "Integer, None or expression expected for range value"
+            ),
+            replace_context=err,
+        )
+    except TypeError:
+        return bound
+    else:
+        return RANGE_CURRENT if lower == 0 else lower
+
+
+def _interpret_range(self, range_):
+    if not isinstance(range_, tuple) or len(range_) != 2:
+        raise sa.exc.ArgumentError("2-tuple expected for range/rows")
+
+    lower = _reinterpret_range_bound(range_[0])
+    upper = _reinterpret_range_bound(range_[1])
+    return lower, upper
+
+
+# monkeypatch to allow expressions in range and rows bounds
+sa.sql.elements.Over._interpret_range = _interpret_range
+
+
+def _compile_bounds(processor, left, right) -> str:
+    if left is RANGE_UNBOUNDED:
+        left = "UNBOUNDED PRECEDING"
+    elif left is RANGE_CURRENT:
+        left = "CURRENT ROW"
+    else:
+        left = f"{processor(left)} PRECEDING"
+
+    if right is RANGE_UNBOUNDED:
+        right = "UNBOUNDED FOLLOWING"
+    elif right is RANGE_CURRENT:
+        right = "CURRENT ROW"
+    else:
+        right = f"{processor(right)} FOLLOWING"
+
+    return f"BETWEEN {left} AND {right}"
 
-    raise com.TranslationError("Window boundaries must be literal values")
+
+@compiles(sa.sql.elements.Over)
+def compile_over(over, compiler, **kw) -> str:
+    processor = functools.partial(compiler.process, **kw)
+
+    text = processor(over.element)
+
+    if over.range_:
+        bounds = _compile_bounds(processor, *over.range_)
+        range_ = f"RANGE {bounds}"
+    elif over.rows:
+        bounds = _compile_bounds(processor, *over.rows)
+        range_ = f"ROWS {bounds}"
+    else:
+        range_ = None
+
+    args = [
+        f"{word} BY {processor(clause)}"
+        for word, clause in (
+            ("PARTITION", over.partition_by),
+            ("ORDER", over.order_by),
+        )
+        if clause is not None and len(clause)
+    ]
+
+    if range_ is not None:
+        args.append(range_)
+
+    return f"{text} OVER ({' '.join(args)})"
 
 
 def _window_function(t, window):
@@ -347,22 +415,26 @@ def _window_function(t, window):
     else:
         raise NotImplementedError(type(window.frame))
 
-    if t._forbids_frame_clause and isinstance(func, t._forbids_frame_clause):
-        # some functions on some backends don't support frame clauses
-        additional_params = {}
-    else:
-        start = _translate_window_boundary(window.frame.start)
-        end = _translate_window_boundary(window.frame.end)
-        additional_params = {how: (start, end)}
+    additional_params = {}
+
+    # some functions on some backends don't support frame clauses
+    if not t._forbids_frame_clause or not isinstance(func, t._forbids_frame_clause):
+        if (start := window.frame.start) is not None:
+            start = t.translate(start.value)
+
+        if (end := window.frame.end) is not None:
+            end = t.translate(end.value)
+
+        additional_params[how] = (start, end)
 
     result = sa.over(
         reduction, partition_by=partition_by, order_by=order_by, **additional_params
     )
 
     if isinstance(func, (ops.RowNumber, ops.DenseRank, ops.MinRank, ops.NTile)):
-        return result - 1
-    else:
-        return result
+        result -= 1
+
+    return result
 
 
 def _lag(t, op):


Test patch:
diff --git a/ibis/backends/tests/test_window.py b/ibis/backends/tests/test_window.py
--- a/ibis/backends/tests/test_window.py
+++ b/ibis/backends/tests/test_window.py
@@ -15,12 +15,6 @@
 import ibis.expr.datatypes as dt
 from ibis.legacy.udf.vectorized import analytic, reduction
 
-try:
-    from py4j.protocol import Py4JJavaError
-except ImportError:
-    Py4JJavaError = None
-
-
 pytestmark = pytest.mark.notimpl(
     ["druid"],
     raises=(
@@ -49,6 +43,17 @@
 except ImportError:
     HiveServer2Error = None
 
+try:
+    from py4j.protocol import Py4JJavaError
+except ImportError:
+    Py4JJavaError = None
+
+
+try:
+    from google.api_core.exceptions import BadRequest as GoogleBadRequest
+except ImportError:
+    GoogleBadRequest = None
+
 
 # adapted from https://gist.github.com/xmnlab/2c1f93df1a6c6bde4e32c8579117e9cc
 def pandas_ntile(x, bucket: int):
@@ -1186,3 +1191,61 @@ def test_first_last(backend):
         }
     )
     tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.notyet(
+    ["bigquery"], raises=GoogleBadRequest, reason="not supported by BigQuery"
+)
+@pytest.mark.notyet(
+    ["clickhouse"],
+    raises=ClickHouseOperationalError,
+    reason="not supported by ClickHouse",
+)
+@pytest.mark.notyet(
+    ["datafusion"], raises=Exception, reason="not supported by DataFusion"
+)
+@pytest.mark.notyet(
+    ["impala"], raises=HiveServer2Error, reason="not supported by Impala"
+)
+@pytest.mark.notyet(
+    ["mysql"], raises=sa.exc.ProgrammingError, reason="not supported by MySQL"
+)
+@pytest.mark.notyet(
+    ["mssql", "oracle", "polars", "snowflake", "sqlite"],
+    raises=com.OperationNotDefinedError,
+    reason="not support by the backend",
+)
+@pytest.mark.notyet(["flink"], raises=Py4JJavaError, reason="not supported by Flink")
+def test_range_expression_bounds(backend):
+    t = ibis.memtable(
+        {
+            "time": pd.to_datetime(
+                [
+                    "2016-05-25 13:30:00.023",
+                    "2016-05-25 13:30:00.023",
+                    "2016-05-25 13:30:00.030",
+                    "2016-05-25 13:30:00.041",
+                    "2016-05-25 13:30:00.048",
+                    "2016-05-25 13:30:00.049",
+                    "2016-05-25 13:30:00.072",
+                    "2016-05-25 13:30:00.075",
+                ],
+                utc=True,
+            ),
+            "ticker": ["GOOG", "MSFT", "MSFT", "MSFT", "GOOG", "AAPL", "GOOG", "MSFT"],
+            "bid": [720.50, 51.95, 51.97, 51.99, 720.50, 97.99, 720.50, 52.01],
+            "ask": [720.93, 51.96, 51.98, 52.00, 720.93, 98.01, 720.88, 52.03],
+        }
+    )
+    expr = t.select(
+        "ticker",
+        total_bid_amount=lambda t: t.bid.sum().over(
+            range=(-ibis.interval(seconds=10), 0), order_by=t.time
+        ),
+    )
+
+    con = backend.connection
+    result = con.execute(expr)
+
+    assert not result.empty
+    assert len(result) == con.execute(t.count())