Repository: ibis-project/ibis

Issue:
bug: `ibis.range_window()` has incorrect behavior on Pandas, PySpark, CSV, and Parquet backends
When using an `ibis.range_window()` spec, the Pandas, PySpark, CSV, and Parquet backends seem to treat it as a non-range window. They treat the values of `preceding` and `following` as **the number of rows** to include in the window (this is incorrect), rather than treating them as **the range of values** in the `order_by` column to accept.

# How to reproduce
## Set up a backend
### Pandas
```
from pathlib import Path

import ibis
from ibis.backends.pandas.tests.conftest import TestConf as PandasTestConf

ibis_backend = PandasTestConf.connect(Path('../ci/ibis-testing-data'))
```

### PySpark
```
from pathlib import Path

import ibis
from ibis.backends.tests.base import get_pyspark_testing_client

ibis_backend = get_pyspark_testing_client(Path('../ci/ibis-testing-data'))
```

### CSV
```
from pathlib import Path

import ibis
from ibis.backends.csv.tests.conftest import TestConf as CsvTestConf

ibis_backend = CsvTestConf.connect(Path('../ci/ibis-testing-data'))
```

### Parquet
```
from pathlib import Path

import ibis
from ibis.backends.parquet.tests.conftest import TestConf as ParquetTestConf

ibis_backend = ParquetTestConf.connect(Path('../ci/ibis-testing-data'))
```

## Main code to execute (same for all four backends)

```
## Setup test data
alltypes = ibis_backend.table('functional_alltypes')

## Execute ibis.range_window() on Pandasbackend
window = ibis.range_window(
    preceding=10, following=0, order_by='id', group_by='string_col',
)
expr = alltypes.mutate(val=alltypes.double_col.sum().over(window))
result = expr.execute().set_index('id').sort_index()

df = alltypes.execute()
expected = (
    df.assign(
        val=df.apply(
            lambda x: df.double_col[
                (df.string_col == x.string_col)
                & ((x.id-10) <= df.id)
                & (df.id <= x.id)
            ].sum(),
            axis=1,
        )
        .sort_index(level=1)
        .reset_index(drop=True)
    )
    .set_index('id')
    .sort_index()
)

comparison_df = pd.concat([
    result.val.rename('val_result'),
    expected.val.rename('val_expected'),
], axis=1)

print(comparison_df[0:30].to_markdown())
```

## Output (same for all four backends)

|   id |   val_result |   val_expected |
|-----:|-------------:|---------------:|
|    0 |          0   |            0   |
|    1 |         10.1 |           10.1 |
|    2 |         20.2 |           20.2 |
|    3 |         30.3 |           30.3 |
|    4 |         40.4 |           40.4 |
|    5 |         50.5 |           50.5 |
|    6 |         60.6 |           60.6 |
|    7 |         70.7 |           70.7 |
|    8 |         80.8 |           80.8 |
|    9 |         90.9 |           90.9 |
|   10 |          0   |            0   |
|   11 |         20.2 |           20.2 |
|   12 |         40.4 |           40.4 |
|   13 |         60.6 |           60.6 |
|   14 |         80.8 |           80.8 |
|   15 |        101   |          101   |
|   16 |        121.2 |          121.2 |
|   17 |        141.4 |          141.4 |
|   18 |        161.6 |          161.6 |
|   19 |        181.8 |          181.8 |
|   20 |          0   |            0   |
|   21 |         30.3 |           20.2 |
|   22 |         60.6 |           40.4 |
|   23 |         90.9 |           60.6 |
|   24 |        121.2 |           80.8 |
|   25 |        151.5 |          101   |
|   26 |        181.8 |          121.2 |
|   27 |        212.1 |          141.4 |
|   28 |        242.4 |          161.6 |
|   29 |        272.7 |          181.8 |



Discussion:
@timothydijamco This is another great issue, thank you for that. This is indeed a problem.
I think at the very least for 3.0.0 we can fail here as opposed to giving incorrect results.

Resolving patch:
diff --git a/ibis/backends/pandas/execution/window.py b/ibis/backends/pandas/execution/window.py
--- a/ibis/backends/pandas/execution/window.py
+++ b/ibis/backends/pandas/execution/window.py
@@ -10,6 +10,7 @@
 from pandas.core.groupby import SeriesGroupBy
 
 import ibis.common.exceptions as com
+import ibis.expr.datatypes as dt
 import ibis.expr.operations as ops
 import ibis.expr.window as win
 from ibis.expr.scope import Scope
@@ -264,6 +265,14 @@ def execute_window_op(
     clients=None,
     **kwargs,
 ):
+    if window.how == "range" and any(
+        not isinstance(ob.type(), (dt.Time, dt.Date, dt.Timestamp))
+        for ob in window._order_by
+    ):
+        raise NotImplementedError(
+            "The pandas backend only implements range windows with temporal "
+            "ordering keys"
+        )
     operand = op.expr
     # pre execute "manually" here because otherwise we wouldn't pickup
     # relevant scope changes from the child operand since we're managing
diff --git a/ibis/backends/pyspark/compiler.py b/ibis/backends/pyspark/compiler.py
--- a/ibis/backends/pyspark/compiler.py
+++ b/ibis/backends/pyspark/compiler.py
@@ -1246,8 +1246,10 @@ def compile_window_op(t, expr, scope, timecontext, **kwargs):
                 t, window.following, scope, timecontext, **kwargs
             )
 
-        if isinstance(window.preceding, ir.IntervalScalar) or isinstance(
-            window.following, ir.IntervalScalar
+        if (
+            isinstance(window.preceding, ir.IntervalScalar)
+            or isinstance(window.following, ir.IntervalScalar)
+            or window.how == "range"
         ):
             pyspark_window = pyspark_window.rangeBetween(start, end)
         else:
diff --git a/ibis/expr/types/sortkeys.py b/ibis/expr/types/sortkeys.py
--- a/ibis/expr/types/sortkeys.py
+++ b/ibis/expr/types/sortkeys.py
@@ -1,7 +1,12 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from public import public
 
+if TYPE_CHECKING:
+    import ibis.expr.datatypes as dt
+
 from .generic import Expr
 
 
@@ -9,3 +14,6 @@
 class SortExpr(Expr):
     def get_name(self) -> str | None:
         return self.op().resolve_name()
+
+    def type(self) -> dt.DataType:
+        return self.op().expr.type()


Test patch:
diff --git a/ibis/backends/tests/test_window.py b/ibis/backends/tests/test_window.py
--- a/ibis/backends/tests/test_window.py
+++ b/ibis/backends/tests/test_window.py
@@ -515,9 +515,7 @@ def test_ungrouped_unbounded_window(
     backend.assert_series_equal(left, right)
 
 
-@pytest.mark.notimpl(
-    ["clickhouse", "dask", "datafusion", "impala", "pandas", "pyspark"]
-)
+@pytest.mark.notimpl(["clickhouse", "dask", "datafusion", "impala", "pandas"])
 def test_grouped_bounded_range_window(backend, alltypes, df):
     # Explanation of the range window spec below:
     #
@@ -527,9 +525,9 @@ def test_grouped_bounded_range_window(backend, alltypes, df):
     # `group_by='string_col'`:
     #     The window at a particular row will only contain other rows that
     #     have the same 'string_col' value.
-    #
+    preceding = 10
     window = ibis.range_window(
-        preceding=10,
+        preceding=preceding,
         following=0,
         order_by='id',
         group_by='string_col',
@@ -538,21 +536,22 @@ def test_grouped_bounded_range_window(backend, alltypes, df):
     result = expr.execute().set_index('id').sort_index()
 
     def gb_fn(df):
-        indices = np.searchsorted(
-            df.id,
-            [
-                df.id - 10,
-                # add 1 to get the upper bound without having to make two
-                # searchsorted calls
-                df.id + 1,
-            ],
-            side="left",
-        ).T
+        indices = np.searchsorted(df.id, [df["prec"], df["foll"]], side="left")
         double_col = df.double_col.values
-        result = [double_col[start:stop].sum() for start, stop in indices]
-        return pd.Series(result, index=df.index)
+        return pd.Series(
+            [double_col[start:stop].sum() for start, stop in indices.T],
+            index=df.index,
+        )
 
-    res = df.sort_values("id").groupby("string_col").apply(gb_fn).droplevel(0)
+    res = (
+        # add 1 to get the upper bound without having to make two
+        # searchsorted calls
+        df.assign(prec=lambda t: t.id - preceding, foll=lambda t: t.id + 1)
+        .sort_values("id")
+        .groupby("string_col")
+        .apply(gb_fn)
+        .droplevel(0)
+    )
     expected = (
         df.assign(
             # Mimic our range window spec using .apply()
@@ -562,9 +561,7 @@ def gb_fn(df):
         .sort_index()
     )
 
-    left, right = result.val, expected.val
-
-    backend.assert_series_equal(left, right)
+    backend.assert_series_equal(result.val, expected.val)
 
 
 @pytest.mark.notimpl(["clickhouse", "dask", "datafusion", "pyspark"])
@@ -575,6 +572,4 @@ def test_percent_rank_whole_table_no_order_by(backend, alltypes, df):
     column = df.id.rank(method="min").sub(1).div(len(df) - 1)
     expected = df.assign(val=column).set_index('id').sort_index()
 
-    left, right = result.val, expected.val
-
-    backend.assert_series_equal(left, right)
+    backend.assert_series_equal(result.val, expected.val)