Repository: ibis-project/ibis

Issue:
bug: invalid values returned by .first().over(w) or .last().over(w) when using the pandas backend
Minimal reproducer:

```python
import ibis
import pandas as pd

df = pd.DataFrame(
    {

        "g": ["a", "a", "a", "a", "a"],
        "x": [0, 1, 2, 3, 4],
        "y": [3, 2, 0, 1, 1],
    }
)
df.to_parquet("test.parquet")
t_pandas = ibis.pandas.connect({"t": df}).table("t")
t_duckdb = ibis.duckdb.connect().register("test.parquet", table_name="t")


def simple_window_ops(t):
    w = ibis.window(
        group_by=t.g,
        order_by=[t.x, t.y],
        preceding=1,
        following=0,
    )
    return t.mutate(
        x_first=t.x.first().over(w),
        x_last=t.x.last().over(w),
        y_first=t.y.first().over(w),
        y_last=t.y.last().over(w),
    )
```

Then pandas does not seem to take the `preceding` and `following` window boundaries into account:

```python
>>> print(simple_window_ops(t_pandas).execute())
   g  x  y  x_first  x_last  y_first  y_last
0  a  0  3        0       4        3       1
1  a  1  2        0       4        3       1
2  a  2  0        0       4        3       1
3  a  3  1        0       4        3       1
4  a  4  1        0       4        3       1
```

while duckdb works as expected:

```python
>>> print(simple_window_ops(t_duckdb).execute())
   g  x  y  x_first  x_last  y_first  y_last
0  a  0  3        0       0        3       3
1  a  1  2        0       1        3       2
2  a  2  0        1       2        2       0
3  a  3  1        2       3        0       1
4  a  4  1        3       4        1       1
```


Discussion:
Also, when I omit the `group_by` argument in the window spec:

```python
def simple_window_ops(t):
    w = ibis.window(
        # group_by=t.g,     # <-- commented out
        order_by=[t.x, t.y],
        preceding=1,
        following=0,
    )
    return t.mutate(
        x_first=t.x.first().over(w),
        x_last=t.x.last().over(w),
        y_first=t.y.first().over(w),
        y_last=t.y.last().over(w),
    )
```


then the pandas function fails with:

<details>

```python-traceback
Traceback (most recent call last):
  File "/Users/ogrisel/tmp/ibis_pandas_window.py", line 34, in <module>
    print(simple_window_ops(t_pandas).execute())
  File "/Users/ogrisel/code/ibis/ibis/expr/types/core.py", line 274, in execute
    return self._find_backend(use_default=True).execute(
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/__init__.py", line 233, in execute
    return execute_and_reset(node, params=params, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 494, in execute_and_reset
    result = execute(
  File "/Users/ogrisel/mambaforge/envs/dev/lib/python3.10/site-packages/multipledispatch/dispatcher.py", line 278, in __call__
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/trace.py", line 136, in traced_func
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 438, in main_execute
    return execute_with_scope(
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 218, in execute_with_scope
    result = execute_until_in_scope(
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/trace.py", line 136, in traced_func
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 360, in execute_until_in_scope
    result = execute_node(
  File "/Users/ogrisel/mambaforge/envs/dev/lib/python3.10/site-packages/multipledispatch/dispatcher.py", line 278, in __call__
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/trace.py", line 136, in traced_func
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/execution/selection.py", line 315, in execute_selection_dataframe
    result = build_df_from_projection(
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/execution/selection.py", line 279, in build_df_from_projection
    data_pieces = [
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/execution/selection.py", line 280, in <listcomp>
    compute_projection(node, op, data, **kwargs) for node in selection_exprs
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/execution/selection.py", line 119, in compute_projection
    result = execute(node, scope=scope, timecontext=timecontext, **kwargs)
  File "/Users/ogrisel/mambaforge/envs/dev/lib/python3.10/site-packages/multipledispatch/dispatcher.py", line 278, in __call__
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/trace.py", line 136, in traced_func
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 438, in main_execute
    return execute_with_scope(
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 218, in execute_with_scope
    result = execute_until_in_scope(
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/trace.py", line 136, in traced_func
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 332, in execute_until_in_scope
    scopes = [
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 333, in <listcomp>
    execute_until_in_scope(
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/trace.py", line 136, in traced_func
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/core.py", line 360, in execute_until_in_scope
    result = execute_node(
  File "/Users/ogrisel/mambaforge/envs/dev/lib/python3.10/site-packages/multipledispatch/dispatcher.py", line 278, in __call__
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/trace.py", line 136, in traced_func
    return func(*args, **kwargs)
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/execution/window.py", line 390, in execute_window_op
    result = post_process(
  File "/Users/ogrisel/code/ibis/ibis/backends/pandas/execution/window.py", line 89, in _post_process_order_by
    series = series.reorder_levels(names)
AttributeError: 'numpy.int64' object has no attribute 'reorder_levels'
```

</details>

while duckdb still works as expected.
Thanks for reporting this, @ogrisel.  This is definitely two bugs.  We'll try to get pandas to return valid results, or at the very least, raise a meaningful error if it can't.
I don't think we can handle arbitrary window frame boundaries with pandas without a more generic interface to the rolling and expanding APIs that would allow us to definite exactly how to calculate the bounds.
I think ibis should at least raise NotImplementedError in those cases instead of using pandas to compute erroneous results.
Note that in this particular case, it would be possible to use pandas' `shift` method to get the correct results efficiently.

Update: and using ibis' `lag` works as expected.
The plan here is to prevent these kinds of computations from returning anything from the pandas backend. Fixing them is out of scope right now.

I think the failure case is caused by incorrect execution in the pandas backend when all the following things are true:

1. `group_by` is set
2. `order_by` is set
3. `preceding` or `following` are not the defaults (perhaps `following` being 0 and and `preceding` being the default may work)
Indeed raising a `NotImplementedError` or similar would be fine. Maybe the error message could be extra helpful and state that the pandas backend does not support generic window operations but that some specific window operation via the `lag` and `lead`  methods.
BTW, what about the second problem documented in https://github.com/ibis-project/ibis/issues/4676#issuecomment-1283756388

Do you want me to open a dedicated issue or both problems are likely to be solved by the same PR?

Resolving patch:
diff --git a/ibis/backends/bigquery/udf/core.py b/ibis/backends/bigquery/udf/core.py
--- a/ibis/backends/bigquery/udf/core.py
+++ b/ibis/backends/bigquery/udf/core.py
@@ -569,8 +569,8 @@ def range(n):
 
         i = 0
         foo = 2
-        bar = lambda x: x  # noqa: E731
-        bazel = lambda x: y  # noqa: E731
+        bar = lambda x: x
+        bazel = lambda x: y
         while i < n:
             foo = bar(bazel(10))
             i += 1
diff --git a/ibis/backends/impala/pandas_interop.py b/ibis/backends/impala/pandas_interop.py
--- a/ibis/backends/impala/pandas_interop.py
+++ b/ibis/backends/impala/pandas_interop.py
@@ -18,6 +18,7 @@
 import tempfile
 from posixpath import join as pjoin
 
+import ibis.backends.pandas.client  # noqa: F401
 import ibis.common.exceptions as com
 import ibis.expr.schema as sch
 import ibis.util as util
diff --git a/ibis/backends/pandas/execution/window.py b/ibis/backends/pandas/execution/window.py
--- a/ibis/backends/pandas/execution/window.py
+++ b/ibis/backends/pandas/execution/window.py
@@ -6,6 +6,7 @@
 import re
 from typing import TYPE_CHECKING, Any, Callable, NoReturn
 
+import numpy as np
 import pandas as pd
 import toolz
 from multipledispatch import Dispatcher
@@ -155,12 +156,18 @@ def get_aggcontext_window(
     output_type = operand.output_dtype
 
     aggcontext: agg_ctx.AggregationContext
+
     if not group_by and not order_by:
         aggcontext = agg_ctx.Summarize(parent=parent, output_type=output_type)
-    elif (
-        isinstance(operand, (ops.Reduction, ops.CumulativeOp, ops.Any, ops.All))
-        and order_by
-    ):
+    elif group_by and not order_by:
+        # groupby transform (window with a partition by clause in SQL parlance)
+        aggcontext = agg_ctx.Transform(
+            parent=parent,
+            group_by=group_by,
+            order_by=order_by,
+            output_type=output_type,
+        )
+    else:
         # XXX(phillipc): What a horror show
         preceding = window.preceding
         if preceding is not None:
@@ -182,14 +189,6 @@ def get_aggcontext_window(
                 order_by=order_by,
                 output_type=output_type,
             )
-    else:
-        # groupby transform (window with a partition by clause in SQL parlance)
-        aggcontext = agg_ctx.Transform(
-            parent=parent,
-            group_by=group_by,
-            order_by=order_by,
-            output_type=output_type,
-        )
 
     return aggcontext
 
@@ -514,9 +513,13 @@ def execute_series_first_value(op, data, **kwargs):
     return data.values[0]
 
 
+def _getter(x: pd.Series | np.ndarray, idx: int):
+    return getattr(x, "values", x)[idx]
+
+
 @execute_node.register(ops.FirstValue, SeriesGroupBy)
 def execute_series_group_by_first_value(op, data, aggcontext=None, **kwargs):
-    return aggcontext.agg(data, 'first')
+    return aggcontext.agg(data, lambda x: _getter(x, 0))
 
 
 @execute_node.register(ops.LastValue, pd.Series)
@@ -526,7 +529,7 @@ def execute_series_last_value(op, data, **kwargs):
 
 @execute_node.register(ops.LastValue, SeriesGroupBy)
 def execute_series_group_by_last_value(op, data, aggcontext=None, **kwargs):
-    return aggcontext.agg(data, 'last')
+    return aggcontext.agg(data, lambda x: _getter(x, -1))
 
 
 @execute_node.register(ops.MinRank, (pd.Series, SeriesGroupBy))


Test patch:
diff --git a/ibis/backends/bigquery/tests/conftest.py b/ibis/backends/bigquery/tests/conftest.py
--- a/ibis/backends/bigquery/tests/conftest.py
+++ b/ibis/backends/bigquery/tests/conftest.py
@@ -16,7 +16,7 @@
 from ibis.backends.bigquery.datatypes import ibis_type_to_bigquery_type
 from ibis.backends.conftest import TEST_TABLES
 from ibis.backends.tests.base import BackendTest, RoundAwayFromZero, UnorderedComparator
-from ibis.backends.tests.data import non_null_array_types, struct_types
+from ibis.backends.tests.data import non_null_array_types, struct_types, win
 
 DATASET_ID = "ibis_gbq_testing"
 DEFAULT_PROJECT_ID = "ibis-gbq"
@@ -129,7 +129,7 @@ def _load_data(data_dir: Path, script_dir: Path, **_: Any) -> None:
         client.create_table(date_table, exists_ok=True)
 
         write_disposition = bq.WriteDisposition.WRITE_TRUNCATE
-        make_job = lambda func, *a, **kw: func(*a, **kw).result()  # noqa: E731
+        make_job = lambda func, *a, **kw: func(*a, **kw).result()
 
         futures = []
         with concurrent.futures.ThreadPoolExecutor() as e:
@@ -168,6 +168,7 @@ def _load_data(data_dir: Path, script_dir: Path, **_: Any) -> None:
                     ),
                 )
             )
+
             futures.append(
                 e.submit(
                     make_job,
@@ -223,6 +224,21 @@ def _load_data(data_dir: Path, script_dir: Path, **_: Any) -> None:
                 )
             )
 
+            futures.append(
+                e.submit(
+                    make_job,
+                    client.load_table_from_dataframe,
+                    win,
+                    bq.TableReference(testing_dataset, "win"),
+                    job_config=bq.LoadJobConfig(
+                        write_disposition=write_disposition,
+                        schema=ibis_schema_to_bq_schema(
+                            dict(g="string", x="int64", y="int64")
+                        ),
+                    ),
+                )
+            )
+
             for table, schema in TEST_TABLES.items():
                 futures.append(
                     e.submit(
diff --git a/ibis/backends/bigquery/tests/unit/udf/test_core.py b/ibis/backends/bigquery/tests/unit/udf/test_core.py
--- a/ibis/backends/bigquery/tests/unit/udf/test_core.py
+++ b/ibis/backends/bigquery/tests/unit/udf/test_core.py
@@ -187,7 +187,7 @@ def sum(sequence):
                 total += value
             return total
 
-        splat_sum = lambda *args: sum(args)  # noqa: E731
+        splat_sum = lambda *args: sum(args)
         return splat_sum(1, 2, 3)
 
     js = compile(f)
diff --git a/ibis/backends/clickhouse/tests/test_select.py b/ibis/backends/clickhouse/tests/test_select.py
--- a/ibis/backends/clickhouse/tests/test_select.py
+++ b/ibis/backends/clickhouse/tests/test_select.py
@@ -304,7 +304,7 @@ def test_timestamp_scalar_in_filter(alltypes):
 def test_named_from_filter_groupby(snapshot):
     t = ibis.table([('key', 'string'), ('value', 'double')], name='t0')
     gb = t.filter(t.value == 42).group_by(t.key)
-    sum_expr = lambda t: (t.value + 1 + 2 + 3).sum()  # noqa: E731
+    sum_expr = lambda t: (t.value + 1 + 2 + 3).sum()
     expr = gb.aggregate(abc=sum_expr)
     result = ibis.clickhouse.compile(expr)
     snapshot.assert_match(result, "out1.sql")
diff --git a/ibis/backends/dask/tests/conftest.py b/ibis/backends/dask/tests/conftest.py
--- a/ibis/backends/dask/tests/conftest.py
+++ b/ibis/backends/dask/tests/conftest.py
@@ -7,6 +7,7 @@
 
 import ibis
 from ibis.backends.pandas.tests.conftest import TestConf as PandasTest
+from ibis.backends.tests.data import win
 
 dd = pytest.importorskip("dask.dataframe")
 
@@ -50,18 +51,22 @@ def connect(data_directory: Path):
                     pd.read_csv(str(data_directory / 'diamonds.csv')),
                     npartitions=NPARTITIONS,
                 ),
-                'json_t': pd.DataFrame(
-                    {
-                        "js": [
-                            '{"a": [1,2,3,4], "b": 1}',
-                            '{"a":null,"b":2}',
-                            '{"a":"foo", "c":null}',
-                            "null",
-                            "[42,47,55]",
-                            "[]",
-                        ]
-                    }
+                'json_t': dd.from_pandas(
+                    pd.DataFrame(
+                        {
+                            "js": [
+                                '{"a": [1,2,3,4], "b": 1}',
+                                '{"a":null,"b":2}',
+                                '{"a":"foo", "c":null}',
+                                "null",
+                                "[42,47,55]",
+                                "[]",
+                            ]
+                        }
+                    ),
+                    npartitions=NPARTITIONS,
                 ),
+                "win": dd.from_pandas(win, npartitions=NPARTITIONS),
             }
         )
 
diff --git a/ibis/backends/impala/tests/conftest.py b/ibis/backends/impala/tests/conftest.py
--- a/ibis/backends/impala/tests/conftest.py
+++ b/ibis/backends/impala/tests/conftest.py
@@ -19,6 +19,7 @@
 from ibis.backends.conftest import TEST_TABLES, _random_identifier
 from ibis.backends.impala.compiler import ImpalaCompiler, ImpalaExprTranslator
 from ibis.backends.tests.base import BackendTest, RoundAwayFromZero, UnorderedComparator
+from ibis.backends.tests.data import win
 from ibis.tests.expr.mocks import MockBackend
 
 
@@ -493,6 +494,12 @@ def impala_create_test_database(con, env):
         ),
         database=env.test_data_db,
     )
+    con.create_table(
+        "win",
+        schema=ibis.schema(dict(g="string", x="int64", y="int64")),
+        database=env.test_data_db,
+    )
+    con.table("win", database=env.test_data_db).insert(win, overwrite=True)
 
 
 PARQUET_SCHEMAS = {
diff --git a/ibis/backends/impala/tests/test_exprs.py b/ibis/backends/impala/tests/test_exprs.py
--- a/ibis/backends/impala/tests/test_exprs.py
+++ b/ibis/backends/impala/tests/test_exprs.py
@@ -711,7 +711,7 @@ def test_filter_with_analytic(snapshot):
 def test_named_from_filter_group_by(snapshot):
     t = ibis.table([('key', 'string'), ('value', 'double')], name='t0')
     gb = t.filter(t.value == 42).group_by(t.key)
-    sum_expr = lambda t: (t.value + 1 + 2 + 3).sum()  # noqa: E731
+    sum_expr = lambda t: (t.value + 1 + 2 + 3).sum()
     expr = gb.aggregate(abc=sum_expr)
     snapshot.assert_match(ibis.impala.compile(expr), "abc.sql")
 
diff --git a/ibis/backends/pandas/tests/conftest.py b/ibis/backends/pandas/tests/conftest.py
--- a/ibis/backends/pandas/tests/conftest.py
+++ b/ibis/backends/pandas/tests/conftest.py
@@ -6,7 +6,7 @@
 import ibis.expr.operations as ops
 import ibis.expr.types as ir
 from ibis.backends.tests.base import BackendTest, RoundHalfToEven
-from ibis.backends.tests.data import array_types, json_types, struct_types
+from ibis.backends.tests.data import array_types, json_types, struct_types, win
 
 
 class TestConf(BackendTest, RoundHalfToEven):
@@ -35,6 +35,7 @@ def connect(data_directory: Path):
                 'struct': struct_types,
                 'json_t': json_types,
                 'array_types': array_types,
+                'win': win,
             }
         )
 
diff --git a/ibis/backends/polars/tests/conftest.py b/ibis/backends/polars/tests/conftest.py
--- a/ibis/backends/polars/tests/conftest.py
+++ b/ibis/backends/polars/tests/conftest.py
@@ -5,7 +5,7 @@
 import ibis
 import ibis.expr.types as ir
 from ibis.backends.tests.base import BackendTest, RoundAwayFromZero
-from ibis.backends.tests.data import array_types, struct_types
+from ibis.backends.tests.data import array_types, struct_types, win
 
 pl = pytest.importorskip("polars")
 
@@ -46,6 +46,7 @@ def connect(data_directory: Path):
         client.register(data_directory / 'diamonds.csv', table_name='diamonds')
         client.register(array_types, table_name='array_types')
         client.register(struct_types, table_name='struct')
+        client.register(win, table_name="win")
 
         return client
 
diff --git a/ibis/backends/pyspark/tests/conftest.py b/ibis/backends/pyspark/tests/conftest.py
--- a/ibis/backends/pyspark/tests/conftest.py
+++ b/ibis/backends/pyspark/tests/conftest.py
@@ -12,6 +12,7 @@
 from ibis.backends.conftest import TEST_TABLES, _random_identifier
 from ibis.backends.pyspark.datatypes import spark_dtype
 from ibis.backends.tests.base import BackendTest, RoundAwayFromZero
+from ibis.backends.tests.data import win
 
 pytest.importorskip("pyspark")
 
@@ -201,6 +202,9 @@ def get_common_spark_testing_client(data_directory, connect):
     )
     df_json_t.createOrReplaceTempView("json_t")
 
+    win_t = s.createDataFrame(win)
+    win_t.createOrReplaceTempView("win")
+
     return _spark_testing_client
 
 
diff --git a/ibis/backends/tests/data.py b/ibis/backends/tests/data.py
--- a/ibis/backends/tests/data.py
+++ b/ibis/backends/tests/data.py
@@ -114,3 +114,11 @@
         ]
     }
 )
+
+win = pd.DataFrame(
+    {
+        "g": ["a", "a", "a", "a", "a"],
+        "x": [0, 1, 2, 3, 4],
+        "y": [3, 2, 0, 1, 1],
+    }
+)
diff --git a/ibis/backends/tests/test_temporal.py b/ibis/backends/tests/test_temporal.py
--- a/ibis/backends/tests/test_temporal.py
+++ b/ibis/backends/tests/test_temporal.py
@@ -413,8 +413,8 @@ def test_temporal_binop(backend, con, alltypes, df, expr_fn, expected_fn):
     backend.assert_series_equal(result, expected)
 
 
-plus = lambda t, td: t.timestamp_col + pd.Timedelta(td)  # noqa: E731
-minus = lambda t, td: t.timestamp_col - pd.Timedelta(td)  # noqa: E731
+plus = lambda t, td: t.timestamp_col + pd.Timedelta(td)
+minus = lambda t, td: t.timestamp_col - pd.Timedelta(td)
 
 
 @pytest.mark.parametrize(
diff --git a/ibis/backends/tests/test_window.py b/ibis/backends/tests/test_window.py
--- a/ibis/backends/tests/test_window.py
+++ b/ibis/backends/tests/test_window.py
@@ -1,5 +1,6 @@
 import numpy as np
 import pandas as pd
+import pandas.testing as tm
 import pytest
 from pytest import param
 
@@ -796,3 +797,29 @@ def test_mutate_window_filter(backend, alltypes, df):
     res = expr.execute()
     sol = pd.DataFrame({"int_col": [1, 1, 1], "next_int": [2, 2, 2]})
     backend.assert_frame_equal(res, sol, check_dtype=False)
+
+
+@pytest.mark.notimpl(["dask", "datafusion", "polars"])
+@pytest.mark.broken(["impala"], reason="the database returns incorrect results")
+def test_first_last(con):
+    t = con.table("win")
+    w = ibis.window(group_by=t.g, order_by=[t.x, t.y], preceding=1, following=0)
+    expr = t.mutate(
+        x_first=t.x.first().over(w),
+        x_last=t.x.last().over(w),
+        y_first=t.y.first().over(w),
+        y_last=t.y.last().over(w),
+    )
+    result = expr.execute()
+    expected = pd.DataFrame(
+        {
+            "g": ["a"] * 5,
+            "x": range(5),
+            "y": [3, 2, 0, 1, 1],
+            "x_first": [0, 0, 1, 2, 3],
+            "x_last": range(5),
+            "y_first": [3, 3, 2, 0, 1],
+            "y_last": [3, 2, 0, 1, 1],
+        }
+    )
+    tm.assert_frame_equal(result, expected)