Repository: ibis-project/ibis

Issue:
bug: pandas window operator without group_by raises AttributeError: 'numpy.int64' object has no attribute 'reorder_levels'
### What happened?

This is a follow-up for #4676 that was recently fixed. But there remains a problem initially reported as a comment: https://github.com/ibis-project/ibis/issues/4676#issuecomment-1283756388

Here is the reproducer for that part:

```python
import ibis
import pandas as pd

df = pd.DataFrame(
    {
        "x": [0, 1, 2, 3, 4],
        "y": [3, 2, 0, 1, 1],
    }
)
t_duckdb = ibis.memtable(df)
t_pandas = ibis.pandas.connect({"t": df}).table("t")


def simple_window_ops(t):
    w = ibis.window(          # no groub_by here!
        order_by=[t.x, t.y],
        preceding=1,
        following=0,
    )
    return t.mutate(
        x_first=t.x.first().over(w),
        x_last=t.x.last().over(w),
        y_first=t.y.first().over(w),
        y_last=t.y.last().over(w),
    )
```

Here is the traceback when executing:

```python
>>> simple_window_ops(t_pandas).execute()
```
<details>

```python-traceback
Traceback (most recent call last):
  Cell In[22], line 1
    simple_window_ops(t_pandas).execute()
  File ~/code/ibis/ibis/expr/types/core.py:300 in execute
    return self._find_backend(use_default=True).execute(
  File ~/code/ibis/ibis/backends/pandas/__init__.py:240 in execute
    return execute_and_reset(node, params=params, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:491 in execute_and_reset
    result = execute(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:437 in main_execute
    return execute_with_scope(
  File ~/code/ibis/ibis/backends/pandas/core.py:224 in execute_with_scope
    result = execute_until_in_scope(
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:356 in execute_until_in_scope
    result = execute_node(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/execution/selection.py:287 in execute_selection_dataframe
    result = build_df_from_projection(
  File ~/code/ibis/ibis/backends/pandas/execution/selection.py:251 in build_df_from_projection
    data_pieces = [
  File ~/code/ibis/ibis/backends/pandas/execution/selection.py:252 in <listcomp>
    compute_projection(node, op, data, **kwargs) for node in selection_exprs
  File ~/code/ibis/ibis/backends/pandas/execution/selection.py:108 in compute_projection
    result = execute(node, scope=scope, timecontext=timecontext, **kwargs)
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:437 in main_execute
    return execute_with_scope(
  File ~/code/ibis/ibis/backends/pandas/core.py:224 in execute_with_scope
    result = execute_until_in_scope(
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:328 in execute_until_in_scope
    scopes = [
  File ~/code/ibis/ibis/backends/pandas/core.py:329 in <listcomp>
    execute_until_in_scope(
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:356 in execute_until_in_scope
    result = execute_node(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/execution/window.py:389 in execute_window_op
    result = post_process(
  File ~/code/ibis/ibis/backends/pandas/execution/window.py:93 in _post_process_order_by
    series = series.reorder_levels(names)
AttributeError: 'numpy.int64' object has no attribute 'reorder_levels'
```

</details>

while this same query runs fine with the duckdb backend:

```python
>>> simple_window_ops(t_duckdb).execute()
   x  y  x_first  x_last  y_first  y_last
0  0  3        0       0        3       3
1  1  2        0       1        3       2
2  2  0        1       2        2       0
3  3  1        2       3        0       1
4  4  1        3       4        1       1
```

### What version of ibis are you using?

Today's master branch.

### What backend(s) are you using, if any?

pandas (and duckdb as reference)

### Relevant log output

_No response_

### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Note, that it's possible to further simplify this reproducer to:

```python
import ibis
import pandas as pd

df = pd.DataFrame({"x": [0, 1, 2, 3, 4]})
t_pandas = ibis.pandas.connect({"t": df}).table("t")


def simple_window_ops(t):
    w = ibis.window(          # no groub_by here!
        order_by=t.x,
        preceding=1,
        following=0,
    )
    return t.mutate(
        x_first=t.x.first().over(w),
    )

simple_window_ops(t_pandas).execute()
```

and then get the following slightly different exception instead:


```python-traceback
Traceback (most recent call last):
  Cell In[26], line 1
    simple_window_ops(t_pandas).execute()
  File ~/code/ibis/ibis/expr/types/core.py:300 in execute
    return self._find_backend(use_default=True).execute(
  File ~/code/ibis/ibis/backends/pandas/__init__.py:240 in execute
    return execute_and_reset(node, params=params, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:491 in execute_and_reset
    result = execute(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:437 in main_execute
    return execute_with_scope(
  File ~/code/ibis/ibis/backends/pandas/core.py:224 in execute_with_scope
    result = execute_until_in_scope(
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:356 in execute_until_in_scope
    result = execute_node(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/execution/selection.py:287 in execute_selection_dataframe
    result = build_df_from_projection(
  File ~/code/ibis/ibis/backends/pandas/execution/selection.py:251 in build_df_from_projection
    data_pieces = [
  File ~/code/ibis/ibis/backends/pandas/execution/selection.py:252 in <listcomp>
    compute_projection(node, op, data, **kwargs) for node in selection_exprs
  File ~/code/ibis/ibis/backends/pandas/execution/selection.py:108 in compute_projection
    result = execute(node, scope=scope, timecontext=timecontext, **kwargs)
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:437 in main_execute
    return execute_with_scope(
  File ~/code/ibis/ibis/backends/pandas/core.py:224 in execute_with_scope
    result = execute_until_in_scope(
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:328 in execute_until_in_scope
    scopes = [
  File ~/code/ibis/ibis/backends/pandas/core.py:329 in <listcomp>
    execute_until_in_scope(
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/core.py:356 in execute_until_in_scope
    result = execute_node(
  File ~/mambaforge/envs/ibisdev/lib/python3.11/site-packages/multipledispatch/dispatcher.py:278 in __call__
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/trace.py:136 in traced_func
    return func(*args, **kwargs)
  File ~/code/ibis/ibis/backends/pandas/execution/window.py:389 in execute_window_op
    result = post_process(
  File ~/code/ibis/ibis/backends/pandas/execution/window.py:94 in _post_process_order_by
    series = series.iloc[index.argsort(kind='mergesort')]
AttributeError: 'numpy.int64' object has no attribute 'iloc'
```
If this operation cannot easily or efficiently mapped to pandas operations, it would be fine to raise `NotImplementedError` instead.

But then reporting the `Window` operation as supported for pandas in the [backends matrix](https://ibis-project.org/docs/4.1.0/backends/support_matrix/) would be a bit misleading.

Thanks @ogrisel!

Let's see if @kszucs's #5014 PR fixes this issue.
I just gave it a try and, at this time (at commit 0346e0bfc0efef2f85c2c2290b03bfc691796935), I can still reproduce the originally reported problem.

Resolving patch:
diff --git a/ibis/backends/pandas/execution/generic.py b/ibis/backends/pandas/execution/generic.py
--- a/ibis/backends/pandas/execution/generic.py
+++ b/ibis/backends/pandas/execution/generic.py
@@ -772,13 +772,21 @@ def execute_reduction_series_mask(op, data, mask, aggcontext=None, **kwargs):
 @execute_node.register(ops.First, pd.Series, (pd.Series, type(None)))
 def execute_first_series_mask(op, data, mask, aggcontext=None, **kwargs):
     operand = data[mask] if mask is not None else data
-    return aggcontext.agg(operand, lambda x: x.iloc[0])
+
+    def _first(x):
+        return getattr(x, "iloc", x)[0]
+
+    return aggcontext.agg(operand, _first)
 
 
 @execute_node.register(ops.Last, pd.Series, (pd.Series, type(None)))
 def execute_last_series_mask(op, data, mask, aggcontext=None, **kwargs):
     operand = data[mask] if mask is not None else data
-    return aggcontext.agg(operand, lambda x: x.iloc[-1])
+
+    def _last(x):
+        return getattr(x, "iloc", x)[-1]
+
+    return aggcontext.agg(operand, _last)
 
 
 @execute_node.register(
diff --git a/ibis/backends/pandas/execution/window.py b/ibis/backends/pandas/execution/window.py
--- a/ibis/backends/pandas/execution/window.py
+++ b/ibis/backends/pandas/execution/window.py
@@ -87,9 +87,17 @@ def _post_process_order_by(
     assert order_by and not group_by
     indexed_parent = parent.set_index(order_by)
     index = indexed_parent.index
-    names = index.names
-    if len(names) > 1:
-        series = series.reorder_levels(names)
+
+    # get the names of the levels that will be in the result
+    series_index_names = frozenset(series.index.names)
+
+    # get the levels common to series.index, in the order that they occur in
+    # the parent's index
+    reordered_levels = [name for name in index.names if name in series_index_names]
+
+    if len(reordered_levels) > 1:
+        series = series.reorder_levels(reordered_levels)
+
     series = series.iloc[index.argsort(kind='mergesort')]
     return series
 


Test patch:
diff --git a/ibis/backends/pandas/tests/execution/test_window.py b/ibis/backends/pandas/tests/execution/test_window.py
--- a/ibis/backends/pandas/tests/execution/test_window.py
+++ b/ibis/backends/pandas/tests/execution/test_window.py
@@ -118,6 +118,23 @@ def test_last(t, df):
     assert result == expected
 
 
+def test_first_and_last_over_window(t):
+    def simple_window_ops(table):
+        w = ibis.window(
+            order_by=[table.plain_uint64, table.dup_ints],
+            preceding=1,
+            following=0,
+        )
+        return table.mutate(
+            x_first=t.plain_uint64.first().over(w),
+            x_last=t.plain_uint64.last().over(w),
+            y_first=t.dup_ints.first().over(w),
+            y_last=t.dup_ints.last().over(w),
+        )
+
+    assert simple_window_ops(t).execute() is not None
+
+
 def test_group_by_mutate_analytic(t, df):
     gb = t.group_by(t.dup_strings)
     expr = gb.mutate(