Repository: ibis-project/ibis

Issue:
bug(exprs): chain mutate() for window op and non window op throws error for pyspark backend
Currently, chain mutate() for window will fail for pyspark backend:
```
table = client.table('time_indexed_table')
    context = (
        pd.Timestamp('20170102 07:00:00', tz='UTC'),
        pd.Timestamp('20170105', tz='UTC'),
    )
    window = ibis.trailing_window(
        preceding=ibis.interval(hours=1), order_by='time', group_by='key'
    )
result_pd = (
        table.mutate(
            count_1h=table['value'].count().over(window),
        ).mutate(count=table['value'].count())
        .execute(timecontext=context)
    )
```
failed with AnalysisException:
```
E               pyspark.sql.utils.AnalysisException: "grouping expressions sequence is empty, and 'time_indexed_table.`time`' is not an aggregate function. Wrap '(count(time_indexed_table.`value`) AS `count_yoyo`)' in windowing function(s) or wrap 'time_indexed_table.`time`' in first() (or first_value) if you don't care which value you get.;;\nProject [time#19, key#20L, value#21, count#52L, count_yoyo#53L]\n+- Project [time#19, key#20L, value#21, count_yoyo#53L, _w0#54L, count#52L, count#52L]\n   +- Window [count(value#21) windowspecdefinition(key#20L, _w0#54L ASC NULLS FIRST, specifiedwindowframe(RangeFrame, currentrow$(), 3600)) AS count#52L], [key#20L], [_w0#54L ASC NULLS FIRST]\n      +- Aggregate [time#19, key#20L, value#21, count(value#21) AS count_yoyo#53L, cast(time#19 as bigint) AS _w0#54L]\n         +- Filter ((time#19 >= 1483340400000000) && (time#19 < 1483405200000000))\n            +- SubqueryAlias `time_indexed_table`\n               +- LogicalRDD [time#19, key#20L, value#21], false\n"
```



Discussion:
@LeeTZ Can you still reproduce this?

Resolving patch:
diff --git a/ibis/expr/analysis.py b/ibis/expr/analysis.py
--- a/ibis/expr/analysis.py
+++ b/ibis/expr/analysis.py
@@ -664,6 +664,14 @@ def try_fusion(self, root):
                 ]
             except (AttributeError, IbisTypeError):
                 resolved = clean_exprs
+            else:
+                # if any expressions aren't exactly equivalent then don't try
+                # to fuse them
+                if any(
+                    not res_root_root.equals(res_root)
+                    for res_root_root, res_root in zip(resolved, clean_exprs)
+                ):
+                    return None
         else:
             # joins cannot be used to resolve expressions, but we still may be
             # able to fuse columns from a projection off of a join. In that


Test patch:
diff --git a/ibis/backends/impala/tests/test_sql.py b/ibis/backends/impala/tests/test_sql.py
--- a/ibis/backends/impala/tests/test_sql.py
+++ b/ibis/backends/impala/tests/test_sql.py
@@ -196,16 +196,19 @@ def test_nested_join_multiple_ctes():
 WHERE t2.`movieid` IN (
   SELECT `movieid`
   FROM (
-    SELECT t1.*
-    FROM t1
-    WHERE (t1.`userid` = 118205) AND
-          (extract(t1.`datetime`, 'year') > 2001) AND
-          (t1.`userid` = 118205) AND
-          (extract(t1.`datetime`, 'year') < 2009)
+    SELECT `movieid`
+    FROM (
+      SELECT t1.*
+      FROM t1
+      WHERE (t1.`userid` = 118205) AND
+            (extract(t1.`datetime`, 'year') > 2001) AND
+            (t1.`userid` = 118205) AND
+            (extract(t1.`datetime`, 'year') < 2009)
+    ) t5
   ) t4
 )"""
-    compiled_result = ImpalaCompiler.to_sql(result)
-    assert compiled_result == expected
+    result = ImpalaCompiler.to_sql(result)
+    assert result == expected
 
 
 def test_logically_negate_complex_boolean_expr():
diff --git a/ibis/backends/pyspark/tests/test_window_context_adjustment.py b/ibis/backends/pyspark/tests/test_window_context_adjustment.py
--- a/ibis/backends/pyspark/tests/test_window_context_adjustment.py
+++ b/ibis/backends/pyspark/tests/test_window_context_adjustment.py
@@ -351,11 +351,6 @@ def test_rolling_with_non_window_op(client, ibis_windows, spark_range):
     tm.assert_frame_equal(result_pd, expected)
 
 
-@pytest.mark.xfail(
-    reason='Issue #2453 chain mutate() for window op and'
-    'non window op throws error for pyspark backend',
-    strict=True,
-)
 def test_complex_window(client):
     """Test window with different sizes
     mix context adjustment for window op that require context
diff --git a/ibis/tests/expr/test_analysis.py b/ibis/tests/expr/test_analysis.py
--- a/ibis/tests/expr/test_analysis.py
+++ b/ibis/tests/expr/test_analysis.py
@@ -290,3 +290,11 @@ def test_no_filter_means_no_selection():
     t = ibis.table(dict(a="string"))
     proj = t.filter([])
     assert proj.equals(t)
+
+
+def test_mutate_overwrites_existing_column():
+    t = ibis.table(dict(a="string"))
+    mut = t.mutate(a=42).select(["a"])
+    sel = mut.op().selections[0].op().table.op().selections[0].op().arg
+    assert isinstance(sel.op(), ops.Literal)
+    assert sel.op().value == 42
diff --git a/ibis/tests/sql/test_select_sql.py b/ibis/tests/sql/test_select_sql.py
--- a/ibis/tests/sql/test_select_sql.py
+++ b/ibis/tests/sql/test_select_sql.py
@@ -423,7 +423,23 @@ def test_projection_filter_fuse(projection_fuse_filter):
     sql3 = Compiler.to_sql(expr3)
 
     assert sql1 == sql2
-    assert sql1 == sql3
+
+    # ideally sql1 == sql3 but the projection logic has been a mess for a long
+    # time and causes bugs like
+    #
+    # https://github.com/ibis-project/ibis/issues/4003
+    #
+    # so we're conservative in fusing projections and filters
+    #
+    # even though it may seem obvious what to do, it's not
+    expected_sql3 = """\
+SELECT `a`, `b`, `c`
+FROM (
+  SELECT *
+  FROM foo
+  WHERE `a` > 0
+) t0"""
+    assert sql3 == expected_sql3
 
 
 def test_bug_project_multiple_times(customer, nation, region):