Repository: ibis-project/ibis

Issue:
bug: `asof_join` doesn't work when there are overlapping columns
### What happened?

Based on the documentation, it sounds like `asof_join` should work even when there are overlapping columns between the left and right tables. In the case where left and right tables have overlapping columns, left and right suffixes will be used to rename overlapping columns. However, the following code raises `ValueError: left and right DataFrame columns overlap on frozenset({'unix_time'}) in a join. Please specify the columns you want to select from the join, e.g., join[left.column1, right.column2, ...]`:

Code block:
```python
import random
import pandas as pd
import numpy as np
import ibis

size = 10
left_df = pd.DataFrame(
    list(zip(
        random.choices(["1", "2"], k=size),
        np.random.randint(0, 10, size=size),
        np.random.randint(0, 100, size=size),
        np.random.randint(0, 10, size=size),
        random.choices(["a", "b", "c"], k=size),
    )),
    columns=["cc_num", "unix_time", "amt", "event_time", "partition"]
)
right_df = pd.DataFrame(
    list(zip(
        random.choices(["1", "2"], k=size),
        np.random.randint(0, 10, size=size),
        np.random.randint(0, 100, size=size),
        np.random.randint(0, 10, size=size),
        random.choices(["a", "b", "c"], k=size),
    )),
    columns=["cc_num", "unix_time", "average_amt", "event_time", "partition"]
)
backend = ibis.pandas.connect({"left": left_df, "right": right_df})
l_expr = backend.table("left")
r_expr = backend.table("right")
expr = l_expr.asof_join(r_expr, predicates=[("event_time", "event_time")], by=[('cc_num', 'cc_num'), ('partition', 'partition')])
expr.execute()
```

Screenshot:
<img width="1135" alt="Screenshot 2023-06-02 at 1 05 22 PM" src="https://github.com/ibis-project/ibis/assets/57872406/63674aad-0e0a-41ff-bbd6-b6056af2b14b">
<img width="1194" alt="Screenshot 2023-06-02 at 1 09 19 PM" src="https://github.com/ibis-project/ibis/assets/57872406/f0062448-6e3b-4656-a125-f4f7c23c78c2">


### What version of ibis are you using?

5.1.0

### What backend(s) are you using, if any?

pandas

### Relevant log output

```sh
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[1], line 31
     29 r_expr = backend.table("right")
     30 expr = l_expr.asof_join(r_expr, predicates=[("event_time", "event_time")], by=[('cc_num', 'cc_num'), ('partition', 'partition')])
---> 31 expr.execute()

File ~/Desktop/ibis-public/ibis/expr/types/core.py:296, in Expr.execute(self, limit, timecontext, params, **kwargs)
    269 def execute(
    270     self,
    271     limit: int | str | None = 'default',
   (...)
    274     **kwargs: Any,
    275 ):
    276     """Execute an expression against its backend if one exists.
    277 
    278     Parameters
   (...)
    294         Keyword arguments
    295     """
--> 296     return self._find_backend(use_default=True).execute(
    297         self, limit=limit, timecontext=timecontext, params=params, **kwargs
    298     )

File ~/Desktop/ibis-public/ibis/backends/pandas/__init__.py:294, in Backend.execute(self, query, params, limit, **kwargs)
    289 else:
    290     params = {
    291         k.op() if isinstance(k, ir.Expr) else k: v for k, v in params.items()
    292     }
--> 294 return execute_and_reset(node, params=params, **kwargs)

File ~/Desktop/ibis-public/ibis/backends/pandas/core.py:482, in execute_and_reset(node, params, scope, timecontext, aggcontext, **kwargs)
    438 def execute_and_reset(
    439     node,
    440     params=None,
   (...)
    444     **kwargs,
    445 ):
    446     """Execute an expression against data that are bound to it.
    447 
    448     If no data are bound, raise an Exception.
   (...)
    480         * If no data are bound to the input expression
    481     """
--> 482     result = execute(
    483         node,
    484         params=params,
    485         scope=scope,
    486         timecontext=timecontext,
    487         aggcontext=aggcontext,
    488         **kwargs,
    489     )
    490     return _apply_schema(node, result)

File ~/anaconda3/envs/ibis/lib/python3.10/site-packages/multipledispatch/dispatcher.py:278, in Dispatcher.__call__(self, *args, **kwargs)
    276     self._cache[types] = func
    277 try:
--> 278     return func(*args, **kwargs)
    280 except MDNotImplementedError:
    281     funcs = self.dispatch_iter(*types)

File ~/Desktop/ibis-public/ibis/backends/pandas/trace.py:136, in trace..traced_func(*args, **kwargs)
    133 ibis.pandas  # noqa: B018
    135 if not options.pandas.enable_trace:
--> 136     return func(*args, **kwargs)
    137 else:
    138     start = datetime.now()

File ~/Desktop/ibis-public/ibis/backends/pandas/core.py:428, in main_execute(node, params, scope, timecontext, aggcontext, cache, **kwargs)
    425     cache = {}
    427 scope = scope.merge_scope(Scope(params, timecontext))
--> 428 return execute_with_scope(
    429     node,
    430     scope,
    431     timecontext=timecontext,
    432     aggcontext=aggcontext,
    433     cache=cache,
    434     **kwargs,
    435 )

File ~/Desktop/ibis-public/ibis/backends/pandas/core.py:218, in execute_with_scope(node, scope, timecontext, aggcontext, clients, **kwargs)
    209 pre_executed_scope = pre_execute(
    210     node,
    211     *clients,
   (...)
    215     **kwargs,
    216 )
    217 new_scope = scope.merge_scope(pre_executed_scope)
--> 218 result = execute_until_in_scope(
    219     node,
    220     new_scope,
    221     timecontext=timecontext,
    222     aggcontext=aggcontext,
    223     clients=clients,
    224     # XXX: we *explicitly* pass in scope and not new_scope here so that
    225     # post_execute sees the scope of execute_with_scope, not the scope of
    226     # execute_until_in_scope
    227     post_execute_=functools.partial(
    228         post_execute,
    229         scope=scope,
    230         timecontext=timecontext,
    231         aggcontext=aggcontext,
    232         clients=clients,
    233         **kwargs,
    234     ),
    235     **kwargs,
    236 ).get_value(node, timecontext)
    237 return result

File ~/Desktop/ibis-public/ibis/backends/pandas/trace.py:136, in trace..traced_func(*args, **kwargs)
    133 ibis.pandas  # noqa: B018
    135 if not options.pandas.enable_trace:
--> 136     return func(*args, **kwargs)
    137 else:
    138     start = datetime.now()

File ~/Desktop/ibis-public/ibis/backends/pandas/core.py:322, in execute_until_in_scope(node, scope, timecontext, aggcontext, clients, post_execute_, **kwargs)
    316 if len(arg_timecontexts) != len(computable_args):
    317     raise com.IbisError(
    318         'arg_timecontexts differ with computable_arg in length '
    319         f'for type:\n{type(node).__name__}.'
    320     )
--> 322 scopes = [
    323     execute_until_in_scope(
    324         arg,
    325         new_scope,
    326         timecontext=timecontext,
    327         aggcontext=aggcontext,
    328         post_execute_=post_execute_,
    329         clients=clients,
    330         **kwargs,
    331     )
    332     if isinstance(arg, ops.Node)
    333     else Scope({arg: arg}, timecontext)
    334     for (arg, timecontext) in zip(computable_args, arg_timecontexts)
    335 ]
    337 # if we're unable to find data then raise an exception
    338 if not scopes and computable_args:

File ~/Desktop/ibis-public/ibis/backends/pandas/core.py:323, in (.0)
    316 if len(arg_timecontexts) != len(computable_args):
    317     raise com.IbisError(
    318         'arg_timecontexts differ with computable_arg in length '
    319         f'for type:\n{type(node).__name__}.'
    320     )
    322 scopes = [
--> 323     execute_until_in_scope(
    324         arg,
    325         new_scope,
    326         timecontext=timecontext,
    327         aggcontext=aggcontext,
    328         post_execute_=post_execute_,
    329         clients=clients,
    330         **kwargs,
    331     )
    332     if isinstance(arg, ops.Node)
    333     else Scope({arg: arg}, timecontext)
    334     for (arg, timecontext) in zip(computable_args, arg_timecontexts)
    335 ]
    337 # if we're unable to find data then raise an exception
    338 if not scopes and computable_args:

File ~/Desktop/ibis-public/ibis/backends/pandas/trace.py:136, in trace..traced_func(*args, **kwargs)
    133 ibis.pandas  # noqa: B018
    135 if not options.pandas.enable_trace:
--> 136     return func(*args, **kwargs)
    137 else:
    138     start = datetime.now()

File ~/Desktop/ibis-public/ibis/backends/pandas/core.py:350, in execute_until_in_scope(node, scope, timecontext, aggcontext, clients, post_execute_, **kwargs)
    345 # pass our computed arguments to this node's execute_node implementation
    346 data = [
    347     new_scope.get_value(arg, timecontext) if isinstance(arg, ops.Node) else arg
    348     for (arg, timecontext) in zip(computable_args, arg_timecontexts)
    349 ]
--> 350 result = execute_node(
    351     node,
    352     *data,
    353     scope=scope,
    354     timecontext=timecontext,
    355     aggcontext=aggcontext,
    356     clients=clients,
    357     **kwargs,
    358 )
    359 computed = post_execute_(
    360     node, result, timecontext=timecontext, aggcontext=aggcontext, **kwargs
    361 )
    362 return Scope({node: computed}, timecontext)

File ~/anaconda3/envs/ibis/lib/python3.10/site-packages/multipledispatch/dispatcher.py:278, in Dispatcher.__call__(self, *args, **kwargs)
    276     self._cache[types] = func
    277 try:
--> 278     return func(*args, **kwargs)
    280 except MDNotImplementedError:
    281     funcs = self.dispatch_iter(*types)

File ~/Desktop/ibis-public/ibis/backends/pandas/trace.py:136, in trace..traced_func(*args, **kwargs)
    133 ibis.pandas  # noqa: B018
    135 if not options.pandas.enable_trace:
--> 136     return func(*args, **kwargs)
    137 else:
    138     start = datetime.now()

File ~/Desktop/ibis-public/ibis/backends/pandas/execution/join.py:133, in execute_asof_join(op, left, right, by, tolerance, predicates, **kwargs)
    131 left_on, right_on = _extract_predicate_names(predicates)
    132 left_by, right_by = _extract_predicate_names(by)
--> 133 _validate_columns(overlapping_columns, left_on, right_on, left_by, right_by)
    135 return pd.merge_asof(
    136     left=left,
    137     right=right,
   (...)
    142     tolerance=tolerance,
    143 )

File ~/Desktop/ibis-public/ibis/backends/pandas/execution/join.py:164, in _validate_columns(orig_columns, *key_lists)
    160 overlapping_columns = orig_columns.difference(
    161     item for sublist in key_lists for item in sublist
    162 )
    163 if overlapping_columns:
--> 164     raise ValueError(
    165         'left and right DataFrame columns overlap on {} in a join. '
    166         'Please specify the columns you want to select from the join, '
    167         'e.g., join[left.column1, right.column2, ...]'.format(overlapping_columns)
    168     )

ValueError: left and right DataFrame columns overlap on frozenset({'unix_time'}) in a join. Please specify the columns you want to select from the join, e.g., join[left.column1, right.column2, ...]
```


### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
It looks like the pandas backend is enforcing this limitation. I'm not sure why that is (this was implemented a while ago). We can add your test case and if removing the check passes your test and doesn't break anything then we can make that change!
There's a few small extra wrinkles in here:
1. Pandas requires the predicates in a `merge_asof` to be sorted -- I think for now we can leave that alone (rather than imposing implicit sort behavior)
2. We weren't passing join suffixes to `merge_asof` -- I'll put up a PR in a few fixing that and removing the unnecessary column validation check.

Resolving patch:
diff --git a/ibis/backends/dask/execution/join.py b/ibis/backends/dask/execution/join.py
--- a/ibis/backends/dask/execution/join.py
+++ b/ibis/backends/dask/execution/join.py
@@ -10,7 +10,6 @@
 from ibis.backends.pandas.execution.join import (
     _compute_join_column,
     _extract_predicate_names,
-    _validate_columns,
 )
 
 
@@ -23,10 +22,8 @@
     tuple,
 )
 def execute_asof_join(op, left, right, by, tolerance, predicates, **kwargs):
-    overlapping_columns = frozenset(left.columns) & frozenset(right.columns)
     left_on, right_on = _extract_predicate_names(predicates)
     left_by, right_by = _extract_predicate_names(by)
-    _validate_columns(overlapping_columns, left_on, right_on, left_by, right_by)
 
     assert 0 <= len(left_on) <= 1, f"len(left_on) == {len(left_on)}"
     assert 0 <= len(right_on) <= 1, f"len(right_on) == {len(right_on)}"
@@ -45,6 +42,7 @@ def execute_asof_join(op, left, right, by, tolerance, predicates, **kwargs):
         left_by=left_by or None,
         right_by=right_by or None,
         tolerance=tolerance,
+        suffixes=constants.JOIN_SUFFIXES,
     )
 
 
diff --git a/ibis/backends/pandas/execution/join.py b/ibis/backends/pandas/execution/join.py
--- a/ibis/backends/pandas/execution/join.py
+++ b/ibis/backends/pandas/execution/join.py
@@ -127,10 +127,8 @@ def execute_join(op, left, right, predicates, **kwargs):
     tuple,
 )
 def execute_asof_join(op, left, right, by, tolerance, predicates, **kwargs):
-    overlapping_columns = frozenset(left.columns) & frozenset(right.columns)
     left_on, right_on = _extract_predicate_names(predicates)
     left_by, right_by = _extract_predicate_names(by)
-    _validate_columns(overlapping_columns, left_on, right_on, left_by, right_by)
 
     return pd.merge_asof(
         left=left,
@@ -140,6 +138,7 @@ def execute_asof_join(op, left, right, by, tolerance, predicates, **kwargs):
         left_by=left_by or None,
         right_by=right_by or None,
         tolerance=tolerance,
+        suffixes=constants.JOIN_SUFFIXES,
     )
 
 
@@ -154,15 +153,3 @@ def _extract_predicate_names(predicates):
         lefts.append(left_name)
         rights.append(right_name)
     return lefts, rights
-
-
-def _validate_columns(orig_columns, *key_lists):
-    overlapping_columns = orig_columns.difference(
-        item for sublist in key_lists for item in sublist
-    )
-    if overlapping_columns:
-        raise ValueError(
-            'left and right DataFrame columns overlap on {} in a join. '
-            'Please specify the columns you want to select from the join, '
-            'e.g., join[left.column1, right.column2, ...]'.format(overlapping_columns)
-        )


Test patch:
diff --git a/ibis/backends/dask/tests/execution/test_join.py b/ibis/backends/dask/tests/execution/test_join.py
--- a/ibis/backends/dask/tests/execution/test_join.py
+++ b/ibis/backends/dask/tests/execution/test_join.py
@@ -386,6 +386,33 @@ def test_keyed_asof_join_with_tolerance(
     )
 
 
+@merge_asof_minversion
+def test_asof_join_overlapping_non_predicate(
+    time_keyed_left, time_keyed_right, time_keyed_df1, time_keyed_df2
+):
+    # Add a junk column with a colliding name
+    time_keyed_left = time_keyed_left.mutate(
+        collide=time_keyed_left.key + time_keyed_left.value
+    )
+    time_keyed_right = time_keyed_right.mutate(
+        collide=time_keyed_right.key + time_keyed_right.other_value
+    )
+    time_keyed_df1.assign(collide=time_keyed_df1["key"] + time_keyed_df1["value"])
+    time_keyed_df2.assign(collide=time_keyed_df2["key"] + time_keyed_df2["other_value"])
+
+    expr = time_keyed_left.asof_join(
+        time_keyed_right, predicates=[("time", "time")], by=[("key", "key")]
+    )
+    result = expr.compile()
+    expected = dd.merge_asof(
+        time_keyed_df1, time_keyed_df2, on='time', by='key', suffixes=("", "_right")
+    )
+    tm.assert_frame_equal(
+        result[expected.columns].compute(scheduler='single-threaded'),
+        expected.compute(scheduler='single-threaded'),
+    )
+
+
 @pytest.mark.parametrize(
     "how",
     [
diff --git a/ibis/backends/pandas/tests/execution/test_join.py b/ibis/backends/pandas/tests/execution/test_join.py
--- a/ibis/backends/pandas/tests/execution/test_join.py
+++ b/ibis/backends/pandas/tests/execution/test_join.py
@@ -302,6 +302,30 @@ def test_keyed_asof_join_with_tolerance(
     tm.assert_frame_equal(result[expected.columns], expected)
 
 
+@merge_asof_minversion
+def test_asof_join_overlapping_non_predicate(
+    time_keyed_left, time_keyed_right, time_keyed_df1, time_keyed_df2
+):
+    # Add a junk column with a colliding name
+    time_keyed_left = time_keyed_left.mutate(
+        collide=time_keyed_left.key + time_keyed_left.value
+    )
+    time_keyed_right = time_keyed_right.mutate(
+        collide=time_keyed_right.key + time_keyed_right.other_value
+    )
+    time_keyed_df1.assign(collide=time_keyed_df1["key"] + time_keyed_df1["value"])
+    time_keyed_df2.assign(collide=time_keyed_df2["key"] + time_keyed_df2["other_value"])
+
+    expr = time_keyed_left.asof_join(
+        time_keyed_right, predicates=[("time", "time")], by=[("key", "key")]
+    )
+    result = expr.execute()
+    expected = pd.merge_asof(
+        time_keyed_df1, time_keyed_df2, on='time', by='key', suffixes=("", "_right")
+    )
+    tm.assert_frame_equal(result[expected.columns], expected)
+
+
 @pytest.mark.parametrize(
     "how",
     [