Repository: ibis-project/ibis

Issue:
pandas backend: can't join a join-derived table expression to another table expression
# Issue
I'm trying to join three table expressions using the pandas backend.
Let `harvest`, `fields`, and `tracts` be table expressions from a pandas backend using three dataframes.

```python
fielded = harvest.inner_join(fields, predicates=[harvest['harvest_field_id'] == fields['field_id']])
tracted = fielded.inner_join(tracts, predicates=[fielded['field_tract_id'] == tracts['tract_id']])
tracted.execute()
```
Expected: `tracted` is a table expression from joining the join-derived table expression `fielded` (`harvest` inner join `fields`) to the table expression `tracts`.

Actual: error, ST:

```python
<snip>
File ~/dev/repos/ibis/ibis/backends/pandas/execution/join.py:19, in _compute_join_column(column_expr, **kwargs)
     17 else:
     18     new_column = execute(column_expr, **kwargs)
---> 19 (root_table,) = column_op.root_tables()
     20 return new_column, root_table

ValueError: too many values to unpack (expected 1)
```
# Source
Problem line in [join.py](https://github.com/ibis-project/ibis/blob/master/ibis/backends/pandas/execution/join.py):

[_compute_join_column](https://github.com/ibis-project/ibis/blob/master/ibis/backends/pandas/execution/join.py#L19)

# Setup:
```python
import ibis
import pandas as pd


def df(vals, cols):
    return pd.DataFrame(vals, columns=cols)

cols = ['farm_id', 'farm_name', 'farm_address', 'farm_origin_x', 'farm_origin_y']
vals = [
    [1, 'North Farm', '100 North St., City00, State00 00000', 1, 7]
    ,[2, 'East Farm', '100 East St., City00, State00 00000', 4, 2]
    ,[3, 'South Farm', '200 South St., City01, State01 00001', -2, -6]
    ,[4, 'West Farm', '300 West St., City02, State02 00002', -5, 2]
]

df_farms = df(vals, cols)

cols = ['tract_id', 'tract_farm_id']
vals = [
    [1, 1]
    ,[2, 1]
    ,[3, 2]
    ,[4, 2]
    ,[5, 3]
    ,[6, 4]
]

df_tracts = df(vals, cols)

cols = ['field_id', 'field_tract_id', 'field_vertices']
vals = [
    [1, 1, '[(0, 2), (1, 3), (2, 0), (3, 1)]']
    ,[2, 1, '[(2, 2), (3, 2), (3, 1)]']
    ,[3, 2, '[(0, 1), (-1, 0), (-2, 0), (-2, 1)]']
    ,[4, 3, '[(0, 1), (1, 1), (1, 2), (0, 2)]']
    ,[5, 3, '[(1, 0), (2, 0), (2, 3), (1, 3)]']
    ,[6, 3, '[(2, 0), (3, 0), (3, 2), (2, 2)]']
    ,[7, 4, '[(-1, -1), (0, -1), (0, -2)]']
    ,[8, 4, '[(1, 0), (1, -2), (0, -2), (0, -1)]']
    ,[9, 5, '[(1, 0), (2, 0), (1, -1), (1, -2), (-1, -2), (-1, -1), (-2, 0), (-1, 0), (0, -1)]']
    ,[10, 6, '[(-1, 2), (0, 2), (0, 0), (-1, 0)]']
    ,[11, 6, '[(0, 2), (1, 2), (1, 1), (0, 1)]']
]

df_fields = df(vals, cols)

cols = ['harvest_id', 'harvest_field_id', 'harvest_farmer_group_id', 'harvest_crop_id', 'harvest_date_id', 'harvest_value']
vals = [
    [1, 1, 1, 1, 1, 65.80]
    ,[2, 2, 1, 2, 2, 5750.00]
    ,[3, 3, 1, 1, 1, 59.85]
    ,[4, 4, 2, 2, 2, 10100.00]
    ,[5, 5, 2, 1, 1, 90.30]
    ,[6, 6, 2, 2, 2, 21000.00]
    ,[7, 7, 2, 2, 2, 5150.00]
    ,[8, 8, 2, 1, 1, 53.55]
    ,[9, 9, 3, 1, 1, 147.00]
    ,[10, 10, 4, 1, 1, 70.70]
    ,[11, 11, 4, 2, 2, 9600.00]
    ,[12, 1, 1, 2, 4, 22800.00]
    ,[13, 2, 1, 1, 3, 19.25]
    ,[14, 3, 1, 2, 4, 13050.00]
    ,[15, 4, 2, 1, 3, 31.15]
    ,[16, 5, 2, 2, 4, 33000.00]
    ,[17, 6, 2, 1, 3, 64.40]
    ,[18, 7, 2, 1, 3, 16.45]
    ,[19, 8, 2, 2, 4, 15000.00]
    ,[20, 9, 3, 2, 4, 38400.00]
    ,[21, 10, 4, 2, 4, 19800.00]
    ,[22, 11, 4, 1, 3, 34.30]
]

df_harvest = df(vals, cols)

conn = ibis.pandas.connect({
    'farms': df_farms
    ,'tracts': df_tracts
    ,'fields': df_fields
    ,'harvest': df_harvest
})

farms = conn.table("farms")
tracts = conn.table("tracts")
fields = conn.table("fields")
harvest = conn.table("harvest")
```
# Line by Line
```python
import operator
import pandas as pd

import ibis.backends.pandas.execution.constants as constants
import ibis.expr.operations as ops

fielded = harvest.inner_join(fields, predicates=[harvest['harvest_field_id'] == fields['field_id']])

## Want
# tracted = fielded.inner_join(tracts, predicates=[fielded['field_tract_id'] == tracts['tract_id']])

preds = [fielded['field_tract_id'] == tracts['tract_id']]

op = ibis.backends.pandas.execution.ops.InnerJoin(fielded, tracts, preds)

op_type = type(op)

print('\nop_type:', op_type)

how = constants.JOIN_TYPES[op_type]

print('\nhow:', how)

# left_on, right_on = _construct_join_predicate_columns(
#     op, predicates, **kwargs
# )

left_op = op.left.op()
right_op = op.right.op()

print('\nleft op:', left_op)

print('\nright_op:', right_op)

on = {left_op: [], right_op: []}

ixrb = [k for k in map(operator.methodcaller('op'), preds)]

predicate = ixrb[0]

print('\npredicate:', predicate)

print('\nops.Equals:', isinstance(ixrb[0], ops.Equals))

column_op = predicate.left.op()

print(column_op.root_tables())

print('\npush', len(column_op.root_tables()), 'values into (root_table,)')
```

# Output

```python
op_type: <class 'ibis.expr.operations.relations.InnerJoin'>

how: inner

left op: InnerJoin(left=PandasTable: harvest
  harvest_id              int64
  harvest_field_id        int64
  harvest_farmer_group_id int64
  harvest_crop_id         int64
  harvest_date_id         int64
  harvest_value           float64, right=PandasTable: fields
  field_id       int64
  field_tract_id int64
  field_vertices string, predicates=(r0 := PandasTable: fields
  field_id       int64
  field_tract_id int64
  field_vertices string

r1 := PandasTable: harvest
  harvest_id              int64
  harvest_field_id        int64
  harvest_farmer_group_id int64
  harvest_crop_id         int64
  harvest_date_id         int64
  harvest_value           float64

r1.harvest_field_id == r0.field_id,))

right_op: PandasTable(name='tracts', schema=ibis.Schema {
  tract_id       int64
  tract_farm_id  int64
}, source=<ibis.backends.pandas.Backend object at 0x7fbb30186c70>)

predicate: Equals(left=r0 := PandasTable: fields
  field_id       int64
  field_tract_id int64
  field_vertices string

r1 := PandasTable: harvest
  harvest_id              int64
  harvest_field_id        int64
  harvest_farmer_group_id int64
  harvest_crop_id         int64
  harvest_date_id         int64
  harvest_value           float64

r2 := InnerJoin[r1, r0] r1.harvest_field_id == r0.field_id

field_tract_id: r2.field_tract_id, right=r0 := PandasTable: tracts
  tract_id      int64
  tract_farm_id int64

tract_id: r0.tract_id)

ops.Equals: True
[PandasTable(name='harvest', schema=ibis.Schema {
  harvest_id               int64
  harvest_field_id         int64
  harvest_farmer_group_id  int64
  harvest_crop_id          int64
  harvest_date_id          int64
  harvest_value            float64
}, source=<ibis.backends.pandas.Backend object at 0x7fbb30186c70>), PandasTable(name='fields', schema=ibis.Schema {
  field_id        int64
  field_tract_id  int64
  field_vertices  string
}, source=<ibis.backends.pandas.Backend object at 0x7fbb30186c70>)]

push 2 values into (root_table,)
```


Discussion:
Yes, this looks to be a real bug.  We'll add a generic backend test for multi-table joins.  Thanks for filing it.

Resolving patch:
diff --git a/ibis/backends/pandas/execution/selection.py b/ibis/backends/pandas/execution/selection.py
--- a/ibis/backends/pandas/execution/selection.py
+++ b/ibis/backends/pandas/execution/selection.py
@@ -1,16 +1,15 @@
 """Dispatching code for Selection operations.
 """
 
-
 import functools
 import operator
-from collections import OrderedDict, defaultdict
+from collections import defaultdict
 from operator import methodcaller
 from typing import List, Optional
 
 import pandas as pd
 from multipledispatch import Dispatcher
-from toolz import compose, concat, concatv, first, unique
+from toolz import compose, concat, concatv, unique
 
 import ibis.expr.operations as ops
 import ibis.expr.types as ir
@@ -140,7 +139,7 @@ def compute_projection_column_expr(
 
 
 @compute_projection.register(ir.Table, ops.Selection, pd.DataFrame)
-def compute_projection_table_expr(expr, parent, data, **kwargs):
+def compute_projection_table_expr(expr, parent, data, **_):
     if expr.equals(parent.table):
         return data
 
@@ -159,22 +158,22 @@ def compute_projection_table_expr(expr, parent, data, **kwargs):
 
 
 def remap_overlapping_column_names(table_op, root_table, data_columns):
-    """Return an ``OrderedDict`` mapping possibly suffixed column names to
+    """Return a mapping of possibly-suffixed column names to
     column names without suffixes.
 
     Parameters
     ----------
-    table_op : TableNode
-        The ``TableNode`` we're selecting from.
-    root_table : TableNode
+    table_op
+        The `TableNode` we're selecting from.
+    root_table
         The root table of the expression we're selecting from.
-    data_columns : set or frozenset
+    data_columns
         The available columns to select from
 
     Returns
     -------
-    mapping : OrderedDict[str, str]
-        A map from possibly-suffixed column names to column names without
+    dict[str, str]
+        A mapping from possibly-suffixed column names to column names without
         suffixes.
     """
     if not isinstance(table_op, ops.Join):
@@ -185,20 +184,33 @@ def remap_overlapping_column_names(table_op, root_table, data_columns):
         left_root: constants.LEFT_JOIN_SUFFIX,
         right_root: constants.RIGHT_JOIN_SUFFIX,
     }
+
+    # if we're selecting from the root table and that's not the left or right
+    # child, don't add a suffix
+    #
+    # this can happen when selecting directly from a join as opposed to
+    # explicitly referencing the left or right tables
+    #
+    # we use setdefault here because the root_table can be the left/right table
+    # which we may have already put into `suffixes`
+    suffixes.setdefault(root_table, "")
+
+    suffix = suffixes[root_table]
+
     column_names = [
-        ({name, name + suffixes[root_table]} & data_columns, name)
+        ({name, f"{name}{suffix}"} & data_columns, name)
         for name in root_table.schema.names
     ]
-    mapping = OrderedDict(
-        (first(col_name), final_name)
-        for col_name, final_name in column_names
-        if col_name
-    )
-    return mapping
+
+    return {
+        col_names.pop(): final_name
+        for col_names, final_name in column_names
+        if col_names
+    }
 
 
 def map_new_column_names_to_data(mapping, df):
-    if mapping is not None:
+    if mapping:
         return df.loc[:, mapping.keys()].rename(columns=mapping)
     return df
 
diff --git a/ibis/expr/operations/relations.py b/ibis/expr/operations/relations.py
--- a/ibis/expr/operations/relations.py
+++ b/ibis/expr/operations/relations.py
@@ -11,7 +11,7 @@
 from ibis.expr import rules as rlz
 from ibis.expr import schema as sch
 from ibis.expr import types as ir
-from ibis.expr.operations.core import Node, distinct_roots
+from ibis.expr.operations.core import Node
 from ibis.expr.operations.logical import ExistsSubquery, NotExistsSubquery
 from ibis.expr.operations.sortkeys import _maybe_convert_sort_keys
 
@@ -52,6 +52,9 @@ def is_ancestor(self, other):  # pragma: no cover
 
         return _is_ancestor(self, other)
 
+    def root_tables(self):
+        return [self]
+
 
 @public
 class PhysicalTable(TableNode, sch.HasSchema):
@@ -184,13 +187,6 @@ def schema(self):
     def has_schema(self):
         return not set(self.left.columns) & set(self.right.columns)
 
-    def root_tables(self):
-        if util.all_of([self.left.op(), self.right.op()], (Join, Selection)):
-            # Unraveling is not possible
-            return [self.left.op(), self.right.op()]
-        else:
-            return distinct_roots(self.left, self.right)
-
 
 @public
 class InnerJoin(Join):
@@ -306,9 +302,6 @@ def schema(self):
     def has_schema(self):
         return self.table.op().has_schema()
 
-    def root_tables(self):
-        return [self]
-
 
 @public
 class SelfReference(TableNode, sch.HasSchema):
@@ -318,12 +311,6 @@ class SelfReference(TableNode, sch.HasSchema):
     def schema(self):
         return self.table.schema()
 
-    def root_tables(self):
-        # The dependencies of this operation are not walked, which makes the
-        # table expression holding this relationally distinct from other
-        # expressions, so things like self-joins are possible
-        return [self]
-
     def blocks(self):
         return True
 
@@ -443,9 +430,6 @@ def blocks(self):
     def substitute_table(self, table_expr):  # pragma: no cover
         return Selection(table_expr, self.selections)
 
-    def root_tables(self):
-        return [self]
-
     @util.deprecated(instead="", version="4.0.0")
     def can_add_filters(self, wrapped_expr, predicates):  # pragma: no cover
         pass


Test patch:
diff --git a/ibis/backends/pandas/tests/execution/test_join.py b/ibis/backends/pandas/tests/execution/test_join.py
--- a/ibis/backends/pandas/tests/execution/test_join.py
+++ b/ibis/backends/pandas/tests/execution/test_join.py
@@ -92,7 +92,7 @@ def test_join_with_invalid_predicates(how, left, right):
 
 @mutating_join_type
 @pytest.mark.xfail(reason='Hard to detect this case')
-def test_join_with_duplicate_non_key_columns(how, left, right, df1, df2):
+def test_join_with_duplicate_non_key_columns(how, left, right):
     left = left.mutate(x=left.value * 2)
     right = right.mutate(x=right.other_value * 3)
     expr = left.join(right, left.key == right.key, how=how)
@@ -464,3 +464,125 @@ def test_mutate_after_join():
     )
     result = joined.execute()
     tm.assert_frame_equal(result, expected)
+
+
+@pytest.fixture
+def tracts_df():
+    return pd.DataFrame(
+        [[1, 1], [2, 1], [3, 2], [4, 2], [5, 3], [6, 4]],
+        columns=["tract_id", "tract_farm_id"],
+    )
+
+
+@pytest.fixture
+def fields_df():
+    vals = [
+        [1, 1, "[(0, 2), (1, 3), (2, 0), (3, 1)]"],
+        [2, 1, "[(2, 2), (3, 2), (3, 1)]"],
+        [3, 2, "[(0, 1), (-1, 0), (-2, 0), (-2, 1)]"],
+        [4, 3, "[(0, 1), (1, 1), (1, 2), (0, 2)]"],
+        [5, 3, "[(1, 0), (2, 0), (2, 3), (1, 3)]"],
+        [6, 3, "[(2, 0), (3, 0), (3, 2), (2, 2)]"],
+        [7, 4, "[(-1, -1), (0, -1), (0, -2)]"],
+        [8, 4, "[(1, 0), (1, -2), (0, -2), (0, -1)]"],
+        [
+            9,
+            5,
+            str(
+                [
+                    (1, 0),
+                    (2, 0),
+                    (1, -1),
+                    (1, -2),
+                    (-1, -2),
+                    (-1, -1),
+                    (-2, 0),
+                    (-1, 0),
+                    (0, -1),
+                ]
+            ),
+        ],
+        [10, 6, "[(-1, 2), (0, 2), (0, 0), (-1, 0)]"],
+        [11, 6, "[(0, 2), (1, 2), (1, 1), (0, 1)]"],
+    ]
+    return pd.DataFrame(
+        vals,
+        columns=["field_id", "field_tract_id", "field_vertices"],
+    )
+
+
+@pytest.fixture
+def harvest_df():
+    vals = [
+        [1, 1, 1, 1, 1, 65.80],
+        [2, 2, 1, 2, 2, 5750.00],
+        [3, 3, 1, 1, 1, 59.85],
+        [4, 4, 2, 2, 2, 10100.00],
+        [5, 5, 2, 1, 1, 90.30],
+        [6, 6, 2, 2, 2, 21000.00],
+        [7, 7, 2, 2, 2, 5150.00],
+        [8, 8, 2, 1, 1, 53.55],
+        [9, 9, 3, 1, 1, 147.00],
+        [10, 10, 4, 1, 1, 70.70],
+        [11, 11, 4, 2, 2, 9600.00],
+        [12, 1, 1, 2, 4, 22800.00],
+        [13, 2, 1, 1, 3, 19.25],
+        [14, 3, 1, 2, 4, 13050.00],
+        [15, 4, 2, 1, 3, 31.15],
+        [16, 5, 2, 2, 4, 33000.00],
+        [17, 6, 2, 1, 3, 64.40],
+        [18, 7, 2, 1, 3, 16.45],
+        [19, 8, 2, 2, 4, 15000.00],
+        [20, 9, 3, 2, 4, 38400.00],
+        [21, 10, 4, 2, 4, 19800.00],
+        [22, 11, 4, 1, 3, 34.30],
+    ]
+
+    return pd.DataFrame(
+        vals,
+        columns=[
+            "harvest_id",
+            "harvest_field_id",
+            "harvest_farmer_group_id",
+            "harvest_crop_id",
+            "harvest_date_id",
+            "harvest_value",
+        ],
+    )
+
+
+def test_multijoin(tracts_df, fields_df, harvest_df):
+    conn = ibis.pandas.connect(
+        dict(
+            tracts=tracts_df,
+            fields=fields_df,
+            harvest=harvest_df,
+        )
+    )
+
+    tracts, fields, harvest = map(conn.table, "tracts fields harvest".split())
+
+    fielded = harvest.inner_join(
+        fields,
+        harvest.harvest_field_id == fields.field_id,
+    )
+    tracted = fielded.inner_join(
+        tracts,
+        fielded.field_tract_id == tracts.tract_id,
+    )
+    result = tracted.execute()
+
+    fielded_df = pd.merge(
+        harvest_df,
+        fields_df,
+        left_on="harvest_field_id",
+        right_on="field_id",
+    )
+    expected = pd.merge(
+        fielded_df,
+        tracts_df,
+        left_on="field_tract_id",
+        right_on="tract_id",
+    )
+
+    tm.assert_frame_equal(result, expected)