Repository: blaze/blaze

Issue:
Failing joins on field that is nullable in one table but not in the other
Given following tables (postgresql db):

```
example> \d table1
+----------+---------+-------------+
| Column   | Type    | Modifiers   |
|----------+---------+-------------|
| id       | integer | not null    |
| field    | integer |             |
+----------+---------+-------------+
Indexes:
    "table1_pkey" PRIMARY KEY, btree (id)

example> \d table2
+----------+---------+-------------+
| Column   | Type    | Modifiers   |
|----------+---------+-------------|
| id       | integer | not null    |
| field    | integer | not null    |
+----------+---------+-------------+
Indexes:
    "table2_pkey" PRIMARY KEY, btree (id)
```

Following code fails:

```
join(table1, table2, 'field')
```

With this error message:

```
TypeError                                 Traceback (most recent call last)
<ipython-input-14-2a191a07eec9> in <module>()
----> 1 join(database.table1, database.table2, 'field')

/home/user/anaconda/envs/env/lib/python2.7/site-packages/blaze/expr/collections.pyc in join(lhs, rhs, on_left, on_right, how, suffixes)
    435                          (lhs, rhs))
    436     if types_of_fields(on_left, lhs) != types_of_fields(on_right, rhs):
--> 437         raise TypeError("Schema's of joining columns do not match")
    438     _on_left = tuple(on_left) if isinstance(on_left, list) else on_left
    439     _on_right = (tuple(on_right) if isinstance(on_right, list)

TypeError: Schema's of joining columns do not match
```

Types returned by types_of_fields function:

```
types_of_fields('field', database.table1) #  ?int32
types_of_fields('field', database.table2) #  ctype("int32")  
```



Discussion:

Resolving patch:
diff --git a/blaze/compute/python.py b/blaze/compute/python.py
--- a/blaze/compute/python.py
+++ b/blaze/compute/python.py
@@ -535,17 +535,40 @@ def compute_up(t, seq, **kwargs):
     return tuple(keyfunc(k) + valfunc(v) for k, v in d.items())
 
 
-def pair_assemble(t):
+def pair_assemble(t, on_left=None, on_right=None):
     """ Combine a pair of records into a single record
 
     This is mindful to shared columns as well as missing records
+
+    Parameters
+    ----------
+    t : Join
+        The join to combine.
+    on_left : list of column indicies, optional
+        The column indicies of the left columns.
+    on_right : list of column indicies, optional
+        The column indicies of the right columns.
+
+    Returns
+    -------
+    assemble : callable
+        A function that assembles the data from a row of the
+        left and right data sources.
     """
     try:
         from cytoolz import get  # not curried version
     except:
         from toolz import get
-    on_left = [t.lhs.fields.index(col) for col in listpack(t.on_left)]
-    on_right = [t.rhs.fields.index(col) for col in listpack(t.on_right)]
+    on_left = (
+        on_left
+        if on_left is not None else
+        [t.lhs.fields.index(col) for col in listpack(t.on_left)]
+    )
+    on_right = (
+        on_right
+        if on_right is not None else
+        [t.rhs.fields.index(col) for col in listpack(t.on_right)]
+    )
 
     left_self_columns = [t.lhs.fields.index(c) for c in t.lhs.fields
                          if c not in listpack(t.on_left)]
@@ -603,7 +626,7 @@ def compute_up(t, lhs, rhs, **kwargs):
                        left_default=left_default,
                        right_default=right_default)
 
-    assemble = pair_assemble(t)
+    assemble = pair_assemble(t, on_left, on_right)
 
     return map(assemble, pairs)
 
diff --git a/blaze/expr/collections.py b/blaze/expr/collections.py
--- a/blaze/expr/collections.py
+++ b/blaze/expr/collections.py
@@ -1,18 +1,25 @@
 from __future__ import absolute_import, division, print_function
 
+from functools import partial
+from itertools import chain
 
 import datashape
-from datashape import DataShape, Option, Record, Unit, dshape, var, Fixed, Var
+from datashape import (
+    DataShape, Option, Record, Unit, dshape, var, Fixed, Var, promote, object_,
+)
 from datashape.predicates import isscalar, iscollection, isrecord
 from toolz import (
-    isdistinct, frequencies, concat as tconcat, unique, get, first,
+    isdistinct, frequencies, concat as tconcat, unique, get, first, compose,
+    keymap,
 )
+import toolz.curried.operator as op
 from odo.utils import copydoc
 
 from .core import common_subexpression
 from .expressions import Expr, ElemWise, label, Field
 from .expressions import dshape_method_list
 from ..compatibility import zip_longest, _strtypes
+from ..utils import listpack
 
 
 __all__ = ['Sort', 'Distinct', 'Head', 'Merge', 'IsIn', 'isin', 'distinct',
@@ -398,17 +405,17 @@ class Join(Expr):
 
     @property
     def on_left(self):
-        if isinstance(self._on_left, tuple):
-            return list(self._on_left)
-        else:
-            return self._on_left
+        on_left = self._on_left
+        if isinstance(on_left, tuple):
+            return list(on_left)
+        return on_left
 
     @property
     def on_right(self):
-        if isinstance(self._on_right, tuple):
-            return list(self._on_right)
-        else:
-            return self._on_right
+        on_right = self._on_right
+        if isinstance(on_right, tuple):
+            return list(on_right)
+        return on_right
 
     @property
     def schema(self):
@@ -435,35 +442,57 @@ def schema(self):
         """
         option = lambda dt: dt if isinstance(dt, Option) else Option(dt)
 
-        joined = [[name, dt] for name, dt in self.lhs.schema[0].parameters[0]
-                  if name in self.on_left]
+        on_left = self.on_left
+        if not isinstance(on_left, list):
+            on_left = on_left,
+
+        on_right = self.on_right
+        if not isinstance(on_right, list):
+            on_right = on_right,
 
-        left = [[name, dt] for name, dt in
-                zip(self.lhs.fields, types_of_fields(
-                    self.lhs.fields, self.lhs))
-                if name not in self.on_left]
+        right_types = keymap(
+            dict(zip(on_right, on_left)).get,
+            self.rhs.dshape.measure.dict,
+        )
+        joined = (
+            (name, promote(dt, right_types[name], promote_option=False))
+            for n, (name, dt) in enumerate(filter(
+                compose(op.contains(on_left), first),
+                self.lhs.dshape.measure.fields,
+            ))
+        )
 
-        right = [[name, dt] for name, dt in
-                 zip(self.rhs.fields, types_of_fields(
-                     self.rhs.fields, self.rhs))
-                 if name not in self.on_right]
+        left = [
+            (name, dt) for name, dt in zip(
+                self.lhs.fields,
+                types_of_fields(self.lhs.fields, self.lhs)
+            ) if name not in on_left
+        ]
+
+        right = [
+            (name, dt) for name, dt in zip(
+                self.rhs.fields,
+                types_of_fields(self.rhs.fields, self.rhs)
+            ) if name not in on_right
+        ]
 
         # Handle overlapping but non-joined case, e.g.
-        left_other = [name for name, dt in left if name not in self.on_left]
-        right_other = [name for name, dt in right if name not in self.on_right]
-        overlap = set.intersection(set(left_other), set(right_other))
+        left_other = set(name for name, dt in left if name not in on_left)
+        right_other = set(name for name, dt in right if name not in on_right)
+        overlap = left_other & right_other
+
         left_suffix, right_suffix = self.suffixes
-        left = [[name + left_suffix if name in overlap else name, dt]
-                for name, dt in left]
-        right = [[name + right_suffix if name in overlap else name, dt]
-                 for name, dt in right]
+        left = ((name + left_suffix if name in overlap else name, dt)
+                for name, dt in left)
+        right = ((name + right_suffix if name in overlap else name, dt)
+                 for name, dt in right)
 
         if self.how in ('right', 'outer'):
-            left = [[name, option(dt)] for name, dt in left]
+            left = ((name, option(dt)) for name, dt in left)
         if self.how in ('left', 'outer'):
-            right = [[name, option(dt)] for name, dt in right]
+            right = ((name, option(dt)) for name, dt in right)
 
-        return dshape(Record(joined + left + right))
+        return dshape(Record(chain(joined, left, right)))
 
     @property
     def dshape(self):
@@ -492,7 +521,7 @@ def types_of_fields(fields, expr):
     else:
         if isinstance(fields, (tuple, list, set)):
             assert len(fields) == 1
-            fields = fields[0]
+            fields, = fields
         assert fields == expr._name
         return expr.dshape.measure
 
@@ -511,10 +540,28 @@ def join(lhs, rhs, on_left=None, on_right=None,
     if isinstance(on_right, tuple):
         on_right = list(on_right)
     if not on_left or not on_right:
-        raise ValueError("Can not Join.  No shared columns between %s and %s" %
-                         (lhs, rhs))
-    if types_of_fields(on_left, lhs) != types_of_fields(on_right, rhs):
-        raise TypeError("Schema's of joining columns do not match")
+        raise ValueError(
+            "Can not Join.  No shared columns between %s and %s" % (lhs, rhs),
+        )
+    left_types = listpack(types_of_fields(on_left, lhs))
+    right_types = listpack(types_of_fields(on_right, rhs))
+    if len(left_types) != len(right_types):
+        raise ValueError(
+            'Length of on_left=%d not equal to length of on_right=%d' % (
+                len(left_types), len(right_types),
+            ),
+        )
+
+    for n, promotion in enumerate(map(partial(promote, promote_option=False),
+                                      left_types,
+                                      right_types)):
+        if promotion == object_:
+            raise ValueError(
+                "Schema's of joining columns do not match,"
+                ' no promotion found for %s=%s and %s=%s' % (
+                    on_left[n], left_types[n], on_right[n], right_types[n],
+                ),
+            )
     _on_left = tuple(on_left) if isinstance(on_left, list) else on_left
     _on_right = (tuple(on_right) if isinstance(on_right, list)
                  else on_right)


Test patch:
diff --git a/blaze/compute/tests/test_pandas_compute.py b/blaze/compute/tests/test_pandas_compute.py
--- a/blaze/compute/tests/test_pandas_compute.py
+++ b/blaze/compute/tests/test_pandas_compute.py
@@ -311,6 +311,20 @@ def test_join_suffixes():
     tm.assert_frame_equal(result, expected)
 
 
+def test_join_promotion():
+    a_data = pd.DataFrame([[0.0, 1.5], [1.0, 2.5]], columns=list('ab'))
+    b_data = pd.DataFrame([[0, 1], [1, 2]], columns=list('ac'))
+    a = symbol('a', discover(a_data))
+    b = symbol('b', discover(b_data))
+
+    joined = join(a, b, 'a')
+    assert joined.dshape == dshape('var * {a: float64, b: ?float64, c: int64}')
+
+    expected = pd.merge(a_data, b_data, on='a')
+    result = compute(joined, {a: a_data, b: b_data})
+    tm.assert_frame_equal(result, expected)
+
+
 def test_sort():
     tm.assert_frame_equal(compute(t.sort('amount'), df),
                           df.sort('amount'))
diff --git a/blaze/compute/tests/test_python_compute.py b/blaze/compute/tests/test_python_compute.py
--- a/blaze/compute/tests/test_python_compute.py
+++ b/blaze/compute/tests/test_python_compute.py
@@ -358,16 +358,28 @@ def test_graph_double_join():
     t_arc = symbol('t_arc', 'var * {a: int32, b: int32}')
     t_wanted = symbol('t_wanted', 'var * {name: string}')
 
+    # >>> compute(join(t_idx, t_arc, 'b'), {t_idx: idx, t_arc: arc})
+    # [[1, A, 3],
+    #  [1, A, 2],
+    #  [1, A, 5],
+    #  [3, C, 1],
+    #  [3, C, 2],
+    #  [3, C, 4],
+    #  [3, C, 5],
+    #  [6, F, 1],
+    #  [6, F, 2],
+    #  [6, F, 4]]
+
     j = join(join(t_idx, t_arc, 'b'), t_wanted, 'name')[['name', 'b', 'a']]
 
     result = compute(j, {t_idx: idx, t_arc: arc, t_wanted: wanted})
     result = sorted(map(tuple, result))
-    expected = sorted([('A', 3, 1),
-                    ('A', 2, 1),
-                    ('A', 5, 1),
-                    ('F', 1, 6),
-                    ('F', 2, 6),
-                    ('F', 4, 6)])
+    expected = sorted([('A', 1, 3),
+                       ('A', 1, 2),
+                       ('A', 1, 5),
+                       ('F', 6, 1),
+                       ('F', 6, 2),
+                       ('F', 6, 4)])
 
     assert result == expected
 
diff --git a/blaze/expr/tests/test_collections.py b/blaze/expr/tests/test_collections.py
--- a/blaze/expr/tests/test_collections.py
+++ b/blaze/expr/tests/test_collections.py
@@ -83,6 +83,36 @@ def test_raise_error_if_join_on_no_columns():
     assert raises(ValueError, lambda: join(a, b))
 
 
+def test_join_option_types():
+    a = symbol('a', 'var * {x: ?int}')
+    b = symbol('b', 'var * {x: int}')
+
+    assert join(a, b, 'x').dshape == dshape('var * {x: int}')
+    assert join(b, a, 'x').dshape == dshape('var * {x: int}')
+
+
+def test_join_mismatched_schema():
+    a = symbol('a', 'var * {x: int}')
+    b = symbol('b', 'var * {x: string}')
+
+    with pytest.raises(ValueError):
+        join(a, b, 'x')
+
+
+def test_join_type_promotion():
+    a = symbol('a', 'var * {x: int32}')
+    b = symbol('b', 'var * {x: int64}')
+
+    assert join(a, b, 'x').dshape == dshape('var * {x: int64}')
+
+
+def test_join_type_promotion_option():
+    a = symbol('a', 'var * {x: ?int32}')
+    b = symbol('b', 'var * {x: int64}')
+
+    assert join(a, b, 'x').dshape == dshape('var * {x: int64}')
+
+
 def test_isin():
     a = symbol('a', 'var * {x: int, y: string}')
     assert hasattr(a.x, 'isin')
diff --git a/blaze/expr/tests/test_table.py b/blaze/expr/tests/test_table.py
--- a/blaze/expr/tests/test_table.py
+++ b/blaze/expr/tests/test_table.py
@@ -241,7 +241,7 @@ def test_join():
     assert join(t, s, 'name').on_right == 'name'
 
     assert join(t, r, ('name', 'amount')).on_left == ['name', 'amount']
-    with pytest.raises(TypeError):
+    with pytest.raises(ValueError):
         join(t, q, 'name')
     with pytest.raises(ValueError):
         join(t, s, how='upside_down')