Repository: blaze/blaze

Issue:
DataShape not NumPy-compatible prevents joining tables
Using foreign-keys in a database is a very common thing to do however it results in a map type in datashape which apparently prevents performing even a simple join



Discussion:
**Repro:**

``` python
from itertools import chain

import blaze as bz
import numpy as np
from numpy.random import choice, randint
from odo import odo
import pandas as pd
import sqlalchemy as sa

engine = sa.create_engine('sqlite://')
metadata = sa.MetaData(bind=engine)
T1 = sa.Table(
    'demo',
    metadata,
    sa.Column('timestamp', sa.String, primary_key=True),
    sa.Column('pkid', sa.Integer, sa.ForeignKey("labels.pkid"), nullable=False),
)
T2 = sa.Table(
    'labels',
    metadata,
    sa.Column('pkid', sa.Integer, primary_key=True),
    sa.Column('label', sa.String),
)

metadata.create_all()


labels = pd.DataFrame(
    list(zip(range(4), ['A','B','C','D'])),
    columns=['pkid', 'label']
)
odo(labels, T2)


dates = pd.date_range('01-Jan-2017', '01-Apr-2017', freq='D')
df = pd.DataFrame({'timestamp': list(map(str, dates))})
df['pkid'] = randint(0, labels.shape[0], dates.size)
odo(df, T1)
data1 = bz.data(T1)
data2 = bz.data(T2)
```

``` python
In [13]: odo(data2, pd.DataFrame)
Out[13]: 
   pkid label
0     0     A
1     1     B
2     2     C
3     3     D

In [14]: data1.head()
Out[14]: 
             timestamp  pkid
0  2017-01-01 00:00:00     2
1  2017-01-02 00:00:00     0
2  2017-01-03 00:00:00     2
3  2017-01-04 00:00:00     3
4  2017-01-05 00:00:00     1
5  2017-01-06 00:00:00     2
6  2017-01-07 00:00:00     1
7  2017-01-08 00:00:00     3
8  2017-01-09 00:00:00     3
9  2017-01-10 00:00:00     3

In [15]: bz.join(data1, data2, 'pkid', 'pkid')
Traceback (most recent call last):

  File "<ipython-input-15-7a95ddc62146>", line 1, in <module>
    bz.join(data1, data2, 'pkid', 'pkid')

  File "c:\dev\packages\blaze\blaze\expr\collections.py", line 639, in join
    right_types)):

  File "c:\dev\packages\datashape\datashape\promote.py", line 49, in promote
    np_res_type = np.result_type(datashape.to_numpy_dtype(left),

  File "c:\dev\packages\datashape\datashape\coretypes.py", line 1279, in to_numpy_dtype
    return to_numpy(ds.measure)[1]

  File "c:\dev\packages\datashape\datashape\coretypes.py", line 1312, in to_numpy
    return tuple(shape), msr.to_numpy_dtype()

  File "c:\dev\packages\datashape\datashape\coretypes.py", line 173, in to_numpy_dtype
    raise TypeError('DataShape %s is not NumPy-compatible' % self)

TypeError: DataShape map[int32, {pkid: int32, label: ?string}] is not NumPy-compatible
```

@dhirschfeld Thanks for the report -- indeed the semantics to determine the new datashape when `join`ing tables is too simple, and as you see here, uses NumPy's type coercion rules, which work for scalar types, but not for heterogeneous types.  Will look into it.


Resolving patch:
diff --git a/blaze/expr/collections.py b/blaze/expr/collections.py
--- a/blaze/expr/collections.py
+++ b/blaze/expr/collections.py
@@ -538,7 +538,7 @@ def _schema(self):
             self.rhs.dshape.measure.dict,
         )
         joined = (
-            (name, promote(dt, right_types[name], promote_option=False))
+            (name, promote(extract_key(dt), extract_key(right_types[name]), promote_option=False))
             for n, (name, dt) in enumerate(filter(
                 compose(op.contains(on_left), first),
                 self.lhs.dshape.measure.fields,
@@ -608,6 +608,10 @@ def types_of_fields(fields, expr):
         return expr.dshape.measure
 
 
+def extract_key(m):
+    return m.measure.key if isinstance(m.measure, datashape.coretypes.Map) else m
+
+
 @copydoc(Join)
 def join(lhs, rhs, on_left=None, on_right=None,
          how='inner', suffixes=('_left', '_right')):
@@ -627,6 +631,9 @@ def join(lhs, rhs, on_left=None, on_right=None,
         )
     left_types = listpack(types_of_fields(on_left, lhs))
     right_types = listpack(types_of_fields(on_right, rhs))
+    # Replace map[x, y] with x to resolve foreign keys.
+    left_types = list(map(extract_key, left_types))
+    right_types = list(map(extract_key, right_types))
     if len(left_types) != len(right_types):
         raise ValueError(
             'Length of on_left=%d not equal to length of on_right=%d' % (


Test patch:
diff --git a/blaze/tests/test_sql.py b/blaze/tests/test_sql.py
--- a/blaze/tests/test_sql.py
+++ b/blaze/tests/test_sql.py
@@ -2,13 +2,14 @@
 
 pytest.importorskip('sqlalchemy')
 
+from datashape import dshape
 from functools import partial
 
 from toolz import first
 from sqlalchemy.exc import OperationalError
 from odo import into, drop
 
-from blaze import data as bz_data
+from blaze import data as bz_data, join, symbol
 from blaze import create_index
 
 
@@ -97,3 +98,9 @@ def test_ignore_existing(sql, cols):
 
     # Shouldn't error
     create_call(ignore_existing=True)
+
+
+def test_join_foreign_key():
+    a = symbol('a', "var * {timestamp: string, pkid: map[int32, {pkid: int32, label: ?string}]}")
+    b = symbol('a', "var * {pkid: int32, label: ?string}")
+    assert join(a, b, 'pkid', 'pkid').dshape == dshape("var * {pkid: int32, timestamp: string, label: ?string}")