Repository: vaexio/vaex

Issue:
Using ds.join twice with masked values / and inner join
When joining ds1 with ds3 (right join to get ds31),
ds31 = ds1.join(ds3, left_on='a', right_on='c', how='right'):

\# | c | z | a | x
-- | -- | -- | -- | --
0 | 'a' | 1 | 'a' | 1
1 | 'b' | 8 | 'b' | 2
2 | 'e' | 27 | masked | masked
3 | 'f' | 64 | masked | masked

for ds1:

\# | a | x
-- | -- | --
0 | 'a' | 1
1 | 'b' | 2
2 | 'c' | 3


for ds3:

\# | c | z | a | x
-- | -- | -- | -- | --
0 | 'a' | 1 | 'a' | 1
1 | 'b' | 8 | 'b' | 2
2 | 'e' | 27 | masked | masked
3 | 'f' | 64 | masked | masked

Now I want to join (left) ds31 with ds4:
ds314 = ds31.join(ds4, left_on='a', right_on='d', how='left', inplace='False')

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-178-7b64de6bfa10> in <module>()
----> 1 ds314 = ds31.join(ds4, left_on='a', right_on='d', how='left', inplace='False')

/net/gaia/data/users/breddels/python/anaconda3/lib/python3.6/site-packages/vaex/dataset.py in join(self, other, on, left_on, right_on, lsuffix, rsuffix, how, inplace)
   5168             right_values = right.evaluate(right_on)
   5169             # maps from the left_values to row #
-> 5170             index_left = dict(zip(left_values, range(N)))
   5171             # idem for right
   5172             index_other = dict(zip(right_values, range(N_other)))

TypeError: unhashable type: 'MaskedConstant'

for ds4:

\# | d | z4
-- | -- | --
0 | 'a' | 1
1 | 'b' | 16
2 | 'c' | 81
3 | 'd' | 256
4 | 'e' | 625



In other words, vaex tries to find matches in ds4 for the masked rows of ds31.
I would like to have a resulting Table in which all ds4 is matched to ds31. Values for ds314 should be masked (or Nan) if the corresponding rows in ds31 are masked.

The same error occurs when:
ds431 = ds4.join(ds31, left_on='d', right_on='a', how='left', inplace='False')

So: **ds.join can't handle 'masked' values.**

In addition: a feature for **inner join** would be useful too.








Discussion:
Thanks @jhjhagen for the great bug reports, I can confirm this is still an issue.

Resolving patch:
diff --git a/packages/vaex-core/vaex/column.py b/packages/vaex-core/vaex/column.py
--- a/packages/vaex-core/vaex/column.py
+++ b/packages/vaex-core/vaex/column.py
@@ -76,6 +76,30 @@ def __init__(self, df, indices, name):
         max_index = self.indices.max()
         assert max_index < self.df._length_original
 
+    @staticmethod
+    def index(df, column, name, indices, direct_indices_map=None):
+        """Creates a new column indexed by indices which avoids nested indices
+
+        :param df: Dataframe where column comes from
+        :param column: Column object or numpy array
+        :param name: name of column
+        :param indices: ndarray with indices which rows to take
+        :param direct_indices_map: cache of the nested indices (pass a dict for better performance), defaults to None
+        :return: A column object which avoids two levels of indirection
+        :rtype: ColumnIndexed
+        """
+        direct_indices_map = direct_indices_map or {}
+        if isinstance(column, ColumnIndexed):
+            # TODO: think about what happpens when the indices are masked.. ?
+            if id(column.indices) not in direct_indices_map:
+                direct_indices = column.indices[indices]
+                direct_indices_map[id(column.indices)] = direct_indices
+            else:
+                direct_indices = direct_indices_map[id(column.indices)]
+            return ColumnIndexed(column.df, direct_indices, column.name)
+        else:
+            return ColumnIndexed(df, indices, name)
+
     def __len__(self):
         return len(self.indices)
 
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -3674,16 +3674,7 @@ def take(self, indices, unfiltered=False):
             if column is not None:
                 # we optimize this somewhere, so we don't do multiple
                 # levels of indirection
-                if isinstance(column, ColumnIndexed):
-                    # TODO: think about what happpens when the indices are masked.. ?
-                    if id(column.indices) not in direct_indices_map:
-                        direct_indices = column.indices[indices]
-                        direct_indices_map[id(column.indices)] = direct_indices
-                    else:
-                        direct_indices = direct_indices_map[id(column.indices)]
-                    df.columns[name] = ColumnIndexed(column.df, direct_indices, column.name)
-                else:
-                    df.columns[name] = ColumnIndexed(df_trimmed, indices, name)
+                df.columns[name] = ColumnIndexed.index(df_trimmed, column, name, indices, direct_indices_map)
         df._length_original = len(indices)
         df._length_unfiltered = df._length_original
         df._cached_filtered_length = None
@@ -5041,6 +5032,7 @@ def reduce(a, b):
                 left = left.take(left_indices_matched)
             else:
                 lookup = np.ma.array(lookup, mask=lookup==-1)
+            direct_indices_map = {}  # for performance, keeps a cache of two levels of indirection of indices
             for name in right:
                 right_name = name
                 if name in left:
@@ -5049,7 +5041,8 @@ def reduce(a, b):
                 if name in right.virtual_columns:
                     left.add_virtual_column(right_name, right.virtual_columns[name])
                 else:
-                    left.add_column(right_name, ColumnIndexed(right, lookup, name))
+                    column = ColumnIndexed.index(right, right.columns[name], name, lookup, direct_indices_map)
+                    left.add_column(right_name, column)
         return left
 
     def export(self, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):


Test patch:
diff --git a/tests/join_test.py b/tests/join_test.py
--- a/tests/join_test.py
+++ b/tests/join_test.py
@@ -53,6 +53,11 @@ def test_left_a_b():
     assert df.evaluate('y').tolist() == [0, None, 2]
     assert df.evaluate('y_r').tolist() == [None, 1, None]
 
+def test_join_indexed():
+    df = df_a.join(other=df_b, left_on='a', right_on='b', rsuffix='_r')
+    df_X = df_a.join(df, left_on='a', right_on='b', rsuffix='_r')
+    assert df_X.evaluate('b').tolist() == ['A', 'B', None]
+
 
 def test_left_a_b_filtered():
     df_af = df_a[df_a.x > 0]