Repository: vaexio/vaex

Issue:
[BUG-REPORT] Joining using virtual columns throws an exception
Joining using virtual columns throws an exception. For example:

```python
import vaex

df1 = vaex.from_arrays(numbers1=['t_h_r_e_e', 't_w_o', 'o_n_e'])
df2 = vaex.from_arrays(numbers2=['one', 'one', 'four'])

print('df1')
print(df1)

def transform(text):
    return '_'.join(char for char in text)

df2['numbers2x'] = df2['numbers2'].apply(transform, multiprocessing=False)

print('df2')
print(df2)

joined = df1.join(df2, left_on='numbers1', right_on='numbers2x', how='inner', allow_duplication=True)

print('joined')
print(joined)
```

Produces this output:

```
ERROR:MainThread:vaex:error evaluating: numbers2x at rows 0-2
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/vaex/scopes.py", line 102, in evaluate
    result = self[expression]
  File "/usr/local/lib/python3.9/site-packages/vaex/scopes.py", line 159, in __getitem__
    raise KeyError("Unknown variables or column: %r" % (variable,))
KeyError: "Unknown variables or column: 'lambda_function(numbers2)'"

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/vaex/dataframe.py", line 1968, in data_type
    data = self.evaluate(expression, 0, 1, filtered=False, array_type=array_type, parallel=False)
  File "/usr/local/lib/python3.9/site-packages/vaex/dataframe.py", line 2642, in evaluate
    return self._evaluate_implementation(expression, i1=i1, i2=i2, out=out, selection=selection, filtered=filtered, array_type=array_type, parallel=parallel, chunk_size=chunk_size)
  File "/usr/local/lib/python3.9/site-packages/vaex/dataframe.py", line 5676, in _evaluate_implementation
    value = scope.evaluate(expression)
  File "/usr/local/lib/python3.9/site-packages/vaex/scopes.py", line 102, in evaluate
    result = self[expression]
  File "/usr/local/lib/python3.9/site-packages/vaex/scopes.py", line 152, in __getitem__
    values = self.evaluate(expression)
  File "/usr/local/lib/python3.9/site-packages/vaex/scopes.py", line 108, in evaluate
    result = eval(expression, expression_namespace, self)
  File "<string>", line 1, in <module>
NameError: name 'lambda_function' is not defined
```

This is repeated a few times. Then:

```
df1
  #  numbers1
  0  't_h_r_e_e'
  1  't_w_o'
  2  'o_n_e'
df2
  #  numbers2    numbers2x
  0  'one'       o_n_e
  1  'one'       o_n_e
  2  'four'      f_o_u_r
joined
  #  numbers1    numbers2    numbers2x
  0  'o_n_e'     one         error
  1  'o_n_e'     one         error
```

Vaex version: `{'vaex': '4.1.0', 'vaex-core': '4.1.0', 'vaex-viz': '0.5.0', 'vaex-hdf5': '0.7.0', 'vaex-server': '0.4.0', 'vaex-astro': '0.8.0', 'vaex-jupyter': '0.6.0', 'vaex-ml': '0.11.1'}`

Installed with Pip
Python 3.9.1 on MacOS 10.15.7



Discussion:
Hi @maxharlow, even I encountered with this issue while trying to join on virtual column. I'm just checking whether you found a way around this or still stuck with the same error? 

If yes, please let me know how did you solve this.

Thanks and Regards,

Vijay

Resolving patch:
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -3733,7 +3733,9 @@ def column_count(self, hidden=False):
     def get_names(self, hidden=False):
         """Return a list of column names and variable names."""
         names = self.get_column_names(hidden=hidden)
-        return names + [k for k in self.variables.keys() if not hidden or not k.startswith('__')]
+        return names +\
+            [k for k in self.variables.keys() if not hidden or not k.startswith('__')] +\
+            [k for k in self.functions.keys() if not hidden or not k.startswith('__')]
 
     def get_column_names(self, virtual=True, strings=True, hidden=False, regex=None):
         """Return a list of column names
@@ -5962,6 +5964,10 @@ def mangle_name(prefix, name, suffix):
                 left.set_variable(name, right.variables[name])
             elif column_name in right.virtual_columns:
                 left.add_virtual_column(name, right.virtual_columns[column_name])
+            elif column_name in right.functions:
+                if name in left.functions:
+                    raise NameError(f'Name collision for function {name}')
+                left.functions[name] = right.functions[name]
             else:
                 right_columns.append(name)
                 # we already add the column name here to get the same order


Test patch:
diff --git a/tests/join_test.py b/tests/join_test.py
--- a/tests/join_test.py
+++ b/tests/join_test.py
@@ -302,6 +302,21 @@ def test_join_variables():
     assert df.r_z_rhs.values[0] == 2*3 + 3*4
 
 
+
+def test_join_functions():
+    df1 = vaex.from_scalars(j=444, x=1, y=2)
+    df2 = vaex.from_scalars(k=555, x=1)
+    # df2['x'] = df2.apply(lambda y: y-1, arguments=[df2.y])
+    df2['z'] = df2.apply(lambda x: x+10, arguments=[df1.x])
+    df = df1.join(df2, on='x')
+    assert 'lambda_function' in df.get_names()
+    assert df.x.tolist() == [1]
+    assert df.y.tolist() == [2]
+    assert df.z.tolist() == [11]
+    assert df.j.tolist() == [444]
+    assert df.k.tolist() == [555]
+
+
 def test_with_masked_no_short_circuit():
     # this test that the full table is joined, in some rare condition
     # it can happen that the left table has a value not present in the right