Repository: vaexio/vaex

Issue:
Superjoin: join now uses the new hash tables
Joining is now using the same hash table structures. Benchmarks should follow soon.


Discussion:

Resolving patch:
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -209,6 +209,7 @@ def __init__(self, name, column_names, executor=None):
         self._selection_mask_caches = collections.defaultdict(dict)
         self._selection_masks = {}  # maps to vaex.superutils.Mask object
         self._renamed_columns = []
+        self._column_aliases = {}  # maps from invalid variable names to valid ones
 
     def __getattr__(self, name):
         # will support the hidden methods
@@ -2137,7 +2138,8 @@ def clean(value):
                      units=units,
                      descriptions=descriptions,
                      description=self.description,
-                     active_range=[self._index_start, self._index_end])
+                     active_range=[self._index_start, self._index_end],
+                     column_aliases=self._column_aliases)
         return state
 
     def state_set(self, state, use_active_range=False, trusted=True):
@@ -2197,6 +2199,7 @@ def state_set(self, state, use_active_range=False, trusted=True):
             for name, value in state['virtual_columns'].items():
                 self[name] = self._expr(value)
         self.variables = state['variables']
+        self._column_aliases = state.get('column_aliases', {})
         import astropy  # TODO: make this dep optional?
         units = {key: astropy.units.Unit(value) for key, value in state["units"].items()}
         self.units.update(units)
@@ -2641,6 +2644,7 @@ def copy_metadata(self, other):
                 self.descriptions[name] = other.descriptions[name]
             if name in other.ucds:
                 self.ucds[name] = other.ucds[name]
+        self._column_aliases = dict(other._column_aliases)
         self.description = other.description
 
     @docsubst
@@ -2758,20 +2762,23 @@ def add_column(self, name, f_or_array, dtype=None):
                         raise ValueError("Array is of length %s, while the length of the DataFrame is %s due to the filtering, the (unfiltered) length is %s." % (len(ar), len(self), self.length_unfiltered()))
                 raise ValueError("array is of length %s, while the length of the DataFrame is %s" % (len(ar), self.length_original()))
             # assert self.length_unfiltered() == len(data), "columns should be of equal length, length should be %d, while it is %d" % ( self.length_unfiltered(), len(data))
-            self.columns[name] = f_or_array
-            if name not in self.column_names:
-                self.column_names.append(name)
+            valid_name = vaex.utils.find_valid_name(name)
+            if name != valid_name:
+                self._column_aliases[name] = valid_name
+            self.columns[valid_name] = f_or_array
+            if valid_name not in self.column_names:
+                self.column_names.append(valid_name)
             ar = f_or_array
             if dtype is not None:
-                self._dtypes_override[name] = dtype
+                self._dtypes_override[valid_name] = dtype
             else:
                 if isinstance(ar, np.ndarray) and ar.dtype.kind == 'O':
                     types = list({type(k) for k in ar if k == k and k is not None})
                     if len(types) == 1 and issubclass(types[0], six.string_types):
-                        self._dtypes_override[name] = str_type
+                        self._dtypes_override[valid_name] = str_type
         else:
             raise ValueError("functions not yet implemented")
-        self._save_assign_expression(name, Expression(self, name))
+        self._save_assign_expression(valid_name, Expression(self, valid_name))
 
     def _sparse_matrix(self, column):
         column = _ensure_string_from_expression(column)
@@ -4230,6 +4237,8 @@ def __getitem__(self, item):
                 return getattr(self, item)
             # if item in self.virtual_columns:
             #   return Expression(self, self.virtual_columns[item])
+            if item in self._column_aliases:
+                item = self._column_aliases[item]  # translate the alias name into the real name
             return Expression(self, item)  # TODO we'd like to return the same expression if possible
         elif isinstance(item, Expression):
             expression = item.expression
@@ -4517,6 +4526,7 @@ def copy(self, column_names=None, virtual=True):
         df._index_start = self._index_start
         df._active_fraction = self._active_fraction
         df._renamed_columns = list(self._renamed_columns)
+        df._column_aliases = dict(self._column_aliases)
         df.units.update(self.units)
         df.variables.update(self.variables)
         df._categories.update(self._categories)
diff --git a/packages/vaex-core/vaex/utils.py b/packages/vaex-core/vaex/utils.py
--- a/packages/vaex-core/vaex/utils.py
+++ b/packages/vaex-core/vaex/utils.py
@@ -13,6 +13,7 @@
 import time
 import warnings
 import numbers
+import keyword
 
 import numpy as np
 import progressbar
@@ -560,6 +561,8 @@ def unlistify(waslist, *args):
 def find_valid_name(name, used=[]):
     first, rest = name[0], name[1:]
     name = re.sub("[^a-zA-Z_]", "_", first) + re.sub("[^a-zA-Z_0-9]", "_", rest)
+    if keyword.iskeyword(name):
+        name += '_'
     if name in used:
         nr = 1
         while name + ("_%d" % nr) in used:
diff --git a/packages/vaex-hdf5/vaex/hdf5/dataset.py b/packages/vaex-hdf5/vaex/hdf5/dataset.py
--- a/packages/vaex-hdf5/vaex/hdf5/dataset.py
+++ b/packages/vaex-hdf5/vaex/hdf5/dataset.py
@@ -107,6 +107,11 @@ def write_meta(self):
                                         h5dataset = column
                 if h5dataset is None:
                     raise ValueError('column {} not found'.format(column_name))
+
+                aliases = [key for key, value in self._column_aliases.items() if value == column_name]
+                aliases.sort()
+                if aliases:
+                    h5dataset.attrs['alias'] = aliases[0]
                 for name, values in [("ucd", self.ucds), ("unit", self.units), ("description", self.descriptions)]:
                     if column_name in values:
                         value = ensure_string(values[column_name], cast=True)
@@ -293,6 +298,8 @@ def check_format(self, *args, **kwargs):
                 column = h5columns[column_name]
                 if "ucd" in column.attrs:
                     self.ucds[column_name] = ensure_string(column.attrs["ucd"])
+                if "alias" in column.attrs:
+                    self._column_aliases[ensure_string(column.attrs["alias"])] = column_name
                 if "description" in column.attrs:
                     self.descriptions[column_name] = ensure_string(column.attrs["description"])
                 if "unit" in column.attrs:


Test patch:
diff --git a/tests/column_names_test.py b/tests/column_names_test.py
--- a/tests/column_names_test.py
+++ b/tests/column_names_test.py
@@ -1,17 +1,37 @@
 import vaex
 from common import *
 
+
+x = np.arange(10)
+
+
 def test_column_names(ds_local):
-	ds = ds_local
-	columns_names = ds.get_column_names(virtual=True)
-	ds['__x'] = ds.x
-	assert columns_names == ds.get_column_names(virtual=True)
-	assert '__x' in ds.get_column_names(virtual=True, hidden=True)
-	assert len(columns_names) == len(ds.get_column_names(virtual=True, hidden=True))-1
-
-	ds = vaex.example()
-	ds['__x'] = ds['x'] + 1
-	assert 'FeH' in ds.get_column_names(regex='e*')
-	assert 'FeH' not in ds.get_column_names(regex='e')
-	assert '__x' not in ds.get_column_names(regex='__x')
-	assert '__x' in ds.get_column_names(regex='__x', hidden=True)
+    ds = ds_local
+    columns_names = ds.get_column_names(virtual=True)
+    ds['__x'] = ds.x
+    assert columns_names == ds.get_column_names(virtual=True)
+    assert '__x' in ds.get_column_names(virtual=True, hidden=True)
+    assert len(columns_names) == len(ds.get_column_names(virtual=True, hidden=True))-1
+
+    ds = vaex.example()
+    ds['__x'] = ds['x'] + 1
+    assert 'FeH' in ds.get_column_names(regex='e*')
+    assert 'FeH' not in ds.get_column_names(regex='e')
+    assert '__x' not in ds.get_column_names(regex='__x')
+    assert '__x' in ds.get_column_names(regex='__x', hidden=True)
+
+
+def test_add_invalid_name(tmpdir):
+    # support invalid names and keywords
+    df = vaex.from_dict({'X!1': x, 'class': x*2})
+    assert df['X!1'].tolist() == x.tolist()
+    assert (df['X!1']*2).tolist() == (x*2).tolist()
+    assert (df['class']).tolist() == (x*2).tolist()
+    assert 'X!1' in df._column_aliases
+    assert (df.copy()['X!1']*2).tolist() == (x*2).tolist()
+
+    path = str(tmpdir.join('test.hdf5'))
+    df.export(path)
+    df = vaex.open(path)
+    assert df['X!1'].tolist() == x.tolist()
+    assert (df.copy()['X!1']*2).tolist() == (x*2).tolist()