Repository: vaexio/vaex

Issue:
Better handle unicode column names
Creating a dataframe with unicode chars like this:
```python
df = vaex.from_dict({'ÚÚ': vaex.vrange(0, 100)})
```
Will replace the two characters with two underscores, which effectively hides the column (column names beginning with `__` are hidden).
This was mentioned here:
https://stackoverflow.com/questions/59738879/python-vaex-how-to-create-dataframe-from-a-csv-file


Discussion:
I think this will be obvious (to you!) but since I am here anyway:

```python
df = vaex.from_dict({'./label': vaex.vrange(0,100)})
```

In our case these characters have special meaning for us, so mutating the string is pretty inconvenient.  I'm hoping the solution involves preserving the original string!
Thanks for sharing that.

To give some understanding, vaex uses the column names as variable names, thus they have to be valid identifiers (like variable names), since it's all built on valid Python expressions.  In #370 we improved this to provide automatic translations, but it seems we need to do some more work.

Resolving patch:
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -2924,7 +2924,7 @@ def add_column(self, name, f_or_array, dtype=None):
                         raise ValueError("Array is of length %s, while the length of the DataFrame is %s due to the filtering, the (unfiltered) length is %s." % (len(ar), len(self), self.length_unfiltered()))
                 raise ValueError("array is of length %s, while the length of the DataFrame is %s" % (len(ar), self.length_original()))
             # assert self.length_unfiltered() == len(data), "columns should be of equal length, length should be %d, while it is %d" % ( self.length_unfiltered(), len(data))
-            valid_name = vaex.utils.find_valid_name(name)
+            valid_name = vaex.utils.find_valid_name(name, used=self.get_column_names(hidden=True))
             if name != valid_name:
                 self._column_aliases[name] = valid_name
             ar = f_or_array
@@ -2975,10 +2975,13 @@ def add_columns(self, names, columns):
             if len(names) != columns.shape[1]:
                 raise ValueError('number of columns ({}) does not match number of column names ({})'.format(columns.shape[1], len(names)))
             for i, name in enumerate(names):
-                self.columns[name] = ColumnSparse(columns, i)
-                self.column_names.append(name)
-                self._sparse_matrices[name] = columns
-                self._save_assign_expression(name, Expression(self, name))
+                valid_name = vaex.utils.find_valid_name(name, used=self.get_column_names(hidden=True))
+                if name != valid_name:
+                    self._column_aliases[name] = valid_name
+                self.columns[valid_name] = ColumnSparse(columns, i)
+                self.column_names.append(valid_name)
+                self._sparse_matrices[valid_name] = columns
+                self._save_assign_expression(valid_name, Expression(self, valid_name))
         else:
             raise ValueError('only scipy.sparse.csr_matrix is supported')
 
@@ -3250,21 +3253,26 @@ def add_virtual_column(self, name, expression, unique=False):
                 expression = expression.copy(self)
         column_position = len(self.column_names)
         # if the current name is an existing column name....
-        if name in self.get_column_names():
+        if name in self.get_column_names(hidden=True):
             column_position = self.column_names.index(name)
             renamed = vaex.utils.find_valid_name('__' +name, used=self.get_column_names(hidden=True))
             # we rewrite all existing expressions (including the passed down expression argument)
             self._rename(name, renamed)
         expression = _ensure_string_from_expression(expression)
 
-        name = vaex.utils.find_valid_name(name, used=[] if not unique else self.get_column_names())
+        if vaex.utils.find_valid_name(name) != name:
+            # if we have to rewrite the name, we need to make it unique
+            unique = True
+        valid_name = vaex.utils.find_valid_name(name, used=[] if not unique else self.get_column_names(hidden=True))
+        if name != valid_name:
+            self._column_aliases[name] = valid_name
 
-        self.virtual_columns[name] = expression
-        self._virtual_expressions[name] = Expression(self, expression)
+        self.virtual_columns[valid_name] = expression
+        self._virtual_expressions[valid_name] = Expression(self, expression)
         if name not in self.column_names:
-            self.column_names.insert(column_position, name)
-        self._save_assign_expression(name)
-        self.signal_column_changed.emit(self, name, "add")
+            self.column_names.insert(column_position, valid_name)
+        self._save_assign_expression(valid_name)
+        self.signal_column_changed.emit(self, valid_name, "add")
         # self.write_virtual_meta()
 
     def rename(self, name, new_name, unique=False):
@@ -3523,12 +3531,14 @@ def _as_table(self, i1, i2, j1=None, j2=None, format='html'):
         parts = []  # """<div>%s (length=%d)</div>""" % (self.name, len(self))]
         parts += ["<table class='table-striped'>"]
 
-        column_names = self.get_column_names()
+        aliases_reverse = {value: key for key, value in self._column_aliases.items()}
+        # we need to get the underlying names since we use df.evaluate
+        column_names = self.get_column_names(alias=False)
         values_list = []
         values_list.append(['#', []])
         # parts += ["<thead><tr>"]
         for name in column_names:
-            values_list.append([name, []])
+            values_list.append([aliases_reverse.get(name, name), []])
             # parts += ["<th>%s</th>" % name]
         # parts += ["</tr></thead>"]
 
@@ -3699,7 +3709,7 @@ def get_names(self, hidden=False):
         names = self.get_column_names(hidden=hidden)
         return names + [k for k in self.variables.keys() if not hidden or not k.startswith('__')]
 
-    def get_column_names(self, virtual=True, strings=True, hidden=False, regex=None):
+    def get_column_names(self, virtual=True, strings=True, hidden=False, regex=None, alias=True):
         """Return a list of column names
 
         Example:
@@ -3718,19 +3728,10 @@ def get_column_names(self, virtual=True, strings=True, hidden=False, regex=None)
         :param hidden: If False, skip hidden columns
         :param strings: If False, skip string columns
         :param regex: Only return column names matching the (optional) regular expression
+        :param alias: Return the alias (True) or internal name (False).
         :rtype: list of str
-
-        Example:
-        >>> import vaex
-        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
-        >>> df['r'] = (df.x**2 + df.y**2)**2
-        >>> df.get_column_names()
-        ['x', 'x2', 'y', 's', 'r']
-        >>> df.get_column_names(virtual=False)
-        ['x', 'x2', 'y', 's']
-        >>> df.get_column_names(regex='x.*')
-        ['x', 'x2']
         """
+        aliases_reverse = {value: key for key, value in self._column_aliases.items()} if alias else {}
         def column_filter(name):
             '''Return True if column with specified name should be returned'''
             if regex and not re.match(regex, name):
@@ -3742,11 +3743,11 @@ def column_filter(name):
             if not hidden and name.startswith('__'):
                 return False
             return True
-        if hidden and virtual and regex is None:
+        if hidden and virtual and regex is None and not alias:
             return list(self.column_names)  # quick path
-        if not hidden and virtual and regex is None:
+        if not hidden and virtual and regex is None and not alias:
             return [k for k in self.column_names if not k.startswith('__')]  # also a quick path
-        return [name for name in self.column_names if column_filter(name)]
+        return [aliases_reverse.get(name, name) for name in self.column_names if column_filter(name)]
 
     def __bool__(self):
         return True  # we are always true :) otherwise Python might call __len__, which can be expensive
@@ -4546,12 +4547,12 @@ def __getitem__(self, item):
             names = self.get_column_names()
             return [self.evaluate(name, item, item+1)[0] for name in names]
         elif isinstance(item, six.string_types):
+            if item in self._column_aliases:
+                item = self._column_aliases[item]  # translate the alias name into the real name
             if hasattr(self, item) and isinstance(getattr(self, item), Expression):
                 return getattr(self, item)
             # if item in self.virtual_columns:
             #   return Expression(self, self.virtual_columns[item])
-            if item in self._column_aliases:
-                item = self._column_aliases[item]  # translate the alias name into the real name
             # if item in self._virtual_expressions:
             #     return self._virtual_expressions[item]
             self.validate_expression(item)
@@ -4930,8 +4931,8 @@ def copy(self, column_names=None, virtual=True):
         df.variables.update(self.variables)  # we add all, could maybe only copy used
         df._categories.update(self._categories)
         if column_names is None:
-            column_names = self.get_column_names(hidden=True)
-        all_column_names = self.get_column_names(hidden=True)
+            column_names = self.get_column_names(hidden=True, alias=False)
+        all_column_names = self.get_column_names(hidden=True, alias=False)
 
         # put in the selections (thus filters) in place
         # so drop moves instead of really dropping it
diff --git a/packages/vaex-core/vaex/utils.py b/packages/vaex-core/vaex/utils.py
--- a/packages/vaex-core/vaex/utils.py
+++ b/packages/vaex-core/vaex/utils.py
@@ -538,7 +538,20 @@ def unlistify(waslist, *args):
 
 def find_valid_name(name, used=[]):
     first, rest = name[0], name[1:]
-    name = re.sub("[^a-zA-Z_]", "_", first) + re.sub("[^a-zA-Z_0-9]", "_", rest)
+    if not first.isidentifier():
+        if ('col_' + first).isidentifier():
+            first = 'col_' + first
+        else:
+            first = 'col_'
+    name = first
+    for char in rest:
+        # we test if it is an identifier with _ prefixed, since not every first character
+        # and following character are treated the same
+        # https://docs.python.org/3/reference/lexical_analysis.html#identifiers
+        if not ('_' + char).isidentifier():
+            name += '_'
+        else:
+            name += char
     if keyword.iskeyword(name):
         name += '_'
     if name in used:
diff --git a/packages/vaex-hdf5/vaex/hdf5/export.py b/packages/vaex-hdf5/vaex/hdf5/export.py
--- a/packages/vaex-hdf5/vaex/hdf5/export.py
+++ b/packages/vaex-hdf5/vaex/hdf5/export.py
@@ -135,7 +135,7 @@ def export_hdf5(dataset, path, column_names=None, byteorder="=", shuffle=False,
         logger.debug("virtual=%r", virtual)
         logger.debug("exporting %d rows to file %s" % (N, path))
         # column_names = column_names or (dataset.get_column_names() + (list(dataset.virtual_columns.keys()) if virtual else []))
-        column_names = column_names or dataset.get_column_names(virtual=virtual, strings=True)
+        column_names = column_names or dataset.get_column_names(virtual=virtual, strings=True, alias=False)
 
         logger.debug("exporting columns(hdf5): %r" % column_names)
         sparse_groups = collections.defaultdict(list)
@@ -148,7 +148,7 @@ def export_hdf5(dataset, path, column_names=None, byteorder="=", shuffle=False,
                 sparse_matrices[id(sparse_matrix)] = sparse_matrix
                 continue
             dtype = dataset.dtype(column_name)
-            if column_name in dataset.get_column_names(virtual=False):
+            if column_name in dataset.get_column_names(virtual=False, alias=False):
                 column = dataset.columns[column_name]
                 shape = (N,) + column.shape[1:]
             else:


Test patch:
diff --git a/tests/column_names_test.py b/tests/column_names_test.py
--- a/tests/column_names_test.py
+++ b/tests/column_names_test.py
@@ -24,6 +24,8 @@ def test_column_names(ds_local):
 def test_add_invalid_name(tmpdir):
     # support invalid names and keywords
     df = vaex.from_dict({'X!1': x, 'class': x*2})
+    assert df.get_column_names() == ['X!1', 'class']
+    assert df.get_column_names(alias=False) != ['X!1', 'class']
     assert df['X!1'].tolist() == x.tolist()
     assert (df['X!1']*2).tolist() == (x*2).tolist()
     assert (df['class']).tolist() == (x*2).tolist()
@@ -36,6 +38,30 @@ def test_add_invalid_name(tmpdir):
     assert df['X!1'].tolist() == x.tolist()
     assert (df.copy()['X!1']*2).tolist() == (x*2).tolist()
 
+
+def test_add_invalid_virtual_columns(df_local):
+    df = df_local
+     # TODO: support for remote
+    df['1'] = df.x
+    assert df['1'].expression != '1'
+    df['2'] = df.y
+    assert df['1'].tolist() == df.x.tolist()
+    assert df['2'].tolist() == df.y.tolist()
+
+
+def test_not_hide_invalid_name():
+    x = np.arange(10)
+    df = vaex.from_dict({'./bla': x})
+    assert len(df.get_column_names()) == 1
+    assert df['./bla'].tolist() == x.tolist()
+
+
+def test_unicode_names():
+    x = np.arange(10)
+    df = vaex.from_dict({'远': x})
+    assert df.远.tolist() == x.tolist()
+
+
 def test_invalid_name_read(tmpdir):
     # earlier version of vaex could write invalid names, check if we can read those
     df = vaex.from_dict({'x': x})
diff --git a/tests/repr_test.py b/tests/repr_test.py
--- a/tests/repr_test.py
+++ b/tests/repr_test.py
@@ -3,6 +3,12 @@
 import datetime
 
 
+def test_repr_invalid_name(df):
+    df['is'] = df.x * 1
+    code = df._repr_mimebundle_()['text/plain']
+    assert '_is' not in code, "the repr should show the aliased name"
+
+
 def test_repr_default(df):
     code = df._repr_mimebundle_()['text/plain']
     assert 'x' in code
diff --git a/tests/sparse_test.py b/tests/sparse_test.py
--- a/tests/sparse_test.py
+++ b/tests/sparse_test.py
@@ -2,14 +2,26 @@
 from scipy.sparse import csc_matrix, csr_matrix
 
 
+x = np.arange(3)
+s = csr_matrix([[0, 1], [1, 0], [2, 3]])
+
+
 def test_sparse_basics():
-    x = np.arange(3)
-    s = csr_matrix([[0, 1], [1, 0], [2, 3]])
-    ds = vaex.from_arrays(x=x)
-    ds.add_columns(['s1', 's2'], s)
-    assert ds.s1.tolist() == [0, 1, 2]
-    assert ds.s2.tolist() == [1, 0, 3]
-    assert "error" not in repr(ds)
+    df = vaex.from_arrays(x=x)
+    df.add_columns(['s1', 's2'], s)
+    assert df.s1.tolist() == [0, 1, 2]
+    assert df.s2.tolist() == [1, 0, 3]
+    assert "error" not in repr(df)
+
+
+def test_sparse_repr():
+    df = vaex.from_arrays(x=x)
+    df.add_columns(['is', '9'], s)
+    assert df['is'].tolist() == [0, 1, 2]
+    assert df['9'].tolist() == [1, 0, 3]
+    assert "error" not in repr(df)
+    assert "_is" not in repr(df)
+
 
 
 def test_sparse_export(tmpdir):