Repository: vaexio/vaex

Issue:
Better handling when a non-existend column is used.
We need a better way to catch an error when a user makes a typo and queries a column that does not exist in a `DataFrame`. For example:
```
import vaex
ds = vaex.example()
ds[['x', 'xy']]   # gives a not so friendly error
```

On the other hand:
```
import vaex
ds = vaex.example()
ds['xy']   # gives a helpful error message
```



Discussion:

Resolving patch:
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -646,6 +646,9 @@ def _compute_agg(self, name, expression, binby=[], limits=None, shape=default_sh
         if extra_expressions:
             extra_expressions = _ensure_strings_from_expressions(extra_expressions)
         expression_waslist, [expressions,] = vaex.utils.listify(expression)
+        for expression in expressions:
+            if expression and expression != "*":
+                self.validate_expression(expression)
         assert self._aggregator_nest_count == 0, "detected nested aggregator call"
         # Instead of 'expression is not None', we would like to have 'not virtual'
         # but in agg.py we do some casting, which results in calling .dtype(..) with a non-column
@@ -2849,9 +2852,16 @@ def getitem(df, item):
     def validate_expression(self, expression):
         """Validate an expression (may throw Exceptions)"""
         # return self.evaluate(expression, 0, 2)
-        vars = set(self.get_column_names()) | set(self.variables.keys())
-        funcs = set(expression_namespace.keys())
-        return vaex.expresso.validate_expression(expression, vars, funcs)
+        if str(expression) in self.virtual_columns:
+            return
+        if self.is_local() and str(expression) in self.columns:
+            return
+        vars = set(self.get_names(hidden=True))
+        funcs = set(expression_namespace.keys())  | set(self.functions.keys())
+        try:
+            return vaex.expresso.validate_expression(expression, vars, funcs)
+        except NameError as e:
+            raise NameError(str(e)) from None
 
     def _block_scope(self, i1, i2):
         variables = {key: self.evaluate_variable(key) for key in self.variables.keys()}
@@ -4497,6 +4507,7 @@ def __getitem__(self, item):
                 item = self._column_aliases[item]  # translate the alias name into the real name
             # if item in self._virtual_expressions:
             #     return self._virtual_expressions[item]
+            self.validate_expression(item)
             return Expression(self, item)  # TODO we'd like to return the same expression if possible
         elif isinstance(item, Expression):
             expression = item.expression
@@ -4512,6 +4523,8 @@ def __getitem__(self, item):
                 elif isinstance(item[1], slice):
                     names = self.get_column_names().__getitem__(item[1])
                     return df[names]
+            for expression in item:
+                self.validate_expression(expression)
             df = self.copy(column_names=item)
             return df
         elif isinstance(item, slice):
@@ -4722,6 +4735,9 @@ def _create_grid(self, binby, limits, shape, delay=False):
         else:
             binbys = [binby]
         binbys = _ensure_strings_from_expressions(binbys)
+        for expression in binbys:
+            if expression:
+                self.validate_expression(expression)
         binners = []
         if len(binbys):
             limits = _expand_limits(limits, len(binbys))
@@ -4910,6 +4926,7 @@ def copy(self, column_names=None, virtual=True):
                         depending.update(deps)
                 else:
                     # this might be an expression, create a valid name
+                    self.validate_expression(name)
                     expression = name
                     name = vaex.utils.find_valid_name(name)
                     # add the expression
diff --git a/packages/vaex-core/vaex/expresso.py b/packages/vaex-core/vaex/expresso.py
--- a/packages/vaex-core/vaex/expresso.py
+++ b/packages/vaex-core/vaex/expresso.py
@@ -10,6 +10,8 @@
 import sys
 import six
 import copy
+import difflib
+
 
 logger = logging.getLogger("expr")
 logger.setLevel(logging.ERROR)
@@ -82,11 +84,17 @@ def validate_expression(expr, variable_set, function_set=[], names=None):
     elif isinstance(expr, _ast.Name):
         validate_id(expr.id)
         if expr.id not in variable_set:
-            raise NameError("variable %r is not defined (available are: %s)" % (
-                expr.id, ", ".join(list(variable_set))))
+            matches = difflib.get_close_matches(expr.id, list(variable_set))
+            msg = "Column or variable %r does not exist." % expr.id
+            if matches:
+                msg += ' Did you mean: ' + " or ".join(map(repr, matches))
+
+            raise NameError(msg)
         names.append(expr.id)
     elif isinstance(expr, _ast.Num):
         pass  # numbers are fine
+    elif isinstance(expr, _ast.Str):
+        pass  # as well as strings
     elif isinstance(expr, _ast.Call):
         validate_func(expr.func, function_set)
         last_func = expr


Test patch:
diff --git a/tests/copy_test.py b/tests/copy_test.py
--- a/tests/copy_test.py
+++ b/tests/copy_test.py
@@ -1,4 +1,6 @@
 from common import *
+import pytest
+
 
 def test_copy(df):
     df = df
@@ -10,4 +12,9 @@ def test_copy(df):
     assert 'v' in dfc.get_column_names()
     assert 'v' in dfc.virtual_columns
     assert 'myvar' in dfc.variables
-    dfc.x.values
\ No newline at end of file
+    dfc.x.values
+
+def test_non_existing_column(df_local):
+    df = df_local
+    with pytest.raises(NameError, match='.*Did you.*'):
+        df.copy(column_names=['x', 'x_'])
diff --git a/tests/getattr_test.py b/tests/getattr_test.py
--- a/tests/getattr_test.py
+++ b/tests/getattr_test.py
@@ -92,3 +92,8 @@ def test_access_data_after_virtual_column_creation(ds_local):
     ds['virtual'] = ds.x * 2
     # it should also work after we added a virtual column
     assert ds[['x']].values[:,0].tolist() == ds.x.values.tolist()
+
+def test_non_existing_column(df_local):
+    df = df_local
+    with pytest.raises(NameError, match='.*Did you.*'):
+        df['x_']