Repository: modin-project/modin

Issue:
Local variables not yet supported in eval.
I encounter this message when trying to use modin 
Local variables not yet supported in eval.
To request implementation, send an email to feature_requests@modin.org

here's a code 
![Screenshot (314)](https://user-images.githubusercontent.com/35177448/92144852-9b934580-ee17-11ea-8b1a-cb57298ab752.png)




Discussion:
Thanks @AhmedSalam22 for the request! 

This is related to (but not the same as) #1369. Since the data is distributed, not all workers will have access to the variable so it must be copied/broadcasted. This may not be desired in many cases for performance reasons, so we're still working on some way to solve this issue generally.
@anmyachev Anatoly, can you please update and close this issue?

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -1957,8 +1957,8 @@ def quantile_builder(df, **kwargs):
         return result.transpose() if axis == 1 else result
 
     def query(self, expr, **kwargs):
-        def query_builder(df, **kwargs):
-            return df.query(expr, inplace=False, **kwargs)
+        def query_builder(df, **modin_internal_kwargs):
+            return df.query(expr, inplace=False, **kwargs, **modin_internal_kwargs)
 
         return self.__constructor__(
             self._modin_frame.filter_full_axis(1, query_builder)
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -26,6 +26,7 @@
 from pandas._libs.lib import no_default
 from pandas._typing import StorageOptions
 
+import re
 import itertools
 import functools
 import numpy as np
@@ -819,12 +820,50 @@ def explode(
             pandas.DataFrame.explode, column, ignore_index=ignore_index
         )
 
+    def _update_var_dicts_in_kwargs(self, expr, kwargs):
+        """
+        Copy variables with "@" prefix in `local_dict` and `global_dict` keys of kwargs.
+
+        Parameters
+        ----------
+        expr : str
+            The expression string to search variables with "@" prefix.
+        kwargs : dict
+            See the documentation for eval() for complete details on the keyword arguments accepted by query().
+        """
+        if "@" not in expr:
+            return
+        frame = sys._getframe()
+        try:
+            f_locals = frame.f_back.f_back.f_locals
+            f_globals = frame.f_back.f_back.f_globals
+        finally:
+            del frame
+        local_names = set(re.findall(r"@([\w]+)", expr))
+        local_dict = {}
+        global_dict = {}
+
+        for name in local_names:
+            for dct_out, dct_in in ((local_dict, f_locals), (global_dict, f_globals)):
+                try:
+                    dct_out[name] = dct_in[name]
+                except KeyError:
+                    pass
+
+        if local_dict:
+            local_dict.update(kwargs.get("local_dict") or {})
+            kwargs["local_dict"] = local_dict
+        if global_dict:
+            global_dict.update(kwargs.get("global_dict") or {})
+            kwargs["global_dict"] = global_dict
+
     def eval(self, expr, inplace=False, **kwargs):  # noqa: PR01, RT01, D200
         """
         Evaluate a string describing operations on ``DataFrame`` columns.
         """
         self._validate_eval_query(expr, **kwargs)
         inplace = validate_bool_kwarg(inplace, "inplace")
+        self._update_var_dicts_in_kwargs(expr, kwargs)
         new_query_compiler = self._query_compiler.eval(expr, **kwargs)
         return_type = type(
             pandas.DataFrame(columns=self.columns)
@@ -1677,6 +1716,7 @@ def query(self, expr, inplace=False, **kwargs):  # noqa: PR01, RT01, D200
         Query the columns of a ``DataFrame`` with a boolean expression.
         """
         ErrorMessage.non_verified_udf()
+        self._update_var_dicts_in_kwargs(expr, kwargs)
         self._validate_eval_query(expr, **kwargs)
         inplace = validate_bool_kwarg(inplace, "inplace")
         new_query_compiler = self._query_compiler.query(expr, **kwargs)
@@ -2855,9 +2895,6 @@ def _validate_eval_query(self, expr, **kwargs):
         if isinstance(expr, str) and expr == "":
             raise ValueError("expr cannot be an empty string")
 
-        if isinstance(expr, str) and "@" in expr:
-            ErrorMessage.not_implemented("Local variables not yet supported in eval.")
-
         if isinstance(expr, str) and "not" in expr:
             if "parser" in kwargs and kwargs["parser"] == "python":
                 ErrorMessage.not_implemented(


Test patch:
diff --git a/modin/pandas/test/dataframe/test_udf.py b/modin/pandas/test/dataframe/test_udf.py
--- a/modin/pandas/test/dataframe/test_udf.py
+++ b/modin/pandas/test/dataframe/test_udf.py
@@ -227,6 +227,19 @@ def test_eval_df_arithmetic_subexpression():
     df_equals(modin_df, df)
 
 
+TEST_VAR = 2
+
+
+@pytest.mark.parametrize("method", ["query", "eval"])
+@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+@pytest.mark.parametrize("local_var", [2])
+def test_eval_and_query_with_local_and_global_var(method, data, local_var):
+    modin_df, pandas_df = pd.DataFrame(data), pandas.DataFrame(data)
+    op = "+" if method == "eval" else "<"
+    for expr in (f"col1 {op} @local_var", f"col1 {op} @TEST_VAR"):
+        df_equals(getattr(modin_df, method)(expr), getattr(pandas_df, method)(expr))
+
+
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_filter(data):
     modin_df = pd.DataFrame(data)
@@ -289,12 +302,6 @@ def test_query(data, funcs):
     modin_df = pd.DataFrame(data)
     pandas_df = pandas.DataFrame(data)
 
-    with pytest.raises(ValueError):
-        modin_df.query("")
-    with pytest.raises(NotImplementedError):
-        x = 2  # noqa F841
-        modin_df.query("col1 < @x")
-
     try:
         pandas_result = pandas_df.query(funcs)
     except Exception as e:
@@ -305,6 +312,13 @@ def test_query(data, funcs):
         df_equals(modin_result, pandas_result)
 
 
+def test_empty_query():
+    modin_df = pd.DataFrame([1, 2, 3, 4, 5])
+
+    with pytest.raises(ValueError):
+        modin_df.query("")
+
+
 def test_query_after_insert():
     modin_df = pd.DataFrame({"x": [-1, 0, 1, None], "y": [1, 2, None, 3]})
     modin_df["z"] = modin_df.eval("x / y")