Repository: statsmodels/statsmodels

Issue:
PatsyError with custom function when trying to plot partial regression
#### Describe the bug

I am experiencing an `PatsyError` where `patsy` dose not recognize the custom function `lg()` in the name space. This happened with the `statsmodels.api.graphics.plot_partregress()` function.

#### Code Sample, a copy-pastable example if possible


```python
import statsmodels
import statsmodels.api as sm
import patsy
from patsy import dmatrix, dmatrices
import pandas as pd
import numpy as np

@np.vectorize
def lg(x):
    return np.log10(x) if x > 0 else 0

df = pd.DataFrame(
    dict(
        a=np.random.random(size=10),
        b=np.random.random(size=10),
        c=np.random.random(size=10),
    )
)

sm.graphics.plot_partregress(
    "a", "lg(b)", ["c"], obs_labels=False, data=df
)
```
<details>
I tried to add kwarg `eval_env=1` (as well as 2 and 3), but the error remains.

``` python
    ---------------------------------------------------------------------------
    NameError                                 Traceback (most recent call last)
    /usr/local/lib/python3.7/dist-packages/patsy/compat.py in call_and_wrap_exc(msg, origin, f, *args, **kwargs)
         35     try:
    ---> 36         return f(*args, **kwargs)
         37     except Exception as e:
    
    11 frames
    /usr/local/lib/python3.7/dist-packages/patsy/eval.py in eval(self, expr, source_name, inner_namespace)
        165         return eval(code, {}, VarLookupDict([inner_namespace]
    --> 166                                             + self._namespaces))
        167 
    
    <string> in <module>()
    
    NameError: name 'lg' is not defined
    
    The above exception was the direct cause of the following exception:
    
    PatsyError                                Traceback (most recent call last)
    <ipython-input-11-d075df7a2fe3> in <module>()
         19 
         20 sm.graphics.plot_partregress(
    ---> 21     "a", "lg(b)", ["c"], obs_labels=False, data=df, eval_env=1
         22 )
    
    /usr/local/lib/python3.7/dist-packages/statsmodels/graphics/regressionplots.py in plot_partregress(endog, exog_i, exog_others, data, title_kwargs, obs_labels, label_kwargs, ax, ret_coords, **kwargs)
        402         RHS_isemtpy = True
        403     if isinstance(exog_i, str):
    --> 404         exog_i = dmatrix(exog_i + "-1", data)
        405 
        406     # all arrays or pandas-like
    
    /usr/local/lib/python3.7/dist-packages/patsy/highlevel.py in dmatrix(formula_like, data, eval_env, NA_action, return_type)
        289     eval_env = EvalEnvironment.capture(eval_env, reference=1)
        290     (lhs, rhs) = _do_highlevel_design(formula_like, data, eval_env,
    --> 291                                       NA_action, return_type)
        292     if lhs.shape[1] != 0:
        293         raise PatsyError("encountered outcome variables for a model "
    
    /usr/local/lib/python3.7/dist-packages/patsy/highlevel.py in _do_highlevel_design(formula_like, data, eval_env, NA_action, return_type)
        163         return iter([data])
        164     design_infos = _try_incr_builders(formula_like, data_iter_maker, eval_env,
    --> 165                                       NA_action)
        166     if design_infos is not None:
        167         return build_design_matrices(design_infos, data,
    
    /usr/local/lib/python3.7/dist-packages/patsy/highlevel.py in _try_incr_builders(formula_like, data_iter_maker, eval_env, NA_action)
         68                                       data_iter_maker,
         69                                       eval_env,
    ---> 70                                       NA_action)
         71     else:
         72         return None
    
    /usr/local/lib/python3.7/dist-packages/patsy/build.py in design_matrix_builders(termlists, data_iter_maker, eval_env, NA_action)
        694                                                    factor_states,
        695                                                    data_iter_maker,
    --> 696                                                    NA_action)
        697     # Now we need the factor infos, which encapsulate the knowledge of
        698     # how to turn any given factor into a chunk of data:
    
    /usr/local/lib/python3.7/dist-packages/patsy/build.py in _examine_factor_types(factors, factor_states, data_iter_maker, NA_action)
        441     for data in data_iter_maker():
        442         for factor in list(examine_needed):
    --> 443             value = factor.eval(factor_states[factor], data)
        444             if factor in cat_sniffers or guess_categorical(value):
        445                 if factor not in cat_sniffers:
    
    /usr/local/lib/python3.7/dist-packages/patsy/eval.py in eval(self, memorize_state, data)
        564         return self._eval(memorize_state["eval_code"],
        565                           memorize_state,
    --> 566                           data)
        567 
        568     __getstate__ = no_pickling
    
    /usr/local/lib/python3.7/dist-packages/patsy/eval.py in _eval(self, code, memorize_state, data)
        549                                  memorize_state["eval_env"].eval,
        550                                  code,
    --> 551                                  inner_namespace=inner_namespace)
        552 
        553     def memorize_chunk(self, state, which_pass, data):
    
    /usr/local/lib/python3.7/dist-packages/patsy/compat.py in call_and_wrap_exc(msg, origin, f, *args, **kwargs)
         41                                  origin)
         42             # Use 'exec' to hide this syntax from the Python 2 parser:
    ---> 43             exec("raise new_exc from e")
         44         else:
         45             # In python 2, we just let the original exception escape -- better
    
    /usr/local/lib/python3.7/dist-packages/patsy/compat.py in <module>()
    
    PatsyError: Error evaluating factor: NameError: name 'lg' is not defined
        lg(b)-1
        ^^^^^
 ```


</details>


#### Expected Output

I expect the `lg()` function can be recognized by `patsy`.

#### Output of ``import statsmodels.api as sm; sm.show_versions()``

<details>
INSTALLED VERSIONS
------------------
Python: 3.7.11.final.0
OS: Linux 5.4.104+ #1 SMP Sat Jun 5 09:50:34 PDT 2021 x86_64
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8

statsmodels
===========

Installed: 0.12.2 (/usr/local/lib/python3.7/dist-packages/statsmodels)

Required Dependencies
=====================

cython: 0.29.24 (/usr/local/lib/python3.7/dist-packages/Cython)
numpy: 1.19.5 (/usr/local/lib/python3.7/dist-packages/numpy)
scipy: 1.4.1 (/usr/local/lib/python3.7/dist-packages/scipy)
pandas: 1.1.5 (/usr/local/lib/python3.7/dist-packages/pandas)
    dateutil: 2.8.2 (/usr/local/lib/python3.7/dist-packages/dateutil)
patsy: 0.5.1 (/usr/local/lib/python3.7/dist-packages/patsy)

Optional Dependencies
=====================

matplotlib: 3.2.2 (/usr/local/lib/python3.7/dist-packages/matplotlib)
    backend: module://ipykernel.pylab.backend_inline 
cvxopt: 1.2.6 (/usr/local/lib/python3.7/dist-packages/cvxopt)
joblib: 1.0.1 (/usr/local/lib/python3.7/dist-packages/joblib)

Developer Tools
================

IPython: 5.5.0 (/usr/local/lib/python3.7/dist-packages/IPython)
    jinja2: 2.11.3 (/usr/local/lib/python3.7/dist-packages/jinja2)
sphinx: 1.8.5 (/usr/local/lib/python3.7/dist-packages/sphinx)
    pygments: 2.6.1 (/usr/local/lib/python3.7/dist-packages/pygments)
pytest: 3.6.4 (/usr/local/lib/python3.7/dist-packages)
virtualenv: Not installed

</details>



Discussion:
Does not work for me either.
I don't understand why adding `eval_env` to the dmatrix calls does not help. :(

(I'm running the example in a notebook.)
I still have no idea why patsy doesn't find it.

However, putting the call to plot_partialregress into a unit test function and eval_env=1, works with a pytest run of the unit test

Also putting the code in a function in the notebook works

```
def plot_pr():
    @np.vectorize
    def lg(x):
        return np.log10(x) if x > 0 else 0  
    fig = sm.graphics.plot_partregress(
            "a", "lg(b)", ["c"], obs_labels=False, data=df, eval_env=1
        )
    return fig
plot_pr();
```

But I have no idea why a plain script (or notebook) doesn't work.

another version that I managed to get to work is to put `lg` inside the np namespace

```
np.lg = lg

sm.graphics.plot_partregress(
    "a", "np.lg(b)", ["c"], obs_labels=False, data=df, eval_env=1
)
```

(I'm using python 3.9.2 )
I still have no idea why patsy doesn't find it.

However, putting the call to plot_partialregress into a unit test function and eval_env=1, works with a pytest run of the unit test

Also putting the code in a function in the notebook works

```
def plot_pr():
    @np.vectorize
    def lg(x):
        return np.log10(x) if x > 0 else 0  
    fig = sm.graphics.plot_partregress(
            "a", "lg(b)", ["c"], obs_labels=False, data=df, eval_env=1
        )
    return fig
plot_pr();
```

But I have no idea why a plain script (or notebook) doesn't work.

another version that I managed to get to work is to put `lg` inside the np namespace

```
np.lg = lg

sm.graphics.plot_partregress(
    "a", "np.lg(b)", ["c"], obs_labels=False, data=df, eval_env=1
)
```

(I'm using python 3.9.2 )
The easiest workaround is to put the transformed variable into a dataframe column.
Thanks @josef-pkt for the effort and the 2 possible workarounds. I didn't know that you can assign a function into `numpy`'s name space.
trying again today, the eval_env option also works in notebook directly.
(not sure what was going on)

PR #7673

Resolving patch:
diff --git a/statsmodels/graphics/regressionplots.py b/statsmodels/graphics/regressionplots.py
--- a/statsmodels/graphics/regressionplots.py
+++ b/statsmodels/graphics/regressionplots.py
@@ -301,7 +301,7 @@ def _partial_regression(endog, exog_i, exog_others):
 
 def plot_partregress(endog, exog_i, exog_others, data=None,
                      title_kwargs={}, obs_labels=True, label_kwargs={},
-                     ax=None, ret_coords=False, **kwargs):
+                     ax=None, ret_coords=False, eval_env=1, **kwargs):
     """Plot partial regression for a single regressor.
 
     Parameters
@@ -337,6 +337,9 @@ def plot_partregress(endog, exog_i, exog_others, data=None,
     ret_coords : bool
         If True will return the coordinates of the points in the plot. You
         can use this to add your own annotations.
+    eval_env : int
+        Patsy eval environment if user functions and formulas are used in
+        defining endog or exog.
     **kwargs
         The keyword arguments passed to plot for the points.
 
@@ -384,16 +387,16 @@ def plot_partregress(endog, exog_i, exog_others, data=None,
     #NOTE: there is no interaction between possible missing data and
     #obs_labels yet, so this will need to be tweaked a bit for this case
     fig, ax = utils.create_mpl_ax(ax)
-
+    print("eval_env:", eval_env)
     # strings, use patsy to transform to data
     if isinstance(endog, str):
-        endog = dmatrix(endog + "-1", data)
+        endog = dmatrix(endog + "-1", data, eval_env=eval_env)
 
     if isinstance(exog_others, str):
-        RHS = dmatrix(exog_others, data)
+        RHS = dmatrix(exog_others, data, eval_env=eval_env)
     elif isinstance(exog_others, list):
         RHS = "+".join(exog_others)
-        RHS = dmatrix(RHS, data)
+        RHS = dmatrix(RHS, data, eval_env=eval_env)
     else:
         RHS = exog_others
     RHS_isemtpy = False
@@ -402,7 +405,7 @@ def plot_partregress(endog, exog_i, exog_others, data=None,
     elif isinstance(RHS, pd.DataFrame) and RHS.empty:
         RHS_isemtpy = True
     if isinstance(exog_i, str):
-        exog_i = dmatrix(exog_i + "-1", data)
+        exog_i = dmatrix(exog_i + "-1", data, eval_env=eval_env)
 
     # all arrays or pandas-like
 


Test patch:
diff --git a/statsmodels/graphics/tests/test_regressionplots.py b/statsmodels/graphics/tests/test_regressionplots.py
--- a/statsmodels/graphics/tests/test_regressionplots.py
+++ b/statsmodels/graphics/tests/test_regressionplots.py
@@ -348,3 +348,25 @@ def test_ceres_poisson(self, close_figures):
                 ax.set_title(ti + "\nPoisson regression\n" +
                              effect_str)
                 close_or_save(pdf, fig)
+
+
+@pytest.mark.matplotlib
+def test_partregress_formula_env():
+    # test that user function in formulas work, see #7672
+
+    @np.vectorize
+    def lg(x):
+        return np.log10(x) if x > 0 else 0
+
+    df = DataFrame(
+        dict(
+            a=np.random.random(size=10),
+            b=np.random.random(size=10),
+            c=np.random.random(size=10),
+            )
+        )
+    sm.graphics.plot_partregress(
+        "a", "lg(b)", ["c"], obs_labels=False, data=df, eval_env=1)
+
+    sm.graphics.plot_partregress(
+        "a", "lg(b)", ["c"], obs_labels=False, data=df)