Repository: statsmodels/statsmodels

Issue:
ENH: Add xvals param to lowess smoother
xvals allows evaluation of the lowess smoother at
x-values other than those used in the smoothing, for example
fitting from some points and using that to 'predict' others

- [x] closes #6543
- [x] tests added / passed. 
- [x] code/documentation is well formatted.  
- [x] properly formatted commit message. See 
      [NumPy's guide](https://docs.scipy.org/doc/numpy-1.15.1/dev/gitwash/development_workflow.html#writing-the-commit-message). 

<details>


**Notes**:

* It is essential that you add a test when making code changes. Tests are not 
  needed for doc changes.
* When adding a new function, test values should usually be verified in another package (e.g., R/SAS/Stata).
* When fixing a bug, you must add a test that would produce the bug in master and
  then show that it is fixed with the new code.
* New code additions must be well formatted. Changes should pass flake8. If on Linux or OSX, you can
  verify you changes are well formatted by running 
  ```
  git diff upstream/master -u -- "*.py" | flake8 --diff --isolated
  ```
  assuming `flake8` is installed. This command is also available on Windows 
  using the Windows System for Linux once `flake8` is installed in the 
  local Linux environment. While passing this test is not required, it is good practice and it help 
  improve code quality in `statsmodels`.
* Docstring additions must render correctly, including escapes and LaTeX.

</details>



Discussion:

Resolving patch:
diff --git a/statsmodels/nonparametric/smoothers_lowess.py b/statsmodels/nonparametric/smoothers_lowess.py
--- a/statsmodels/nonparametric/smoothers_lowess.py
+++ b/statsmodels/nonparametric/smoothers_lowess.py
@@ -10,7 +10,7 @@
 import numpy as np
 from ._smoothers_lowess import lowess as _lowess
 
-def lowess(endog, exog, frac=2.0/3.0, it=3, delta=0.0, is_sorted=False,
+def lowess(endog, exog, frac=2.0/3.0, it=3, delta=0.0, xvals=None, is_sorted=False,
            missing='drop', return_sorted=True):
     '''LOWESS (Locally Weighted Scatterplot Smoothing)
 
@@ -32,10 +32,14 @@ def lowess(endog, exog, frac=2.0/3.0, it=3, delta=0.0, is_sorted=False,
     delta : float
         Distance within which to use linear-interpolation
         instead of weighted regression.
+    xvals: 1-D numpy array
+        Values of the exogenous variable at which to evaluate the regression.
+        If supplied, cannot use delta.
     is_sorted : bool
         If False (default), then the data will be sorted by exog before
         calculating lowess. If True, then it is assumed that the data is
-        already sorted by exog.
+        already sorted by exog. If xvals is specified, then it too must be
+        sorted if is_sorted is True.
     missing : str
         Available options are 'none', 'drop', and 'raise'. If 'none', no nan
         checking is done. If 'drop', any observations with nans are dropped.
@@ -56,6 +60,9 @@ def lowess(endog, exog, frac=2.0/3.0, it=3, delta=0.0, is_sorted=False,
         the associated estimated y (endog) values.
         If return_sorted is False, then only the fitted values are returned,
         and the observations will be in the same order as the input arrays.
+        If xvals is provided, then return_sorted is ignored and the returned
+        array is always one dimensional, containing the y values fitted at
+        the x values provided by xvals.
 
     Notes
     -----
@@ -89,6 +96,10 @@ def lowess(endog, exog, frac=2.0/3.0, it=3, delta=0.0, is_sorted=False,
     Judicious choice of delta can cut computation time considerably
     for large data (N > 5000). A good choice is ``delta = 0.01 * range(exog)``.
 
+    If `xvals` is provided, the regression is then computed at those points
+    and the fit values are returned. Otherwise, the regression is run
+    at points of `exog`.
+
     Some experimentation is likely required to find a good
     choice of `frac` and `iter` for a particular dataset.
 
@@ -127,6 +138,9 @@ def lowess(endog, exog, frac=2.0/3.0, it=3, delta=0.0, is_sorted=False,
     endog = np.asarray(endog, float)
     exog = np.asarray(exog, float)
 
+    # Whether xvals argument was provided
+    given_xvals = (xvals is not None)
+
     # Inputs should be vectors (1-D arrays) of the
     # same length.
     if exog.ndim != 1:
@@ -137,7 +151,6 @@ def lowess(endog, exog, frac=2.0/3.0, it=3, delta=0.0, is_sorted=False,
         raise ValueError('exog and endog must have same length')
 
     if missing in ['drop', 'raise']:
-        # Cut out missing values
         mask_valid = (np.isfinite(exog) & np.isfinite(endog))
         all_valid = np.all(mask_valid)
         if all_valid:
@@ -162,26 +175,81 @@ def lowess(endog, exog, frac=2.0/3.0, it=3, delta=0.0, is_sorted=False,
         x = np.array(x[sort_index])
         y = np.array(y[sort_index])
 
-    res = _lowess(y, x, frac=frac, it=it, delta=delta)
+    if not given_xvals:
+        # If given no explicit x values, we use the x-values in the exog array
+        xvals = exog
+        xvalues = x
+
+        xvals_all_valid = all_valid
+        if missing == 'drop':
+            xvals_mask_valid = mask_valid
+    else:
+        if delta != 0.0:
+            raise ValueError("Cannot have non-zero 'delta' and 'xvals' values")
+            # TODO: allow this again
+
+        # With explicit xvals, we ignore 'return_sorted' and always
+        # use the order provided
+        return_sorted = False
+
+        if missing in ['drop', 'raise']:
+            xvals_mask_valid = np.isfinite(xvals)
+            xvals_all_valid = np.all(xvals_mask_valid)
+            if xvals_all_valid:
+                xvalues = xvals
+            else:
+                if missing == 'drop':
+                    xvalues = xvals[xvals_mask_valid]
+                else:
+                    raise ValueError("nan or inf found in xvals")
+
+        if not is_sorted:
+            sort_index = np.argsort(xvalues)
+            xvalues = np.array(xvalues[sort_index])
+        else:
+            xvals_all_valid = True
+
+    if not given_xvals:
+        # Run LOWESS on the data points
+        print("Not giving xvalues")
+        res, _ = _lowess(y, x, x, np.ones_like(x),
+                        frac=frac, it=it, delta=delta, given_xvals=False)
+    else:
+        # First run LOWESS on the data points to get the weights of the data points
+        # using it-1 iterations, last iter done next
+        if it > 0:
+            print("Not giving xvalues")
+            _, weights = _lowess(y, x, x, np.ones_like(x),
+                                frac=frac, it=it-1, delta=delta, given_xvals=False)
+        else:
+            weights = np.ones_like(x)
+
+        # Then run once more using those supplied weights at the points provided by xvals
+        # No extra iterations are performed here since weights are fixed
+        print("giving xvalues")
+        res, _ = _lowess(y, x, xvalues, weights,
+                        frac=frac, it=0, delta=delta, given_xvals=True)
+
     _, yfitted = res.T
 
     if return_sorted:
         return res
     else:
+
         # rebuild yfitted with original indices
         # a bit messy: y might have been selected twice
         if not is_sorted:
-            yfitted_ = np.empty_like(y)
+            yfitted_ = np.empty_like(xvalues)
             yfitted_.fill(np.nan)
             yfitted_[sort_index] = yfitted
             yfitted = yfitted_
         else:
             yfitted = yfitted
 
-        if not all_valid:
-            yfitted_ = np.empty_like(endog)
+        if not xvals_all_valid:
+            yfitted_ = np.empty_like(xvals)
             yfitted_.fill(np.nan)
-            yfitted_[mask_valid] = yfitted
+            yfitted_[xvals_mask_valid] = yfitted
             yfitted = yfitted_
 
         # we do not need to return exog anymore


Test patch:
diff --git a/statsmodels/nonparametric/tests/test_lowess.py b/statsmodels/nonparametric/tests/test_lowess.py
--- a/statsmodels/nonparametric/tests/test_lowess.py
+++ b/statsmodels/nonparametric/tests/test_lowess.py
@@ -120,6 +120,12 @@ def test_options(self):
         assert_almost_equal(actual_lowess, actual_lowess1, decimal=13)
         assert_(actual_lowess.dtype is np.dtype(float))
 
+        # Test specifying xvals explicitly
+        perm_idx = np.arange(len(x)//2)
+        np.random.shuffle(perm_idx)
+        actual_lowess2 = lowess(y, x, xvals = x[perm_idx], return_sorted=False)
+        assert_almost_equal(actual_lowess[perm_idx,1], actual_lowess2, decimal=13)
+
         # check with nans,  this changes the arrays
         y[[5, 6]] = np.nan
         x[3] = np.nan
@@ -147,6 +153,11 @@ def test_options(self):
         yhat = yhat[np.isfinite(yhat)]
         assert_almost_equal(yhat, actual_lowess2[:,1], decimal=13)
 
+        # Test specifying xvals explicitly, now with nans
+        perm_idx = np.arange(actual_lowess.shape[0])
+        actual_lowess4 = lowess(y, x, xvals = actual_lowess[perm_idx,0], return_sorted=False)
+        assert_almost_equal(actual_lowess[perm_idx,1], actual_lowess4, decimal=13)
+
 
 def test_returns_inputs():
     # see 1960