Repository: pandas-dev/pandas

Issue:
Pandas Interpolate Does not Handle Empty Values at Front of Series
[This code](https://github.com/pydata/pandas/blob/7d6fb510c1dfdbe7342f32f05ca5fd69b7854081/pandas/core/common.py#L1646) means the interpolator won't handle extrapolating to the front of the Series even though the underlying implementations may have no problem with the extrapolation. 

See for example [`UnivariateSpline`](http://docs.scipy.org/doc/scipy-0.15.1/reference/generated/scipy.interpolate.UnivariateSpline.html) whose default behavior is extrapolation.

Interpolation works fine at the end of the Series.

For example:

``` python
s = pd.Series([1, 2, 3, 4, np.nan, 6, np.nan])
s.interpolate(method='spline', order=1)
0    1
1    2
2    3
3    4
4    5
5    6
6    7
dtype: float64
```

but:

``` python
s = pd.Series([np.nan, 2, 3, 4, np.nan, 6, 7])
s.interpolate(method='spline', order=1)
0   NaN
1     2
2     3
3     4
4     5
5     6
6     7
dtype: float64
```



Discussion:
hmm, I suppose that if the underlying can handle it then would be ok to change this. Try it and see if anything blows up. You might need to expand the tests a bit to make sure that this is all tested.

This seems related to #9218 -- the only reason the values aren't filled at the beginning of the DataFrame/Series is that the fill for interpolate only happens forward.

Sure. This lines in the referenced ticket seems to sum it up:

> The limit refers to forward filling only, even though interpolation is not inherently directional. 

Agreed.

Also see #10691 for an ongoing pull request/discussion.


Resolving patch:
diff --git a/pandas/core/common.py b/pandas/core/common.py
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -1589,6 +1589,7 @@ def _clean_interp_method(method, **kwargs):
 
 
 def interpolate_1d(xvalues, yvalues, method='linear', limit=None,
+                   limit_direction='forward',
                    fill_value=None, bounds_error=False, order=None, **kwargs):
     """
     Logic for the 1-d interpolation.  The result should be 1-d, inputs
@@ -1602,9 +1603,15 @@ def interpolate_1d(xvalues, yvalues, method='linear', limit=None,
     invalid = isnull(yvalues)
     valid = ~invalid
 
-    valid_y = yvalues[valid]
-    valid_x = xvalues[valid]
-    new_x = xvalues[invalid]
+    if not valid.any():
+        # have to call np.asarray(xvalues) since xvalues could be an Index
+        # which cant be mutated
+        result = np.empty_like(np.asarray(xvalues), dtype=np.float64)
+        result.fill(np.nan)
+        return result
+
+    if valid.all():
+        return yvalues
 
     if method == 'time':
         if not getattr(xvalues, 'is_all_dates', None):
@@ -1614,33 +1621,54 @@ def interpolate_1d(xvalues, yvalues, method='linear', limit=None,
                              'DatetimeIndex')
         method = 'values'
 
-    def _interp_limit(invalid, limit):
-        """mask off values that won't be filled since they exceed the limit"""
+    def _interp_limit(invalid, fw_limit, bw_limit):
+        "Get idx of values that won't be forward-filled b/c they exceed the limit."
         all_nans = np.where(invalid)[0]
         if all_nans.size == 0: # no nans anyway
             return []
-        violate = [invalid[x:x + limit + 1] for x in all_nans]
-        violate = np.array([x.all() & (x.size > limit) for x in violate])
-        return all_nans[violate] + limit
+        violate = [invalid[max(0, x - bw_limit):x + fw_limit + 1] for x in all_nans]
+        violate = np.array([x.all() & (x.size > bw_limit + fw_limit) for x in violate])
+        return all_nans[violate] + fw_limit - bw_limit
+
+    valid_limit_directions = ['forward', 'backward', 'both']
+    limit_direction = limit_direction.lower()
+    if limit_direction not in valid_limit_directions:
+        msg = 'Invalid limit_direction: expecting one of %r, got %r.' % (
+            valid_limit_directions, limit_direction)
+        raise ValueError(msg)
 
-    xvalues = getattr(xvalues, 'values', xvalues)
-    yvalues = getattr(yvalues, 'values', yvalues)
+    from pandas import Series
+    ys = Series(yvalues)
+    start_nans = set(range(ys.first_valid_index()))
+    end_nans = set(range(1 + ys.last_valid_index(), len(valid)))
+
+    # This is a list of the indexes in the series whose yvalue is currently NaN,
+    # but whose interpolated yvalue will be overwritten with NaN after computing
+    # the interpolation. For each index in this list, one of these conditions is
+    # true of the corresponding NaN in the yvalues:
+    #
+    # a) It is one of a chain of NaNs at the beginning of the series, and either
+    #    limit is not specified or limit_direction is 'forward'.
+    # b) It is one of a chain of NaNs at the end of the series, and limit is
+    #    specified and limit_direction is 'backward' or 'both'.
+    # c) Limit is nonzero and it is further than limit from the nearest non-NaN
+    #    value (with respect to the limit_direction setting).
+    #
+    # The default behavior is to fill forward with no limit, ignoring NaNs at
+    # the beginning (see issues #9218 and #10420)
+    violate_limit = sorted(start_nans)
 
     if limit:
-        violate_limit = _interp_limit(invalid, limit)
-    if valid.any():
-        firstIndex = valid.argmax()
-        valid = valid[firstIndex:]
-        invalid = invalid[firstIndex:]
-        result = yvalues.copy()
-        if valid.all():
-            return yvalues
-    else:
-        # have to call np.array(xvalues) since xvalues could be an Index
-        # which cant be mutated
-        result = np.empty_like(np.array(xvalues), dtype=np.float64)
-        result.fill(np.nan)
-        return result
+        if limit_direction == 'forward':
+            violate_limit = sorted(start_nans | set(_interp_limit(invalid, limit, 0)))
+        if limit_direction == 'backward':
+            violate_limit = sorted(end_nans | set(_interp_limit(invalid, 0, limit)))
+        if limit_direction == 'both':
+            violate_limit = _interp_limit(invalid, limit, limit)
+
+    xvalues = getattr(xvalues, 'values', xvalues)
+    yvalues = getattr(yvalues, 'values', yvalues)
+    result = yvalues.copy()
 
     if method in ['linear', 'time', 'index', 'values']:
         if method in ('values', 'index'):
@@ -1648,32 +1676,27 @@ def _interp_limit(invalid, limit):
             # hack for DatetimeIndex, #1646
             if issubclass(inds.dtype.type, np.datetime64):
                 inds = inds.view(np.int64)
-
             if inds.dtype == np.object_:
                 inds = lib.maybe_convert_objects(inds)
         else:
             inds = xvalues
-
-        inds = inds[firstIndex:]
-
-        result[firstIndex:][invalid] = np.interp(inds[invalid], inds[valid],
-                                                 yvalues[firstIndex:][valid])
-
-        if limit:
-            result[violate_limit] = np.nan
+        result[invalid] = np.interp(inds[invalid], inds[valid], yvalues[valid])
+        result[violate_limit] = np.nan
         return result
 
     sp_methods = ['nearest', 'zero', 'slinear', 'quadratic', 'cubic',
                   'barycentric', 'krogh', 'spline', 'polynomial',
                   'piecewise_polynomial', 'pchip']
     if method in sp_methods:
-        new_x = new_x[firstIndex:]
-
-        result[firstIndex:][invalid] = _interpolate_scipy_wrapper(
-            valid_x, valid_y, new_x, method=method, fill_value=fill_value,
+        inds = np.asarray(xvalues)
+        # hack for DatetimeIndex, #1646
+        if issubclass(inds.dtype.type, np.datetime64):
+            inds = inds.view(np.int64)
+        result[invalid] = _interpolate_scipy_wrapper(
+            inds[valid], yvalues[valid], inds[invalid], method=method,
+            fill_value=fill_value,
             bounds_error=bounds_error, order=order, **kwargs)
-        if limit:
-            result[violate_limit] = np.nan
+        result[violate_limit] = np.nan
         return result
 
 
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -2964,7 +2964,7 @@ def replace(self, to_replace=None, value=None, inplace=False, limit=None,
             return self._constructor(new_data).__finalize__(self)
 
     def interpolate(self, method='linear', axis=0, limit=None, inplace=False,
-                    downcast=None, **kwargs):
+                    limit_direction='forward', downcast=None, **kwargs):
         """
         Interpolate values according to different methods.
 
@@ -3001,6 +3001,12 @@ def interpolate(self, method='linear', axis=0, limit=None, inplace=False,
             * 1: fill row-by-row
         limit : int, default None.
             Maximum number of consecutive NaNs to fill.
+        limit_direction : {'forward', 'backward', 'both'}, defaults to 'forward'
+            If limit is specified, consecutive NaNs will be filled in this
+            direction.
+
+            .. versionadded:: 0.17.0
+
         inplace : bool, default False
             Update the NDFrame in place if possible.
         downcast : optional, 'infer' or None, defaults to None
@@ -3071,6 +3077,7 @@ def interpolate(self, method='linear', axis=0, limit=None, inplace=False,
             index=index,
             values=_maybe_transposed_self,
             limit=limit,
+            limit_direction=limit_direction,
             inplace=inplace,
             downcast=downcast,
             **kwargs
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -747,6 +747,7 @@ def putmask(self, mask, new, align=True, inplace=False,
 
     def interpolate(self, method='pad', axis=0, index=None,
                     values=None, inplace=False, limit=None,
+                    limit_direction='forward',
                     fill_value=None, coerce=False, downcast=None, **kwargs):
 
         def check_int_bool(self, inplace):
@@ -790,6 +791,7 @@ def check_int_bool(self, inplace):
                                      values=values,
                                      axis=axis,
                                      limit=limit,
+                                     limit_direction=limit_direction,
                                      fill_value=fill_value,
                                      inplace=inplace,
                                      downcast=downcast,
@@ -829,6 +831,7 @@ def _interpolate_with_fill(self, method='pad', axis=0, inplace=False,
 
     def _interpolate(self, method=None, index=None, values=None,
                      fill_value=None, axis=0, limit=None,
+                     limit_direction='forward',
                      inplace=False, downcast=None, **kwargs):
         """ interpolate using scipy wrappers """
 
@@ -855,6 +858,7 @@ def func(x):
             # should the axis argument be handled below in apply_along_axis?
             # i.e. not an arg to com.interpolate_1d
             return com.interpolate_1d(index, x, method=method, limit=limit,
+                                      limit_direction=limit_direction,
                                       fill_value=fill_value,
                                       bounds_error=False, **kwargs)
 


Test patch:
diff --git a/pandas/tests/test_generic.py b/pandas/tests/test_generic.py
--- a/pandas/tests/test_generic.py
+++ b/pandas/tests/test_generic.py
@@ -857,10 +857,79 @@ def test_interp_scipy_basic(self):
 
     def test_interp_limit(self):
         s = Series([1, 3, np.nan, np.nan, np.nan, 11])
+
         expected = Series([1., 3., 5., 7., np.nan, 11.])
         result = s.interpolate(method='linear', limit=2)
         assert_series_equal(result, expected)
 
+    def test_interp_limit_forward(self):
+        s = Series([1, 3, np.nan, np.nan, np.nan, 11])
+
+        # Provide 'forward' (the default) explicitly here.
+        expected = Series([1., 3., 5., 7., np.nan, 11.])
+
+        result = s.interpolate(
+            method='linear', limit=2, limit_direction='forward')
+        assert_series_equal(result, expected)
+
+        result = s.interpolate(
+            method='linear', limit=2, limit_direction='FORWARD')
+        assert_series_equal(result, expected)
+
+    def test_interp_limit_bad_direction(self):
+        s = Series([1, 3, np.nan, np.nan, np.nan, 11])
+        expected = Series([1., 3., 5., 7., 9., 11.])
+
+        self.assertRaises(ValueError, s.interpolate,
+                          method='linear', limit=2,
+                          limit_direction='abc')
+
+        # raises an error even if no limit is specified.
+        self.assertRaises(ValueError, s.interpolate,
+                          method='linear',
+                          limit_direction='abc')
+
+    def test_interp_limit_direction(self):
+        # These tests are for issue #9218 -- fill NaNs in both directions.
+        s = Series([1, 3, np.nan, np.nan, np.nan, 11])
+
+        expected = Series([1., 3., np.nan, 7., 9., 11.])
+        result = s.interpolate(
+            method='linear', limit=2, limit_direction='backward')
+        assert_series_equal(result, expected)
+
+        expected = Series([1., 3., 5., np.nan, 9., 11.])
+        result = s.interpolate(
+            method='linear', limit=1, limit_direction='both')
+        assert_series_equal(result, expected)
+
+        # Check that this works on a longer series of nans.
+        s = Series([1, 3, np.nan, np.nan, np.nan, 7, 9, np.nan, np.nan, 12, np.nan])
+
+        expected = Series([1., 3., 4., 5., 6., 7., 9., 10., 11., 12., 12.])
+        result = s.interpolate(
+            method='linear', limit=2, limit_direction='both')
+        assert_series_equal(result, expected)
+
+        expected = Series([1., 3., 4., np.nan, 6., 7., 9., 10., 11., 12., 12.])
+        result = s.interpolate(
+            method='linear', limit=1, limit_direction='both')
+        assert_series_equal(result, expected)
+
+    def test_interp_limit_to_ends(self):
+        # These test are for issue #10420 -- flow back to beginning.
+        s = Series([np.nan, np.nan, 5, 7, 9, np.nan])
+
+        expected = Series([5., 5., 5., 7., 9., np.nan])
+        result = s.interpolate(
+            method='linear', limit=2, limit_direction='backward')
+        assert_series_equal(result, expected)
+
+        expected = Series([5., 5., 5., 7., 9., 9.])
+        result = s.interpolate(
+            method='linear', limit=2, limit_direction='both')
+        assert_series_equal(result, expected)
+
     def test_interp_all_good(self):
         # scipy
         tm._skip_if_no_scipy()