Repository: pandas-dev/pandas
Pull request: BUG: fix reindexing with a tz-aware index and method='nearest'
- [x] closes #26683
- [x] tests added / passed
- [x] passes `black pandas`
- [x] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
- [x] whatsnew entry

First pandas PR, happy to make changes as needed.  Is a whatsnew entry needed?  And if so, I assume it should go in v1.0.1.rst? 

Problem statement:
Warning while reindexing tz aware index with method='nearest'
Reindexing a tz aware dataframe using method='nearest' raise an internal warning.
```python
from pandas.util.testing import makeTimeDataFrame
df = makeTimeDataFrame(freq='1h')
df = df.tz_localize('UTC')
df.reindex(df.index[1:4], method='nearest')
```
raises a warning:
```
/lib/python3.7/site-packages/pandas/core/indexes/base.py:2820: FutureWarning: Converting timezone-aware DatetimeArray to timezone-naive ndarray with 'datetime64[ns]' dtype. In the future, this will return an ndarray with 'object' dtype where each element is a 'pandas.Timestamp' with the correct 'tz'.
        To accept the future behavior, pass 'dtype=object'.
        To keep the old behavior, pass 'dtype="datetime64[ns]"'.
  target = np.asarray(target)
```
in pandas 0.24.2

Hints from issue discussion:
I think we need seomthingn like

```diff
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 8538687ca..1aa461b69 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -2904,9 +2904,9 @@ class Index(IndexOpsMixin, PandasObject):
         left_indexer = self.get_indexer(target, 'pad', limit=limit)
         right_indexer = self.get_indexer(target, 'backfill', limit=limit)

-        target = np.asarray(target)
-        left_distances = abs(self.values[left_indexer] - target)
-        right_distances = abs(self.values[right_indexer] - target)
+        # target = np.asarray(target)
+        left_distances = abs(self[left_indexer] - target)
+        right_distances = abs(self[right_indexer] - target)

         op = operator.lt if self.is_monotonic_increasing else operator.le
         indexer = np.where(op(left_distances, right_distances) |
```

That runs into https://github.com/pandas-dev/pandas/issues/26689.

We'll also want an ASV benchmark for this.

In 1.0.0 reindexing a tz-aware index with method='nearest' raises an error:
<details>
  <summary>Click to expand</summary>

```
In [7]: from pandas.util.testing import makeTimeDataFrame
   ...: df = makeTimeDataFrame(freq='1h')
   ...: df = df.tz_localize('UTC')
   ...: df.reindex(df.index[1:4], method='nearest')
C:\Users\KANDERSO\Software\Anaconda3\envs\rdtools_temp\Scripts\ipython:1: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
---------------------------------------------------------------------------
UFuncTypeError                            Traceback (most recent call last)
<ipython-input-7-d8f9897c3a34> in <module>
      2 df = makeTimeDataFrame(freq='1h')
      3 df = df.tz_localize('UTC')
----> 4 df.reindex(df.index[1:4], method='nearest')

c:\users\kanderso\software\anaconda3\envs\rdtools_temp\lib\site-packages\pandas\util\_decorators.py in wrapper(*args, **kwargs)
    225         @wraps(func)
    226         def wrapper(*args, **kwargs) -> Callable[..., Any]:
--> 227             return func(*args, **kwargs)
    228
    229         kind = inspect.Parameter.POSITIONAL_OR_KEYWORD

c:\users\kanderso\software\anaconda3\envs\rdtools_temp\lib\site-packages\pandas\core\frame.py in reindex(self, *args, **kwargs)
   3851         kwargs.pop("axis", None)
   3852         kwargs.pop("labels", None)
-> 3853         return self._ensure_type(super().reindex(**kwargs))
   3854
   3855     def drop(

c:\users\kanderso\software\anaconda3\envs\rdtools_temp\lib\site-packages\pandas\core\generic.py in reindex(self, *args, **kwargs)
   4541         # perform the reindex on the axes
   4542         return self._reindex_axes(
-> 4543             axes, level, limit, tolerance, method, fill_value, copy
   4544         ).__finalize__(self)
   4545

c:\users\kanderso\software\anaconda3\envs\rdtools_temp\lib\site-packages\pandas\core\frame.py in _reindex_axes(self, axes, level, limit, tolerance, method, fill_value, copy)
   3739         if index is not None:
   3740             frame = frame._reindex_index(
-> 3741                 index, method, copy, level, fill_value, limit, tolerance
   3742             )
   3743

c:\users\kanderso\software\anaconda3\envs\rdtools_temp\lib\site-packages\pandas\core\frame.py in _reindex_index(self, new_index, method, copy, level, fill_value, limit, tolerance)
   3755     ):
   3756         new_index, indexer = self.index.reindex(
-> 3757             new_index, method=method, level=level, limit=limit, tolerance=tolerance
   3758         )
   3759         return self._reindex_with_indexers(

c:\users\kanderso\software\anaconda3\envs\rdtools_temp\lib\site-packages\pandas\core\indexes\base.py in reindex(self, target, method, level, limit, tolerance)
   3143                 if self.is_unique and not getattr(self, "is_overlapping", False):
   3144                     indexer = self.get_indexer(
-> 3145                         target, method=method, limit=limit, tolerance=tolerance
   3146                     )
   3147                 else:

c:\users\kanderso\software\anaconda3\envs\rdtools_temp\lib\site-packages\pandas\core\indexes\base.py in get_indexer(self, target, method, limit, tolerance)
   2738             indexer = self._get_fill_indexer(target, method, limit, tolerance)
   2739         elif method == "nearest":
-> 2740             indexer = self._get_nearest_indexer(target, limit, tolerance)
   2741         else:
   2742             if tolerance is not None:

c:\users\kanderso\software\anaconda3\envs\rdtools_temp\lib\site-packages\pandas\core\indexes\base.py in _get_nearest_indexer(self, target, limit, tolerance)
   2819
   2820         target = np.asarray(target)
-> 2821         left_distances = abs(self.values[left_indexer] - target)
   2822         right_distances = abs(self.values[right_indexer] - target)
   2823

UFuncTypeError: ufunc 'subtract' cannot use operands with types dtype('<M8[ns]') and dtype('O')
```

</details>

I am happy to help with a fix but just FYI I have no experience with pandas's test suite or ASV at all.  Is @TomAugspurger's above suggestion still a good place to start?

Commit message: BUG: fix reindexing a tz-aware index with method='nearest' (GH26683)

Patch to review:
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 22ba317e78e63..b6fd5d31c03e8 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -3061,9 +3061,8 @@ def _get_nearest_indexer(self, target: "Index", limit, tolerance) -> np.ndarray:
         left_indexer = self.get_indexer(target, "pad", limit=limit)
         right_indexer = self.get_indexer(target, "backfill", limit=limit)
 
-        target = np.asarray(target)
-        left_distances = abs(self.values[left_indexer] - target)
-        right_distances = abs(self.values[right_indexer] - target)
+        left_distances = abs(self[left_indexer] - target)
+        right_distances = abs(self[right_indexer] - target)
 
         op = operator.lt if self.is_monotonic_increasing else operator.le
         indexer = np.where(
diff --git a/pandas/tests/frame/indexing/test_indexing.py b/pandas/tests/frame/indexing/test_indexing.py
index cbb9dd09bbede..7a0a3aca9eb51 100644
--- a/pandas/tests/frame/indexing/test_indexing.py
+++ b/pandas/tests/frame/indexing/test_indexing.py
@@ -1599,6 +1599,14 @@ def test_reindex_methods_nearest_special(self):
         actual = df.reindex(target, method="nearest", tolerance=[0.5, 0.01, 0.4, 0.1])
         tm.assert_frame_equal(expected, actual)
 
+    def test_reindex_nearest_tz(self):
+        idx = pd.date_range("2019-01-01", periods=5, tz="US/Eastern")
+        df = pd.DataFrame({"x": list(range(5))}, index=idx)
+
+        expected = df.head(3)
+        actual = df.reindex(idx[:3], method="nearest")
+        tm.assert_frame_equal(expected, actual)
+
     def test_reindex_frame_add_nat(self):
         rng = date_range("1/1/2000 00:00:00", periods=10, freq="10s")
         df = DataFrame({"A": np.random.randn(len(rng)), "B": rng})


Code review comments:
diff_hunk: @@ -1599,6 +1599,14 @@ def test_reindex_methods_nearest_special(self):
         actual = df.reindex(target, method="nearest", tolerance=[0.5, 0.01, 0.4, 0.1])
         tm.assert_frame_equal(expected, actual)
 
+    def test_reindex_nearest_tz(self):
+        idx = pd.date_range("2019-01-01", periods=5, tz="US/Eastern")
path: pandas/tests/frame/indexing/test_indexing.py
text: @user1:
There are some fixtures defined in `conftest.py`, I think here you might want to use `tz_aware_fixture`:
```
def test_reindex_nearest_tz(self, tz_aware_fixture):
    tz = tz_aware_fixture
    idx = pd.date_range("2019-01-01", periods=5, tz=tz)
```