Repository: pandas-dev/pandas

Issue:
AmbiguousTimeError merging two timezone-aware DataFrames with DST change
When merging two DataFrames by a timezone-aware datetime column, if the datetime values doesn't include a DST change, there's no problem:

```python

df1 = pd.DataFrame([pd.to_datetime('2017-10-30 02:00:00+01:00'),
                    pd.to_datetime('2017-10-30 03:00:00+01:00'),
                    pd.to_datetime('2017-10-30 04:00:00+01:00')],columns=['date'])
df1['date'] = df1['date'].dt.tz_localize('UTC').dt.tz_convert('Europe/Madrid')
df1['value'] = 1

df2 = pd.DataFrame([pd.to_datetime('2017-10-30 04:00:00+01:00'),
                    pd.to_datetime('2017-10-30 05:00:00+01:00'),
                    pd.to_datetime('2017-10-30 06:00:00+01:00')],columns=['date'])
df2['date'] = df2['date'].dt.tz_localize('UTC').dt.tz_convert('Europe/Madrid')
df2['value'] = 2

pd.merge(df1, df2, how='outer', on='date')

```

### Output

```
                       date  value_x  value_y
0 2017-10-30 02:00:00+01:00      1.0      NaN
1 2017-10-30 03:00:00+01:00      1.0      NaN
2 2017-10-30 04:00:00+01:00      1.0      2.0
3 2017-10-30 05:00:00+01:00      NaN      2.0
4 2017-10-30 06:00:00+01:00      NaN      2.0
```

This is correct. But if the datetime values include a date with DST change, we get an AmbiguousTimeError exception:

```python
df1 = pd.DataFrame([pd.to_datetime('2017-10-29 02:00:00+02:00'),
                    pd.to_datetime('2017-10-29 02:00:00+01:00'),
                    pd.to_datetime('2017-10-29 03:00:00+01:00')],columns=['date'])
df1['date'] = df1['date'].dt.tz_localize('UTC').dt.tz_convert('Europe/Madrid')
df1['value'] = 1

df2 = pd.DataFrame([pd.to_datetime('2017-10-29 03:00:00+01:00'),
                    pd.to_datetime('2017-10-29 04:00:00+01:00'),
                    pd.to_datetime('2017-10-29 05:00:00+01:00')],columns=['date'])
df2['date'] = df2['date'].dt.tz_localize('UTC').dt.tz_convert('Europe/Madrid')
df2['value'] = 2

pd.merge(df1, df2, how='outer', on='date')
```

### Expected output

```
                       date  value_x   value_y
0 2017-10-29 02:00:00+02:00      1.0       NaN
1 2017-10-29 02:00:00+01:00      1.0       NaN
2 2017-10-29 03:00:00+01:00      1.0       2.0
3 2017-10-29 04:00:00+01:00      NaN       2.0
4 2017-10-29 05:00:00+01:00      NaN       2.0
```

### Actual output

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "(...)/venv/lib/python3.6/site-packages/pandas/core/reshape/merge.py", line 58, in merge
    return op.get_result()
  File "(...)//venv/lib/python3.6/site-packages/pandas/core/reshape/merge.py", line 604, in get_result
    self._maybe_add_join_keys(result, left_indexer, right_indexer)
  File "(...)//venv/lib/python3.6/site-packages/pandas/core/reshape/merge.py", line 715, in _maybe_add_join_keys
    key_col = Index(lvals).where(~mask, rvals)
  File "(...)//venv/lib/python3.6/site-packages/pandas/core/indexes/datetimelike.py", line 809, in where
    result = self._ensure_localized(result)
  File "(...)//venv/lib/python3.6/site-packages/pandas/core/indexes/datetimelike.py", line 230, in _ensure_localized
    result = result.tz_localize(self.tz)
  File "(...)//venv/lib/python3.6/site-packages/pandas/util/_decorators.py", line 118, in wrapper
    return func(*args, **kwargs)
  File "(...)//venv/lib/python3.6/site-packages/pandas/core/indexes/datetimes.py", line 1858, in tz_localize
    errors=errors)
  File "pandas/_libs/tslib.pyx", line 3593, in pandas._libs.tslib.tz_localize_to_utc
pytz.exceptions.AmbiguousTimeError: Cannot infer dst time from Timestamp('2017-10-29 02:00:00'), try using the 'ambiguous' argument
```

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.6.2.final.0
python-bits: 64
OS: Darwin
OS-release: 17.3.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: es_ES.UTF-8
LOCALE: es_ES.UTF-8

pandas: 0.21.1
pytest: 3.2.5
pip: 9.0.1
setuptools: 36.8.0
Cython: None
numpy: 1.13.3
scipy: None
pyarrow: None
xarray: None
IPython: None
sphinx: 1.5.3
patsy: None
dateutil: 2.6.1
pytz: 2017.3
blosc: None
bottleneck: None
tables: None
numexpr: None
feather: None
matplotlib: None
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: 0.9.6
lxml: None
bs4: None
html5lib: None
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: 2.9.6
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None

</details>


Discussion:
yeah, we are joining indicies and do a ``.where()`` on them. we drop the tz, do the op in i8, then localize to the original zone.

what we need is an attribute for ``Timestamp`` and ``DatetimeIndex`` like ``is_ambiguous``, then we could record the ambiguous so we can recreate properly.

interested in a PR?

cc @jbrockmendel 
I'd love to, but I don't know the pandas/numpy internals, and `merge()` doesn't sound like an easy place to start :-) Maybe with some guidance...

Resolving patch:
diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py
--- a/pandas/core/indexes/datetimelike.py
+++ b/pandas/core/indexes/datetimelike.py
@@ -277,7 +277,7 @@ def _evaluate_compare(self, other, op):
         except TypeError:
             return result
 
-    def _ensure_localized(self, result, ambiguous='raise'):
+    def _ensure_localized(self, arg, ambiguous='raise', from_utc=False):
         """
         ensure that we are re-localized
 
@@ -286,9 +286,11 @@ def _ensure_localized(self, result, ambiguous='raise'):
 
         Parameters
         ----------
-        result : DatetimeIndex / i8 ndarray
-        ambiguous : str, bool, or bool-ndarray
-            default 'raise'
+        arg : DatetimeIndex / i8 ndarray
+        ambiguous : str, bool, or bool-ndarray, default 'raise'
+        from_utc : bool, default False
+            If True, localize the i8 ndarray to UTC first before converting to
+            the appropriate tz. If False, localize directly to the tz.
 
         Returns
         -------
@@ -297,10 +299,13 @@ def _ensure_localized(self, result, ambiguous='raise'):
 
         # reconvert to local tz
         if getattr(self, 'tz', None) is not None:
-            if not isinstance(result, ABCIndexClass):
-                result = self._simple_new(result)
-            result = result.tz_localize(self.tz, ambiguous=ambiguous)
-        return result
+            if not isinstance(arg, ABCIndexClass):
+                arg = self._simple_new(arg)
+            if from_utc:
+                arg = arg.tz_localize('UTC').tz_convert(self.tz)
+            else:
+                arg = arg.tz_localize(self.tz, ambiguous=ambiguous)
+        return arg
 
     def _box_values_as_index(self):
         """
@@ -622,11 +627,11 @@ def repeat(self, repeats, *args, **kwargs):
 
     @Appender(_index_shared_docs['where'] % _index_doc_kwargs)
     def where(self, cond, other=None):
-        other = _ensure_datetimelike_to_i8(other)
-        values = _ensure_datetimelike_to_i8(self)
+        other = _ensure_datetimelike_to_i8(other, to_utc=True)
+        values = _ensure_datetimelike_to_i8(self, to_utc=True)
         result = np.where(cond, values, other).astype('i8')
 
-        result = self._ensure_localized(result)
+        result = self._ensure_localized(result, from_utc=True)
         return self._shallow_copy(result,
                                   **self._get_attributes_dict())
 
@@ -695,23 +700,37 @@ def astype(self, dtype, copy=True):
         return super(DatetimeIndexOpsMixin, self).astype(dtype, copy=copy)
 
 
-def _ensure_datetimelike_to_i8(other):
-    """ helper for coercing an input scalar or array to i8 """
+def _ensure_datetimelike_to_i8(other, to_utc=False):
+    """
+    helper for coercing an input scalar or array to i8
+
+    Parameters
+    ----------
+    other : 1d array
+    to_utc : bool, default False
+        If True, convert the values to UTC before extracting the i8 values
+        If False, extract the i8 values directly.
+
+    Returns
+    -------
+    i8 1d array
+    """
     if is_scalar(other) and isna(other):
-        other = iNaT
+        return iNaT
     elif isinstance(other, ABCIndexClass):
         # convert tz if needed
         if getattr(other, 'tz', None) is not None:
-            other = other.tz_localize(None).asi8
-        else:
-            other = other.asi8
+            if to_utc:
+                other = other.tz_convert('UTC')
+            else:
+                other = other.tz_localize(None)
     else:
         try:
-            other = np.array(other, copy=False).view('i8')
+            return np.array(other, copy=False).view('i8')
         except TypeError:
             # period array cannot be coerces to int
-            other = Index(other).asi8
-    return other
+            other = Index(other)
+    return other.asi8
 
 
 def wrap_arithmetic_op(self, other, result):


Test patch:
diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py
--- a/pandas/tests/indexing/test_coercion.py
+++ b/pandas/tests/indexing/test_coercion.py
@@ -590,11 +590,9 @@ def test_where_series_datetime64(self, fill_val, exp_dtype):
                          pd.Timestamp('2011-01-03'), values[3]])
         self._assert_where_conversion(obj, cond, values, exp, exp_dtype)
 
-    @pytest.mark.parametrize("fill_val,exp_dtype", [
-        (pd.Timestamp('2012-01-01'), 'datetime64[ns]'),
-        (pd.Timestamp('2012-01-01', tz='US/Eastern'), np.object)],
-        ids=['datetime64', 'datetime64tz'])
-    def test_where_index_datetime(self, fill_val, exp_dtype):
+    def test_where_index_datetime(self):
+        fill_val = pd.Timestamp('2012-01-01')
+        exp_dtype = 'datetime64[ns]'
         obj = pd.Index([pd.Timestamp('2011-01-01'),
                         pd.Timestamp('2011-01-02'),
                         pd.Timestamp('2011-01-03'),
@@ -613,13 +611,33 @@ def test_where_index_datetime(self, fill_val, exp_dtype):
                         pd.Timestamp('2011-01-03'),
                         pd.Timestamp('2012-01-04')])
 
-        if fill_val.tz:
-            self._assert_where_conversion(obj, cond, values, exp,
-                                          'datetime64[ns]')
-            pytest.xfail("ToDo: do not ignore timezone, must be object")
         self._assert_where_conversion(obj, cond, values, exp, exp_dtype)
-        pytest.xfail("datetime64 + datetime64 -> datetime64 must support"
-                     " scalar")
+
+    @pytest.mark.xfail(
+        reason="GH 22839: do not ignore timezone, must be object")
+    def test_where_index_datetimetz(self):
+        fill_val = pd.Timestamp('2012-01-01', tz='US/Eastern')
+        exp_dtype = np.object
+        obj = pd.Index([pd.Timestamp('2011-01-01'),
+                        pd.Timestamp('2011-01-02'),
+                        pd.Timestamp('2011-01-03'),
+                        pd.Timestamp('2011-01-04')])
+        assert obj.dtype == 'datetime64[ns]'
+        cond = pd.Index([True, False, True, False])
+
+        msg = ("Index\\(\\.\\.\\.\\) must be called with a collection "
+               "of some kind")
+        with tm.assert_raises_regex(TypeError, msg):
+            obj.where(cond, fill_val)
+
+        values = pd.Index(pd.date_range(fill_val, periods=4))
+        exp = pd.Index([pd.Timestamp('2011-01-01'),
+                        pd.Timestamp('2012-01-02', tz='US/Eastern'),
+                        pd.Timestamp('2011-01-03'),
+                        pd.Timestamp('2012-01-04', tz='US/Eastern')],
+                       dtype=exp_dtype)
+
+        self._assert_where_conversion(obj, cond, values, exp, exp_dtype)
 
     def test_where_index_complex128(self):
         pass
diff --git a/pandas/tests/reshape/merge/test_merge.py b/pandas/tests/reshape/merge/test_merge.py
--- a/pandas/tests/reshape/merge/test_merge.py
+++ b/pandas/tests/reshape/merge/test_merge.py
@@ -601,6 +601,30 @@ def test_merge_on_datetime64tz(self):
         assert result['value_x'].dtype == 'datetime64[ns, US/Eastern]'
         assert result['value_y'].dtype == 'datetime64[ns, US/Eastern]'
 
+    def test_merge_datetime64tz_with_dst_transition(self):
+        # GH 18885
+        df1 = pd.DataFrame(pd.date_range(
+            '2017-10-29 01:00', periods=4, freq='H', tz='Europe/Madrid'),
+            columns=['date'])
+        df1['value'] = 1
+        df2 = pd.DataFrame({
+            'date': pd.to_datetime([
+                '2017-10-29 03:00:00', '2017-10-29 04:00:00',
+                '2017-10-29 05:00:00'
+            ]),
+            'value': 2
+        })
+        df2['date'] = df2['date'].dt.tz_localize('UTC').dt.tz_convert(
+            'Europe/Madrid')
+        result = pd.merge(df1, df2, how='outer', on='date')
+        expected = pd.DataFrame({
+            'date': pd.date_range(
+                '2017-10-29 01:00', periods=7, freq='H', tz='Europe/Madrid'),
+            'value_x': [1] * 4 + [np.nan] * 3,
+            'value_y': [np.nan] * 4 + [2] * 3
+        })
+        assert_frame_equal(result, expected)
+
     def test_merge_non_unique_period_index(self):
         # GH #16871
         index = pd.period_range('2016-01-01', periods=16, freq='M')