Repository: pandas-dev/pandas

Issue:
BUG/API: Fix .replace dtype conversion rules
#### Code Sample, a copy-pastable example if possible
##### 1. replace bool with int (#12780)

NG, should be int Series

```
pd.Series([True]).replace({True: 3})
#0    True
# dtype: bool
```
##### 2. replace datetime/timedelta with int  (#12780)

NG, should be int Series

```
pd.Series([pd.Timestamp('2011-01-01')]).replace({pd.Timestamp('2011-01-01'): 3})
#0   1970-01-01 00:00:00.000000003
# dtype: datetime64[ns]

pd.Series([pd.Timedelta('1 days')]).replace({pd.Timedelta('1 days'): 3})
#0   00:00:00.000000
# dtype: timedelta64[ns]
```
##### 3. replace datetime/timedelta with object  (#12780)

NG, coerced to internal repr unexpectedly.

```
rep = {pd.Timestamp('2011-01-01'): 'a', pd.Timestamp('2011-01-02'): 'b'}
pd.Series([pd.Timestamp('2011-01-01'), pd.Timestamp('2011-01-02')]).replace(rep)
#0    1293840000000000000
#1                      b
# dtype: object
```
##### 4. replace with datetimetz (#12780)

NG, coerced to GMT

```
pd.Series(['a']).replace({'a': pd.Timestamp('2011-01-01', tz='US/Eastern')})
#0   2011-01-01 05:00:00
# dtype: datetime64[ns]
```

##### 5. replacement of string (open, see #15743)

```
In [37]: pd.Series([1,2,3]).replace('1', np.nan)
Out[37]: 
0    NaN
1    2.0
2    3.0
dtype: float64
```

#### output of `pd.show_versions()`

Latest master.

BUG: replace coerces incorrect dtype
- [x] closes #12747
- [x] tests added / passed
- [x] passes `git diff upstream/master | flake8 --diff`
- [x] whatsnew entry



Discussion:

Found this affects more functions than I thought. Let me do it in 4 separate PRs.

1 Add numeric related tests to clarify the current behaviour (and future change)
2. Fix numeric coercion.
3. Add datetime-like tests.
4. Fix datetime-like coercion.

## [Current coverage](https://codecov.io/gh/pandas-dev/pandas/pull/12780?src=pr) is 85.97% (diff: 89.65%)
> Merging [#12780](https://codecov.io/gh/pandas-dev/pandas/pull/12780?src=pr) into [master](https://codecov.io/gh/pandas-dev/pandas/branch/master?src=pr) will decrease coverage by **<.01%**

```diff
@@             master     #12780   diff @@
==========================================
  Files           140        140          
  Lines         51254      51275    +21   
  Methods           0          0          
  Messages          0          0          
  Branches          0          0          
==========================================
+ Hits          44068      44085    +17   
- Misses         7186       7190     +4   
  Partials          0          0          
```

> Powered by [Codecov](https://codecov.io?src=pr). Last update [a62fdf8...f9154e8](https://codecov.io/gh/pandas-dev/pandas/compare/a62fdf8b85bcb0fcedbe9b5269625d50f4b2cdd7...f9154e8a70e128cb0b07edcc325b4444930adcf5?src=pr)
after #13147 this should be slightly more standard

@sinhrks looks pretty good, just a couple of small comments.
@sinhrks status? (lgtm so far)
Updated to fix the above points, and now ready for review.
looks fine. still some internal things ideally should clean up. but can do later.

though couple of comments if you can address.
@sinhrks I am rebasing this and will merge after testing.

Resolving patch:
diff --git a/pandas/core/internals.py b/pandas/core/internals.py
--- a/pandas/core/internals.py
+++ b/pandas/core/internals.py
@@ -1894,8 +1894,11 @@ def convert(self, *args, **kwargs):
                 blocks.append(newb)
 
         else:
-            values = fn(
-                self.values.ravel(), **fn_kwargs).reshape(self.values.shape)
+            values = fn(self.values.ravel(), **fn_kwargs)
+            try:
+                values = values.reshape(self.values.shape)
+            except NotImplementedError:
+                pass
             blocks.append(make_block(values, ndim=self.ndim,
                                      placement=self.mgr_locs))
 
@@ -3238,6 +3241,16 @@ def comp(s):
             return _possibly_compare(values, getattr(s, 'asm8', s),
                                      operator.eq)
 
+        def _cast_scalar(block, scalar):
+            dtype, val = _infer_dtype_from_scalar(scalar, pandas_dtype=True)
+            if not is_dtype_equal(block.dtype, dtype):
+                dtype = _find_common_type([block.dtype, dtype])
+                block = block.astype(dtype)
+                # use original value
+                val = scalar
+
+            return block, val
+
         masks = [comp(s) for i, s in enumerate(src_list)]
 
         result_blocks = []
@@ -3260,7 +3273,8 @@ def comp(s):
                         # particular block
                         m = masks[i][b.mgr_locs.indexer]
                         if m.any():
-                            new_rb.extend(b.putmask(m, d, inplace=True))
+                            b, val = _cast_scalar(b, d)
+                            new_rb.extend(b.putmask(m, val, inplace=True))
                         else:
                             new_rb.append(b)
                 rb = new_rb
diff --git a/pandas/types/cast.py b/pandas/types/cast.py
--- a/pandas/types/cast.py
+++ b/pandas/types/cast.py
@@ -21,7 +21,7 @@
                      _ensure_int32, _ensure_int64,
                      _NS_DTYPE, _TD_DTYPE, _INT64_DTYPE,
                      _POSSIBLY_CAST_DTYPES)
-from .dtypes import ExtensionDtype
+from .dtypes import ExtensionDtype, DatetimeTZDtype, PeriodDtype
 from .generic import ABCDatetimeIndex, ABCPeriodIndex, ABCSeries
 from .missing import isnull, notnull
 from .inference import is_list_like
@@ -312,8 +312,17 @@ def _maybe_promote(dtype, fill_value=np.nan):
     return dtype, fill_value
 
 
-def _infer_dtype_from_scalar(val):
-    """ interpret the dtype from a scalar """
+def _infer_dtype_from_scalar(val, pandas_dtype=False):
+    """
+    interpret the dtype from a scalar
+
+    Parameters
+    ----------
+    pandas_dtype : bool, default False
+        whether to infer dtype including pandas extension types.
+        If False, scalar belongs to pandas extension types is inferred as
+        object
+    """
 
     dtype = np.object_
 
@@ -336,13 +345,20 @@ def _infer_dtype_from_scalar(val):
 
         dtype = np.object_
 
-    elif isinstance(val, (np.datetime64,
-                          datetime)) and getattr(val, 'tzinfo', None) is None:
-        val = lib.Timestamp(val).value
-        dtype = np.dtype('M8[ns]')
+    elif isinstance(val, (np.datetime64, datetime)):
+        val = tslib.Timestamp(val)
+        if val is tslib.NaT or val.tz is None:
+            dtype = np.dtype('M8[ns]')
+        else:
+            if pandas_dtype:
+                dtype = DatetimeTZDtype(unit='ns', tz=val.tz)
+            else:
+                # return datetimetz as object
+                return np.object_, val
+        val = val.value
 
     elif isinstance(val, (np.timedelta64, timedelta)):
-        val = lib.Timedelta(val).value
+        val = tslib.Timedelta(val).value
         dtype = np.dtype('m8[ns]')
 
     elif is_bool(val):
@@ -363,6 +379,11 @@ def _infer_dtype_from_scalar(val):
     elif is_complex(val):
         dtype = np.complex_
 
+    elif pandas_dtype:
+        if lib.is_period(val):
+            dtype = PeriodDtype(freq=val.freq)
+            val = val.ordinal
+
     return dtype, val
 
 


Test patch:
diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py
--- a/pandas/tests/indexing/test_coercion.py
+++ b/pandas/tests/indexing/test_coercion.py
@@ -1153,12 +1153,27 @@ def setUp(self):
         self.rep['float64'] = [1.1, 2.2]
         self.rep['complex128'] = [1 + 1j, 2 + 2j]
         self.rep['bool'] = [True, False]
+        self.rep['datetime64[ns]'] = [pd.Timestamp('2011-01-01'),
+                                      pd.Timestamp('2011-01-03')]
+
+        for tz in ['UTC', 'US/Eastern']:
+            # to test tz => different tz replacement
+            key = 'datetime64[ns, {0}]'.format(tz)
+            self.rep[key] = [pd.Timestamp('2011-01-01', tz=tz),
+                             pd.Timestamp('2011-01-03', tz=tz)]
+
+        self.rep['timedelta64[ns]'] = [pd.Timedelta('1 day'),
+                                       pd.Timedelta('2 day')]
 
     def _assert_replace_conversion(self, from_key, to_key, how):
         index = pd.Index([3, 4], name='xxx')
         obj = pd.Series(self.rep[from_key], index=index, name='yyy')
         self.assertEqual(obj.dtype, from_key)
 
+        if (from_key.startswith('datetime') and to_key.startswith('datetime')):
+            # different tz, currently mask_missing raises SystemError
+            return
+
         if how == 'dict':
             replacer = dict(zip(self.rep[from_key], self.rep[to_key]))
         elif how == 'series':
@@ -1175,17 +1190,12 @@ def _assert_replace_conversion(self, from_key, to_key, how):
             pytest.skip("windows platform buggy: {0} -> {1}".format
                         (from_key, to_key))
 
-        if ((from_key == 'float64' and
-             to_key in ('bool', 'int64')) or
-
+        if ((from_key == 'float64' and to_key in ('bool', 'int64')) or
             (from_key == 'complex128' and
              to_key in ('bool', 'int64', 'float64')) or
 
-            (from_key == 'int64' and
-             to_key in ('bool')) or
-
-            # TODO_GH12747 The result must be int?
-                (from_key == 'bool' and to_key == 'int64')):
+            # GH12747 The result must be int?
+           (from_key == 'int64' and to_key in ('bool'))):
 
             # buggy on 32-bit
             if tm.is_platform_32bit():
@@ -1248,13 +1258,31 @@ def test_replace_series_bool(self):
             self._assert_replace_conversion(from_key, to_key, how='series')
 
     def test_replace_series_datetime64(self):
-        pass
+        from_key = 'datetime64[ns]'
+        for to_key in self.rep:
+            self._assert_replace_conversion(from_key, to_key, how='dict')
+
+        from_key = 'datetime64[ns]'
+        for to_key in self.rep:
+            self._assert_replace_conversion(from_key, to_key, how='series')
 
     def test_replace_series_datetime64tz(self):
-        pass
+        from_key = 'datetime64[ns, US/Eastern]'
+        for to_key in self.rep:
+            self._assert_replace_conversion(from_key, to_key, how='dict')
+
+        from_key = 'datetime64[ns, US/Eastern]'
+        for to_key in self.rep:
+            self._assert_replace_conversion(from_key, to_key, how='series')
 
     def test_replace_series_timedelta64(self):
-        pass
+        from_key = 'timedelta64[ns]'
+        for to_key in self.rep:
+            self._assert_replace_conversion(from_key, to_key, how='dict')
+
+        from_key = 'timedelta64[ns]'
+        for to_key in self.rep:
+            self._assert_replace_conversion(from_key, to_key, how='series')
 
     def test_replace_series_period(self):
         pass
diff --git a/pandas/tests/series/test_replace.py b/pandas/tests/series/test_replace.py
--- a/pandas/tests/series/test_replace.py
+++ b/pandas/tests/series/test_replace.py
@@ -132,8 +132,8 @@ def check_replace(to_rep, val, expected):
             tm.assert_series_equal(expected, r)
             tm.assert_series_equal(expected, sc)
 
-        # should NOT upcast to float
-        e = pd.Series([0, 1, 2, 3, 4])
+        # MUST upcast to float
+        e = pd.Series([0., 1., 2., 3., 4.])
         tr, v = [3], [3.0]
         check_replace(tr, v, e)