Repository: pandas-dev/pandas

Issue:
BUG: inconsistent state of DatetimeIndex._data
Depending on how a DatetimeIndex is constructed, the underlying `._data` attribute is a DatetimeIndex or a datetime64 ndarray:

```
In [1]: idx1 = pd.DatetimeIndex(start="2012-01-01", periods=3, freq='D') # date_range kind of construction

In [2]: idx1._data
Out[2]: DatetimeIndex(['2012-01-01', '2012-01-02', '2012-01-03'], dtype='datetime64[ns]', freq=None)

In [3]: idx2 = pd.DatetimeIndex(idx1)

In [4]: idx2._data
Out[4]: 
array(['2012-01-01T00:00:00.000000000', '2012-01-02T00:00:00.000000000',
       '2012-01-03T00:00:00.000000000'], dtype='datetime64[ns]')
```

I think this should always be a numpy array? (it clearly doesn't hurt, but I don't see any reason to have it sometimes as a DatetimeIndex)

This came out of fixing warnings in https://github.com/pandas-dev/pandas/pull/20721, and is due to how `_generate_regular_range` and `_simple_new` on DatetimeIndex are implemented. From the code of `_simple_new`, I suspect that it assumes the input is always an ndarray and not DatetimeIndex, but in several places (like `_generate_regular_range`) an already constructed DatetimeIndex is passed to `_simple_new`.


Discussion:

Resolving patch:
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -506,6 +506,10 @@ def _shallow_copy(self, values=None, **kwargs):
         attributes.update(kwargs)
         if not len(values) and 'dtype' not in kwargs:
             attributes['dtype'] = self.dtype
+
+        # _simple_new expects an ndarray
+        values = getattr(values, 'values', values)
+
         return self._simple_new(values, **attributes)
 
     def _shallow_copy_with_infer(self, values=None, **kwargs):
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -12,7 +12,6 @@
 from pandas.core.dtypes.common import (
     _INT64_DTYPE,
     _NS_DTYPE,
-    is_object_dtype,
     is_datetime64_dtype,
     is_datetimetz,
     is_dtype_equal,
@@ -551,10 +550,11 @@ def _generate(cls, start, end, periods, name, freq,
                     index = _generate_regular_range(start, end, periods, freq)
 
                 if tz is not None and getattr(index, 'tz', None) is None:
-                    index = conversion.tz_localize_to_utc(_ensure_int64(index),
-                                                          tz,
-                                                          ambiguous=ambiguous)
-                    index = index.view(_NS_DTYPE)
+                    arr = conversion.tz_localize_to_utc(_ensure_int64(index),
+                                                        tz,
+                                                        ambiguous=ambiguous)
+
+                    index = DatetimeIndex(arr)
 
                     # index is localized datetime64 array -> have to convert
                     # start/end as well to compare
@@ -575,7 +575,9 @@ def _generate(cls, start, end, periods, name, freq,
             index = index[1:]
         if not right_closed and len(index) and index[-1] == end:
             index = index[:-1]
-        index = cls._simple_new(index, name=name, freq=freq, tz=tz)
+
+        index = cls._simple_new(index.values, name=name, freq=freq, tz=tz)
+
         return index
 
     def _convert_for_op(self, value):
@@ -606,12 +608,14 @@ def _simple_new(cls, values, name=None, freq=None, tz=None,
                            dtype=dtype, **kwargs)
             values = np.array(values, copy=False)
 
-        if is_object_dtype(values):
-            return cls(values, name=name, freq=freq, tz=tz,
-                       dtype=dtype, **kwargs).values
-        elif not is_datetime64_dtype(values):
+        if not is_datetime64_dtype(values):
             values = _ensure_int64(values).view(_NS_DTYPE)
 
+        values = getattr(values, 'values', values)
+
+        assert isinstance(values, np.ndarray), "values is not an np.ndarray"
+        assert is_datetime64_dtype(values)
+
         result = super(DatetimeIndex, cls)._simple_new(values, freq, tz,
                                                        **kwargs)
         result.name = name
@@ -1000,7 +1004,7 @@ def unique(self, level=None):
         else:
             naive = self
         result = super(DatetimeIndex, naive).unique(level=level)
-        return self._simple_new(result, name=self.name, tz=self.tz,
+        return self._simple_new(result.values, name=self.name, tz=self.tz,
                                 freq=self.freq)
 
     def union(self, other):
@@ -1855,7 +1859,7 @@ def _generate_regular_range(start, end, periods, freq):
                              "if a 'period' is given.")
 
         data = np.arange(b, e, stride, dtype=np.int64)
-        data = DatetimeIndex._simple_new(data, None, tz=tz)
+        data = DatetimeIndex._simple_new(data.view(_NS_DTYPE), None, tz=tz)
     else:
         if isinstance(start, Timestamp):
             start = start.to_pydatetime()
diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py
--- a/pandas/io/pytables.py
+++ b/pandas/io/pytables.py
@@ -2472,7 +2472,8 @@ def _get_index_factory(self, klass):
         if klass == DatetimeIndex:
             def f(values, freq=None, tz=None):
                 # data are already in UTC, localize and convert if tz present
-                result = DatetimeIndex._simple_new(values, None, freq=freq)
+                result = DatetimeIndex._simple_new(values.values, None,
+                                                   freq=freq)
                 if tz is not None:
                     result = result.tz_localize('UTC').tz_convert(tz)
                 return result


Test patch:
diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py
--- a/pandas/tests/indexes/test_base.py
+++ b/pandas/tests/indexes/test_base.py
@@ -329,7 +329,7 @@ def test_index_ctor_infer_periodindex(self):
     ])
     def test_constructor_simple_new(self, vals, dtype):
         index = Index(vals, name=dtype)
-        result = index._simple_new(index, dtype)
+        result = index._simple_new(index.values, dtype)
         tm.assert_index_equal(result, index)
 
     @pytest.mark.parametrize("vals", [