Repository: pandas-dev/pandas
Pull request #55410 (2023-10-05 02:11:22)

TYP: read_csv and read_excel fixes
read_csv's date_format accepts dicts
read_excel's names can be a Sequence with any Hashable types inside


edit: should probably enable type-checking of some tests (`def test_xxx(...) -> None` should be enough for mypy to check what is happening inside the function) 


Patch:
diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
index 4749ceec4a..7743f762d8 100644
--- a/doc/source/whatsnew/v2.2.0.rst
+++ b/doc/source/whatsnew/v2.2.0.rst
@@ -296,7 +296,7 @@ Datetimelike
 
 Timedelta
 ^^^^^^^^^
--
+- Bug in rendering (``__repr__``) of :class:`TimedeltaIndex` and :class:`Series` with timedelta64 values with non-nanosecond resolution entries that are all multiples of 24 hours failing to use the compact representation used in the nanosecond cases (:issue:`55405`)
 -
 
 Timezones
diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py
index 5b4a2a524f..5059f5d000 100644
--- a/pandas/core/arrays/categorical.py
+++ b/pandas/core/arrays/categorical.py
@@ -2167,11 +2167,11 @@ class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMi
         )
         if len(self.categories) > max_categories:
             num = max_categories // 2
-            head = format_array(self.categories[:num])
-            tail = format_array(self.categories[-num:])
+            head = format_array(self.categories[:num]._values)
+            tail = format_array(self.categories[-num:]._values)
             category_strs = head + ["..."] + tail
         else:
-            category_strs = format_array(self.categories)
+            category_strs = format_array(self.categories._values)
 
         # Strip all leading spaces, which format_array adds for columns...
         category_strs = [x.strip() for x in category_strs]
diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py
index 52596f29ff..a2960a2870 100644
--- a/pandas/core/arrays/datetimelike.py
+++ b/pandas/core/arrays/datetimelike.py
@@ -41,6 +41,7 @@ from pandas._libs.tslibs import (
     iNaT,
     ints_to_pydatetime,
     ints_to_pytimedelta,
+    periods_per_day,
     to_offset,
 )
 from pandas._libs.tslibs.fields import (
@@ -2312,6 +2313,30 @@ class TimelikeOps(DatetimeLikeArrayMixin):
             return self
         return type(self)._simple_new(out_data, dtype=self.dtype)
 
+    # --------------------------------------------------------------
+    # Unsorted
+
+    @property
+    def _is_dates_only(self) -> bool:
+        """
+        Check if we are round times at midnight (and no timezone), which will
+        be given a more compact __repr__ than other cases. For TimedeltaArray
+        we are checking for multiples of 24H.
+        """
+        if not lib.is_np_dtype(self.dtype):
+            # i.e. we have a timezone
+            return False
+
+        values_int = self.asi8
+        consider_values = values_int != iNaT
+        reso = get_unit_from_dtype(self.dtype)
+        ppd = periods_per_day(reso)
+
+        # TODO: can we reuse is_date_array_normalized?  would need a skipna kwd
+        #  (first attempt at this was less performant than this implementation)
+        even_days = np.logical_and(consider_values, values_int % ppd != 0).sum() == 0
+        return even_days
+
 
 # -------------------------------------------------------------------
 # Shared Constructor Helpers
diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index fae42f170a..9544a61635 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -28,14 +28,12 @@ from pandas._libs.tslibs import (
     get_resolution,
     get_supported_reso,
     get_unit_from_dtype,
-    iNaT,
     ints_to_pydatetime,
     is_date_array_normalized,
     is_supported_unit,
     is_unitless,
     normalize_i8_timestamps,
     npy_unit_to_abbrev,
-    periods_per_day,
     timezones,
     to_offset,
     tz_convert_from_utc,
@@ -745,25 +743,6 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):  # type: ignore[misc]
             self.asi8, tz=self.tz, format=date_format, na_rep=na_rep, reso=self._creso
         )
 
-    @property
-    def _is_dates_only(self) -> bool:
-        """
-        Check if we are round times at midnight (and no timezone), which will
-        be given a more compact __repr__ than other cases.
-        """
-        if self.tz is not None:
-            return False
-
-        values_int = self.asi8
-        consider_values = values_int != iNaT
-        dtype = cast(np.dtype, self.dtype)  # since we checked tz above
-        reso = get_unit_from_dtype(dtype)
-        ppd = periods_per_day(reso)
-
-        # TODO: can we reuse is_date_array_normalized?  would need a skipna kwd
-        even_days = np.logical_and(consider_values, values_int % ppd != 0).sum() == 0
-        return even_days
-
     # -----------------------------------------------------------------
     # Comparison Methods
 
diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py
index cd6a488394..5ce3dd33ee 100644
--- a/pandas/core/indexes/timedeltas.py
+++ b/pandas/core/indexes/timedeltas.py
@@ -336,8 +336,7 @@ def timedelta_range(
     **Specify a unit**
 
     >>> pd.timedelta_range("1 Day", periods=3, freq="100000D", unit="s")
-    TimedeltaIndex(['1 days 00:00:00', '100001 days 00:00:00',
-                    '200001 days 00:00:00'],
+    TimedeltaIndex(['1 days', '100001 days', '200001 days'],
                    dtype='timedelta64[s]', freq='100000D')
     """
     if freq is None and com.any_none(periods, start, end):
diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py
index 073115cab8..6def0024d9 100644
--- a/pandas/io/excel/_base.py
+++ b/pandas/io/excel/_base.py
@@ -80,6 +80,7 @@ if TYPE_CHECKING:
         IntStrT,
         ReadBuffer,
         Self,
+        SequenceNotStr,
         StorageOptions,
         WriteExcelBuffer,
     )
@@ -387,7 +388,7 @@ def read_excel(
     sheet_name: str | int = ...,
     *,
     header: int | Sequence[int] | None = ...,
-    names: list[str] | None = ...,
+    names: SequenceNotStr[Hashable] | range | None = ...,
     index_col: int | Sequence[int] | None = ...,
     usecols: int
     | str
@@ -426,7 +427,7 @@ def read_excel(
     sheet_name: list[IntStrT] | None,
     *,
     header: int | Sequence[int] | None = ...,
-    names: list[str] | None = ...,
+    names: SequenceNotStr[Hashable] | range | None = ...,
     index_col: int | Sequence[int] | None = ...,
     usecols: int
     | str
@@ -465,7 +466,7 @@ def read_excel(
     sheet_name: str | int | list[IntStrT] | None = 0,
     *,
     header: int | Sequence[int] | None = 0,
-    names: list[str] | None = None,
+    names: SequenceNotStr[Hashable] | range | None = None,
     index_col: int | Sequence[int] | None = None,
     usecols: int
     | str
@@ -730,7 +731,7 @@ class BaseExcelReader(Generic[_WorkbookT]):
         self,
         sheet_name: str | int | list[int] | list[str] | None = 0,
         header: int | Sequence[int] | None = 0,
-        names=None,
+        names: SequenceNotStr[Hashable] | range | None = None,
         index_col: int | Sequence[int] | None = None,
         usecols=None,
         dtype: DtypeArg | None = None,
@@ -1589,7 +1590,7 @@ class ExcelFile:
         self,
         sheet_name: str | int | list[int] | list[str] | None = 0,
         header: int | Sequence[int] | None = 0,
-        names=None,
+        names: SequenceNotStr[Hashable] | range | None = None,
         index_col: int | Sequence[int] | None = None,
         usecols=None,
         converters=None,
diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py
index 356db34918..c6c09c2636 100644
--- a/pandas/io/formats/format.py
+++ b/pandas/io/formats/format.py
@@ -41,7 +41,6 @@ from pandas._libs.tslibs import (
     NaT,
     Timedelta,
     Timestamp,
-    iNaT,
 )
 from pandas._libs.tslibs.nattype import NaTType
 
@@ -103,7 +102,6 @@ if TYPE_CHECKING:
         SequenceNotStr,
         StorageOptions,
         WriteBuffer,
-        npt,
     )
 
     from pandas import (
@@ -1775,15 +1773,7 @@ def get_format_timedelta64(
 
     If box, then show the return in quotes
     """
-    values_int = values.view(np.int64)
-    values_int = cast("npt.NDArray[np.int64]", values_int)
-
-    consider_values = values_int != iNaT
-
-    one_day_nanos = 86400 * 10**9
-    not_midnight = values_int % one_day_nanos != 0
-    both = np.logical_and(consider_values, not_midnight)
-    even_days = both.sum() == 0
+    even_days = values._is_dates_only
 
     if even_days:
         format = None
diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
index 32f8a1fe81..df5b08029a 100644
--- a/pandas/io/parsers/readers.py
+++ b/pandas/io/parsers/readers.py
@@ -666,7 +666,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser: Callable | lib.NoDefault = ...,
-    date_format: str | None = ...,
+    date_format: str | dict[Hashable, str] | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: Literal[True],
@@ -726,7 +726,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser: Callable | lib.NoDefault = ...,
-    date_format: str | None = ...,
+    date_format: str | dict[Hashable, str] | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: bool = ...,
@@ -786,7 +786,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser: Callable | lib.NoDefault = ...,
-    date_format: str | None = ...,
+    date_format: str | dict[Hashable, str] | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: Literal[False] = ...,
@@ -846,7 +846,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser: Callable | lib.NoDefault = ...,
-    date_format: str | None = ...,
+    date_format: str | dict[Hashable, str] | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: bool = ...,
@@ -920,7 +920,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = lib.no_default,
     keep_date_col: bool = False,
     date_parser: Callable | lib.NoDefault = lib.no_default,
-    date_format: str | None = None,
+    date_format: str | dict[Hashable, str] | None = None,
     dayfirst: bool = False,
     cache_dates: bool = True,
     # Iteration
@@ -1009,7 +1009,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser: Callable | lib.NoDefault = ...,
-    date_format: str | None = ...,
+    date_format: str | dict[Hashable, str] | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: Literal[True],
@@ -1066,7 +1066,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser: Callable | lib.NoDefault = ...,
-    date_format: str | None = ...,
+    date_format: str | dict[Hashable, str] | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: bool = ...,
@@ -1123,7 +1123,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser: Callable | lib.NoDefault = ...,
-    date_format: str | None = ...,
+    date_format: str | dict[Hashable, str] | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: Literal[False] = ...,
@@ -1180,7 +1180,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser: Callable | lib.NoDefault = ...,
-    date_format: str | None = ...,
+    date_format: str | dict[Hashable, str] | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: bool = ...,
@@ -1253,7 +1253,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = lib.no_default,
     keep_date_col: bool = False,
     date_parser: Callable | lib.NoDefault = lib.no_default,
-    date_format: str | None = None,
+    date_format: str | dict[Hashable, str] | None = None,
     dayfirst: bool = False,
     cache_dates: bool = True,
     # Iteration


Test patch:
diff --git a/pandas/tests/indexes/timedeltas/test_formats.py b/pandas/tests/indexes/timedeltas/test_formats.py
index 751f9e4cc9..ee090bd0aa 100644
--- a/pandas/tests/indexes/timedeltas/test_formats.py
+++ b/pandas/tests/indexes/timedeltas/test_formats.py
@@ -8,6 +8,18 @@ from pandas import (
 
 
 class TestTimedeltaIndexRendering:
+    def test_repr_round_days_non_nano(self):
+        # GH#55405
+        # we should get "1 days", not "1 days 00:00:00" with non-nano
+        tdi = TimedeltaIndex(["1 days"], freq="D").as_unit("s")
+        result = repr(tdi)
+        expected = "TimedeltaIndex(['1 days'], dtype='timedelta64[s]', freq='D')"
+        assert result == expected
+
+        result2 = repr(Series(tdi))
+        expected2 = "0   1 days\ndtype: timedelta64[s]"
+        assert result2 == expected2
+
     @pytest.mark.parametrize("method", ["__repr__", "__str__"])
     def test_representation(self, method):
         idx1 = TimedeltaIndex([], freq="D")
diff --git a/pandas/tests/io/formats/test_format.py b/pandas/tests/io/formats/test_format.py
index 0087149021..fe6ddd3aeb 100644
--- a/pandas/tests/io/formats/test_format.py
+++ b/pandas/tests/io/formats/test_format.py
@@ -3186,7 +3186,7 @@ class TestRepr_timedelta64:
 
 class TestTimedelta64Formatter:
     def test_days(self):
-        x = pd.to_timedelta(list(range(5)) + [NaT], unit="D")
+        x = pd.to_timedelta(list(range(5)) + [NaT], unit="D")._values
         result = fmt._Timedelta64Formatter(x, box=True).get_result()
         assert result[0].strip() == "'0 days'"
         assert result[1].strip() == "'1 days'"
@@ -3202,48 +3202,48 @@ class TestTimedelta64Formatter:
         assert result[0].strip() == "1 days"
 
     def test_days_neg(self):
-        x = pd.to_timedelta(list(range(5)) + [NaT], unit="D")
+        x = pd.to_timedelta(list(range(5)) + [NaT], unit="D")._values
         result = fmt._Timedelta64Formatter(-x, box=True).get_result()
         assert result[0].strip() == "'0 days'"
         assert result[1].strip() == "'-1 days'"
 
     def test_subdays(self):
-        y = pd.to_timedelta(list(range(5)) + [NaT], unit="s")
+        y = pd.to_timedelta(list(range(5)) + [NaT], unit="s")._values
         result = fmt._Timedelta64Formatter(y, box=True).get_result()
         assert result[0].strip() == "'0 days 00:00:00'"
         assert result[1].strip() == "'0 days 00:00:01'"
 
     def test_subdays_neg(self):
-        y = pd.to_timedelta(list(range(5)) + [NaT], unit="s")
+        y = pd.to_timedelta(list(range(5)) + [NaT], unit="s")._values
         result = fmt._Timedelta64Formatter(-y, box=True).get_result()
         assert result[0].strip() == "'0 days 00:00:00'"
         assert result[1].strip() == "'-1 days +23:59:59'"
 
     def test_zero(self):
-        x = pd.to_timedelta(list(range(1)) + [NaT], unit="D")
+        x = pd.to_timedelta(list(range(1)) + [NaT], unit="D")._values
         result = fmt._Timedelta64Formatter(x, box=True).get_result()
         assert result[0].strip() == "'0 days'"
 
-        x = pd.to_timedelta(list(range(1)), unit="D")
+        x = pd.to_timedelta(list(range(1)), unit="D")._values
         result = fmt._Timedelta64Formatter(x, box=True).get_result()
         assert result[0].strip() == "'0 days'"
 
 
 class Test_Datetime64Formatter:
     def test_mixed(self):
-        x = Series([datetime(2013, 1, 1), datetime(2013, 1, 1, 12), NaT])
+        x = Series([datetime(2013, 1, 1), datetime(2013, 1, 1, 12), NaT])._values
         result = fmt._Datetime64Formatter(x).get_result()
         assert result[0].strip() == "2013-01-01 00:00:00"
         assert result[1].strip() == "2013-01-01 12:00:00"
 
     def test_dates(self):
-        x = Series([datetime(2013, 1, 1), datetime(2013, 1, 2), NaT])
+        x = Series([datetime(2013, 1, 1), datetime(2013, 1, 2), NaT])._values
         result = fmt._Datetime64Formatter(x).get_result()
         assert result[0].strip() == "2013-01-01"
         assert result[1].strip() == "2013-01-02"
 
     def test_date_nanos(self):
-        x = Series([Timestamp(200)])
+        x = Series([Timestamp(200)])._values
         result = fmt._Datetime64Formatter(x).get_result()
         assert result[0].strip() == "1970-01-01 00:00:00.000000200"
 
@@ -3252,41 +3252,41 @@ class Test_Datetime64Formatter:
         # make sure that we are consistently display date formatting
         x = Series(date_range("20130101 09:00:00", periods=5, freq="D"))
         x.iloc[1] = np.nan
-        result = fmt._Datetime64Formatter(x).get_result()
+        result = fmt._Datetime64Formatter(x._values).get_result()
         assert result[0].strip() == "2013-01-01 09:00:00"
         assert result[1].strip() == "NaT"
         assert result[4].strip() == "2013-01-05 09:00:00"
 
         x = Series(date_range("20130101 09:00:00", periods=5, freq="s"))
         x.iloc[1] = np.nan
-        result = fmt._Datetime64Formatter(x).get_result()
+        result = fmt._Datetime64Formatter(x._values).get_result()
         assert result[0].strip() == "2013-01-01 09:00:00"
         assert result[1].strip() == "NaT"
         assert result[4].strip() == "2013-01-01 09:00:04"
 
         x = Series(date_range("20130101 09:00:00", periods=5, freq="ms"))
         x.iloc[1] = np.nan
-        result = fmt._Datetime64Formatter(x).get_result()
+        result = fmt._Datetime64Formatter(x._values).get_result()
         assert result[0].strip() == "2013-01-01 09:00:00.000"
         assert result[1].strip() == "NaT"
         assert result[4].strip() == "2013-01-01 09:00:00.004"
 
         x = Series(date_range("20130101 09:00:00", periods=5, freq="us"))
         x.iloc[1] = np.nan
-        result = fmt._Datetime64Formatter(x).get_result()
+        result = fmt._Datetime64Formatter(x._values).get_result()
         assert result[0].strip() == "2013-01-01 09:00:00.000000"
         assert result[1].strip() == "NaT"
         assert result[4].strip() == "2013-01-01 09:00:00.000004"
 
         x = Series(date_range("20130101 09:00:00", periods=5, freq="ns"))
         x.iloc[1] = np.nan
-        result = fmt._Datetime64Formatter(x).get_result()
+        result = fmt._Datetime64Formatter(x._values).get_result()
         assert result[0].strip() == "2013-01-01 09:00:00.000000000"
         assert result[1].strip() == "NaT"
         assert result[4].strip() == "2013-01-01 09:00:00.000000004"
 
     def test_datetime64formatter_yearmonth(self):
-        x = Series([datetime(2016, 1, 1), datetime(2016, 2, 2)])
+        x = Series([datetime(2016, 1, 1), datetime(2016, 2, 2)])._values
 
         def format_func(x):
             return x.strftime("%Y-%m")
@@ -3298,7 +3298,7 @@ class Test_Datetime64Formatter:
     def test_datetime64formatter_hoursecond(self):
         x = Series(
             pd.to_datetime(["10:10:10.100", "12:12:12.120"], format="%H:%M:%S.%f")
-        )
+        )._values
 
         def format_func(x):
             return x.strftime("%H:%M")