Repository: aeturrell/skimpy

Issue:
adding support for datetime.date object types
Hi,

The package is superuseful. However, it seems like the support for some key datatypes frequently used with pandas is missing.
It would be great if you could add support for datetime.date, datetime.month, datetime.year and so on.

for example, it supports datetime64 but if one wants to keep only date part
dt['date'] = dt['datetime'].dt.date

It will give an error "data type 'date' not understood"

Thank you



Discussion:
Great idea! Will add this to list of features to implement.

Resolving patch:
diff --git a/src/skimpy/__init__.py b/src/skimpy/__init__.py
--- a/src/skimpy/__init__.py
+++ b/src/skimpy/__init__.py
@@ -1,4 +1,5 @@
 """skimpy provides summary statistics about variables in pandas data frames."""
+import datetime
 import re
 from collections import defaultdict
 from itertools import chain
@@ -38,6 +39,11 @@
 console = Console()
 QUANTILES = [0, 0.25, 0.5, 0.75, 1]
 HIST_BINS = 6
+# NB: unicode 1/2 and 8/8 blocks have inconsistent widths depending on font
+# systems, so, to make skimpy work on all systems, hist blocks are pinned to
+# nearest available consistent width block. This produces some distortion in
+# histogram but these are just visual indicators anyway and shouldn't be used
+# for deep inference.
 UNICODE_HIST = {
     0: " ",
     1 / 8: "▁",
@@ -56,6 +62,13 @@
 COMPLETE_COL = "NA %"
 MISSING_COL = "NA"
 
+UNSUPPORTED_INFERRED_TYPES = [
+    "mixed-integer",
+    "mixed",
+    "mixed-integer-float",
+    "unknown-array",
+]
+
 
 @typechecked
 def _infer_datatypes(df: pd.DataFrame) -> pd.DataFrame:
@@ -82,8 +95,6 @@ def _infer_datatypes(df: pd.DataFrame) -> pd.DataFrame:
             data_type = "float64"
         elif col[1] == "timedelta64":
             data_type = "timedelta64[ns]"
-        elif col[1] == "timedelta64[ns]":
-            data_type = "timedelta64[ns]"
         elif col[1] == "datetime64":
             data_type = "datetime64[ns]"
         elif col[1] == "categorical":
@@ -92,8 +103,12 @@ def _infer_datatypes(df: pd.DataFrame) -> pd.DataFrame:
             data_type = "bool"
         elif col[1] == "complex":
             data_type = "complex128"
-        else:
-            data_type = col[1]
+        elif col[1] == "date":
+            # do nothing: "date" isn't supported, so we leave as an object.
+            # later on, we will still be able to pick up "date" using
+            # .select_dtypes, which is the important bit.
+            continue
+        # There is no else statement here because logic should never get to this point.
         df[col[0]] = df[col[0]].astype(data_type)
     return df
 
@@ -279,7 +294,8 @@ def _create_unicode_hist(series: pd.Series) -> pd.Series:
 
     Given a pandas series of numerical values, returns a series with one
     entry, the original series name, and a histogram made up of unicode
-    characters.
+    characters. However, note that the histogram is very approximate, partly
+    due to limitations in how unicode is displayed across systems.
 
     Args:
         series (pd.Series): Numeric column of data frame for analysis
@@ -492,6 +508,33 @@ def _datetime_variable_summary_table(xf: pd.DataFrame) -> pd.DataFrame:
     return summary_df
 
 
+@typechecked
+def _delete_unsupported_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """This will remove the pd.api.types.infer_dtype types that are not
+    supported.
+
+    Args:
+        df (pd.DataFrame): _description_
+
+    Returns:
+        pd.DataFrame: _description_
+    """
+    df_types = (
+        pd.DataFrame(df.apply(pd.api.types.infer_dtype, axis=0))
+        .reset_index()
+        .rename(columns={"index": "column", 0: "type"})
+    )
+    df_types["delete"] = df_types["type"].isin(UNSUPPORTED_INFERRED_TYPES)
+    for _, row in df_types.iterrows():
+        if row["delete"]:
+            df = df.drop(row["column"], axis=1)
+    if df.empty:
+        raise ValueError(
+            f"Your input dataframe only has unsupported column types, eg {', '.join(UNSUPPORTED_INFERRED_TYPES)}"
+        )
+    return df
+
+
 @typechecked
 def skim(
     df_in: pd.DataFrame, header_style: str = "bold cyan", **colour_kwargs: str
@@ -504,6 +547,8 @@ def skim(
     better results from ensuring that you set the datatypes in your dataframe
     you want before running skim.
     The colour_kwargs (str) are defined in _dataframe_to_rich_table.
+    Note that any unknown column types, or mixed column types, will not be
+    processed.
 
     Args:
         df_in (pd.DataFrame): Dataframe to skim
@@ -530,6 +575,8 @@ def skim(
 
     # Make a copy so as not to mess with dataframe
     df = df_in.copy()
+    # remove any columns with types that are not currently supported
+    df = _delete_unsupported_columns(df)
     # Perform inference of datatypes
     df = _infer_datatypes(df)
 
@@ -566,10 +613,12 @@ def skim(
     types_funcs_dict = {
         "number": _numeric_variable_summary_table,
         "category": _category_variable_summary_table,
-        "datetime": _datetime_variable_summary_table,
-        "string": _string_variable_summary_table,
         "bool": _bool_variable_summary_table,
+        "datetime": _datetime_variable_summary_table,
+        # Please note that datetime.date is not directly supported by pandas and registers as being of type "object"
+        datetime.date: _datetime_variable_summary_table,  # Re-use of fn intended.
         "timedelta64[ns]": _timedelta_variable_summary_table,
+        "string": _string_variable_summary_table,
     }
     list_of_tabs = []
     for col_type, summary_func in types_funcs_dict.items():
@@ -580,8 +629,12 @@ def skim(
             xf = df.select_dtypes(col_type)
         if not xf.empty:
             sum_df = summary_func(xf)
+            # for rich tables, we need to stringify
+            # specialised and unsupported col types, such as datetime.date,
+            # that are actually registered as object type
+            col_type_to_rich = str(col_type)
             list_of_tabs.append(
-                _dataframe_to_rich_table(col_type, sum_df)  # , **colour_kwargs
+                _dataframe_to_rich_table(col_type_to_rich, sum_df)  # , **colour_kwargs
             )
     # Put all of the info together
     grid = Table.grid(expand=True)
@@ -839,11 +892,13 @@ def generate_test_data() -> pd.DataFrame:
     df.loc[[3, 5, 8, 9, 14, 22], "text"] = None
     df["text"] = df["text"].astype("string")
     # add a datetime column
-    df["date"] = pd.date_range("2018-01-01", periods=len_df, freq="M")
-    df["date_no_freq"] = rng.choice(
+    df["datetime"] = pd.date_range("2018-01-01", periods=len_df, freq="M")
+    df["datetime_no_freq"] = rng.choice(
         (pd.to_datetime(pd.Series(["01/01/2022", "03/04/2023", "01/05/1992"]))), len_df
     )
     df.loc[[3, 12, 0], "date_no_freq"] = pd.NaT
+    df["datetime.date"] = df["datetime"].dt.date
+    df["datetime.date_no_freq"] = df["datetime_no_freq"].dt.date
     timedelta_array = rng.multinomial(40, [1 / 7] * 5, len_df).ravel()
     df["time diff"] = pd.Series([pd.Timedelta(x, "d") for x in timedelta_array])
     df.loc[[22, 1, 13, 65, 120], "time diff"] = pd.NaT


Test patch:
diff --git a/tests/test_main.py b/tests/test_main.py
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -13,6 +13,7 @@
 from skimpy import _convert_case
 from skimpy import _infer_datatypes
 from skimpy import _map_row_positions_to_text_style
+from skimpy import _replace_values
 from skimpy import _round_series
 from skimpy import _simplify_datetimes_in_array
 from skimpy import _string_variable_summary_table
@@ -82,6 +83,7 @@ def test_005_inference_datatypes() -> None:
     df["booly"] = [True, True, False]
     # as example that isn't supported
     df["complex"] = np.array([[1 + 1j], [1 + 1j], [1 + 1j]])
+    df["actual_date"] = df["date2"].dt.date
     df = _infer_datatypes(df)
     dtypes = df.dtypes.apply(lambda x: str(x)).str.split(r"(\d+)").str[0]
     assert list(dtypes) == [
@@ -93,6 +95,7 @@ def test_005_inference_datatypes() -> None:
         "datetime",
         "bool",
         "complex",
+        "object",
     ]
 
 
@@ -516,3 +519,62 @@ def test_24_round_series():
     assert list(_round_series(pd.Series(doubles_to_round)).values) == list(
         np.array([10000.0, 0.99, -0.64, 20.0])
     )
+
+
+def test_25_column_with_mixed_type():
+    """Columns of residual object type, or mixed columns, alongside useful ones."""
+    data_list = [
+        "How are you?",
+        23,
+        True,
+        304.92048,
+    ]
+    df = pd.DataFrame(data_list, columns=["object_col"], dtype="object")
+    # useful to throw in a date here as it's also of type object
+    df["date"] = pd.to_datetime("2003-01-01 00:00:00")
+    df["date"] = df["date"].dt.date
+    df["other_real_data"] = 55.008
+    skim(df)
+
+
+def test_26_only_unsupported_columns():
+    """test that an error is raised when only unsupported columns are passed."""
+    with pytest.raises(ValueError):
+        data_list = [
+            "How are you?",
+            23,
+            True,
+            304.92048,
+        ]
+        df = pd.DataFrame(data_list, columns=["object_col"], dtype="object")
+        skim(df)
+
+
+def test_27_missing_case_entered():
+    """Test for value error when case is misspecified."""
+    with pytest.raises(ValueError):
+        df = pd.DataFrame(
+            {
+                "FirstNom": ["Philip", "Turanga"],
+                "lastName": ["Fry", "Leela"],
+                "Téléphone": ["555-234-5678", "(604) 111-2335"],
+            }
+        )
+        clean_columns(df, case="FAKECASE", replace={"Nom": "Name"})
+
+
+def test_28_special_name_values():
+    """There are special null column names that complicate replace name ops."""
+    # yes, pandas lets you do this.
+    df = pd.DataFrame(
+        {
+            np.nan: ["Philip", "Turanga"],
+            None: ["Fry", "Leela"],
+            "": ["555-234-5678", "(604) 111-2335"],
+        }
+    )
+    ans_one = _replace_values(np.nan, {"Philip": "new_name"})
+    ans_two = _replace_values(None, {"Fry": "new_name"})
+    ans_three = _replace_values("", {"555-234-5678": "new_name"})
+    ans_list = [ans_one, ans_two, ans_three]
+    assert ans_list == [np.nan, None, ""]