Repository: aeturrell/skimpy

Issue:
Skimpy ignores other data types except for Float and Integer
Hi,

I've tried Skimpy first time today and it seems like I found a bug. I used Skimpy on my sample dataframe and it only returns the summary for the Float and Integer columns while others were ignored.

**This is the sample code:**
```
from skimpy import skim
import datetime
import pandas as pd

data = ([datetime.datetime(2021, 1, 1), None, 'as', 6],
        [datetime.datetime(2021, 1, 2), 5.2, 'asd', 7],
        [None, 6.3, 'adasda', 8])

df = pd.DataFrame(data, columns=['date', 'float', 'string', 'integer'])

skim(df)
```
**This is the result I got:**
![image](https://user-images.githubusercontent.com/62826247/136162782-386a0946-8f49-4e1e-81c9-28d5962df326.png)



Discussion:
Will take a look at this as soon as I can.

Resolving patch:
diff --git a/src/skimpy/__init__.py b/src/skimpy/__init__.py
--- a/src/skimpy/__init__.py
+++ b/src/skimpy/__init__.py
@@ -1,5 +1,6 @@
 """skimpy provides summary statistics about variables in pandas data frames."""
 from collections import defaultdict
+from itertools import chain
 
 import numpy as np
 import pandas as pd
@@ -29,6 +30,45 @@
 }
 
 
+@typechecked
+def infer_datatypes(df: pd.DataFrame) -> pd.DataFrame:
+    """Infers the, and applies new, datatypes of dataframe columns.
+
+    :param df: input dataframe of ambiguous col type
+    :type df: pd.DataFrame
+    :return: dataframe with column datatypes set to best of knowledge
+    :rtype: pd.DataFrame
+    """
+    df_types = (
+        pd.DataFrame(df.apply(pd.api.types.infer_dtype, axis=0))
+        .reset_index()
+        .rename(columns={"index": "column", 0: "type"})
+    )
+    loop_types = df_types.values.tolist()
+    for col in loop_types:
+        if col[1] == "mixed":
+            pass
+        else:
+            if col[1] == "decimal":
+                data_type = "float64"
+            elif col[1] == "string":
+                data_type = "string"
+            elif col[1] == "integer":
+                data_type = "int"
+            elif col[1] == "floating":
+                data_type = "float64"
+            elif col[1] == "date":
+                data_type = "datetime64"
+            elif col[1] == "categorical":
+                data_type = "category"
+            elif col[1] == "boolean":
+                data_type = "bool"
+            else:
+                data_type = col[1]
+            df[col[0]] = df[col[0]].astype(data_type)
+    return df
+
+
 @typechecked
 def round_dataframe(df: pd.DataFrame) -> pd.DataFrame:
     """Rounds dataframe to 2 s.f.
@@ -114,6 +154,22 @@ def dataframe_to_rich_table(
         zip(range(len(columns)), [type_to_colour[x] for x in columns])
     )
     rows = df.values
+    # find any datetimes
+    if ("first" or "last") in df.columns:
+        timestamp_positions = [
+            [
+                [idx, i]
+                for i, j in enumerate(item)
+                if type(j) == pd._libs.tslibs.timestamps.Timestamp
+            ]
+            for idx, item in enumerate(rows)
+        ]
+        timestamp_positions = list(chain.from_iterable(timestamp_positions))
+        timestamp_pos_tuples = [tuple(entry) for entry in timestamp_positions]
+        for entry in timestamp_pos_tuples:
+            hour, min, sec = rows[entry].hour, rows[entry].minute, rows[entry].second
+            if hour == min == sec == 0:
+                rows[entry] = rows[entry].strftime("%Y-%m-%d")
     for col in columns:
         table.add_column(str(col), overflow="fold")
     for row in rows:
@@ -271,12 +327,23 @@ def string_variable_summary_table(xf: pd.DataFrame) -> pd.DataFrame:
             dict(
                 zip(
                     xf.columns,
-                    [xf[col].str.len().sum() / len(xf) for col in xf.columns],
+                    [
+                        xf[xf.columns[0]].str.count(" ").add(1).sum() / len(xf)
+                        for col in xf.columns
+                    ],
                 )
             )
         ),
         "total words": pd.Series(
-            dict(zip(xf.columns, [xf[col].str.len().sum() for col in xf.columns]))
+            dict(
+                zip(
+                    xf.columns,
+                    [
+                        xf[xf.columns[0]].str.count(" ").add(1).sum()
+                        for col in xf.columns
+                    ],
+                )
+            )
         ),
     }
     summary_df = pd.DataFrame(data_dict)
@@ -345,6 +412,9 @@ def skim(
     else:
         name = "dataframe"
 
+    # Perform inference of datatypes
+    # df = infer_datatypes(df)
+
     # Data summary
     tab_1_data = {"Number of rows": df.shape[0], "Number of columns": df.shape[1]}
     dat_sum_table = Table(
@@ -401,6 +471,8 @@ def skim(
     grid.add_column(justify="left")
     for sum_tab in list_of_tabs:
         grid.add_row(sum_tab)
+    # Weirdly, iteration over list of tabs misses last entry
+    grid.add_row(list_of_tabs[-1])
     console.print(Panel(grid, title="skimpy summary", subtitle="End"))
 
 


Test patch:
diff --git a/tests/test_main.py b/tests/test_main.py
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -1,9 +1,13 @@
 """Test cases for the __main__ module."""
+import datetime
+
+import pandas as pd
 import pytest
 from click.testing import CliRunner
 
 from skimpy import __main__
 from skimpy import generate_test_data
+from skimpy import infer_datatypes
 from skimpy import skim
 
 
@@ -52,3 +56,16 @@ def test_004_when_df_is_named() -> None:
     df = generate_test_data()
     df.name = "Named dataframe"
     skim(df)
+
+
+def test_005_inference_datatypes() -> None:
+    """Tests the inference of datatypes."""
+    data = (
+        [datetime.datetime(2021, 1, 1), None, "as", 6],
+        [datetime.datetime(2021, 1, 2), 5.2, "asd", 7],
+        [None, 6.3, "adasda", 8],
+    )
+    df = pd.DataFrame(data, columns=["date", "float", "string", "integer"])
+    df = infer_datatypes(df)
+    dtypes = df.dtypes.apply(lambda x: str(x)).str.split(r"(\d+)").str[0]
+    assert list(dtypes) == ["datetime", "float", "string", "int"]