Repository: alteryx/woodwork (python)

Issue:
Add additional optional outputs to df.ww.describe()
As a user, I wish Woodwork's `describe` and `describe_dict` methods would optionally include additional stats depending on the column type:

- numeric -> add histogram with specified number of bins
- categorical -> add "top N" items by occurrence
- datetime -> add "N most recent" dates

This would require adding some additional parameters to the methods to control this behavior and provide the necessary inputs. The default behavior would produce results identical to those obtained currently.
#### Code Example

```python
df.ww.describe(extra_stats=True, bins=5, top_n=10, n_most_recent=8)
df.ww.describe_dict(extra_stats=True, bins=5, top_n=10, n_most_recent=8)
```


Pull request description:
Add option to compute extra stats with describe
- Add option to compute extra stats to describe
- Closes #960

Adds the ability to compute extra optional stats with the `describe` method by passing `extra_stats=True` along with other optional parameter to control the calculations. Extra stats include:
- histogram (numeric cols)
- top_values (categorical and certain integer cols)
- most_recent (datetime cols)

Patch:
diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
index 10602c9..15c5ce4 100644
--- a/docs/source/release_notes.rst
+++ b/docs/source/release_notes.rst
@@ -7,6 +7,7 @@ Future Release
 ==============
     * Enhancements
         * Pass additional progress information in callback functions (:pr:`979`)
+        * Add the ability to generate optional extra stats with ``DataFrame.ww.describe_dict`` (:pr:`988`)
     * Fixes
     * Changes
     * Documentation Changes
diff --git a/woodwork/statistics_utils.py b/woodwork/statistics_utils.py
index ee9d9e0..5353596 100644
--- a/woodwork/statistics_utils.py
+++ b/woodwork/statistics_utils.py
@@ -4,14 +4,21 @@ import numpy as np
 import pandas as pd
 from sklearn.metrics.cluster import normalized_mutual_info_score
 
-from woodwork.logical_types import Datetime, Double, LatLong
+from woodwork.logical_types import (
+    Datetime,
+    Double,
+    Integer,
+    IntegerNullable,
+    LatLong
+)
 from woodwork.utils import _update_progress, get_valid_mi_types, import_or_none
 
 dd = import_or_none('dask.dataframe')
 ks = import_or_none('databricks.koalas')
 
 
-def _get_describe_dict(dataframe, include=None, callback=None):
+def _get_describe_dict(dataframe, include=None, callback=None,
+                       extra_stats=False, bins=10, top_x=10, recent_x=10):
     """Calculates statistics for data contained in a DataFrame using Woodwork typing information.
 
     Args:
@@ -29,6 +36,18 @@ def _get_describe_dict(dataframe, include=None, callback=None):
             - unit (str): unit of measurement for progress/total
             - time_elapsed (float): total time in seconds elapsed since start of call
 
+        extra_stats (bool): If True, will calculate a histogram for numeric columns, top values
+            for categorical columns and value counts for the most recent values in datetime columns. Will also
+            calculate value counts within the range of values present for integer columns if the range of
+            values present is less than or equal to than the number of bins used to compute the histogram.
+            Output can be controlled by bins, top_x and recent_x parameters.
+        bins (int): Number of bins to use when calculating histogram for numeric columns. Defaults to 10.
+            Will be ignored unless extra_stats=True.
+        top_x (int): Number of items to return when getting the most frequently occurring values for categorical
+            columns. Defaults to 10. Will be ignored unless extra_stats=True.
+        recent_x (int): Number of values to return when calculating value counts for the most recent dates in
+            datetime columns. Defaults to 10. Will be ignored unless extra_stats=True.
+
     Returns:
         dict[str -> dict]: A dictionary with a key for each column in the data or for each column
         matching the logical types, semantic tags or column names specified in ``include``, paired
@@ -106,6 +125,22 @@ def _get_describe_dict(dataframe, include=None, callback=None):
         values["physical_type"] = series.dtype
         values["logical_type"] = logical_type
         values["semantic_tags"] = semantic_tags
+
+        # Calculate extra detailed stats, if requested
+        if extra_stats:
+            if column.is_numeric:
+                values["histogram"] = _get_histogram_values(series, bins=bins)
+                if isinstance(column.logical_type, (Integer, IntegerNullable)):
+                    _range = range(int(values["min"]), int(values["max"]) + 1)
+                    # Calculate top numeric values if range of values present
+                    # is less than or equal number of histogram bins
+                    if len(_range) <= bins:
+                        values["top_values"] = _get_numeric_value_counts_in_range(series, _range)
+            elif column.is_categorical:
+                values["top_values"] = _get_top_values_categorical(series, top_x)
+            elif column.is_datetime:
+                values["recent_values"] = _get_recent_value_counts(series, recent_x)
+
         results[column_name] = values
         current_progress = _update_progress(start_time, timer(), 1, current_progress, total_loops, unit, callback)
     return results
@@ -306,3 +341,79 @@ def _get_value_counts(dataframe, ascending=False, top_n=10, dropna=False):
         values = list(df.to_dict(orient="index").values())
         val_counts[col] = values
     return val_counts
+
+
+def _get_numeric_value_counts_in_range(series, _range):
+    """Count the number of occurrences of integers present in a series with values defined
+    by a range of integers. Null values will be ignored.
+
+    Args:
+        series (pd.Series): data from which to determine the number of occurrences of values
+        _range (type(range)): sequence of integers defining the values for which counts should be made
+
+    Returns:
+        value_counts (list(dict)): a list of dictionaries with keys `value` and
+            `count`. Output is sorted in descending order based on the value counts.
+    """
+    frequencies = series.value_counts(dropna=True)
+    value_counts = [{"value": i, "count": frequencies[i] if i in frequencies else 0} for i in _range]
+    return sorted(value_counts, key=lambda i: (-i["count"], i["value"]))
+
+
+def _get_top_values_categorical(series, num_x):
+    """Get the most frequent values in a pandas Series. Will exclude null values.
+
+    Args:
+        column (pd.Series): data to use find most frequent values
+        num_x (int): the number of top values to retrieve
+
+    Returns:
+        top_list (list(dict)): a list of dictionary with keys `value` and `count`.
+            Output is sorted in descending order based on the value counts.
+    """
+    frequencies = series.value_counts(dropna=True)
+    df = frequencies.head(num_x).reset_index()
+    df.columns = ["value", "count"]
+    df = df.sort_values(["count", "value"], ascending=[False, True])
+    value_counts = list(df.to_dict(orient="index").values())
+    return value_counts
+
+
+def _get_recent_value_counts(column, num_x):
+    """Get the the number of occurrences of the x most recent values in a datetime column.
+
+    Args:
+        column (pd.Series): data to use find value counts
+        num_x (int): the number of values to retrieve
+
+    Returns:
+        value_counts (list(dict)): a list of dictionary with keys `value` and
+            `count`. Output is sorted in descending order based on the value counts.
+    """
+    datetimes = getattr(column.dt, "date")
+    frequencies = datetimes.value_counts(dropna=False)
+    values = frequencies.sort_index(ascending=False)[:num_x]
+    df = values.reset_index()
+    df.columns = ["value", "count"]
+    df = df.sort_values(["count", "value"], ascending=[False, True])
+    value_counts = list(df.to_dict(orient="index").values())
+    return value_counts
+
+
+def _get_histogram_values(series, bins=10):
+    """Get the histogram for a given numeric column.
+
+    Args:
+        series (pd.Series): data to use for histogram
+        bins (int): the number of bins to use for the histogram
+
+    Returns:
+        histogram (list(dict)): a list of dictionary with keys `bins` and
+            `frequency`
+    """
+    values = (
+        pd.cut(series, bins=bins, duplicates='drop').value_counts().sort_index()
+    )
+    df = values.reset_index()
+    df.columns = ["bins", "frequency"]
+    return list(df.to_dict(orient="index").values())
diff --git a/woodwork/table_accessor.py b/woodwork/table_accessor.py
index 6e17912..42f0dfe 100644
--- a/woodwork/table_accessor.py
+++ b/woodwork/table_accessor.py
@@ -808,7 +808,7 @@ class WoodworkTableAccessor:
         mutual_info = self.mutual_information_dict(num_bins, nrows, include_index, callback)
         return pd.DataFrame(mutual_info)
 
-    def describe_dict(self, include=None, callback=None):
+    def describe_dict(self, include=None, callback=None, extra_stats=False, bins=10, top_x=10, recent_x=10):
         """Calculates statistics for data contained in the DataFrame.
 
         Args:
@@ -825,6 +825,18 @@ class WoodworkTableAccessor:
                 - unit (str): unit of measurement for progress/total
                 - time_elapsed (float): total time in seconds elapsed since start of call
 
+            extra_stats (bool): If True, will calculate a histogram for numeric columns, top values
+                for categorical columns and value counts for the most recent values in datetime columns. Will also
+                calculate value counts within the range of values present for integer columns if the range of
+                values present is less than or equal to than the number of bins used to compute the histogram.
+                Output can be controlled by bins, top_x and recent_x parameters.
+            bins (int): Number of bins to use when calculating histogram for numeric columns. Defaults to 10.
+                Will be ignored unless extra_stats=True.
+            top_x (int): Number of items to return when getting the most frequently occurring values for categorical
+                columns. Defaults to 10. Will be ignored unless extra_stats=True.
+            recent_x (int): Number of values to return when calculating value counts for the most recent dates in
+                datetime columns. Defaults to 10. Will be ignored unless extra_stats=True.
+
         Returns:
             dict[str -> dict]: A dictionary with a key for each column in the data or for each column
             matching the logical types, semantic tags or column names specified in ``include``, paired
@@ -832,7 +844,8 @@ class WoodworkTableAccessor:
         """
         if self._schema is None:
             _raise_init_error()
-        return _get_describe_dict(self._dataframe, include=include, callback=callback)
+        return _get_describe_dict(self._dataframe, include=include, callback=callback,
+                                  extra_stats=extra_stats, bins=bins, top_x=top_x, recent_x=recent_x)
 
     def describe(self, include=None, callback=None):
         """Calculates statistics for data contained in the DataFrame.


Test patch:
diff --git a/woodwork/tests/accessor/test_statistics.py b/woodwork/tests/accessor/test_statistics.py
index 4355b3e..cc2020b 100644
--- a/woodwork/tests/accessor/test_statistics.py
+++ b/woodwork/tests/accessor/test_statistics.py
@@ -1,7 +1,9 @@
+from datetime import datetime
 from inspect import isclass
 
 import numpy as np
 import pandas as pd
+import pytest
 
 from woodwork.logical_types import (
     URL,
@@ -29,7 +31,11 @@ from woodwork.logical_types import (
 )
 from woodwork.statistics_utils import (
     _get_describe_dict,
+    _get_histogram_values,
     _get_mode,
+    _get_numeric_value_counts_in_range,
+    _get_recent_value_counts,
+    _get_top_values_categorical,
     _make_categorical_for_mutual_info,
     _replace_nans_for_mutual_info
 )
@@ -239,6 +245,10 @@ def test_get_describe_dict(describe_df):
     describe_df.ww.init(index='index_col')
 
     stats_dict = _get_describe_dict(describe_df)
+    stats_dict_to_df = pd.DataFrame(stats_dict)
+    for extra in ['histogram', 'top_values', 'recent_values']:
+        assert extra not in stats_dict_to_df.index
+
     index_order = ['physical_type',
                    'logical_type',
                    'semantic_tags',
@@ -255,7 +265,8 @@ def test_get_describe_dict(describe_df):
                    'max',
                    'num_true',
                    'num_false']
-    stats_dict_to_df = pd.DataFrame(stats_dict).reindex(index_order)
+
+    stats_dict_to_df = stats_dict_to_df.reindex(index_order)
     stats_df = describe_df.ww.describe()
     pd.testing.assert_frame_equal(stats_df, stats_dict_to_df)
 
@@ -648,6 +659,45 @@ def test_describe_callback(describe_df):
     assert mock_callback.total_elapsed_time > 0
 
 
+def test_describe_dict_extra_stats(describe_df):
+    describe_df = describe_df.drop(columns=['boolean_col', 'natural_language_col', 'formatted_datetime_col',
+                                            'timedelta_col', 'latlong_col'])
+    describe_df['nullable_integer_col'] = describe_df['numeric_col']
+    describe_df['integer_col'] = describe_df['numeric_col'].fillna(0)
+    describe_df['small_range_col'] = describe_df['numeric_col'].fillna(0) // 10
+
+    ltypes = {
+        'category_col': 'Categorical',
+        'datetime_col': 'Datetime',
+        'numeric_col': 'Double',
+        'nullable_integer_col': 'IntegerNullable',
+        'integer_col': 'Integer',
+        'small_range_col': 'Integer',
+    }
+    describe_df.ww.init(index='index_col', logical_types=ltypes)
+    desc_dict = describe_df.ww.describe_dict(extra_stats=True)
+
+    # category columns should have top_values
+    assert isinstance(desc_dict['category_col']['top_values'], list)
+    assert desc_dict['category_col'].get('histogram') is None
+    assert desc_dict['category_col'].get('recent_values') is None
+
+    # datetime columns should have recent_values
+    assert isinstance(desc_dict['datetime_col']['recent_values'], list)
+    assert desc_dict['datetime_col'].get('histogram') is None
+    assert desc_dict['datetime_col'].get('top_values') is None
+
+    # numeric columns should have histogram
+    for col in ['numeric_col', 'nullable_integer_col', 'integer_col', 'small_range_col']:
+        assert isinstance(desc_dict[col]['histogram'], list)
+        assert desc_dict[col].get('recent_values') is None
+        if col == 'small_range_col':
+            # If values are in a narrow range, top values should be present
+            assert isinstance(desc_dict[col]['top_values'], list)
+        else:
+            assert desc_dict[col].get('top_values') is None
+
+
 def test_value_counts(categorical_df):
     logical_types = {
         'ints': IntegerNullable,
@@ -688,3 +738,95 @@ def test_value_counts(categorical_df):
     val_cts_2 = categorical_df.ww.value_counts(top_n=2)
     for col in val_cts_2:
         assert len(val_cts_2[col]) == 2
+
+
+def test_datetime_get_recent_value_counts():
+    times = pd.Series([
+        datetime(2019, 2, 2, 1, 10, 0, 1),
+        datetime(2019, 4, 2, 2, 20, 1, 0),
+        datetime(2019, 3, 1, 3, 30, 1, 0),
+        datetime(2019, 5, 1, 4, 40, 1, 0),
+        datetime(2019, 1, 1, 5, 50, 1, 0),
+        datetime(2019, 4, 2, 6, 10, 1, 0),
+        datetime(2019, 4, 2, 7, 20, 1, 0),
+        datetime(2019, 5, 1, 8, 30, 0, 0),
+        pd.NaT,
+    ])
+    values = _get_recent_value_counts(times, num_x=3)
+    expected_values = [
+        {"value": datetime(2019, 4, 2).date(), "count": 3},
+        {"value": datetime(2019, 5, 1).date(), "count": 2},
+        {"value": datetime(2019, 3, 1).date(), "count": 1},
+    ]
+    assert values == expected_values
+
+
+def test_numeric_histogram():
+    column = pd.Series(np.random.randn(1000))
+    column.append(pd.Series([np.nan, " ", "test"]))
+    bins = 7
+    values = _get_histogram_values(column, bins=bins)
+    assert len(values) == bins
+    total = 0
+    for info in values:
+        assert "bins" in info
+        assert "frequency" in info
+        freq = info["frequency"]
+        total += freq
+    assert total == 1000
+
+
+@pytest.mark.parametrize(
+    "input_series, expected",
+    [
+        (
+            ["a", "b", "b", "c", "c", "c", np.nan],
+            [
+                {"value": "c", "count": 3},
+                {"value": "b", "count": 2},
+                {"value": "a", "count": 1},
+            ],
+        ),
+        (
+            [1, 2, 2, 3],
+            [
+                {"value": 2, "count": 2},
+                {"value": 1, "count": 1},
+                {"value": 3, "count": 1},
+            ],
+        ),
+    ],
+)
+def test_get_top_values_categorical(input_series, expected):
+    column = pd.Series(input_series)
+    top_values = _get_top_values_categorical(column, 10)
+    assert top_values == expected
+
+
+@pytest.mark.parametrize(
+    "input_series, expected",
+    [
+        (
+            pd.Series([1, 2, 2, 3, 3, 3, pd.NA], dtype='Int64'),
+            [
+                {"value": 3, "count": 3},
+                {"value": 2, "count": 2},
+                {"value": 1, "count": 1},
+                {"value": 0, "count": 0},
+            ],
+        ),
+        (
+            pd.Series([1, 2, 2, 3], dtype='int64'),
+            [
+                {"value": 2, "count": 2},
+                {"value": 1, "count": 1},
+                {"value": 3, "count": 1},
+                {"value": 0, "count": 0},
+            ],
+        ),
+    ],
+)
+def test_get_numeric_value_counts_in_range(input_series, expected):
+    column = input_series
+    top_values = _get_numeric_value_counts_in_range(column, range(4))
+    assert top_values == expected