Repository: databricks/koalas

Issue:
DataFrameGroupBy.describe



Discussion:
I have implemented a version of `DataFrameGroupBy.describe` that "works" for my project. I feel like the implementation is not ideal, but I'm happy to share and make fixes based on feedback to get it right, if that sounds good to you all.
I've patched the behavior in my project (see code below).

The only change in `_spark_groupby` is the `aggfunc` option I added to support `describe`, `"quartiles"`. This is the piece I'm actually least comfortable with, since it's adding functionality that isn't there in `pandas`. I investigated a number of options, but I may have missed the option to implement `DataFrameGroupBy.quantile` (it's been a month or so, I forget if I looked into it). Happy to try and implement `DataFrameGroupBy.quantile` and use that instead, if that sounds like the right way forward.

```python3
import databricks.koalas as ks
from databricks.koalas.groupby import GroupBy
from databricks.koalas.internal import SPARK_INDEX_NAME_FORMAT, _InternalFrame
from databricks.koalas.utils import scol_for
from pyspark.sql import functions as F


def _spark_groupby(kdf, func, groupkeys):
    sdf = kdf._sdf
    groupkey_cols = [s._scol.alias(SPARK_INDEX_NAME_FORMAT(i))
                     for i, s in enumerate(groupkeys)]
    multi_aggs = any(isinstance(v, list) for v in func.values())
    reordered = []
    data_columns = []
    column_index = []
    for key, value in func.items():
        idx = key if isinstance(key, tuple) else (key,)
        for aggfunc in [value] if isinstance(value, str) else value:
            name = kdf._internal.column_name_for(idx)
            data_col = "('{0}', '{1}')".format(name, aggfunc) if multi_aggs else name
            data_columns.append(data_col)
            column_index.append(tuple(list(idx) + [aggfunc]) if multi_aggs else idx)
            if aggfunc == "nunique":
                reordered.append(
                    F.expr('count(DISTINCT `{0}`) as `{1}`'.format(name, data_col)))

            # Implement "quartiles" aggregate function for ``describe``.
            elif aggfunc == "quartiles":
                reordered.append(
                    F.expr('percentile_approx(`{0}`, array(0.25, 0.5, 0.75)) as `{1}`'.format(name, data_col)))

            else:
                reordered.append(F.expr('{1}(`{0}`) as `{2}`'.format(name, aggfunc, data_col)))
    sdf = sdf.groupby(*groupkey_cols).agg(*reordered)
    if len(groupkeys) > 0:
        index_map = [(SPARK_INDEX_NAME_FORMAT(i),
                      s._internal.column_index[0])
                     for i, s in enumerate(groupkeys)]
    else:
        index_map = None
    return _InternalFrame(sdf=sdf,
                          column_index=column_index,
                          column_scols=[scol_for(sdf, col) for col in data_columns],
                          index_map=index_map)


GroupBy._spark_groupby = _spark_groupby


def _describe(self):
    kdf = self.agg(["count", "mean", "std", "min", "quartiles", "max"]).reset_index()

    # Split "quartiles" columns into first, second, and third quartiles.
    for label, content in kdf.iteritems():
        if label[1] == "quartiles":
            exploded = ks.DataFrame(content.tolist())
            exploded.columns = [(label[0], "25%"), (label[0], "50%"), (label[0], "75%")]
            kdf = kdf.drop(label).join(exploded)

    # Reindex the DataFrame to reflect initial grouping and agg columns.
    input_groupnames = [s.name for s in self._groupkeys]
    kdf.set_index([(key, "") for key in input_groupnames], inplace=True)
    kdf.index.names = input_groupnames

    return kdf


GroupBy.describe = _describe
```
Nice to see you @deepyaman ! thanks for sharing your experience :D
If you don't mind, could you create a PR for this?
i think it makes easy for comparing to existing code and getting code review 😸 
Yeah, please go ahead for a PR.

Resolving patch:
diff --git a/databricks/koalas/groupby.py b/databricks/koalas/groupby.py
--- a/databricks/koalas/groupby.py
+++ b/databricks/koalas/groupby.py
@@ -22,6 +22,8 @@
 import inspect
 from collections import Callable, OrderedDict, namedtuple
 from functools import partial
+from itertools import product
+from operator import itemgetter
 from typing import Any, List, Tuple, Union
 
 import numpy as np
@@ -33,6 +35,7 @@
 from pyspark.sql.functions import PandasUDFType, pandas_udf, Column
 
 from databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.
+from databricks.koalas.base import _column_op
 from databricks.koalas.typedef import _infer_return_type
 from databricks.koalas.frame import DataFrame
 from databricks.koalas.internal import (_InternalFrame, HIDDEN_COLUMNS, NATURAL_ORDER_COLUMN_NAME,
@@ -210,6 +213,13 @@ def _spark_groupby(kdf, func, groupkeys):
                 if aggfunc == "nunique":
                     reordered.append(
                         F.expr('count(DISTINCT `{0}`) as `{1}`'.format(name, data_col)))
+
+                # Implement "quartiles" aggregate function for ``describe``.
+                elif aggfunc == "quartiles":
+                    reordered.append(
+                        F.expr('percentile_approx(`{0}`, array(0.25, 0.5, 0.75)) as `{1}`'.format(
+                            name, data_col)))
+
                 else:
                     reordered.append(F.expr('{1}(`{0}`) as `{2}`'.format(name, aggfunc, data_col)))
         sdf = sdf.groupby(*groupkey_cols).agg(*reordered)
@@ -1936,6 +1946,34 @@ def _shift(self, periods, fill_value):
                                                     for c in applied])
         return DataFrame(internal)
 
+    def describe(self):
+        kdf = self.agg(["count", "mean", "std", "min", "quartiles", "max"]).reset_index()
+        formatted_percentiles = ["25%", "50%", "75%"]
+
+        # Split "quartiles" columns into first, second, and third quartiles.
+        for label, content in kdf.iteritems():
+            if label[1] == "quartiles":
+                exploded = ks.DataFrame(
+                    {
+                        (label[0], x): _column_op(itemgetter(i))(content).to_numpy()
+                        for i, x in enumerate(formatted_percentiles)
+                    }
+                )
+                kdf = kdf.drop(label).join(exploded)
+
+        # Reindex the DataFrame to reflect initial grouping and agg columns.
+        input_groupnames = [s.name for s in self._groupkeys]
+        kdf.set_index([(key, "") for key in input_groupnames], inplace=True)
+        kdf.index.names = input_groupnames
+
+        # Reorder columns lexicographically by agg column followed by stats.
+        agg_cols = (col.name for col in self._agg_columns)
+        stats = ["count", "mean", "std", "min"] + formatted_percentiles + ["max"]
+        kdf = kdf[list(product(agg_cols, stats))]
+
+        # Cast columns to ``"float64"`` to match `pandas.DataFrame.groupby`.
+        return kdf.astype("float64")
+
 
 class SeriesGroupBy(GroupBy):
 
diff --git a/databricks/koalas/missing/groupby.py b/databricks/koalas/missing/groupby.py
--- a/databricks/koalas/missing/groupby.py
+++ b/databricks/koalas/missing/groupby.py
@@ -50,7 +50,6 @@ class _MissingPandasLikeDataFrameGroupBy(object):
     # Functions
     boxplot = unsupported_function('boxplot')
     cumcount = unsupported_function('cumcount')
-    describe = unsupported_function('describe')
     get_group = unsupported_function('get_group')
     median = unsupported_function('median')
     ngroup = unsupported_function('ngroup')


Test patch:
diff --git a/databricks/koalas/tests/test_groupby.py b/databricks/koalas/tests/test_groupby.py
--- a/databricks/koalas/tests/test_groupby.py
+++ b/databricks/koalas/tests/test_groupby.py
@@ -17,6 +17,7 @@
 import unittest
 import inspect
 from distutils.version import LooseVersion
+from itertools import product
 
 import numpy as np
 import pandas as pd
@@ -257,6 +258,28 @@ def test_aggregate_relabel(self):
         )
         self.assert_eq(agg_kdf, agg_pdf)
 
+    def test_describe(self):
+        pdf = pd.DataFrame({"a": [1, 1, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
+        kdf = ks.from_pandas(pdf)
+
+        describe_pdf = pdf.groupby("a").describe().sort_index()
+        describe_kdf = kdf.groupby("a").describe().sort_index()
+
+        # Check that non-percentile columns are equal.
+        agg_cols = [col.name for col in kdf.groupby("a")._agg_columns]
+        formatted_percentiles = ["25%", "50%", "75%"]
+        self.assert_eq(describe_kdf.drop(list(product(agg_cols, formatted_percentiles))),
+                       describe_pdf.drop(columns=formatted_percentiles, level=1))
+
+        # Check that percentile columns are equal.
+        percentiles = [0.25, 0.5, 0.75]
+        # The interpolation argument is yet to be implemented in Koalas.
+        quantile_pdf = pdf.groupby("a").quantile(percentiles, interpolation="nearest")
+        quantile_pdf = quantile_pdf.unstack(level=1).astype(float)
+        non_percentile_stats = ["count", "mean", "std", "min", "max"]
+        self.assert_eq(describe_kdf.drop(list(product(agg_cols, non_percentile_stats))),
+                       quantile_pdf.rename(columns="{:.0%}".format, level=1))
+
     def test_all_any(self):
         pdf = pd.DataFrame({'A': [1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
                             'B': [True, True, True, False, False, False, None, True, None, False]})