Repository: databricks/koalas

Issue:
DataFrameGroupBy.describe



Discussion:
I have implemented a version of `DataFrameGroupBy.describe` that "works" for my project. I feel like the implementation is not ideal, but I'm happy to share and make fixes based on feedback to get it right, if that sounds good to you all.
I've patched the behavior in my project (see code below).

The only change in `_spark_groupby` is the `aggfunc` option I added to support `describe`, `"quartiles"`. This is the piece I'm actually least comfortable with, since it's adding functionality that isn't there in `pandas`. I investigated a number of options, but I may have missed the option to implement `DataFrameGroupBy.quantile` (it's been a month or so, I forget if I looked into it). Happy to try and implement `DataFrameGroupBy.quantile` and use that instead, if that sounds like the right way forward.

```python3
import databricks.koalas as ks
from databricks.koalas.groupby import GroupBy
from databricks.koalas.internal import SPARK_INDEX_NAME_FORMAT, _InternalFrame
from databricks.koalas.utils import scol_for
from pyspark.sql import functions as F


def _spark_groupby(kdf, func, groupkeys):
    sdf = kdf._sdf
    groupkey_cols = [s._scol.alias(SPARK_INDEX_NAME_FORMAT(i))
                     for i, s in enumerate(groupkeys)]
    multi_aggs = any(isinstance(v, list) for v in func.values())
    reordered = []
    data_columns = []
    column_index = []
    for key, value in func.items():
        idx = key if isinstance(key, tuple) else (key,)
        for aggfunc in [value] if isinstance(value, str) else value:
            name = kdf._internal.column_name_for(idx)
            data_col = "('{0}', '{1}')".format(name, aggfunc) if multi_aggs else name
            data_columns.append(data_col)
            column_index.append(tuple(list(idx) + [aggfunc]) if multi_aggs else idx)
            if aggfunc == "nunique":
                reordered.append(
                    F.expr('count(DISTINCT `{0}`) as `{1}`'.format(name, data_col)))

            # Implement "quartiles" aggregate function for ``describe``.
            elif aggfunc == "quartiles":
                reordered.append(
                    F.expr('percentile_approx(`{0}`, array(0.25, 0.5, 0.75)) as `{1}`'.format(name, data_col)))

            else:
                reordered.append(F.expr('{1}(`{0}`) as `{2}`'.format(name, aggfunc, data_col)))
    sdf = sdf.groupby(*groupkey_cols).agg(*reordered)
    if len(groupkeys) > 0:
        index_map = [(SPARK_INDEX_NAME_FORMAT(i),
                      s._internal.column_index[0])
                     for i, s in enumerate(groupkeys)]
    else:
        index_map = None
    return _InternalFrame(sdf=sdf,
                          column_index=column_index,
                          column_scols=[scol_for(sdf, col) for col in data_columns],
                          index_map=index_map)


GroupBy._spark_groupby = _spark_groupby


def _describe(self):
    kdf = self.agg(["count", "mean", "std", "min", "quartiles", "max"]).reset_index()

    # Split "quartiles" columns into first, second, and third quartiles.
    for label, content in kdf.iteritems():
        if label[1] == "quartiles":
            exploded = ks.DataFrame(content.tolist())
            exploded.columns = [(label[0], "25%"), (label[0], "50%"), (label[0], "75%")]
            kdf = kdf.drop(label).join(exploded)

    # Reindex the DataFrame to reflect initial grouping and agg columns.
    input_groupnames = [s.name for s in self._groupkeys]
    kdf.set_index([(key, "") for key in input_groupnames], inplace=True)
    kdf.index.names = input_groupnames

    return kdf


GroupBy.describe = _describe
```
Nice to see you @deepyaman ! thanks for sharing your experience :D
If you don't mind, could you create a PR for this?
i think it makes easy for comparing to existing code and getting code review 😸 
Yeah, please go ahead for a PR.
@itholic @HyukjinKwon Done! I've opened #1168, also noting a few of the obvious tasks that I haven't completed yet (only ported the code from my project so far). I've also added a manual test to show the functionality working.
@deepyaman thanks for the contribution! i'll check soon. 😃 

Resolving patch:
diff --git a/databricks/koalas/groupby.py b/databricks/koalas/groupby.py
--- a/databricks/koalas/groupby.py
+++ b/databricks/koalas/groupby.py
@@ -35,7 +35,6 @@
 from pyspark.sql.functions import PandasUDFType, pandas_udf, Column
 
 from databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.
-from databricks.koalas.base import _column_op
 from databricks.koalas.typedef import _infer_return_type
 from databricks.koalas.frame import DataFrame
 from databricks.koalas.internal import (_InternalFrame, HIDDEN_COLUMNS, NATURAL_ORDER_COLUMN_NAME,
@@ -1946,33 +1945,85 @@ def _shift(self, periods, fill_value):
                                                     for c in applied])
         return DataFrame(internal)
 
+    # TODO: Implement 'percentiles', 'include', and 'exclude' arguments.
+    # TODO: Add ``DataFrame.select_dtypes`` to See Also when 'include'
+    #   and 'exclude' arguments are implemented.
     def describe(self):
+        """
+        Generate descriptive statistics that summarize the central tendency,
+        dispersion and shape of a dataset's distribution, excluding
+        ``NaN`` values.
+
+        Analyzes both numeric and object series, as well
+        as ``DataFrame`` column sets of mixed data types. The output
+        will vary depending on what is provided. Refer to the notes
+        below for more detail.
+
+        .. note:: Unlike pandas, the percentiles in Koalas are based upon
+            approximate percentile computation because computing percentiles
+            across a large dataset is extremely expensive.
+
+        Returns
+        -------
+        DataFrame
+            Summary statistics of the DataFrame provided.
+
+        See Also
+        --------
+        DataFrame.count
+        DataFrame.max
+        DataFrame.min
+        DataFrame.mean
+        DataFrame.std
+
+        Examples
+        --------
+        >>> df = ks.DataFrame({'a': [1, 1, 3], 'b': [4, 5, 6], 'c': [7, 8, 9]})
+        >>> df
+           a  b  c
+        0  1  4  7
+        1  1  5  8
+        2  3  6  9
+
+        Describing a ``DataFrame``. By default only numeric fields
+        are returned.
+
+        >>> described = df.groupby('a').describe()
+        >>> described  # doctest: +NORMALIZE_WHITESPACE
+              b                                        c
+          count mean       std min 25% 50% 75% max count mean       std min 25% 50% 75% max
+        a
+        1   2.0  4.5  0.707107 4.0 4.0 4.0 5.0 5.0   2.0  7.5  0.707107 7.0 7.0 7.0 8.0 8.0
+        3   1.0  6.0       NaN 6.0 6.0 6.0 6.0 6.0   1.0  9.0       NaN 9.0 9.0 9.0 9.0 9.0
+
+        """
         kdf = self.agg(["count", "mean", "std", "min", "quartiles", "max"]).reset_index()
+        sdf = kdf._sdf
+        agg_cols = [col.name for col in self._agg_columns]
         formatted_percentiles = ["25%", "50%", "75%"]
 
         # Split "quartiles" columns into first, second, and third quartiles.
-        for label, content in kdf.iteritems():
-            if label[1] == "quartiles":
-                exploded = ks.DataFrame(
-                    {
-                        (label[0], x): _column_op(itemgetter(i))(content).to_numpy()
-                        for i, x in enumerate(formatted_percentiles)
-                    }
-                )
-                kdf = kdf.drop(label).join(exploded)
-
-        # Reindex the DataFrame to reflect initial grouping and agg columns.
-        input_groupnames = [s.name for s in self._groupkeys]
-        kdf.set_index([(key, "") for key in input_groupnames], inplace=True)
-        kdf.index.names = input_groupnames
+        for col in agg_cols:
+            quartiles_col = str((col, "quartiles"))
+            for i, percentile in enumerate(formatted_percentiles):
+                sdf = sdf.withColumn(str((col, percentile)), F.col(quartiles_col)[i])
+            sdf = sdf.drop(quartiles_col)
 
         # Reorder columns lexicographically by agg column followed by stats.
-        agg_cols = (col.name for col in self._agg_columns)
         stats = ["count", "mean", "std", "min"] + formatted_percentiles + ["max"]
-        kdf = kdf[list(product(agg_cols, stats))]
+        column_index = list(product(agg_cols, stats))
+        data_columns = map(str, column_index)
+
+        # Reindex the DataFrame to reflect initial grouping and agg columns.
+        internal = _InternalFrame(sdf=sdf,
+                                  index_map=([(s._internal.data_columns[0],
+                                               s._internal.column_index[0])
+                                              for s in self._groupkeys]),
+                                  column_index=column_index,
+                                  column_scols=[scol_for(sdf, col) for col in data_columns])
 
         # Cast columns to ``"float64"`` to match `pandas.DataFrame.groupby`.
-        return kdf.astype("float64")
+        return DataFrame(internal).astype("float64")
 
 
 class SeriesGroupBy(GroupBy):


Test patch: