Repository: databricks/koalas

Issue:
Return type consistency in docs
It is currently not transparent what the return type of a kolas dataframe / series is.

Take for example
[DataFrame.max()](https://koalas.readthedocs.io/en/latest/reference/api/databricks.koalas.DataFrame.max.html)

> Returns
maxscalar for a Series, and a Series for a DataFrame.

Returned type i receive is a pandas.Series

[Series.value_counts()](https://koalas.readthedocs.io/en/latest/reference/api/databricks.koalas.Series.value_counts.html)

> Returns
counts Series

Returned type i receive is a databricks.koalas.series.Series

I would like to suggest that it is made clear in the docs across all functions what the return type is in all cases

In both these cases it is unclear what type of series is returned


Discussion:
I think we should match it to Koalas Series.

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -464,10 +464,12 @@ def _reduce_for_stat_function(self, sfun, name, axis=None, numeric_only=False):
         """
         from inspect import signature
         from databricks.koalas import Series
+        from databricks.koalas.series import _col
 
         axis = validate_axis(axis)
         if axis == 0:
             exprs = []
+            new_column_labels = []
             num_args = len(signature(sfun).parameters)
             for label in self._internal.column_labels:
                 col_sdf = self._internal.spark_column_for(label)
@@ -491,19 +493,23 @@ def _reduce_for_stat_function(self, sfun, name, axis=None, numeric_only=False):
                         # Pass in both the column and its data type if sfun accepts two args
                         col_sdf = sfun(col_sdf, col_type)
                     exprs.append(col_sdf.alias(name_like_string(label)))
+                    new_column_labels.append(label)
 
             sdf = self._sdf.select(*exprs)
-            pdf = sdf.toPandas()
-
-            if self._internal.column_labels_level > 1:
-                pdf.columns = pd.MultiIndex.from_tuples(self._internal.column_labels)
 
-            assert len(pdf) == 1, (sdf, pdf)
+            # The data is expected to be small so it's fine to transpose/use default index.
+            with ks.option_context(
+                "compute.default_index_type", "distributed", "compute.max_rows", None
+            ):
+                kdf = DataFrame(sdf)
+                internal = _InternalFrame(
+                    kdf._internal.spark_frame,
+                    index_map=kdf._internal.index_map,
+                    column_labels=new_column_labels,
+                    column_label_names=self._internal.column_label_names,
+                )
 
-            row = pdf.iloc[0]
-            row.name = None
-            # TODO: return Koalas series.
-            return row  # Return first row as a Series
+                return _col(DataFrame(internal).transpose())
 
         elif axis == 1:
             # Here we execute with the first 1000 to get the return type.
@@ -3845,7 +3851,7 @@ def cache(self):
         ...
         dogs    4
         cats    4
-        dtype: int64
+        Name: 0, dtype: int64
 
         >>> df = df.cache()
         >>> df.to_pandas().mean(axis=1)
@@ -5890,7 +5896,7 @@ def count(self, axis=None):
         Person    5
         Age       4
         Single    5
-        dtype: int64
+        Name: 0, dtype: int64
 
         >>> df.count(axis=1)
         0    3
@@ -9288,7 +9294,10 @@ def info(self, verbose=None, buf=None, max_cols=None, null_counts=None):
             "display.max_info_columns", sys.maxsize, "display.max_info_rows", sys.maxsize
         ):
             try:
-                self._data = self  # hack to use pandas' info as is.
+                # hack to use pandas' info as is.
+                self._data = self
+                count_func = self.count
+                self.count = lambda: count_func().to_pandas()
                 return pd.DataFrame.info(
                     self,
                     verbose=verbose,
@@ -9299,6 +9308,7 @@ def info(self, verbose=None, buf=None, max_cols=None, null_counts=None):
                 )
             finally:
                 del self._data
+                self.count = count_func
 
     # TODO: fix parameter 'axis' and 'numeric_only' to work same as pandas'
     def quantile(self, q=0.5, axis=0, numeric_only=True, accuracy=10000):
diff --git a/databricks/koalas/generic.py b/databricks/koalas/generic.py
--- a/databricks/koalas/generic.py
+++ b/databricks/koalas/generic.py
@@ -956,7 +956,7 @@ def mean(self, axis=None, numeric_only=True):
         >>> df.mean()
         a    2.0
         b    0.2
-        dtype: float64
+        Name: 0, dtype: float64
 
         >>> df.mean(axis=1)
         0    0.55
@@ -1001,7 +1001,7 @@ def sum(self, axis=None, numeric_only=True):
         >>> df.sum()
         a    6.0
         b    0.6
-        dtype: float64
+        Name: 0, dtype: float64
 
         >>> df.sum(axis=1)
         0    1.1
@@ -1085,7 +1085,7 @@ def kurtosis(self, axis=None, numeric_only=True):
         >>> df.kurtosis()
         a   -1.5
         b   -1.5
-        dtype: float64
+        Name: 0, dtype: float64
 
         On a Series:
 
@@ -1125,7 +1125,7 @@ def min(self, axis=None, numeric_only=None):
         >>> df.min()
         a    1.0
         b    0.1
-        dtype: float64
+        Name: 0, dtype: float64
 
         >>> df.min(axis=1)
         0    0.1
@@ -1170,7 +1170,7 @@ def max(self, axis=None, numeric_only=None):
         >>> df.max()
         a    3.0
         b    0.3
-        dtype: float64
+        Name: 0, dtype: float64
 
         >>> df.max(axis=1)
         0    1.0
@@ -1215,7 +1215,7 @@ def std(self, axis=None, numeric_only=True):
         >>> df.std()
         a    1.0
         b    0.1
-        dtype: float64
+        Name: 0, dtype: float64
 
         >>> df.std(axis=1)
         0    0.636396
@@ -1260,7 +1260,7 @@ def var(self, axis=None, numeric_only=True):
         >>> df.var()
         a    1.00
         b    0.01
-        dtype: float64
+        Name: 0, dtype: float64
 
         >>> df.var(axis=1)
         0    0.405


Test patch:
diff --git a/databricks/koalas/tests/test_stats.py b/databricks/koalas/tests/test_stats.py
--- a/databricks/koalas/tests/test_stats.py
+++ b/databricks/koalas/tests/test_stats.py
@@ -16,9 +16,11 @@
 
 import numpy as np
 import pandas as pd
-
+import unittest
 from distutils.version import LooseVersion
 
+import pyspark
+
 from databricks import koalas as ks
 from databricks.koalas.config import option_context
 from databricks.koalas.testing.utils import ReusedSQLTestCase, SQLTestUtils
@@ -28,8 +30,8 @@ class StatsTest(ReusedSQLTestCase, SQLTestUtils):
     def _test_stat_functions(self, pdf, kdf):
         functions = ["max", "min", "mean", "sum"]
         for funcname in functions:
-            self.assert_eq(getattr(kdf.A, funcname)(), getattr(pdf.A, funcname)())
-            self.assert_eq(getattr(kdf, funcname)(), getattr(pdf, funcname)())
+            self.assert_eq(getattr(kdf.A, funcname)(), getattr(pdf.A, funcname)(), almost=True)
+            self.assert_eq(getattr(kdf, funcname)(), getattr(pdf, funcname)(), almost=True)
 
         functions = ["std", "var"]
         for funcname in functions:
@@ -43,6 +45,10 @@ def _test_stat_functions(self, pdf, kdf):
             getattr(kdf.A, funcname)()
             getattr(kdf, funcname)()
 
+    @unittest.skipIf(
+        LooseVersion(pyspark.__version__) < LooseVersion("2.4"),
+        "transpose() doesn't work property with PySpark<2.4",
+    )
     def test_stat_functions(self):
         pdf = pd.DataFrame({"A": [1, 2, 3, 4], "B": [1.0, 2.1, 3, 4]})
         kdf = ks.from_pandas(pdf)
@@ -67,7 +73,6 @@ def test_abs(self):
         self.assert_eq(kdf.A.abs(), pdf.A.abs())
         self.assert_eq(kdf.B.abs(), pdf.B.abs())
         self.assert_eq(kdf[["B", "C"]].abs(), pdf[["B", "C"]].abs())
-        # self.assert_eq(kdf.select('A', 'B').abs(), pdf[['A', 'B']].abs())
 
     def test_axis_on_dataframe(self):
         # The number of each count is intentionally big
@@ -155,28 +160,32 @@ def test_stats_on_boolean_dataframe(self):
         pdf = pd.DataFrame({"A": [True, False, True], "B": [False, False, True]})
         kdf = ks.from_pandas(pdf)
 
-        pd.testing.assert_series_equal(kdf.min(), pdf.min())
-        pd.testing.assert_series_equal(kdf.max(), pdf.max())
+        self.assert_eq(kdf.min(), pdf.min())
+        self.assert_eq(kdf.max(), pdf.max())
 
-        pd.testing.assert_series_equal(kdf.sum(), pdf.sum())
-        pd.testing.assert_series_equal(kdf.mean(), pdf.mean())
+        self.assert_eq(kdf.sum(), pdf.sum())
+        self.assert_eq(kdf.mean(), pdf.mean())
 
-        pd.testing.assert_series_equal(kdf.var(), pdf.var())
-        pd.testing.assert_series_equal(kdf.std(), pdf.std())
+        self.assert_eq(kdf.var(), pdf.var(), almost=True)
+        self.assert_eq(kdf.std(), pdf.std(), almost=True)
 
     def test_stats_on_boolean_series(self):
         pser = pd.Series([True, False, True])
         kser = ks.from_pandas(pser)
 
-        self.assertEqual(kser.min(), pser.min())
-        self.assertEqual(kser.max(), pser.max())
+        self.assert_eq(kser.min(), pser.min())
+        self.assert_eq(kser.max(), pser.max())
 
-        self.assertEqual(kser.sum(), pser.sum())
-        self.assertEqual(kser.mean(), pser.mean())
+        self.assert_eq(kser.sum(), pser.sum())
+        self.assert_eq(kser.mean(), pser.mean())
 
-        self.assertAlmostEqual(kser.var(), pser.var())
-        self.assertAlmostEqual(kser.std(), pser.std())
+        self.assert_eq(kser.var(), pser.var(), almost=True)
+        self.assert_eq(kser.std(), pser.std(), almost=True)
 
+    @unittest.skip(
+        "PySpark does not allow object type. "
+        "Should we support these case by casting to string? or just disallow?"
+    )
     def test_some_stats_functions_should_discard_non_numeric_columns_by_default(self):
         pdf = pd.DataFrame({"i": [0, 1, 2], "b": [False, False, True], "s": ["x", "y", "z"]})
         kdf = ks.from_pandas(pdf)
@@ -184,8 +193,7 @@ def test_some_stats_functions_should_discard_non_numeric_columns_by_default(self
         # min and max do not discard non-numeric columns by default
         self.assertEqual(len(kdf.min()), len(pdf.min()))
         self.assertEqual(len(kdf.max()), len(pdf.max()))
-
-        # TODO?: self.assertEqual(len(kdf.sum()), len(pdf.sum()))
+        self.assertEqual(len(kdf.sum()), len(pdf.sum()))
 
         # all the others do
         self.assertEqual(len(kdf.mean()), len(pdf.mean()))
@@ -200,6 +208,8 @@ def test_stats_on_non_numeric_columns_should_be_discarded_if_numeric_only_is_tru
         pdf = pd.DataFrame({"i": [0, 1, 2], "b": [False, False, True], "s": ["x", "y", "z"]})
         kdf = ks.from_pandas(pdf)
 
+        self.assertTrue(isinstance(kdf.sum(numeric_only=True), ks.Series))
+
         self.assertEqual(len(kdf.sum(numeric_only=True)), len(pdf.sum(numeric_only=True)))
         self.assertEqual(len(kdf.mean(numeric_only=True)), len(pdf.mean(numeric_only=True)))
 
@@ -209,6 +219,10 @@ def test_stats_on_non_numeric_columns_should_be_discarded_if_numeric_only_is_tru
         self.assertEqual(len(kdf.kurtosis(numeric_only=True)), len(pdf.kurtosis(numeric_only=True)))
         self.assertEqual(len(kdf.skew(numeric_only=True)), len(pdf.skew(numeric_only=True)))
 
+    @unittest.skipIf(
+        LooseVersion(pyspark.__version__) < LooseVersion("2.4"),
+        "transpose() doesn't work property with PySpark<2.4",
+    )
     def test_stats_on_non_numeric_columns_should_not_be_discarded_if_numeric_only_is_false(self):
         pdf = pd.DataFrame({"i": [0, 1, 2], "b": [False, False, True], "s": ["x", "y", "z"]})
         kdf = ks.from_pandas(pdf)