Repository: databricks/koalas

Issue:
Convert global or limit-related variables into configurations.
https://github.com/databricks/koalas/commit/94adf06d622aaac1eb05c03407db8a1abc01632c added configuration base like Pandas. See https://pandas.pydata.org/pandas-docs/stable/user_guide/options.html

Currently we have multiple variables that should be converted into configuration values. We should identify and convert it to configuration _all_.

- [x] Support typed values in configurations (#714)
- [x] Add 'display.max_rows' (#714)
- [x] Add 'compute.shortcut_limit' (#717) https://github.com/databricks/koalas/blob/8360c586f5ad4c8847fe394eefbd77b48d95f687/databricks/koalas/frame.py#L373-L379
- [x] Add 'compute.max_rows'  (#721) https://github.com/databricks/koalas/blob/8360c586f5ad4c8847fe394eefbd77b48d95f687/databricks/koalas/frame.py#L1463-L1470 (I think we should remove `limit` parameter).
- [x] Add 'compute.shortcut_limit' (#717) https://github.com/databricks/koalas/blob/8360c586f5ad4c8847fe394eefbd77b48d95f687/databricks/koalas/groupby.py#L806-L812
- [x] Add 'compute.shortcut_limit' (#717) https://github.com/databricks/koalas/blob/8360c586f5ad4c8847fe394eefbd77b48d95f687/databricks/koalas/groupby.py#L1512-L1518
- [x] Add 'plotting.max_rows'  (#728) https://github.com/databricks/koalas/blob/8360c586f5ad4c8847fe394eefbd77b48d95f687/databricks/koalas/plot.py#L54
- [x] Add 'plotting.sample_ratio' (#737) https://github.com/databricks/koalas/blob/8360c586f5ad4c8847fe394eefbd77b48d95f687/databricks/koalas/plot.py#L85
- [x] Add 'compute.ops_on_diff_frames' (#725) https://github.com/databricks/koalas/blob/8360c586f5ad4c8847fe394eefbd77b48d95f687/databricks/koalas/utils.py#L63
- [x] Add 'compute.default_index_type' (#723) https://github.com/databricks/koalas/blob/8360c586f5ad4c8847fe394eefbd77b48d95f687/databricks/koalas/internal.py#L477
- [x] Add 'compute.max_rows' (#736) https://github.com/databricks/koalas/pull/712/files#diff-5ba644c40e914732959b9b4867f30c53R1644
- [x] Add 'compute.shortcut_limit' (#739) https://github.com/databricks/koalas/pull/732/files#diff-5ba644c40e914732959b9b4867f30c53R1605
- [ ] Add documentation (#743). See https://pandas.pydata.org/pandas-docs/stable/user_guide/options.html


Discussion:
@itholic are you interested in working of some of them? Don't have to do everything in one go.
cc @RainFung and @charlesdong1991 too just in case you guys are interested in this.
nice one! what a coincidence! i asked if `max_rows` should be flexible or not yesterday, and today issue is here 😆  i am working on `scatter` and `hexbin` plots now, but would like to take `Add 'plotting.max_rows'` and `Add 'plotting.sample_ratio'` later if you don't mind. @HyukjinKwon 
Please go ahead for both items! BTW, feel free to suggest different names if you have some kind of strong preferences.

Resolving patch:
diff --git a/databricks/koalas/config.py b/databricks/koalas/config.py
--- a/databricks/koalas/config.py
+++ b/databricks/koalas/config.py
@@ -35,6 +35,11 @@
     # just a truncated repr.
     "display.max_rows": 1000,  # TODO: None should support unlimited.
 
+    # 'compute.shortcut_limit' sets the limit for a shortcut.
+    # It computes specified number of rows and use its schema.
+    # When the dataframe length is larger than this limit, Koalas uses PySpark to compute.
+    "compute.shortcut_limit": 1000,  # TODO: None should support unlimited.
+
     # `plotting.max_rows` sets the visual limit on TopN plots. If it is set to 1000, the first 1000
     # data points will be used for plotting, and this number will be seen on the right up corner.
     "plotting.max_rows": 1000,
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -374,7 +374,7 @@ def _reduce_for_stat_function(self, sfun, name, axis=None, numeric_only=False):
         elif axis in ('columns', 1):
             # Here we execute with the first 1000 to get the return type.
             # If the records were less than 1000, it uses pandas API directly for a shortcut.
-            limit = 1000
+            limit = get_option("compute.shortcut_limit")
             pdf = self.head(limit + 1)._to_internal_pandas()
             pser = getattr(pdf, name)(axis=axis, numeric_only=numeric_only)
             if len(pdf) <= limit:
diff --git a/databricks/koalas/groupby.py b/databricks/koalas/groupby.py
--- a/databricks/koalas/groupby.py
+++ b/databricks/koalas/groupby.py
@@ -38,6 +38,7 @@
 from databricks.koalas.missing.groupby import _MissingPandasLikeDataFrameGroupBy, \
     _MissingPandasLikeSeriesGroupBy
 from databricks.koalas.series import Series, _col
+from databricks.koalas.config import get_option
 
 
 class GroupBy(object):
@@ -803,7 +804,7 @@ def apply(self, func):
         if should_infer_schema:
             # Here we execute with the first 1000 to get the return type.
             # If the records were less than 1000, it uses pandas API directly for a shortcut.
-            limit = 1000
+            limit = get_option("compute.shortcut_limit")
             pdf = self._kdf.head(limit + 1)._to_internal_pandas()
             pdf = pdf.groupby(input_groupnames).apply(func)
             kdf = DataFrame(pdf)
@@ -1509,7 +1510,7 @@ def pandas_transform(pdf):
         if should_infer_schema:
             # Here we execute with the first 1000 to get the return type.
             # If the records were less than 1000, it uses pandas API directly for a shortcut.
-            limit = 1000
+            limit = get_option("compute.shortcut_limit")
             pdf = self._kdf.head(limit + 1)._to_internal_pandas()
             pdf = pdf.groupby(input_groupnames).transform(func)
             kdf = DataFrame(pdf)


Test patch:
diff --git a/databricks/koalas/tests/test_groupby.py b/databricks/koalas/tests/test_groupby.py
--- a/databricks/koalas/tests/test_groupby.py
+++ b/databricks/koalas/tests/test_groupby.py
@@ -19,6 +19,7 @@
 import pandas as pd
 
 from databricks import koalas
+from databricks.koalas.config import set_option, reset_option
 from databricks.koalas.exceptions import PandasNotImplementedError
 from databricks.koalas.missing.groupby import _MissingPandasLikeDataFrameGroupBy, \
     _MissingPandasLikeSeriesGroupBy
@@ -346,80 +347,96 @@ def test_bfill(self):
                        repr(pdf.groupby("A")['B'].bfill()))
 
     def test_apply(self):
-        pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6],
-                            'b': [1, 1, 2, 3, 5, 8],
-                            'c': [1, 4, 9, 16, 25, 36]}, columns=['a', 'b', 'c'])
-        kdf = koalas.DataFrame(pdf)
-        self.assert_eq(kdf.groupby("b").apply(lambda x: x + 1).sort_index(),
-                       pdf.groupby("b").apply(lambda x: x + 1).sort_index())
-        self.assert_eq(kdf.groupby(['a', 'b']).apply(lambda x: x * x).sort_index(),
-                       pdf.groupby(['a', 'b']).apply(lambda x: x * x).sort_index())
-        self.assert_eq(kdf.groupby(['b'])['a'].apply(lambda x: x).sort_index(),
-                       pdf.groupby(['b'])['a'].apply(lambda x: x).sort_index())
-
-        # Data is intentionally big to test when schema inference is on.
-        pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6] * 300,
-                            'b': [1, 1, 2, 3, 5, 8] * 300,
-                            'c': [1, 4, 9, 16, 25, 36] * 300}, columns=['a', 'b', 'c'])
-        kdf = koalas.DataFrame(pdf)
-        self.assert_eq(kdf.groupby("b").apply(lambda x: x + 1).sort_index(),
-                       pdf.groupby("b").apply(lambda x: x + 1).sort_index())
-        self.assert_eq(kdf.groupby(['a', 'b']).apply(lambda x: x * x).sort_index(),
-                       pdf.groupby(['a', 'b']).apply(lambda x: x * x).sort_index())
-        self.assert_eq(kdf.groupby(['b'])['a'].apply(lambda x: x).sort_index(),
-                       pdf.groupby(['b'])['a'].apply(lambda x: x).sort_index())
-        with self.assertRaisesRegex(TypeError, "<class 'int'> object is not callable"):
-            kdf.groupby("b").apply(1)
+        # Less than 'compute.shortcut_limit' will execute a shortcut
+        # by using collected pandas dataframe directly.
+        # now we set the 'compute.shortcut_limit' as 1000 explicitly
+        set_option('compute.shortcut_limit', 1000)
+
+        try:
+            pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6],
+                                'b': [1, 1, 2, 3, 5, 8],
+                                'c': [1, 4, 9, 16, 25, 36]}, columns=['a', 'b', 'c'])
+            kdf = koalas.DataFrame(pdf)
+            self.assert_eq(kdf.groupby("b").apply(lambda x: x + 1).sort_index(),
+                           pdf.groupby("b").apply(lambda x: x + 1).sort_index())
+            self.assert_eq(kdf.groupby(['a', 'b']).apply(lambda x: x * x).sort_index(),
+                           pdf.groupby(['a', 'b']).apply(lambda x: x * x).sort_index())
+            self.assert_eq(kdf.groupby(['b'])['a'].apply(lambda x: x).sort_index(),
+                           pdf.groupby(['b'])['a'].apply(lambda x: x).sort_index())
+
+            # Data is intentionally big to test when schema inference is on.
+            pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6] * 300,
+                                'b': [1, 1, 2, 3, 5, 8] * 300,
+                                'c': [1, 4, 9, 16, 25, 36] * 300}, columns=['a', 'b', 'c'])
+            kdf = koalas.DataFrame(pdf)
+            self.assert_eq(kdf.groupby("b").apply(lambda x: x + 1).sort_index(),
+                           pdf.groupby("b").apply(lambda x: x + 1).sort_index())
+            self.assert_eq(kdf.groupby(['a', 'b']).apply(lambda x: x * x).sort_index(),
+                           pdf.groupby(['a', 'b']).apply(lambda x: x * x).sort_index())
+            self.assert_eq(kdf.groupby(['b'])['a'].apply(lambda x: x).sort_index(),
+                           pdf.groupby(['b'])['a'].apply(lambda x: x).sort_index())
+            with self.assertRaisesRegex(TypeError, "<class 'int'> object is not callable"):
+                kdf.groupby("b").apply(1)
+        finally:
+            reset_option('compute.shortcut_limit')
 
     def test_apply_with_new_dataframe(self):
-        # Less than 1000 records will execute a shortcut by using collected pandas dataframe
-        # directly.
-        pdf = pd.DataFrame({
-            "timestamp": [0.0, 0.5, 1.0, 0.0, 0.5],
-            "car_id": ['A', 'A', 'A', 'B', 'B']
-        })
-        kdf = koalas.DataFrame(pdf)
-
-        self.assert_eq(
-            kdf.groupby('car_id').apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index(),
-            pdf.groupby('car_id').apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index())
-
-        # 1000+ records will only infer the schema.
-        pdf = pd.DataFrame({
-            "timestamp": [0.0, 0.5, 1.0, 0.0, 0.5],
-            "car_id": ['A', 'A', 'A', 'B', 'B']
-        })
-        kdf = koalas.DataFrame(pdf)
-
-        self.assert_eq(
-            kdf.groupby('car_id').apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index(),
-            pdf.groupby('car_id').apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index())
+        set_option('compute.shortcut_limit', 1000)
+
+        try:
+            pdf = pd.DataFrame({
+                "timestamp": [0.0, 0.5, 1.0, 0.0, 0.5],
+                "car_id": ['A', 'A', 'A', 'B', 'B']
+            })
+            kdf = koalas.DataFrame(pdf)
+
+            self.assert_eq(
+                kdf.groupby('car_id').apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index(),
+                pdf.groupby('car_id').apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index())
+
+            # 1000+ records will only infer the schema.
+            pdf = pd.DataFrame({
+                "timestamp": [0.0, 0.5, 1.0, 0.0, 0.5] * 300,
+                "car_id": ['A', 'A', 'A', 'B', 'B'] * 300
+            })
+            kdf = koalas.DataFrame(pdf)
+
+            self.assert_eq(
+                kdf.groupby('car_id').apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index(),
+                pdf.groupby('car_id').apply(lambda _: pd.DataFrame({"column": [0.0]})).sort_index())
+        finally:
+            reset_option('compute.shortcut_limit')
 
     def test_transform(self):
-        pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6],
-                            'b': [1, 1, 2, 3, 5, 8],
-                            'c': [1, 4, 9, 16, 25, 36]}, columns=['a', 'b', 'c'])
-        kdf = koalas.DataFrame(pdf)
-        self.assert_eq(kdf.groupby("b").transform(lambda x: x + 1).sort_index(),
-                       pdf.groupby("b").transform(lambda x: x + 1).sort_index())
-        self.assert_eq(kdf.groupby(['a', 'b']).transform(lambda x: x * x).sort_index(),
-                       pdf.groupby(['a', 'b']).transform(lambda x: x * x).sort_index())
-        self.assert_eq(kdf.groupby(['b'])['a'].transform(lambda x: x).sort_index(),
-                       pdf.groupby(['b'])['a'].transform(lambda x: x).sort_index())
-
-        # Data is intentionally big to test when schema inference is on.
-        pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6] * 300,
-                            'b': [1, 1, 2, 3, 5, 8] * 300,
-                            'c': [1, 4, 9, 16, 25, 36] * 300}, columns=['a', 'b', 'c'])
-        kdf = koalas.DataFrame(pdf)
-        self.assert_eq(kdf.groupby("b").transform(lambda x: x + 1).sort_index(),
-                       pdf.groupby("b").transform(lambda x: x + 1).sort_index())
-        self.assert_eq(kdf.groupby(['a', 'b']).transform(lambda x: x * x).sort_index(),
-                       pdf.groupby(['a', 'b']).transform(lambda x: x * x).sort_index())
-        self.assert_eq(kdf.groupby(['b'])['a'].transform(lambda x: x).sort_index(),
-                       pdf.groupby(['b'])['a'].transform(lambda x: x).sort_index())
-        with self.assertRaisesRegex(TypeError, "<class 'int'> object is not callable"):
-            kdf.groupby("b").transform(1)
+        set_option('compute.shortcut_limit', 1000)
+
+        try:
+            pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6],
+                                'b': [1, 1, 2, 3, 5, 8],
+                                'c': [1, 4, 9, 16, 25, 36]}, columns=['a', 'b', 'c'])
+            kdf = koalas.DataFrame(pdf)
+            self.assert_eq(kdf.groupby("b").transform(lambda x: x + 1).sort_index(),
+                           pdf.groupby("b").transform(lambda x: x + 1).sort_index())
+            self.assert_eq(kdf.groupby(['a', 'b']).transform(lambda x: x * x).sort_index(),
+                           pdf.groupby(['a', 'b']).transform(lambda x: x * x).sort_index())
+            self.assert_eq(kdf.groupby(['b'])['a'].transform(lambda x: x).sort_index(),
+                           pdf.groupby(['b'])['a'].transform(lambda x: x).sort_index())
+
+            # Data is intentionally big to test when schema inference is on.
+            pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6] * 300,
+                                'b': [1, 1, 2, 3, 5, 8] * 300,
+                                'c': [1, 4, 9, 16, 25, 36] * 300}, columns=['a', 'b', 'c'])
+            kdf = koalas.DataFrame(pdf)
+            self.assert_eq(kdf.groupby("b").transform(lambda x: x + 1).sort_index(),
+                           pdf.groupby("b").transform(lambda x: x + 1).sort_index())
+            self.assert_eq(kdf.groupby(['a', 'b']).transform(lambda x: x * x).sort_index(),
+                           pdf.groupby(['a', 'b']).transform(lambda x: x * x).sort_index())
+            self.assert_eq(kdf.groupby(['b'])['a'].transform(lambda x: x).sort_index(),
+                           pdf.groupby(['b'])['a'].transform(lambda x: x).sort_index())
+            with self.assertRaisesRegex(TypeError, "<class 'int'> object is not callable"):
+                kdf.groupby("b").transform(1)
+        finally:
+            reset_option('compute.shortcut_limit')
 
     def test_filter(self):
         pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6],
@@ -431,18 +448,6 @@ def test_filter(self):
         self.assert_eq(kdf.groupby(['a', 'b']).filter(lambda x: any(x.a == 2)).sort_index(),
                        pdf.groupby(['a', 'b']).filter(lambda x: any(x.a == 2)).sort_index())
 
-        # Data is intentionally big to test when schema inference is on.
-        pdf = pd.DataFrame({'a': [1, 2, 3, 4, 5, 6] * 300,
-                            'b': [1, 1, 2, 3, 5, 8] * 300,
-                            'c': [1, 4, 9, 16, 25, 36] * 300}, columns=['a', 'b', 'c'])
-        kdf = koalas.DataFrame(pdf)
-        self.assert_eq(kdf.groupby("b").filter(lambda x: x.b.mean() < 4).sort_index(),
-                       pdf.groupby("b").filter(lambda x: x.b.mean() < 4).sort_index())
-        self.assert_eq(kdf.groupby(['a', 'b']).filter(lambda x: any(x.a == 2)).sort_index(),
-                       pdf.groupby(['a', 'b']).filter(lambda x: any(x.a == 2)).sort_index())
-        with self.assertRaisesRegex(TypeError, "<class 'int'> object is not callable"):
-            kdf.groupby("b").filter(1)
-
     def test_idxmax(self):
         pdf = pd.DataFrame({'a': [1, 1, 2, 2, 3],
                             'b': [1, 2, 3, 4, 5],
diff --git a/databricks/koalas/tests/test_stats.py b/databricks/koalas/tests/test_stats.py
--- a/databricks/koalas/tests/test_stats.py
+++ b/databricks/koalas/tests/test_stats.py
@@ -18,6 +18,7 @@
 import pandas as pd
 
 from databricks import koalas
+from databricks.koalas.config import set_option, reset_option
 from databricks.koalas.testing.utils import ReusedSQLTestCase, SQLTestUtils
 
 
@@ -68,20 +69,28 @@ def test_abs(self):
     def test_axis_on_dataframe(self):
         # The number of each count is intentionally big
         # because when data is small, it executes a shortcut.
-        pdf = pd.DataFrame({'A': [1, -2, 3, -4, 5] * 300,
-                            'B': [1., -2, 3, -4, 5] * 300,
-                            'C': [-6., -7, -8, -9, 10] * 300,
-                            'D': [True, False, True, False, False] * 300})
-        kdf = koalas.from_pandas(pdf)
-        self.assert_eq(kdf.count(axis=1), pdf.count(axis=1))
-        self.assert_eq(kdf.var(axis=1), pdf.var(axis=1))
-        self.assert_eq(kdf.std(axis=1), pdf.std(axis=1))
-        self.assert_eq(kdf.max(axis=1), pdf.max(axis=1))
-        self.assert_eq(kdf.min(axis=1), pdf.min(axis=1))
-        self.assert_eq(kdf.sum(axis=1), pdf.sum(axis=1))
-        self.assert_eq(kdf.kurtosis(axis=1), pdf.kurtosis(axis=1))
-        self.assert_eq(kdf.skew(axis=1), pdf.skew(axis=1))
-        self.assert_eq(kdf.mean(axis=1), pdf.mean(axis=1))
+        # Less than 'compute.shortcut_limit' will execute a shortcut
+        # by using collected pandas dataframe directly.
+        # now we set the 'compute.shortcut_limit' as 1000 explicitly
+        set_option('compute.shortcut_limit', 1000)
+
+        try:
+            pdf = pd.DataFrame({'A': [1, -2, 3, -4, 5] * 300,
+                                'B': [1., -2, 3, -4, 5] * 300,
+                                'C': [-6., -7, -8, -9, 10] * 300,
+                                'D': [True, False, True, False, False] * 300})
+            kdf = koalas.from_pandas(pdf)
+            self.assert_eq(kdf.count(axis=1), pdf.count(axis=1))
+            self.assert_eq(kdf.var(axis=1), pdf.var(axis=1))
+            self.assert_eq(kdf.std(axis=1), pdf.std(axis=1))
+            self.assert_eq(kdf.max(axis=1), pdf.max(axis=1))
+            self.assert_eq(kdf.min(axis=1), pdf.min(axis=1))
+            self.assert_eq(kdf.sum(axis=1), pdf.sum(axis=1))
+            self.assert_eq(kdf.kurtosis(axis=1), pdf.kurtosis(axis=1))
+            self.assert_eq(kdf.skew(axis=1), pdf.skew(axis=1))
+            self.assert_eq(kdf.mean(axis=1), pdf.mean(axis=1))
+        finally:
+            reset_option('compute.shortcut_limit')
 
     def test_corr(self):
         # Disable arrow execution since corr() is using UDT internally which is not supported.