Repository: databricks/koalas

Issue:
ValueError: no signature found for builtin type <class 'dict'> when trying to create object column with dicts 
Hello!

I'm trying to get the same results as I would have with a pandas DataFrame, but I got this error instead.

Is it expected? Any chance to get this working with koalas in the future using an official way?

I'm using
pandas 1.1.0
koalas 1.1.0
spark 2.4.5

Thanks!

```python
test_pdf = pd.DataFrame([[1.*i, 2.*i, 3.*i] for i in range(3)], columns=["col_1", "col_2", "col_3"])
test_kdf = ks.from_pandas(test_pdf)

# pandas goes fine
test_pdf["new_col"] = test_pdf.apply(dict, axis=1)
print(test_pdf)
>>>    col_1  col_2  col_3                                     new_col
>>> 0    0.0    0.0    0.0  {'col_1': 0.0, 'col_2': 0.0, 'col_3': 0.0}
>>> 1    1.0    2.0    3.0  {'col_1': 1.0, 'col_2': 2.0, 'col_3': 3.0}
>>> 2    2.0    4.0    6.0  {'col_1': 2.0, 'col_2': 4.0, 'col_3': 6.0}

# koalas doesn't
test_kdf["new_col"] = test_kdf.apply(dict, axis=1)
print(test_kdf)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/usr/lib/python3.7/inspect.py in getfullargspec(func)
   1125                                        skip_bound_arg=False,
-> 1126                                        sigcls=Signature)
   1127     except Exception as ex:

/usr/lib/python3.7/inspect.py in _signature_from_callable(obj, follow_wrapper_chains, skip_bound_arg, sigcls)
   2363                     raise ValueError(
-> 2364                         'no signature found for builtin type {!r}'.format(obj))
   2365 

ValueError: no signature found for builtin type <class 'dict'>

The above exception was the direct cause of the following exception:

TypeError                                 Traceback (most recent call last)
<command-6884238> in <module>
      6 # display(test_pdf)
      7 
----> 8 test_kdf["new_col"] = test_kdf.apply(dict, axis=1)
      9 print(test_kdf)

/local_disk0/pythonVirtualEnvDirs/virtualEnv-88f94e2c-4907-4af0-adac-903a130015df/lib/python3.7/site-packages/databricks/koalas/usage_logging/__init__.py in wrapper(*args, **kwargs)
    178             start = time.perf_counter()
    179             try:
--> 180                 res = func(*args, **kwargs)
    181                 logger.log_success(
    182                     class_name, function_name, time.perf_counter() - start, signature

/local_disk0/pythonVirtualEnvDirs/virtualEnv-88f94e2c-4907-4af0-adac-903a130015df/lib/python3.7/site-packages/databricks/koalas/frame.py in apply(self, func, axis, args, **kwds)
   2313         axis = validate_axis(axis)
   2314         should_return_series = False
-> 2315         spec = inspect.getfullargspec(func)
   2316         return_sig = spec.annotations.get("return", None)
   2317         should_infer_schema = return_sig is None

/usr/lib/python3.7/inspect.py in getfullargspec(func)
   1130         # else. So to be fully backwards compatible, we catch all
   1131         # possible exceptions here, and reraise a TypeError.
-> 1132         raise TypeError('unsupported callable') from ex
   1133 
   1134     args = []

TypeError: unsupported callable
```


Discussion:
I believe you can do it as `df.apply(lambda *a, **kw: dict(*a, **kw))` for now.
But Koalas does not support the nested type yet. You might have to flatten the data for now.

Resolving patch:
diff --git a/databricks/koalas/accessors.py b/databricks/koalas/accessors.py
--- a/databricks/koalas/accessors.py
+++ b/databricks/koalas/accessors.py
@@ -20,6 +20,7 @@
 from collections import OrderedDict
 from distutils.version import LooseVersion
 from typing import Tuple, Union, TYPE_CHECKING
+import types
 
 import numpy as np
 import pandas as pd
@@ -286,13 +287,20 @@ def apply_batch(self, func, args=(), **kwds):
         1    59069   1048596
         2  9765645  60466196
 
-        You can also use ``np.ufunc`` as input.
+        You can also use ``np.ufunc`` and built-in functions as input.
 
         >>> df.koalas.apply_batch(np.add, args=(10,))
             A   B
         0  11  12
         1  13  14
         2  15  16
+
+        >>> (df * -1).koalas.apply_batch(abs)
+           A  B
+        0  1  2
+        1  3  4
+        2  5  6
+
         """
         # TODO: codes here partially duplicate `DataFrame.apply`. Can we deduplicate?
 
@@ -300,12 +308,11 @@ def apply_batch(self, func, args=(), **kwds):
         from databricks.koalas.frame import DataFrame
         from databricks import koalas as ks
 
-        if isinstance(func, np.ufunc):
+        if not isinstance(func, types.FunctionType):
+            assert callable(func), "the first argument should be a callable function."
             f = func
             func = lambda *args, **kwargs: f(*args, **kwargs)
 
-        assert callable(func), "the first argument should be a callable function."
-
         spec = inspect.getfullargspec(func)
         return_sig = spec.annotations.get("return", None)
         should_infer_schema = return_sig is None
@@ -482,6 +489,12 @@ def transform_batch(self, func, *args, **kwargs):
         1  4  5
         2  6  7
 
+        >>> (df * -1).koalas.transform_batch(abs)
+           A  B
+        0  1  2
+        1  3  4
+        2  5  6
+
         Note that you should not transform the index. The index information will not change.
 
         >>> df.koalas.transform_batch(lambda pdf: pdf.B + 1)
@@ -760,16 +773,20 @@ def transform_batch(self, func, *args, **kwargs) -> "ks.Series":
         2    11
         Name: A, dtype: int64
 
-        You can also use ``np.ufunc`` as input.
+        You can also use ``np.ufunc`` and built-in functions as input.
 
         >>> df.A.koalas.transform_batch(np.add, 10)
         0    11
         1    13
         2    15
         Name: A, dtype: int64
-        """
-        from databricks import koalas as ks
 
+        >>> (df * -1).A.koalas.transform_batch(abs)
+        0    1
+        1    3
+        2    5
+        Name: A, dtype: int64
+        """
         assert callable(func), "the first argument should be a callable function."
 
         return_sig = None
@@ -799,7 +816,7 @@ def _transform_batch(self, func, return_schema):
         from databricks.koalas.series import Series
         from databricks import koalas as ks
 
-        if isinstance(func, np.ufunc):
+        if not isinstance(func, types.FunctionType):
             f = func
             func = lambda *args, **kwargs: f(*args, **kwargs)
 
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -23,6 +23,7 @@
 import warnings
 import inspect
 import json
+import types
 from functools import partial, reduce
 import sys
 from itertools import zip_longest
@@ -2269,6 +2270,12 @@ def apply(self, func, axis=0, args=(), **kwds):
         2    13
         Name: 0, dtype: int64
 
+        >>> df.apply(max, axis=1)
+        0    9
+        1    9
+        2    9
+        Name: 0, dtype: int64
+
         Returning a list-like will result in a Series
 
         >>> df.apply(lambda x: [1, 2], axis=1)
@@ -2303,12 +2310,11 @@ def apply(self, func, axis=0, args=(), **kwds):
         from databricks.koalas.groupby import GroupBy
         from databricks.koalas.series import first_series
 
-        if isinstance(func, np.ufunc):
+        if not isinstance(func, types.FunctionType):
+            assert callable(func), "the first argument should be a callable function."
             f = func
             func = lambda *args, **kwargs: f(*args, **kwargs)
 
-        assert callable(func), "the first argument should be a callable function."
-
         axis = validate_axis(axis)
         should_return_series = False
         spec = inspect.getfullargspec(func)
@@ -2503,12 +2509,12 @@ def transform(self, func, axis=0, *args, **kwargs):
         1  1  4
         2  4  9
 
-        >>> df.transform(lambda x: x ** 2)  # doctest: +NORMALIZE_WHITESPACE
+        >>> (df * -1).transform(abs)  # doctest: +NORMALIZE_WHITESPACE
            X
            A  B
         0  0  1
-        1  1  4
-        2  4  9
+        1  1  2
+        2  2  3
 
         You can also specify extra arguments.
 
@@ -2521,7 +2527,11 @@ def transform(self, func, axis=0, *args, **kwargs):
         1    21   1044
         2  1044  59069
         """
-        assert callable(func), "the first argument should be a callable function."
+        if not isinstance(func, types.FunctionType):
+            assert callable(func), "the first argument should be a callable function."
+            f = func
+            func = lambda *args, **kwargs: f(*args, **kwargs)
+
         axis = validate_axis(axis)
         if axis != 0:
             raise NotImplementedError('axis should be either 0 or "index" currently.')
diff --git a/databricks/koalas/groupby.py b/databricks/koalas/groupby.py
--- a/databricks/koalas/groupby.py
+++ b/databricks/koalas/groupby.py
@@ -927,12 +927,11 @@ def apply(self, func, *args, **kwargs):
         1  1.0  1.0
         2  1.0  1.0
 
-        >>> def pandas_length(x) -> int:
-        ...     return len(x)
-        >>> g.apply(pandas_length).sort_index()  # doctest: +NORMALIZE_WHITESPACE
-        0    1
-        1    2
-        Name: 0, dtype: int32
+        >>> g.apply(len).sort_index()  # doctest: +NORMALIZE_WHITESPACE
+        A
+        a    2
+        b    1
+        Name: 0, dtype: int64
 
         In case of Series, it works as below.
 
@@ -1017,6 +1016,18 @@ def pandas_apply(pdf, *a, **k):
                 pser_or_pdf = pdf.groupby(groupkey_names).apply(pandas_apply, *args, **kwargs)
             kser_or_kdf = ks.from_pandas(pser_or_pdf)
             if len(pdf) <= limit:
+                if isinstance(kser_or_kdf, ks.Series) and not is_series_groupby:
+                    # Restore grouping names as the index name
+                    groupkey_scol_name = zip(
+                        self._groupkeys, kser_or_kdf._internal.index_map.items()
+                    )
+                    internal = kser_or_kdf._internal.copy(
+                        index_map=OrderedDict(
+                            (scol_name, (s.name,) if isinstance(s.name, str) else s.name)
+                            for s, (scol_name, name) in groupkey_scol_name
+                        )
+                    )
+                    kser_or_kdf = first_series(DataFrame(internal))
                 return kser_or_kdf
 
             if isinstance(kser_or_kdf, ks.Series):
@@ -1087,6 +1098,15 @@ def wrapped_func(df, *a, **k):
         if should_infer_schema:
             # If schema is inferred, we can restore indexes too.
             internal = kdf_from_pandas._internal.with_new_sdf(sdf)
+            if should_return_series and not is_series_groupby:
+                # Restore grouping names as the index name
+                groupkey_scol_name = zip(self._groupkeys, internal.index_map.items())
+                internal = internal.copy(
+                    index_map=OrderedDict(
+                        (scol_name, (s.name,) if isinstance(s.name, str) else s.name)
+                        for s, (scol_name, name) in groupkey_scol_name
+                    )
+                )
         else:
             # Otherwise, it loses index.
             internal = InternalFrame(spark_frame=sdf, index_map=None)
@@ -1900,10 +1920,10 @@ def transform(self, func, *args, **kwargs):
         2    6
         Name: B, dtype: int32
 
-        >>> df.B.groupby(df.A).transform(plus_min)
-        0    2
-        1    3
-        2    6
+        >>> (df * -1).B.groupby(df.A).transform(abs)
+        0    1
+        1    2
+        2    3
         Name: B, dtype: int64
 
         You can also specify extra arguments to pass to the function.


Test patch:
diff --git a/databricks/koalas/tests/test_groupby.py b/databricks/koalas/tests/test_groupby.py
--- a/databricks/koalas/tests/test_groupby.py
+++ b/databricks/koalas/tests/test_groupby.py
@@ -1405,6 +1405,9 @@ def test_apply(self):
             kdf.groupby("b").apply(lambda x: x + x.min()).sort_index(),
             pdf.groupby("b").apply(lambda x: x + x.min()).sort_index(),
         )
+        self.assert_eq(
+            kdf.groupby("b").apply(len).sort_index(), pdf.groupby("b").apply(len).sort_index(),
+        )
         self.assert_eq(
             kdf.groupby("b")["a"].apply(lambda x, y, z: x + x.min() + y * z, 10, z=20).sort_index(),
             pdf.groupby("b")["a"].apply(lambda x, y, z: x + x.min() + y * z, 10, z=20).sort_index(),
@@ -1421,6 +1424,10 @@ def test_apply(self):
             kdf.groupby(["b"])["c"].apply(lambda x: 1).sort_index(),
             pdf.groupby(["b"])["c"].apply(lambda x: 1).sort_index(),
         )
+        self.assert_eq(
+            kdf.groupby(["b"])["c"].apply(len).sort_index(),
+            pdf.groupby(["b"])["c"].apply(len).sort_index(),
+        )
         self.assert_eq(
             kdf.groupby(kdf.b // 5).apply(lambda x: x + x.min()).sort_index(),
             pdf.groupby(pdf.b // 5).apply(lambda x: x + x.min()).sort_index(),
@@ -1433,6 +1440,10 @@ def test_apply(self):
             kdf.groupby(kdf.b // 5)[["a"]].apply(lambda x: x + x.min()).sort_index(),
             pdf.groupby(pdf.b // 5)[["a"]].apply(lambda x: x + x.min()).sort_index(),
         )
+        self.assert_eq(
+            kdf.groupby(kdf.b // 5)[["a"]].apply(len).sort_index(),
+            pdf.groupby(pdf.b // 5)[["a"]].apply(len).sort_index(),
+        )
 
         with self.assertRaisesRegex(TypeError, "<class 'int'> object is not callable"):
             kdf.groupby("b").apply(1)
@@ -1450,6 +1461,14 @@ def test_apply(self):
             kdf.groupby([("x", "a"), ("x", "b")]).apply(lambda x: x + x.min()).sort_index(),
             pdf.groupby([("x", "a"), ("x", "b")]).apply(lambda x: x + x.min()).sort_index(),
         )
+        self.assert_eq(
+            kdf.groupby(("x", "b")).apply(len).sort_index(),
+            pdf.groupby(("x", "b")).apply(len).sort_index(),
+        )
+        self.assert_eq(
+            kdf.groupby([("x", "a"), ("x", "b")]).apply(len).sort_index(),
+            pdf.groupby([("x", "a"), ("x", "b")]).apply(len).sort_index(),
+        )
 
     def test_apply_without_shortcut(self):
         with option_context("compute.shortcut_limit", 0):