Repository: databricks/koalas

Issue:
Support for Array type hints in APIs that take Python native functions (e.g., DataFrame.apply)
```python
import databricks.koalas as ks

def tokenizeDF(col1) -> ks.Series[np.array(... ???)]:
    pass
```


Discussion:
Relates to #1143 
Hi there, I was wondering if there has been any progress with this? I'd like to be able to annotate my functions that generate lists, so it doesn't have to infer the type and incur a performance penalty.
```python
from typing import List
import databricks.koalas as ks

# Equivalent Spark UDF return type: ArrayType(StringType())
def to_list(x: str) -> List[str]:
    return [x]

ks.Series(["x", "y"]).apply(to_list)
```
Result:
```
Traceback (most recent call last)
<ipython-input-17-13d49d0c81a1> in <module>
      6     return [x]
      7 
----> 8 ks.Series(["x", "y"]).apply(to_list)

/opt/conda/lib/python3.7/site-packages/databricks/koalas/series.py in apply(self, func, args, **kwds)
   2664                 raise ValueError(
   2665                     "Expected the return type of this function to be of scalar type, "
-> 2666                     "but found type {}".format(sig_return)
   2667                 )
   2668             return_schema = sig_return.tpe

ValueError: Expected the return type of this function to be of scalar type, but found type UnknownType[typing.List[str]]
```
Is there any workaround for this until it fixed?
It's WIP. `np.array[...]` or `List[...]` will be provided to work around soon.
cc @itholic FYI
Will work on this from now on.
Maybe we can do this like `List[...]` one.

Resolving patch:
diff --git a/databricks/koalas/typedef/typehints.py b/databricks/koalas/typedef/typehints.py
--- a/databricks/koalas/typedef/typehints.py
+++ b/databricks/koalas/typedef/typehints.py
@@ -104,8 +104,9 @@ def as_spark_type(tpe) -> types.DataType:
     # TODO: Add "boolean" and "string" types.
     # ArrayType
     if tpe in (np.ndarray,):
-        # TODO: support other child types
         return types.ArrayType(types.StringType())
+    elif hasattr(tpe, "__origin__") and issubclass(tpe.__origin__, list):
+        return types.ArrayType(as_spark_type(tpe.__args__[0]))
     # BinaryType
     elif tpe in (bytes, np.character, np.bytes_, np.string_):
         return types.BinaryType()


Test patch:
diff --git a/databricks/koalas/tests/test_typedef.py b/databricks/koalas/tests/test_typedef.py
--- a/databricks/koalas/tests/test_typedef.py
+++ b/databricks/koalas/tests/test_typedef.py
@@ -16,11 +16,14 @@
 import sys
 import unittest
 import datetime
+import decimal
+from typing import List
 
 import pandas
 import pandas as pd
 import numpy as np
 from pyspark.sql.types import (
+    ArrayType,
     BinaryType,
     BooleanType,
     FloatType,
@@ -31,6 +34,8 @@
     StructType,
     ByteType,
     ShortType,
+    DateType,
+    DecimalType,
     DoubleType,
     TimestampType,
 )
@@ -236,6 +241,35 @@ def test_as_spark_type(self):
             # datetime
             np.datetime64: TimestampType(),
             datetime.datetime: TimestampType(),
+            # DateType
+            datetime.date: DateType(),
+            # DecimalType
+            decimal.Decimal: DecimalType(38, 18),
+            # ArrayType
+            np.ndarray: ArrayType(StringType()),
+            List[bytes]: ArrayType(BinaryType()),
+            List[np.character]: ArrayType(BinaryType()),
+            List[np.bytes_]: ArrayType(BinaryType()),
+            List[np.string_]: ArrayType(BinaryType()),
+            List[bool]: ArrayType(BooleanType()),
+            List[np.bool]: ArrayType(BooleanType()),
+            List[datetime.date]: ArrayType(DateType()),
+            List[np.int8]: ArrayType(ByteType()),
+            List[np.byte]: ArrayType(ByteType()),
+            List[decimal.Decimal]: ArrayType(DecimalType(38, 18)),
+            List[float]: ArrayType(DoubleType()),
+            List[np.float]: ArrayType(DoubleType()),
+            List[np.float64]: ArrayType(DoubleType()),
+            List[np.float32]: ArrayType(FloatType()),
+            List[np.int32]: ArrayType(IntegerType()),
+            List[int]: ArrayType(LongType()),
+            List[np.int]: ArrayType(LongType()),
+            List[np.int64]: ArrayType(LongType()),
+            List[np.int16]: ArrayType(ShortType()),
+            List[str]: ArrayType(StringType()),
+            List[np.unicode_]: ArrayType(StringType()),
+            List[datetime.datetime]: ArrayType(TimestampType()),
+            List[np.datetime64]: ArrayType(TimestampType()),
         }
 
         for numpy_or_python_type, spark_type in type_mapper.items():