Repository: databricks/koalas

Issue:
Nullable string/object columns not supported
When trying to load a pandas DataFrame with a nullable string column, I always get 

```
TypeError: field header: Can not merge type <class 'pyspark.sql.types.StringType'> and <class 'pyspark.sql.types.DoubleType'>
```

It can be reproduced in two different ways. We can create an in-memory Series and load it:

```
import numpy as np
import pandas as pd
import databricks.koalas as ks

ks.from_pandas(pd.Series(['foo', np.nan, 'bar']))
```

Or we can populate a CSV file with at least one NULL:

```
header
value
N/A
nothing above
```

And then try to read it:

```
import pandas as pd
import databricks.koalas as ks

ks.from_pandas(pd.read_csv('foo.csv'))
```

Either way, koalas things that the `np.nan` signifies a float, but it's actually a NULL for an object column.


Discussion:
This is the same issue as #114 , it seems. Thanks for the way to reproduce it @kokes .

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -22,9 +22,10 @@
 
 import numpy as np
 import pandas as pd
+from pandas.api.types import is_datetime64_dtype, is_datetime64tz_dtype
 from pyspark import sql as spark
 from pyspark.sql import functions as F, Column
-from pyspark.sql.types import StructType, to_arrow_type
+from pyspark.sql.types import StructField, StructType, to_arrow_type
 from pyspark.sql.utils import AnalysisException
 
 from databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.
@@ -36,6 +37,7 @@
 from databricks.koalas.missing.frame import _MissingPandasLikeDataFrame
 from databricks.koalas.ml import corr
 from databricks.koalas.selection import SparkDataFrameLocator
+from databricks.koalas.typedef import infer_pd_series_spark_type
 
 
 class DataFrame(_Frame):
@@ -58,7 +60,16 @@ def _init_from_pandas(self, pdf, *args):
         metadata = Metadata.from_pandas(pdf)
         reset_index = pdf.reset_index()
         reset_index.columns = metadata.all_fields
-        self._init_from_spark(default_session().createDataFrame(reset_index), metadata)
+        schema = StructType([StructField(name, infer_pd_series_spark_type(col),
+                                         nullable=bool(col.isnull().any()))
+                             for name, col in reset_index.iteritems()])
+        for name, col in reset_index.iteritems():
+            dt = col.dtype
+            if is_datetime64_dtype(dt) or is_datetime64tz_dtype(dt):
+                continue
+            reset_index[name] = col.replace({np.nan: None})
+        self._init_from_spark(default_session().createDataFrame(reset_index, schema=schema),
+                              metadata)
 
     @__init__.register(spark.DataFrame)
     def _init_from_spark(self, sdf, metadata=None, *args):
@@ -481,7 +492,7 @@ def dtypes(self):
         >>> df.dtypes
         a            object
         b             int64
-        c             int64
+        c              int8
         d           float64
         e              bool
         f    datetime64[ns]
diff --git a/databricks/koalas/typedef.py b/databricks/koalas/typedef.py
--- a/databricks/koalas/typedef.py
+++ b/databricks/koalas/typedef.py
@@ -22,6 +22,9 @@
 from decorator import decorate
 from decorator import getfullargspec
 import numpy as np
+import pandas as pd
+from pandas.api.types import is_datetime64_dtype, is_datetime64tz_dtype
+import pyarrow as pa
 from pyspark.sql import Column
 from pyspark.sql.functions import pandas_udf
 import pyspark.sql.types as types
@@ -133,6 +136,23 @@ def as_python_type(spark_tpe):
     return _py_conversions.get(spark_tpe, None)
 
 
+def infer_pd_series_spark_type(s: pd.Series) -> types.DataType:
+    """Infer Spark DataType from pandas Series dtype.
+
+    :param s: :class:`pandas.Series` to be inferred
+    :return: the inferred Spark data type
+    """
+    dt = s.dtype
+    if dt == np.dtype('object'):
+        if len(s) == 0 or s.isnull().all():
+            raise ValueError("can not infer schema from empty or null dataset")
+        return types.from_arrow_type(pa.Array.from_pandas(s).type)
+    elif is_datetime64_dtype(dt) or is_datetime64tz_dtype(dt):
+        return types.TimestampType()
+    else:
+        return types.from_arrow_type(pa.from_numpy_dtype(dt))
+
+
 def _check_compatible(arg, sig_arg: X):
     if isinstance(sig_arg, _Unknown):
         return arg


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe.py b/databricks/koalas/tests/test_dataframe.py
--- a/databricks/koalas/tests/test_dataframe.py
+++ b/databricks/koalas/tests/test_dataframe.py
@@ -21,14 +21,14 @@
 import pandas as pd
 
 from databricks import koalas
-from databricks.koalas.testing.utils import ReusedSQLTestCase, TestUtils
+from databricks.koalas.testing.utils import ReusedSQLTestCase, SQLTestUtils
 from databricks.koalas.exceptions import PandasNotImplementedError
 from databricks.koalas.missing.frame import _MissingPandasLikeDataFrame
 from databricks.koalas.missing.series import _MissingPandasLikeSeries
 from databricks.koalas.series import Series
 
 
-class DataFrameTest(ReusedSQLTestCase, TestUtils):
+class DataFrameTest(ReusedSQLTestCase, SQLTestUtils):
 
     @property
     def pdf(self):
@@ -71,6 +71,51 @@ def test_dataframe(self):
 
         self.assertEqual(ddf.a.notnull().alias("x").name, "x")
 
+    def test_empty_dataframe(self):
+        a = pd.Series([], dtype='i1')
+        b = pd.Series([], dtype='str')
+        pdf = pd.DataFrame({'a': a, 'b': b})
+
+        self.assert_eq(koalas.from_pandas(a), a)
+        self.assertRaises(ValueError, lambda: koalas.from_pandas(b))
+        self.assertRaises(ValueError, lambda: koalas.from_pandas(pdf))
+
+        with self.sql_conf({'spark.sql.execution.arrow.enabled': False}):
+            self.assert_eq(koalas.from_pandas(a), a)
+            self.assertRaises(ValueError, lambda: koalas.from_pandas(b))
+            self.assertRaises(ValueError, lambda: koalas.from_pandas(pdf))
+
+    def test_all_null_dataframe(self):
+        a = pd.Series([None, None, None], dtype='float64')
+        b = pd.Series([None, None, None], dtype='str')
+        pdf = pd.DataFrame({'a': a, 'b': b})
+
+        self.assert_eq(koalas.from_pandas(a).dtype, a.dtype)
+        self.assertTrue(koalas.from_pandas(a).toPandas().isnull().all())
+        self.assertRaises(ValueError, lambda: koalas.from_pandas(b))
+        self.assertRaises(ValueError, lambda: koalas.from_pandas(pdf))
+
+        with self.sql_conf({'spark.sql.execution.arrow.enabled': False}):
+            self.assert_eq(koalas.from_pandas(a).dtype, a.dtype)
+            self.assertTrue(koalas.from_pandas(a).toPandas().isnull().all())
+            self.assertRaises(ValueError, lambda: koalas.from_pandas(b))
+            self.assertRaises(ValueError, lambda: koalas.from_pandas(pdf))
+
+    def test_nullable_object(self):
+        pdf = pd.DataFrame({'a': list('abc') + [np.nan],
+                            'b': list(range(1, 4)) + [np.nan],
+                            'c': list(np.arange(3, 6).astype('i1')) + [np.nan],
+                            'd': list(np.arange(4.0, 7.0, dtype='float64')) + [np.nan],
+                            'e': [True, False, True, np.nan],
+                            'f': list(pd.date_range('20130101', periods=3)) + [np.nan]})
+
+        kdf = koalas.from_pandas(pdf)
+        self.assert_eq(kdf, pdf)
+
+        with self.sql_conf({'spark.sql.execution.arrow.enabled': False}):
+            kdf = koalas.from_pandas(pdf)
+            self.assert_eq(kdf, pdf)
+
     def test_assign(self):
         kdf = self.kdf.copy()
         pdf = self.pdf.copy()