Repository: databricks/koalas

Issue:
Filtering rows throws error when result is empty
In Pandas, if I filter for rows and there are none that satisfy the condition, it returns an empty DataFrame.

With Koalas, it throws an error. I think it should return an empty dataframe, like Pandas.

Note: data is a koalas dataframe in the screenshot below
![Screen Shot 2019-09-11 at 4 11 21 PM](https://user-images.githubusercontent.com/6416014/64741906-19e03480-d4af-11e9-9a4e-5d8bfbd4f260.png)



Discussion:
Thanks for your issue reporting!

Sorry, but if you don't mind, could you show me the python code you used to make Koalas DataFrame if it possible?

I just tested it with the example you uploaded.

In my case, both results are returning an Empty DataFrame like below:

![스크린샷 2019-09-12 오전 10 14 01](https://user-images.githubusercontent.com/44108233/64746452-89553300-d546-11e9-8bc2-4a2f29757b8c.png)

Thanks for your reporting again, Have a nice day :)
When going from a Pandas DF to Koalas DF, I agree, there are no issues. However, this issue seems to come up when reading directly from a CSV file. I can reproduce this issue with two different CSV sources. Take a look at the notebook linked here and you can see the error: https://databricks-prod-cloudfront.cloud.databricks.com/public/793177bc53e528530b06c78a4fa0e086/0/15446625/100107/latest.html
Hmm, that's interesting. Let me check, too. Thanks for sharing the notebook!
I wrote a simple notebook to investigate this issue.

https://github.com/harupy/temp/blob/master/koalas_filter_issue.ipynb

Seems like the error occurs when pyarrow converts a datetime column. 
Thanks @harupy and @itholic for analysis and the codes!

Resolving patch:
diff --git a/databricks/koalas/base.py b/databricks/koalas/base.py
--- a/databricks/koalas/base.py
+++ b/databricks/koalas/base.py
@@ -26,12 +26,11 @@
 from pandas.api.types import is_list_like
 from pyspark import sql as spark
 from pyspark.sql import functions as F, Window
-from pyspark.sql.types import DoubleType, FloatType, LongType, StringType, TimestampType, \
-    to_arrow_type
+from pyspark.sql.types import DoubleType, FloatType, LongType, StringType, TimestampType
 
 from databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.
 from databricks.koalas.internal import _InternalFrame
-from databricks.koalas.typedef import pandas_wraps
+from databricks.koalas.typedef import pandas_wraps, spark_type_to_pandas_dtype
 from databricks.koalas.utils import align_diff_series, scol_for
 
 
@@ -219,10 +218,7 @@ def dtype(self):
         >>> s.rename("a").to_frame().set_index("a").index.dtype
         dtype('<M8[ns]')
         """
-        if type(self.spark_type) == TimestampType:
-            return np.dtype('datetime64[ns]')
-        else:
-            return np.dtype(to_arrow_type(self.spark_type).to_pandas_dtype())
+        return spark_type_to_pandas_dtype(self.spark_type)
 
     @property
     def empty(self):
diff --git a/databricks/koalas/internal.py b/databricks/koalas/internal.py
--- a/databricks/koalas/internal.py
+++ b/databricks/koalas/internal.py
@@ -32,7 +32,7 @@
 
 from databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.
 from databricks.koalas.config import get_option
-from databricks.koalas.typedef import infer_pd_series_spark_type
+from databricks.koalas.typedef import infer_pd_series_spark_type, spark_type_to_pandas_dtype
 from databricks.koalas.utils import column_index_level, default_session, lazy_property, scol_for
 
 
@@ -634,7 +634,7 @@ def pandas_df(self):
         sdf = self.spark_internal_df
         pdf = sdf.toPandas()
         if len(pdf) == 0 and len(sdf.schema) > 0:
-            pdf = pdf.astype({field.name: to_arrow_type(field.dataType).to_pandas_dtype()
+            pdf = pdf.astype({field.name: spark_type_to_pandas_dtype(field.dataType)
                               for field in sdf.schema})
 
         index_columns = self.index_columns
diff --git a/databricks/koalas/typedef.py b/databricks/koalas/typedef.py
--- a/databricks/koalas/typedef.py
+++ b/databricks/koalas/typedef.py
@@ -145,6 +145,14 @@ def as_spark_type(tpe) -> types.DataType:
     return _known_types.get(tpe, None)
 
 
+def spark_type_to_pandas_dtype(spark_type):
+    """ Return the given Spark DataType to pandas dtype. """
+    if isinstance(spark_type, types.TimestampType):
+        return np.dtype('datetime64[ns]')
+    else:
+        return np.dtype(types.to_arrow_type(spark_type).to_pandas_dtype())
+
+
 def as_python_type(spark_tpe):
     return _py_conversions.get(spark_tpe, None)
 


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe.py b/databricks/koalas/tests/test_dataframe.py
--- a/databricks/koalas/tests/test_dataframe.py
+++ b/databricks/koalas/tests/test_dataframe.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 
+from datetime import date, datetime
 import inspect
 
 import numpy as np
@@ -1642,3 +1643,11 @@ def test_transform(self):
 
         with self.assertRaisesRegex(AssertionError, "the first argument should be a callable"):
             kdf.transform(1)
+
+    def test_empty_timestamp(self):
+        pdf = pd.DataFrame({'t': [datetime(2019, 1, 1, 0, 0, 0),
+                                  datetime(2019, 1, 2, 0, 0, 0),
+                                  datetime(2019, 1, 3, 0, 0, 0)]})
+        kdf = ks.from_pandas(pdf)
+        self.assert_eq(kdf[kdf['t'] != kdf['t']], pdf[pdf['t'] != pdf['t']])
+        self.assert_eq(kdf[kdf['t'] != kdf['t']].dtypes, pdf[pdf['t'] != pdf['t']].dtypes)