Repository: databricks/koalas

Issue:
Series.astype() does not not respect "truthyness" as in pandas
As you can see in the code below, I'm trying to convert a `str` to a `bool` below, as part of a quick way to see if a string is not `None`, empty or only consisting of whitespace. Since Koalas seems to hook directly into Spark's `Column.cast` and Java does not have the concept of "truthy" and "falsey" as in Python, the conversion fails and `None` is inserted instead.

This is a very tricky issue and might be very difficult to solve, so I'm not necessarily expecting a fix, but a note relating to the limitations of `Series.astype` in the documentation is probably necessary.

```python
import pandas as pd
import databricks.koalas as ks

print("koalas version: {}\n".format(ks.__version__))

data = ["hi", "hi ", " ", " \t", "", None]
pds = pd.Series(data)
kss = ks.Series(data)

print("pds:\n{}\n".format(pds.astype(bool)))
print("kss:\n{}\n".format(kss.astype(bool)))

print("pds (stripped):\n{}\n".format(pds.str.strip().astype(bool)))
print("kss (stripped):\n{}".format(kss.str.strip().astype(bool)))
```

Output:
```
koalas version: 0.31.0

pds:
0     True
1     True
2     True
3     True
4    False
5    False
dtype: bool

kss:
0    None
1    None
2    None
3    None
4    None
5    None
Name: 0, dtype: object

pds (stripped):
0     True
1     True
2    False
3    False
4    False
5    False
dtype: bool

kss (stripped):
0    None
1    None
2    None
3    None
4    None
5    None
Name: 0, dtype: object
```


Discussion:
Thanks for the report!
That's a good point, and I guess we should follow pandas' behavior.
I'll be working on this.

Resolving patch:
diff --git a/databricks/koalas/series.py b/databricks/koalas/series.py
--- a/databricks/koalas/series.py
+++ b/databricks/koalas/series.py
@@ -32,7 +32,15 @@
 from databricks.koalas.typedef import as_python_type
 from pyspark import sql as spark
 from pyspark.sql import functions as F, Column
-from pyspark.sql.types import BooleanType, StructType, LongType, IntegerType
+from pyspark.sql.types import (
+    BooleanType,
+    DoubleType,
+    FloatType,
+    StringType,
+    StructType,
+    LongType,
+    IntegerType,
+)
 from pyspark.sql.window import Window
 
 from databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.
@@ -918,7 +926,20 @@ def astype(self, dtype) -> "Series":
         spark_type = as_spark_type(dtype)
         if not spark_type:
             raise ValueError("Type {} not understood".format(dtype))
-        return self._with_new_scol(self._scol.cast(spark_type))
+        if isinstance(spark_type, BooleanType):
+            if isinstance(self.spark_type, StringType):
+                scol = F.when(self._scol.isNull(), F.lit(False)).otherwise(F.length(self._scol) > 0)
+            elif isinstance(self.spark_type, (FloatType, DoubleType)):
+                scol = F.when(self._scol.isNull() | F.isnan(self._scol), F.lit(True)).otherwise(
+                    self._scol.cast(spark_type)
+                )
+            else:
+                scol = F.when(self._scol.isNull(), F.lit(False)).otherwise(
+                    self._scol.cast(spark_type)
+                )
+        else:
+            scol = self._scol.cast(spark_type)
+        return self._with_new_scol(scol)
 
     def getField(self, name):
         if not isinstance(self.spark_type, StructType):
diff --git a/databricks/koalas/typedef.py b/databricks/koalas/typedef.py
--- a/databricks/koalas/typedef.py
+++ b/databricks/koalas/typedef.py
@@ -280,7 +280,7 @@ def clean_fun(*args2):
     series = kser._with_new_scol(scol=col)  # type: 'ks.Series'
     all_name_tokens = name_tokens + sorted(kw_name_tokens)
     name = "{}({})".format(f.__name__, ", ".join(all_name_tokens))
-    series = series.astype(return_type).rename(name)
+    series = series.rename(name)
     return series
 
 


Test patch:
diff --git a/databricks/koalas/tests/test_series.py b/databricks/koalas/tests/test_series.py
--- a/databricks/koalas/tests/test_series.py
+++ b/databricks/koalas/tests/test_series.py
@@ -1025,6 +1025,26 @@ def test_shift(self):
     def test_astype(self):
         pser = pd.Series([10, 20, 15, 30, 45], name="x")
         kser = ks.Series(pser)
+
+        self.assert_eq(kser.astype(int), pser.astype(int))
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+
+        pser = pd.Series([10, 20, 15, 30, 45, None, np.nan], name="x")
+        kser = ks.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+
+        pser = pd.Series(["hi", "hi ", " ", " \t", "", None], name="x")
+        kser = ks.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+        self.assert_eq(kser.str.strip().astype(bool), pser.str.strip().astype(bool))
+
+        pser = pd.Series([True, False, None], name="x")
+        kser = ks.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+
         with self.assertRaisesRegex(ValueError, "Type int63 not understood"):
             kser.astype("int63")