Repository: databricks/koalas

Issue:
Column.isnull and Column.isnan
I believe we should now treat them as aliases like in Pandas:
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.isnull.html#pandas.Series.isnull

@ueshin has an implementation already here:
https://github.com/databricks/spark-pandas/pull/63/files#diff-29cc007174a75f4d0a66a7e9ef40cd74R288


Discussion:

Resolving patch:
diff --git a/databricks/koala/structures.py b/databricks/koala/structures.py
--- a/databricks/koala/structures.py
+++ b/databricks/koala/structures.py
@@ -269,16 +269,20 @@ def to_dataframe(self):
     def toPandas(self):
         return _col(self.to_dataframe().toPandas())
 
-    def isna(self):
+    @derived_from(pd.Series)
+    def isnull(self):
         if isinstance(self.schema[self.name].dataType, (FloatType, DoubleType)):
-            return self.isNull() | F.isnan(self)
+            return anchor_wrap(self, self._spark_isNull() | F._spark_isnan(self))
         else:
-            return self.isNull()
+            return anchor_wrap(self, self._spark_isNull())
+
+    isna = isnull
 
-    isnull = isna
+    @derived_from(pd.Series)
+    def notnull(self):
+        return ~self.isnull()
 
-    def notna(self):
-        return ~self.isna()
+    notna = notnull
 
     @derived_from(pd.Series)
     def dropna(self, axis=0, inplace=False, **kwargs):
@@ -344,7 +348,7 @@ def __getattr__(self, item):
         return anchor_wrap(self, self.getField(item))
 
     def __invert__(self):
-        return self.cast("boolean") == False  # noqa: disable=E712
+        return anchor_wrap(self, self._spark_cast("boolean") == F._spark_lit(False))
 
     def __str__(self):
         return self._pandas_orig_repr()
@@ -391,6 +395,11 @@ def _index_columns(self):
         return [anchor_wrap(self, self._spark_getitem(field))
                 for field in self._metadata.index_fields]
 
+    @derived_from(pd.DataFrame)
+    def iteritems(self):
+        cols = list(self.columns)
+        return list((col_name, self[col_name]) for col_name in cols)
+
     def set_index(self, keys, drop=True, append=False, inplace=False):
         """Set the DataFrame index (row labels) using one or more existing columns. By default
         yields a new object.
@@ -510,6 +519,24 @@ def reset_index(self, level=None, drop=False, inplace=False):
             df.columns = columns
             return df
 
+    @derived_from(pd.DataFrame)
+    def isnull(self):
+        df = self.copy()
+        for name, col in df.iteritems():
+            df[name] = col.isnull()
+        return df
+
+    isna = isnull
+
+    @derived_from(pd.DataFrame)
+    def notnull(self):
+        df = self.copy()
+        for name, col in df.iteritems():
+            df[name] = col.notnull()
+        return df
+
+    notna = notnull
+
     @derived_from(DataFrame)
     def toPandas(self):
         df = self._spark_select(self._metadata.all_fields)


Test patch:
diff --git a/databricks/koala/tests/test_dataframe.py b/databricks/koala/tests/test_dataframe.py
--- a/databricks/koala/tests/test_dataframe.py
+++ b/databricks/koala/tests/test_dataframe.py
@@ -265,6 +265,16 @@ def test_value_counts(self):
                                     "value_counts currently does not support bins"):
             ddf.x.value_counts(bins=3)
 
+    def test_isnull(self):
+        df = pd.DataFrame({'x': [1, 2, 3, 4, None, 6], 'y': list('abdabd')},
+                          index=[10, 20, 30, 40, 50, 60])
+        a = self.spark.from_pandas(df)
+
+        self.assert_eq(a.x.notnull(), df.x.notnull())
+        self.assert_eq(a.x.isnull(), df.x.isnull())
+        self.assert_eq(a.notnull(), df.notnull())
+        self.assert_eq(a.isnull(), df.isnull())
+
     def test_to_datetime(self):
         df = pd.DataFrame({'year': [2015, 2016],
                            'month': [2, 3],