Repository: databricks/koalas

Issue:
tail() cannot resolve new column name after calling apply() method
Hi! I use koalas 1.4.0 with Spark 3.0.1 on Ubuntu 20.04.1 Desktop.

After using 'apply()' method,  when I call the 'tail()' method, I got  the "pyspark.sql.utils.AnalysisException: Cannot resolve column name "`Initial`" among (__index_level_0__, PassengerId, Survived, Pclass, Name);" error.

koalas 1.3.0 with Spark 3.0.1 has the same issue.
Before calling apply() method, tail() is ok.

Seems tail() method can't recognize new column, but head() can recognize new column.

Other python packages:
pyspark 3.0.1
pyarrow 2.0.0
pandas 1.1.4
numpy 1.18.5


```
import pandas as pd
import numpy as np
import databricks.koalas as ks
import re

def extract_initial(x) -> np.str:
    m = re.search(r'([A-Za-z]+)\.', x)
    return m.group(1)
 
ks.set_option("compute.default_index_type", "distributed")
 
df_train = ks.DataFrame(
    {
        'PassengerId': [1, 2, 3, 4, 5],
        'Survived': [0, 1, 1, 1, 0],
        'Pclass': [3, 1, 3, 1, 3],
        'Name': ['Braund, Mr. Owen Harris',
                      'Cumings, Mrs. John Bradley',
                      'Heikkinen, Miss. Laina',
                      'Futrelle, Rev. Jacques Heath',
                      'Allen, Sir. William Henry'
                     ]
    })

print(f'Head:\n{df_train.head()}\n')
print(f'Tail:\n{df_train.tail()}\n')

df_train['Initial']=df_train['Name'].apply(extract_initial)

print(f'Columns: {df_train.columns}')

print(f"Initial head:\n{df_train['Initial'].head(5)}\n")
print(f"Initial tail:\n{df_train['Initial'].tail(5)}\n")
```


Discussion:

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -10599,7 +10599,8 @@ def tail(self, n=5) -> "DataFrame":
             n = len(self) + n
         if n <= 0:
             return ks.DataFrame(self._internal.with_filter(F.lit(False)))
-        sdf = self._internal.spark_frame
+        # Should use `resolved_copy` here for the case like `(kdf + 1).tail()`
+        sdf = self._internal.resolved_copy.spark_frame
         rows = sdf.tail(n)
         new_sdf = default_session().createDataFrame(rows, sdf.schema)
 


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe.py b/databricks/koalas/tests/test_dataframe.py
--- a/databricks/koalas/tests/test_dataframe.py
+++ b/databricks/koalas/tests/test_dataframe.py
@@ -4780,6 +4780,12 @@ def test_tail(self):
         self.assert_eq(pdf.tail(0), kdf.tail(0))
         self.assert_eq(pdf.tail(-1001), kdf.tail(-1001))
         self.assert_eq(pdf.tail(1001), kdf.tail(1001))
+        self.assert_eq((pdf + 1).tail(), (kdf + 1).tail())
+        self.assert_eq((pdf + 1).tail(10), (kdf + 1).tail(10))
+        self.assert_eq((pdf + 1).tail(-990), (kdf + 1).tail(-990))
+        self.assert_eq((pdf + 1).tail(0), (kdf + 1).tail(0))
+        self.assert_eq((pdf + 1).tail(-1001), (kdf + 1).tail(-1001))
+        self.assert_eq((pdf + 1).tail(1001), (kdf + 1).tail(1001))
         with self.assertRaisesRegex(TypeError, "bad operand type for unary -: 'str'"):
             kdf.tail("10")
 
diff --git a/databricks/koalas/tests/test_series.py b/databricks/koalas/tests/test_series.py
--- a/databricks/koalas/tests/test_series.py
+++ b/databricks/koalas/tests/test_series.py
@@ -2137,6 +2137,12 @@ def test_tail(self):
         self.assert_eq(pser.tail(0), kser.tail(0))
         self.assert_eq(pser.tail(1001), kser.tail(1001))
         self.assert_eq(pser.tail(-1001), kser.tail(-1001))
+        self.assert_eq((pser + 1).tail(), (kser + 1).tail())
+        self.assert_eq((pser + 1).tail(10), (kser + 1).tail(10))
+        self.assert_eq((pser + 1).tail(-990), (kser + 1).tail(-990))
+        self.assert_eq((pser + 1).tail(0), (kser + 1).tail(0))
+        self.assert_eq((pser + 1).tail(1001), (kser + 1).tail(1001))
+        self.assert_eq((pser + 1).tail(-1001), (kser + 1).tail(-1001))
         with self.assertRaisesRegex(TypeError, "bad operand type for unary -: 'str'"):
             kser.tail("10")