Repository: databricks/koalas

Issue:
Index.to_series() works not properly
When converting an Index to Series for comparing operation like the below,

there is something problem.

```python
>>> pidx = pd.Index([1, 2, 3, 4, 5])
>>> kidx1 = ks.Index([1, 2, 3, 4, 5])
>>> kidx2 = ks.Index(pidx)
>>> kidx3 = ks.from_pandas(pidx)
>>> kidx1.to_series() == kidx2.to_series() == kidx3.to_series()
Traceback (most recent call last):
...
AssertionError: (1, 0)
```

The existing implementation seems only can convert from index to series properly when the index is came from DataFrame like the below.

```python
>>> df = ks.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
...                   columns=['dogs', 'cats'],
...                   index=list('abcd'))
>>> df['dogs'].index.to_series() == df['cats'].index.to_series()
a    True
b    True
c    True
d    True
Name: 0, dtype: bool
```


Discussion:

Resolving patch:
diff --git a/databricks/koalas/indexing.py b/databricks/koalas/indexing.py
--- a/databricks/koalas/indexing.py
+++ b/databricks/koalas/indexing.py
@@ -28,10 +28,10 @@
 from databricks.koalas.internal import _InternalFrame, NATURAL_ORDER_COLUMN_NAME
 from databricks.koalas.exceptions import SparkPandasIndexingError, SparkPandasNotImplementedError
 from databricks.koalas.utils import (
-    column_labels_level,
     lazy_property,
     name_like_string,
     verify_temp_column_name,
+    scol_for,
 )
 
 
@@ -256,12 +256,27 @@ def __getitem__(self, key):
             index_scols = self._internal.index_scols
             index_map = self._internal.index_map
 
-        if self._internal.column_label_names is None:
-            column_label_names = None
+        if len(column_labels) > 0:
+            column_labels = column_labels.copy()
+            column_labels_level = max(
+                len(label) if label is not None else 1 for label in column_labels
+            )
+            none_column = 0
+            for i, label in enumerate(column_labels):
+                if label is None:
+                    label = (str(none_column),)
+                    none_column += 1
+                if len(label) < column_labels_level:
+                    label = tuple(list(label) + ([""]) * (column_labels_level - len(label)))
+                column_labels[i] = label
+
+            if self._internal.column_label_names is None:
+                column_label_names = None
+            else:
+                # Manage column index names
+                column_label_names = self._internal.column_label_names[-column_labels_level:]
         else:
-            # Manage column index names
-            level = column_labels_level(column_labels)
-            column_label_names = self._internal.column_label_names[-level:]
+            column_label_names = None
 
         try:
             sdf = self._internal._sdf
@@ -273,6 +288,7 @@ def __getitem__(self, key):
                 else:
                     sdf = sdf.limit(sdf.count() + limit)
 
+            data_columns = sdf.select(column_scols).columns
             sdf = sdf.select(index_scols + column_scols)
         except AnalysisException:
             raise KeyError(
@@ -283,6 +299,7 @@ def __getitem__(self, key):
             sdf=sdf,
             index_map=index_map,
             column_labels=column_labels,
+            column_scols=[scol_for(sdf, col) for col in data_columns],
             column_label_names=column_label_names,
         )
         kdf = DataFrame(internal)
diff --git a/databricks/koalas/utils.py b/databricks/koalas/utils.py
--- a/databricks/koalas/utils.py
+++ b/databricks/koalas/utils.py
@@ -279,14 +279,9 @@ def align_diff_series(func, this_series, *args, how="full"):
     cols = [arg for arg in args if isinstance(arg, IndexOpsMixin)]
     combined = combine_frames(this_series.to_frame(), *cols, how=how)
 
-    that_columns = [
-        combined["that"][arg._internal.column_labels[0]]._scol
-        if isinstance(arg, IndexOpsMixin)
-        else arg
-        for arg in args
-    ]
-
-    scol = func(combined["this"][this_series._internal.column_labels[0]]._scol, *that_columns)
+    scol = func(
+        combined["this"]._internal.column_scols[0], *combined["that"]._internal.column_scols
+    )
 
     return Series(
         combined._internal.copy(scol=scol, column_labels=this_series._internal.column_labels),


Test patch:
diff --git a/databricks/koalas/tests/test_ops_on_diff_frames.py b/databricks/koalas/tests/test_ops_on_diff_frames.py
--- a/databricks/koalas/tests/test_ops_on_diff_frames.py
+++ b/databricks/koalas/tests/test_ops_on_diff_frames.py
@@ -647,6 +647,17 @@ def test_multi_index_column_assignment_frame(self):
         with self.assertRaisesRegex(KeyError, "Key length \\(3\\) exceeds index depth \\(2\\)"):
             kdf[("1", "2", "3")] = ks.Series([100, 200, 300, 200])
 
+    def test_to_series_comparison(self):
+        kidx1 = ks.Index([1, 2, 3, 4, 5])
+        kidx2 = ks.Index([1, 2, 3, 4, 5])
+
+        self.assert_eq((kidx1.to_series() == kidx2.to_series()).all(), True)
+
+        kidx1.name = "koalas"
+        kidx2.name = "koalas"
+
+        self.assert_eq((kidx1.to_series() == kidx2.to_series()).all(), True)
+
 
 class OpsOnDiffFramesDisabledTest(ReusedSQLTestCase, SQLTestUtils):
     @classmethod