Repository: databricks/koalas

Issue:
Issue with multiple indexes.
```python
df = ks.DataFrame({"A": ["foo", "foo", "foo", "foo", "foo",
                          "bar", "bar", "bar", "bar"],
                    "B": ["one", "one", "one", "two", "two",
                          "one", "one", "two", "two"],
                    "C": ["small", "large", "large", "small",
                          "small", "large", "small", "small",
                          "large"],
                    "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
                    "E": [2, 4, 5, 5, 6, 6, 8, 9, 9]},
                   columns=['A', 'B', 'C', 'D', 'E'])

table = df.pivot_table(values='D', index=['A', 'B'],
                       columns='C', aggfunc='sum', fill_value=0)
```
If I run ,
```python
>>> table.index
Index([('foo', 'one'), ('foo', 'two'), ('bar', 'two'), ('bar', 'one')], 
dtype='object', name='A')

>>>table.index.to_pandas().tolist()
[Row(A='foo', B='one'),
 Row(A='foo', B='two'),
 Row(A='bar', B='two'),
 Row(A='bar', B='one')]


>>>table.to_pandas().index
MultiIndex([('foo', 'one'),
            ('foo', 'two'),
            ('bar', 'two'),
            ('bar', 'one')],
           names=['A', 'B'])
```

I think `table.index` should have both `names: ['A','B']`.  Is `MultiIndex` currently not supported? I looked into the source and `MultiIndex` was there. 


Discussion:
@thoo Thanks for the report. Yeah, it should behave the same as pandas'.

Resolving patch:
diff --git a/databricks/koalas/indexes.py b/databricks/koalas/indexes.py
--- a/databricks/koalas/indexes.py
+++ b/databricks/koalas/indexes.py
@@ -39,7 +39,7 @@
 
 class Index(IndexOpsMixin):
     """
-    Koala Index that corresponds to Pandas Index logically. This might hold Spark Column
+    Koalas Index that corresponds to Pandas Index logically. This might hold Spark Column
     internally.
 
     :ivar _kdf: The parent dataframe
@@ -61,7 +61,6 @@ class Index(IndexOpsMixin):
     """
 
     def __init__(self, kdf: DataFrame, scol: Optional[spark.Column] = None) -> None:
-        assert len(kdf._internal._index_map) == 1
         if scol is None:
             scol = kdf._internal.index_scols[0]
         internal = kdf._internal.copy(scol=scol,
@@ -99,11 +98,10 @@ def size(self) -> int:
 
     def to_pandas(self) -> pd.Index:
         """
-        Return a pandas Series.
+        Return a pandas Index.
 
         .. note:: This method should only be used if the resulting Pandas object is expected
-                  to be small, as all the data is loaded into the driver's memory. If the input
-                  is large, set max_rows parameter.
+                  to be small, as all the data is loaded into the driver's memory.
 
         Examples
         --------
@@ -329,20 +327,10 @@ def __getattr__(self, item: str) -> Any:
 
     def __repr__(self):
         max_display_count = get_option("display.max_rows")
-        sdf = self._kdf._sdf.select(self._scol)
-
         if max_display_count is None:
-            return repr(DataFrame(self._kdf._internal.copy(
-                sdf=sdf,
-                index_map=[(sdf.schema[0].name, self._kdf._internal.index_names[0])],
-                data_columns=[], column_index=[], column_index_names=None)).index.to_pandas())
+            return repr(self.to_pandas())
 
-        sdf = sdf.limit(max_display_count + 1)
-        internal = self._kdf._internal.copy(
-            sdf=sdf,
-            index_map=[(sdf.schema[0].name, self._kdf._internal.index_names[0])],
-            data_columns=[], column_index=[], column_index_names=None)
-        pindex = DataFrame(internal).index.to_pandas()
+        pindex = self._kdf.head(max_display_count + 1).index._with_new_scol(self._scol).to_pandas()
 
         pindex_length = len(pindex)
         repr_string = repr(pindex[:max_display_count])
@@ -354,6 +342,33 @@ def __repr__(self):
 
 
 class MultiIndex(Index):
+    """
+    Koalas MultiIndex that corresponds to Pandas MultiIndex logically. This might hold Spark Column
+    internally.
+
+    :ivar _kdf: The parent dataframe
+    :type _kdf: DataFrame
+    :ivar _scol: Spark Column instance
+    :type _scol: pyspark.Column
+
+    See Also
+    --------
+    Index : A single-level Index.
+
+    Examples
+    --------
+    >>> ks.DataFrame({'a': ['a', 'b', 'c']}, index=[[1, 2, 3], [4, 5, 6]]).index  # doctest: +SKIP
+    MultiIndex([(1, 4),
+                (2, 5),
+                (3, 6)],
+               )
+
+    >>> ks.DataFrame({'a': [1, 2, 3]}, index=[list('abc'), list('def')]).index  # doctest: +SKIP
+    MultiIndex([('a', 'd'),
+                ('b', 'e'),
+                ('c', 'f')],
+               )
+    """
 
     def __init__(self, kdf: DataFrame):
         assert len(kdf._internal._index_map) > 1
@@ -378,6 +393,32 @@ def name(self) -> str:
     def name(self, name: str) -> None:
         raise PandasNotImplementedError(class_name='pd.MultiIndex', property_name='name')
 
+    def to_pandas(self) -> pd.MultiIndex:
+        """
+        Return a pandas MultiIndex.
+
+        .. note:: This method should only be used if the resulting Pandas object is expected
+                  to be small, as all the data is loaded into the driver's memory.
+
+        Examples
+        --------
+        >>> df = ks.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
+        ...                   columns=['dogs', 'cats'],
+        ...                   index=[list('abcd'), list('efgh')])
+        >>> df['dogs'].index.to_pandas()  # doctest: +SKIP
+        MultiIndex([('a', 'e'),
+                    ('b', 'f'),
+                    ('c', 'g'),
+                    ('d', 'h')],
+                   )
+        """
+        # TODO: We might need to handle internal state change.
+        # So far, we don't have any functions to change the internal state of MultiIndex except for
+        # series-like operations. In that case, it creates new Index object instead of MultiIndex.
+        return self._kdf[[]]._to_internal_pandas().index
+
+    toPandas = to_pandas
+
     def __getattr__(self, item: str) -> Any:
         if hasattr(_MissingPandasLikeMultiIndex, item):
             property_or_func = getattr(_MissingPandasLikeMultiIndex, item)
@@ -389,3 +430,18 @@ def __getattr__(self, item: str) -> Any:
 
     def rename(self, name, inplace=False):
         raise NotImplementedError()
+
+    def __repr__(self):
+        max_display_count = get_option("display.max_rows")
+        if max_display_count is None:
+            return repr(self.to_pandas())
+
+        pindex = self._kdf.head(max_display_count + 1).index.to_pandas()
+
+        pindex_length = len(pindex)
+        repr_string = repr(pindex[:max_display_count])
+
+        if pindex_length > max_display_count:
+            footer = '\nShowing only the first {}'.format(max_display_count)
+            return repr_string + footer
+        return repr_string
diff --git a/databricks/koalas/series.py b/databricks/koalas/series.py
--- a/databricks/koalas/series.py
+++ b/databricks/koalas/series.py
@@ -1203,8 +1203,7 @@ def to_pandas(self):
         Return a pandas Series.
 
         .. note:: This method should only be used if the resulting Pandas object is expected
-                  to be small, as all the data is loaded into the driver's memory. If the input
-                  is large, set max_rows parameter.
+                  to be small, as all the data is loaded into the driver's memory.
 
         Examples
         --------


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe.py b/databricks/koalas/tests/test_dataframe.py
--- a/databricks/koalas/tests/test_dataframe.py
+++ b/databricks/koalas/tests/test_dataframe.py
@@ -1223,6 +1223,29 @@ def test_pivot_table(self):
         #  fill_value=999), pdf.pivot_table(index=['e', 'c'], columns="a", values="b",
         #  fill_value=999))
 
+    def test_pivot_table_and_index(self):
+        # https://github.com/databricks/koalas/issues/805
+        pdf = pd.DataFrame({"A": ["foo", "foo", "foo", "foo", "foo",
+                                  "bar", "bar", "bar", "bar"],
+                            "B": ["one", "one", "one", "two", "two",
+                                  "one", "one", "two", "two"],
+                            "C": ["small", "large", "large", "small",
+                                  "small", "large", "small", "small",
+                                  "large"],
+                            "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+                            "E": [2, 4, 5, 5, 6, 6, 8, 9, 9]},
+                           columns=['A', 'B', 'C', 'D', 'E'])
+        kdf = ks.from_pandas(pdf)
+
+        ptable = pdf.pivot_table(values='D', index=['A', 'B'],
+                                 columns='C', aggfunc='sum', fill_value=0).sort_index()
+        ktable = kdf.pivot_table(values='D', index=['A', 'B'],
+                                 columns='C', aggfunc='sum', fill_value=0).sort_index()
+
+        self.assert_eq(ktable, ptable)
+        self.assert_eq(ktable.index, ptable.index)
+        self.assert_eq(repr(ktable.index), repr(ptable.index))
+
     def test_pivot_errors(self):
         kdf = ks.range(10)
 
diff --git a/databricks/koalas/tests/test_indexes.py b/databricks/koalas/tests/test_indexes.py
--- a/databricks/koalas/tests/test_indexes.py
+++ b/databricks/koalas/tests/test_indexes.py
@@ -46,14 +46,8 @@ def test_index(self):
                                  index=pd.date_range('2011-01-01', freq='D', periods=10)),
                     pd.DataFrame(np.random.randn(10, 5),
                                  columns=list('abcde')).set_index(['a', 'b'])]:
-            if LooseVersion(pyspark.__version__) < LooseVersion('2.4'):
-                # PySpark < 2.4 does not support struct type with arrow enabled.
-                with self.sql_conf({'spark.sql.execution.arrow.enabled': False}):
-                    kdf = ks.from_pandas(pdf)
-                    self.assert_eq(kdf.index, pdf.index)
-            else:
-                kdf = ks.from_pandas(pdf)
-                self.assert_eq(kdf.index, pdf.index)
+            kdf = ks.from_pandas(pdf)
+            self.assert_eq(kdf.index, pdf.index)
 
     def test_index_getattr(self):
         kidx = self.kdf.index