Repository: databricks/koalas

Issue:
Names in MultiIndex is not working as expected
According to the documentation: names for each of the index levels. (name is accepted for compat). 

```python 
        columns = pd.MultiIndex.from_tuples([('X', 'A'), ('X', 'B'), ('Y', 'C'), ('Y', 'D')],
                                            names=['lvl_1', 'lvl_2'])
        kdf3 = ks.DataFrame([[1, 2, 3, 4],
                             [5, 6, 7, 8],
                             [9, 10, 11, 12],
                             [13, 14, 15, 16],
                             [17, 18, 19, 20]], columns=columns)
        print(kdf3)
```
Expected output : 
```
lvl_1     X       Y    
lvl_2     A   B   C   D
0         1   2   3   4
1         5   6   7   8
2         9  10  11  12
3        13  14  15  16
4        17  18  19  20
```
Actual output: 
```
    X       Y    
    A   B   C   D
0   1   2   3   4
1   5   6   7   8
2   9  10  11  12
3  13  14  15  16
4  17  18  19  20
```

I'm wondering if the implementation is not finished yet or it's a bug?


Discussion:

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -3698,7 +3698,11 @@ def pivot(self, index=None, columns=None, values=None):
     def columns(self):
         """The column labels of the DataFrame."""
         if self._internal.column_index is not None:
-            return pd.MultiIndex.from_tuples(self._internal.column_index)
+            if self._internal.column_index_names is not None:
+                return pd.MultiIndex.from_tuples(self._internal.column_index,
+                                                 names=self._internal.column_index_names)
+            else:
+                return pd.MultiIndex.from_tuples(self._internal.column_index)
         else:
             return pd.Index(self._internal.data_columns)
 
diff --git a/databricks/koalas/internal.py b/databricks/koalas/internal.py
--- a/databricks/koalas/internal.py
+++ b/databricks/koalas/internal.py
@@ -22,7 +22,7 @@
 
 import numpy as np
 import pandas as pd
-from pandas.api.types import is_datetime64_dtype, is_datetime64tz_dtype
+from pandas.api.types import is_datetime64_dtype, is_datetime64tz_dtype, is_list_like
 from pyspark import sql as spark
 from pyspark._globals import _NoValue, _NoValueType
 from pyspark.sql.types import DataType, StructField, StructType, to_arrow_type
@@ -340,7 +340,8 @@ def __init__(self, sdf: spark.DataFrame,
                  index_map: Optional[List[IndexMap]] = None,
                  scol: Optional[spark.Column] = None,
                  data_columns: Optional[List[str]] = None,
-                 column_index: Optional[List[Tuple[str]]] = None) -> None:
+                 column_index: Optional[List[Tuple[str]]] = None,
+                 column_index_names: Optional[List[str]] = None) -> None:
         """
         Create a new internal immutable DataFrame to manage Spark DataFrame, column fields and
         index fields and names.
@@ -355,6 +356,7 @@ def __init__(self, sdf: spark.DataFrame,
                               argument is ignored, otherwise if this is None, calculated from sdf.
         :param column_index: list of tuples with the same length
                               The multi-level values in the tuples.
+        :param column_index_names: Names for each of the index levels.
         """
         assert isinstance(sdf, spark.DataFrame)
         assert index_map is None \
@@ -381,6 +383,17 @@ def __init__(self, sdf: spark.DataFrame,
                                         len(set(len(i) for i in column_index)) <= 1)
         self._column_index = column_index
 
+        if column_index_names is not None and not is_list_like(column_index_names):
+            raise ValueError('Column_index_names should be list-like or None for a MultiIndex')
+
+        if isinstance(column_index_names, list):
+            if all(name is None for name in column_index_names):
+                self._column_index_names = None
+            else:
+                self._column_index_names = column_index_names
+        else:
+            self._column_index_names = column_index_names
+
     def scol_for(self, column_name: str) -> spark.Column:
         """ Return Spark Column for the given column name. """
         if self._scol is not None and column_name == self._data_columns[0]:
@@ -449,6 +462,11 @@ def column_index(self) -> Optional[List[Tuple[str]]]:
         """ Return the managed column index. """
         return self._column_index
 
+    @property
+    def column_index_names(self) -> Optional[List[str]]:
+        """ Return names of the index levels. """
+        return self._column_index_names
+
     @lazy_property
     def spark_df(self) -> spark.DataFrame:
         """ Return as Spark DataFrame. """
@@ -473,7 +491,11 @@ def pandas_df(self):
             pdf = pdf[self.data_columns]
 
         if self._column_index is not None:
-            pdf.columns = pd.MultiIndex.from_tuples(self._column_index)
+            if self.column_index_names is not None:
+                pdf.columns = pd.MultiIndex.from_tuples(self._column_index,
+                                                        names=self.column_index_names)
+            else:
+                pdf.columns = pd.MultiIndex.from_tuples(self._column_index)
 
         index_names = self.index_names
         if len(index_names) > 0:
@@ -487,7 +509,8 @@ def copy(self, sdf: Union[spark.DataFrame, _NoValueType] = _NoValue,
              index_map: Union[List[IndexMap], _NoValueType] = _NoValue,
              scol: Union[spark.Column, _NoValueType] = _NoValue,
              data_columns: Union[List[str], _NoValueType] = _NoValue,
-             column_index: Union[List[Tuple[str]], _NoValueType] = _NoValue) -> '_InternalFrame':
+             column_index: Union[List[Tuple[str]], _NoValueType] = _NoValue,
+             column_index_names: Union[List[str], _NoValueType] = _NoValue,) -> '_InternalFrame':
         """ Copy the immutable DataFrame.
 
         :param sdf: the new Spark DataFrame. If None, then the original one is used.
@@ -495,6 +518,7 @@ def copy(self, sdf: Union[spark.DataFrame, _NoValueType] = _NoValue,
         :param scol: the new Spark Column. If None, then the original one is used.
         :param data_columns: the new column field names. If None, then the original ones are used.
         :param column_index: the new column index.
+        :param column_index_names: the new names of the index levels.
         :return: the copied immutable DataFrame.
         """
         if sdf is _NoValue:
@@ -507,8 +531,10 @@ def copy(self, sdf: Union[spark.DataFrame, _NoValueType] = _NoValue,
             data_columns = self._data_columns
         if column_index is _NoValue:
             column_index = self._column_index
+        if column_index_names is _NoValue:
+            column_index_names = self._column_index_names
         return _InternalFrame(sdf, index_map=index_map, scol=scol, data_columns=data_columns,
-                              column_index=column_index)
+                              column_index=column_index, column_index_names=column_index_names)
 
     @staticmethod
     def from_pandas(pdf: pd.DataFrame) -> '_InternalFrame':
@@ -521,8 +547,13 @@ def from_pandas(pdf: pd.DataFrame) -> '_InternalFrame':
         data_columns = [str(col) for col in columns]
         if isinstance(columns, pd.MultiIndex):
             column_index = columns.tolist()
+            if columns.names is not None:
+                column_index_names = columns.names
+            else:
+                column_index_names = None
         else:
             column_index = None
+            column_index_names = None
 
         index = pdf.index
 
@@ -552,4 +583,4 @@ def from_pandas(pdf: pd.DataFrame) -> '_InternalFrame':
             reset_index[name] = col.replace({np.nan: None})
         sdf = default_session().createDataFrame(reset_index, schema=schema)
         return _InternalFrame(sdf=sdf, index_map=index_map, data_columns=data_columns,
-                              column_index=column_index)
+                              column_index=column_index, column_index_names=column_index_names)


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe.py b/databricks/koalas/tests/test_dataframe.py
--- a/databricks/koalas/tests/test_dataframe.py
+++ b/databricks/koalas/tests/test_dataframe.py
@@ -100,6 +100,31 @@ def test_dataframe_multiindex_columns(self):
         self.assert_eq(kdf[('x', 'a')], pdf[('x', 'a')])
         self.assert_eq(kdf[('x', 'a', '1')], pdf[('x', 'a', '1')])
 
+    def test_dataframe_multiindex_names_level(self):
+        columns = pd.MultiIndex.from_tuples([('X', 'A'), ('X', 'B'), ('Y', 'C'), ('Y', 'D')],
+                                            names=['lvl_1', 'lvl_2'])
+        kdf = ks.DataFrame([[1, 2, 3, 4],
+                            [5, 6, 7, 8],
+                            [9, 10, 11, 12],
+                            [13, 14, 15, 16],
+                            [17, 18, 19, 20]], columns=columns)
+
+        pdf = pd.DataFrame([[1, 2, 3, 4],
+                            [5, 6, 7, 8],
+                            [9, 10, 11, 12],
+                            [13, 14, 15, 16],
+                            [17, 18, 19, 20]], columns=columns)
+
+        self.assert_eq(kdf.columns.names, pdf.columns.names)
+        self.assert_eq(kdf.to_pandas().columns.names, pdf.columns.names)
+
+        kdf1 = ks.from_pandas(pdf)
+        self.assert_eq(kdf1.columns.names, pdf.columns.names)
+
+        with self.assertRaisesRegex(ValueError, 'Column_index_names should '
+                                                'be list-like or None for a MultiIndex'):
+            ks.DataFrame(kdf1._internal.copy(column_index_names='level'))
+
     def test_reset_index_with_multiindex_columns(self):
         index = pd.MultiIndex.from_tuples([('bird', 'falcon'),
                                            ('bird', 'parrot'),