Repository: databricks/koalas

Issue:
DataFrame.unstack
Implement `DataFrame.unstack`.

https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.unstack.html


Discussion:
Any update on this issue? 
I investigated this for whole day, and seems we shouldn't implement this API. It creates rows * index columns:

```python
>>> df = pd.DataFrame({'A': ['a', 'b', 'c', 'd'], 'B': [1, 3, 5, 7], 'C': [2, 4, 6, 7]}, index=pd.MultiIndex.from_tuples([('X', 'A'), ('X', 'B'), ('Y', 'C'), ('Y', 'D')]))
>>> df
     A  B  C
X A  a  1  2
  B  b  3  4
Y C  c  5  6
  D  d  7  7
>>> df.unstack()
     A                   B                   C
     A    B    C    D    A    B    C    D    A    B    C    D
X    a    b  NaN  NaN  1.0  3.0  NaN  NaN  2.0  4.0  NaN  NaN
Y  NaN  NaN    c    d  NaN  NaN  5.0  7.0  NaN  NaN  6.0  7.0
```

I don't think this practically works since Spark partitions it row based. This is much worse than `transpose` we had to think multiple times. Let's don't do this for now.

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -7877,6 +7877,109 @@ def melt(self, id_vars=None, value_vars=None, var_name=None, value_name="value")
 
         return DataFrame(exploded_df)
 
+    def unstack(self):
+        """
+        Pivot the (necessarily hierarchical) index labels. The output
+        will be a Series.
+
+        Returns
+        -------
+        Series or DataFrame
+
+        See Also
+        --------
+        DataFrame.pivot : Pivot a table based on column values.
+
+        Examples
+        --------
+        >>> df = ks.DataFrame({"A": {"0": "a", "1": "b", "2": "c"},
+        ...                    "B": {"0": "1", "1": "3", "2": "5"},
+        ...                    "C": {"0": "2", "1": "4", "2": "6"}},
+        ...                   columns=["A", "B", "C"])
+        >>> df
+           A  B  C
+        0  a  1  2
+        1  b  3  4
+        2  c  5  6
+
+        >>> df.unstack().sort_index()
+        A  0    a
+           1    b
+           2    c
+        B  0    1
+           1    3
+           2    5
+        C  0    2
+           1    4
+           2    6
+        Name: 0, dtype: object
+
+        >>> df.columns = pd.MultiIndex.from_tuples([('X', 'A'), ('X', 'B'), ('Y', 'C')])
+        >>> df.unstack().sort_index()
+        X  A  0    a
+              1    b
+              2    c
+           B  0    1
+              1    3
+              2    5
+        Y  C  0    2
+              1    4
+              2    6
+        Name: 0, dtype: object
+        """
+        from databricks.koalas.series import _col
+
+        if len(self._internal.index_columns) > 1:
+            raise NotImplementedError(
+                "Multi-index is not supported. Consider "
+                "using DataFrame.pivot_table or DataFrame.pivot instead."
+            )
+
+        # TODO: Codes here are similar with melt. Should we deduplicate?
+        column_labels = self._internal.column_labels
+        ser_name = "0"
+        sdf = self._sdf
+        new_index_columns = [
+            SPARK_INDEX_NAME_FORMAT(i) for i in range(self._internal.column_labels_level)
+        ]
+
+        new_index_map = []
+        if self._internal.column_label_names is not None:
+            new_index_map.extend(zip(new_index_columns, self._internal.column_label_names))
+        else:
+            new_index_map.extend(zip(new_index_columns, [None] * len(new_index_columns)))
+
+        pairs = F.explode(
+            F.array(
+                *[
+                    F.struct(
+                        *(
+                            [F.lit(c).alias(name) for c, name in zip(idx, new_index_columns)]
+                            + [self._internal.scol_for(idx).alias(ser_name)]
+                        )
+                    )
+                    for idx in column_labels
+                ]
+            )
+        )
+
+        columns = [
+            F.col("pairs.%s" % name)
+            for name in new_index_columns[: self._internal.column_labels_level]
+        ] + [F.col("pairs.%s" % ser_name)]
+
+        new_index_len = len(new_index_columns)
+        existing_index_columns = []
+        for i, index_name in enumerate(self._internal.index_names):
+            new_index_map.append((SPARK_INDEX_NAME_FORMAT(i + new_index_len), index_name))
+            existing_index_columns.append(
+                self._internal.index_scols[i].alias(SPARK_INDEX_NAME_FORMAT(i + new_index_len))
+            )
+
+        exploded_df = sdf.withColumn("pairs", pairs).select(existing_index_columns + columns)
+
+        return _col(DataFrame(_InternalFrame(exploded_df, index_map=new_index_map)))
+
     # TODO: axis, skipna, and many arguments should be implemented.
     def all(self, axis: Union[int, str] = 0) -> bool:
         """
diff --git a/databricks/koalas/missing/frame.py b/databricks/koalas/missing/frame.py
--- a/databricks/koalas/missing/frame.py
+++ b/databricks/koalas/missing/frame.py
@@ -83,8 +83,6 @@ class _MissingPandasLikeDataFrame(object):
     tshift = unsupported_function("tshift")
     tz_convert = unsupported_function("tz_convert")
     tz_localize = unsupported_function("tz_localize")
-    unstack = unsupported_function("unstack")
-
     # Deprecated functions
     convert_objects = unsupported_function("convert_objects", deprecated=True)
     select = unsupported_function("select", deprecated=True)


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe.py b/databricks/koalas/tests/test_dataframe.py
--- a/databricks/koalas/tests/test_dataframe.py
+++ b/databricks/koalas/tests/test_dataframe.py
@@ -1859,6 +1859,15 @@ def test_pivot_table_and_index(self):
         self.assert_eq(ktable.index, ptable.index)
         self.assert_eq(repr(ktable.index), repr(ptable.index))
 
+    def test_unstack_errors(self):
+        kdf = ks.DataFrame(
+            np.random.randn(3, 3),
+            index=pd.MultiIndex.from_tuples([("rg1", "x"), ("rg1", "y"), ("rg2", "z")]),
+        )
+
+        with self.assertRaisesRegex(NotImplementedError, "Multi-index is not supported."):
+            kdf.unstack()
+
     def test_pivot_errors(self):
         kdf = ks.range(10)