Repository: databricks/koalas

Issue:
Support for pivot
Customer requested that we implement pivot. They are migrating to Spark. Their legacy platform involves a lot of pandas code. They said pivot is an important method in their ETL scripts but they haven't provided a use case yet. Will push them a bit more but let me know if you need anymore context. 


Discussion:
(FYI, pivot_table is in progress at https://github.com/databricks/koalas/pull/386)
@HyukjinKwon should we close this ticket now #386 is merged?
There's actually [`DataFrame.pivot`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.pivot.html?highlight=pivot#pandas.DataFrame.pivot) too. Let me take a look today.

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -3036,6 +3036,114 @@ def pivot_table(self, values=None, index=None, columns=None,
 
             return DataFrame(sdf.withColumn(columns, F.lit(index_values))).set_index(columns)
 
+    def pivot(self, index=None, columns=None, values=None):
+        """
+        Return reshaped DataFrame organized by given index / column values.
+
+        Reshape data (produce a "pivot" table) based on column values. Uses
+        unique values from specified `index` / `columns` to form axes of the
+        resulting DataFrame. This function does not support data
+        aggregation.
+
+        Parameters
+        ----------
+        index : string, optional
+            Column to use to make new frame's index. If None, uses
+            existing index.
+        columns : string
+            Column to use to make new frame's columns.
+        values : string, object or a list of the previous
+            Column(s) to use for populating new frame's values.
+        Returns
+        -------
+        DataFrame
+            Returns reshaped DataFrame.
+
+        See Also
+        --------
+        DataFrame.pivot_table : Generalization of pivot that can handle
+            duplicate values for one index/column pair.
+
+        Examples
+        --------
+        >>> df = ks.DataFrame({'foo': ['one', 'one', 'one', 'two', 'two',
+        ...                            'two'],
+        ...                    'bar': ['A', 'B', 'C', 'A', 'B', 'C'],
+        ...                    'baz': [1, 2, 3, 4, 5, 6],
+        ...                    'zoo': ['x', 'y', 'z', 'q', 'w', 't']},
+        ...                   columns=['foo', 'bar', 'baz', 'zoo'])
+        >>> df
+           foo bar  baz zoo
+        0  one   A    1   x
+        1  one   B    2   y
+        2  one   C    3   z
+        3  two   A    4   q
+        4  two   B    5   w
+        5  two   C    6   t
+
+        >>> df.pivot(index='foo', columns='bar', values='baz').sort_index()
+        ... # doctest: +NORMALIZE_WHITESPACE
+             A  B  C
+        foo
+        one  1  2  3
+        two  4  5  6
+
+        >>> df.pivot(columns='bar', values='baz').sort_index()
+        ... # doctest: +NORMALIZE_WHITESPACE
+             A    B    C
+        0  1.0  NaN  NaN
+        1  NaN  2.0  NaN
+        2  NaN  NaN  3.0
+        3  4.0  NaN  NaN
+        4  NaN  5.0  NaN
+        5  NaN  NaN  6.0
+
+        Notice that, unlike pandas raises an ValueError when duplicated values are found,
+        Koalas' pivot still works with its first value it meets during operation because pivot
+        is an expensive operation and it is preferred to permissively execute over failing fast
+        when processing large data.
+
+        >>> df = ks.DataFrame({"foo": ['one', 'one', 'two', 'two'],
+        ...                    "bar": ['A', 'A', 'B', 'C'],
+        ...                    "baz": [1, 2, 3, 4]}, columns=['foo', 'bar', 'baz'])
+        >>> df
+           foo bar  baz
+        0  one   A    1
+        1  one   A    2
+        2  two   B    3
+        3  two   C    4
+
+        >>> df.pivot(index='foo', columns='bar', values='baz').sort_index()
+        ... # doctest: +NORMALIZE_WHITESPACE
+               A    B    C
+        foo
+        one  1.0  NaN  NaN
+        two  NaN  3.0  4.0
+        """
+        if columns is None:
+            raise ValueError("columns should be set.")
+
+        if values is None:
+            raise ValueError("values should be set.")
+
+        should_use_existing_index = index is not None
+        if should_use_existing_index:
+            index = [index]
+        else:
+            index = self._internal.index_columns
+
+        df = self.pivot_table(
+            index=index, columns=columns, values=values, aggfunc='first')
+
+        if should_use_existing_index:
+            return df
+        else:
+            index_columns = df._internal.index_columns
+            # Note that the existing indexing column won't exist in the pivoted DataFrame.
+            internal = df._internal.copy(
+                index_map=[(index_column, None) for index_column in index_columns])
+            return DataFrame(internal)
+
     @property
     def columns(self):
         """The column labels of the DataFrame."""
diff --git a/databricks/koalas/missing/frame.py b/databricks/koalas/missing/frame.py
--- a/databricks/koalas/missing/frame.py
+++ b/databricks/koalas/missing/frame.py
@@ -92,7 +92,6 @@ class _MissingPandasLikeDataFrame(object):
     median = unsupported_function('median')
     mode = unsupported_function('mode')
     pct_change = unsupported_function('pct_change')
-    pivot = unsupported_function('pivot')
     pop = unsupported_function('pop')
     prod = unsupported_function('prod')
     product = unsupported_function('product')


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe.py b/databricks/koalas/tests/test_dataframe.py
--- a/databricks/koalas/tests/test_dataframe.py
+++ b/databricks/koalas/tests/test_dataframe.py
@@ -873,7 +873,16 @@ def test_pivot_table(self):
         #  fill_value=999), pdf.pivot_table(index=['e', 'c'], columns="a", values="b",
         #  fill_value=999))
 
-    def test_pivot_table_erros(self):
+    def test_pivot_errors(self):
+        kdf = ks.range(10)
+
+        with self.assertRaisesRegex(ValueError, "columns should be set"):
+            kdf.pivot(index='id')
+
+        with self.assertRaisesRegex(ValueError, "values should be set"):
+            kdf.pivot(index='id', columns="id")
+
+    def test_pivot_table_errors(self):
 
         pdf = pd.DataFrame({'a': [4, 2, 3, 4, 8, 6],
                             'b': [1, 2, 2, 4, 2, 4],