Repository: modin-project/modin

Issue:
DataFrame.unstack works incorrect when level={all_MultiIndex_levels}
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: All
- **Modin version** (`modin.__version__`): Current master (ff6ff0d)
- **Python version**: 3.7.5
- **Code we can use to reproduce**:
```python
if __name__ == "__main__":
    import modin.pandas as pd
    import pandas
    import numpy as np

    l1 = ["a"] * 5 + ["b"] * 5
    l2 = np.arange(10)
    l3 = np.arange(10, 20)

    ml = pandas.MultiIndex.from_tuples([(fi, se, th) for fi, se, th in zip(l1, l2, l3)])
    data = {"col0": np.arange(len(ml))}

    pd_df = pandas.DataFrame(data, index=ml)
    md_df = pd.DataFrame(data, index=ml)

    pd_res = pd_df.unstack(level=[0, 1, 2])
    md_res = md_df.unstack(level=[0, 1, 2])

    print(f"pd_res:\n{pd_res}")
    print(f"\nmd_res:\n{md_res}")

```

<details><summary>Output</summary>



```
pd_res:
col0  a  0  10    0
         1  11    1
         2  12    2
         3  13    3
         4  14    4
      b  5  15    5
         6  16    6
         7  17    7
         8  18    8
         9  19    9
dtype: int64

md_res:
col0  a  0  10    0.0
            11    NaN
            12    NaN
            13    NaN
            14    NaN
                 ...
      b  9  15    NaN
            16    NaN
            17    NaN
            18    NaN
            19    9.0
Length: 200, dtype: float64
```



</details>

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
In case of `unstack` in Modin, it generates all possible combinations of index levels, when Pandas does some different from that logic



Discussion:
cc @prutskov 
@dchigarev It's a bit strange, because test for this case was added. I'll look on this.

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -1097,12 +1097,21 @@ def unstack(self, level, fill_value):
         def map_func(df):
             return pandas.DataFrame(df.unstack(level=level, fill_value=fill_value))
 
+        is_correct_len = True
+        if isinstance(self.index, pandas.MultiIndex):
+            actual_len = 1
+            for lvl in self.index.levels:
+                actual_len *= len(lvl)
+            if len(self.index) * len(self.columns) != actual_len * len(self.columns):
+                is_correct_len = False
+
         is_all_multi_list = False
         if (
             isinstance(self.index, pandas.MultiIndex)
             and isinstance(self.columns, pandas.MultiIndex)
             and is_list_like(level)
             and len(level) == self.index.nlevels
+            and is_correct_len
         ):
             is_all_multi_list = True
             real_cols_bkp = self.columns
@@ -1116,40 +1125,56 @@ def map_func(df):
         )
         result = self.__constructor__(new_modin_frame)
 
-        if is_all_multi_list:
+        def compute_index(index, columns, consider_index=True, consider_columns=True):
+            def get_unique_level_values(index):
+                return [
+                    index.get_level_values(lvl).unique()
+                    for lvl in np.arange(index.nlevels)
+                ]
+
+            new_index = (
+                get_unique_level_values(index)
+                if consider_index
+                else index
+                if isinstance(index, list)
+                else [index]
+            )
+
+            new_columns = (
+                get_unique_level_values(columns) if consider_columns else [columns]
+            )
+            return pandas.MultiIndex.from_product([*new_columns, *new_index])
+
+        if is_all_multi_list and is_correct_len:
             result = result.sort_index()
             index_level_values = [lvl for lvl in obj.index.levels]
-            columns_level_values = [
-                real_cols_bkp.get_level_values(lvl).unique()
-                for lvl in np.arange(real_cols_bkp.nlevels)
-            ]
-            result.index = pandas.MultiIndex.from_product(
-                [*columns_level_values, *index_level_values]
+
+            result.index = compute_index(
+                index_level_values, real_cols_bkp, consider_index=False
             )
             return result
 
         if need_reindex:
-            if isinstance(self.index, pandas.MultiIndex):
-                index_level_values = [
-                    self.index.get_level_values(lvl).unique()
-                    for lvl in np.arange(self.index.nlevels)
-                ]
-                new_index = pandas.MultiIndex.from_product(
-                    [self.columns, *index_level_values]
+            if is_correct_len:
+                is_recompute_index = isinstance(self.index, pandas.MultiIndex)
+                is_recompute_columns = not is_recompute_index and isinstance(
+                    self.columns, pandas.MultiIndex
+                )
+                new_index = compute_index(
+                    self.index, self.columns, is_recompute_index, is_recompute_columns
                 )
             else:
                 if isinstance(self.columns, pandas.MultiIndex):
-                    columns_level_values = [
-                        self.columns.get_level_values(lvl).unique()
-                        for lvl in np.arange(self.columns.nlevels)
-                    ]
-                    new_index = pandas.MultiIndex.from_product(
-                        [*columns_level_values, self.index]
-                    )
+                    new_index = compute_index(self.index, self.columns)
                 else:
-                    new_index = pandas.MultiIndex.from_product(
-                        [self.columns, self.index]
+                    index = pandas.MultiIndex.from_tuples(
+                        list(self.index) * len(self.columns)
                     )
+                    columns = self.columns.repeat(len(self.index))
+                    index_levels = [
+                        index.get_level_values(i) for i in range(index.nlevels)
+                    ]
+                    new_index = pandas.MultiIndex.from_arrays([columns] + index_levels)
             result = result.reindex(0, new_index)
         return result
 


Test patch:
diff --git a/modin/pandas/test/dataframe/test_default.py b/modin/pandas/test/dataframe/test_default.py
--- a/modin/pandas/test/dataframe/test_default.py
+++ b/modin/pandas/test/dataframe/test_default.py
@@ -964,43 +964,25 @@ def test_unstack(data, is_multi_idx, is_multi_col):
     modin_df = pd.DataFrame(data)
 
     if is_multi_idx:
-        if len(pandas_df.index) == 256:
-            index = pd.MultiIndex.from_product(
-                [
-                    ["a", "b", "c", "d"],
-                    ["x", "y", "z", "last"],
-                    ["i", "j", "k", "index"],
-                    [1, 2, 3, 4],
-                ]
-            )
-        elif len(pandas_df.index) == 100:
-            index = pd.MultiIndex.from_product(
-                [
-                    ["x", "y", "z", "last"],
-                    ["a", "b", "c", "d", "f"],
-                    ["i", "j", "k", "l", "index"],
-                ]
-            )
+        index = pd.MultiIndex.from_product(
+            [
+                ["a", "b", "c", "d"],
+                ["x", "y", "z", "last"],
+                ["i", "j", "k", "index"],
+                [1, 2, 3, 4],
+            ]
+        )
     else:
         index = pandas_df.index
 
     if is_multi_col:
-        if len(pandas_df.columns) == 64:
-            columns = pd.MultiIndex.from_product(
-                [
-                    ["A", "B", "C", "D"],
-                    ["xx", "yy", "zz", "LAST"],
-                    [10, 20, 30, 40],
-                ]
-            )
-        elif len(pandas_df.columns) == 100:
-            columns = pd.MultiIndex.from_product(
-                [
-                    ["xx", "yy", "zz", "LAST"],
-                    ["A", "B", "C", "D", "F"],
-                    ["I", "J", "K", "L", "INDEX"],
-                ]
-            )
+        columns = pd.MultiIndex.from_product(
+            [
+                ["A", "B", "C", "D"],
+                ["xx", "yy", "zz", "LAST"],
+                [10, 20, 30, 40],
+            ]
+        )
     else:
         columns = pandas_df.columns
 
@@ -1016,12 +998,44 @@ def test_unstack(data, is_multi_idx, is_multi_col):
         df_equals(modin_df.unstack(level=1), pandas_df.unstack(level=1))
         df_equals(modin_df.unstack(level=[0, 1]), pandas_df.unstack(level=[0, 1]))
         df_equals(modin_df.unstack(level=[0, 1, 2]), pandas_df.unstack(level=[0, 1, 2]))
+        df_equals(
+            modin_df.unstack(level=[0, 1, 2, 3]), pandas_df.unstack(level=[0, 1, 2, 3])
+        )
 
-        if len(pandas_df.index) == 256:
-            df_equals(
-                modin_df.unstack(level=[0, 1, 2, 3]),
-                pandas_df.unstack(level=[0, 1, 2, 3]),
-            )
+
+@pytest.mark.parametrize("is_multi_col", [True, False], ids=["col_multi", "col_index"])
+def test_unstack_not_unique(is_multi_col):
+    MAX_NROWS = 34
+    MAX_NCOLS = 36
+
+    pandas_df = pandas.DataFrame(test_data["int_data"]).iloc[:MAX_NROWS, :MAX_NCOLS]
+    modin_df = pd.DataFrame(test_data["int_data"]).iloc[:MAX_NROWS, :MAX_NCOLS]
+
+    l0 = ["a"] * 17 + ["b"] * 17
+    l1 = np.arange(34)
+    l2 = np.arange(100, 134)
+    index = pandas.MultiIndex.from_tuples([(i, j, k) for i, j, k in zip(l0, l1, l2)])
+
+    if is_multi_col:
+        columns = pd.MultiIndex.from_product(
+            [
+                ["A", "B", "C", "D", "E", "F"],
+                ["qq", "ww", "xx", "yy", "zz", "LAST"],
+            ]
+        )
+    else:
+        columns = pandas_df.columns
+
+    pandas_df.columns = columns
+    pandas_df.index = index
+
+    modin_df.columns = columns
+    modin_df.index = index
+
+    df_equals(modin_df.unstack(), pandas_df.unstack())
+    df_equals(modin_df.unstack(level=1), pandas_df.unstack(level=1))
+    df_equals(modin_df.unstack(level=[0, 1]), pandas_df.unstack(level=[0, 1]))
+    df_equals(modin_df.unstack(level=[0, 1, 2]), pandas_df.unstack(level=[0, 1, 2]))
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -3218,23 +3218,14 @@ def test_unique(data):
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_unstack(data):
     modin_series, pandas_series = create_test_series(data)
-    if len(pandas_series.index) == 256:
-        index = pd.MultiIndex.from_product(
-            [
-                ["a", "b", "c", "d"],
-                ["x", "y", "z", "last"],
-                ["i", "j", "k", "index"],
-                [1, 2, 3, 4],
-            ]
-        )
-    elif len(pandas_series.index) == 100:
-        index = pd.MultiIndex.from_product(
-            [
-                ["x", "y", "z", "last"],
-                ["a", "b", "c", "d", "f"],
-                ["i", "j", "k", "l", "index"],
-            ]
-        )
+    index = pd.MultiIndex.from_product(
+        [
+            ["a", "b", "c", "d"],
+            ["x", "y", "z", "last"],
+            ["i", "j", "k", "index"],
+            [1, 2, 3, 4],
+        ]
+    )
 
     modin_series = pd.Series(data[next(iter(data.keys()))], index=index)
     pandas_series = pandas.Series(data[next(iter(data.keys()))], index=index)
@@ -3242,12 +3233,9 @@ def test_unstack(data):
     df_equals(modin_series.unstack(), pandas_series.unstack())
     df_equals(modin_series.unstack(level=0), pandas_series.unstack(level=0))
     df_equals(modin_series.unstack(level=[0, 1]), pandas_series.unstack(level=[0, 1]))
-
-    if len(pandas_series.index) == 256:
-        df_equals(
-            modin_series.unstack(level=[0, 1, 2]),
-            pandas_series.unstack(level=[0, 1, 2]),
-        )
+    df_equals(
+        modin_series.unstack(level=[0, 1, 2]), pandas_series.unstack(level=[0, 1, 2])
+    )
 
 
 @pytest.mark.parametrize(