Repository: modin-project/modin

Issue:
DataFrame.unstack works incorrect when level={all_MultiIndex_levels}
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: All
- **Modin version** (`modin.__version__`): Current master (ff6ff0d)
- **Python version**: 3.7.5
- **Code we can use to reproduce**:
```python
if __name__ == "__main__":
    import modin.pandas as pd
    import pandas
    import numpy as np

    l1 = ["a"] * 5 + ["b"] * 5
    l2 = np.arange(10)
    l3 = np.arange(10, 20)

    ml = pandas.MultiIndex.from_tuples([(fi, se, th) for fi, se, th in zip(l1, l2, l3)])
    data = {"col0": np.arange(len(ml))}

    pd_df = pandas.DataFrame(data, index=ml)
    md_df = pd.DataFrame(data, index=ml)

    pd_res = pd_df.unstack(level=[0, 1, 2])
    md_res = md_df.unstack(level=[0, 1, 2])

    print(f"pd_res:\n{pd_res}")
    print(f"\nmd_res:\n{md_res}")

```

<details><summary>Output</summary>



```
pd_res:
col0  a  0  10    0
         1  11    1
         2  12    2
         3  13    3
         4  14    4
      b  5  15    5
         6  16    6
         7  17    7
         8  18    8
         9  19    9
dtype: int64

md_res:
col0  a  0  10    0.0
            11    NaN
            12    NaN
            13    NaN
            14    NaN
                 ...
      b  9  15    NaN
            16    NaN
            17    NaN
            18    NaN
            19    9.0
Length: 200, dtype: float64
```



</details>

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
In case of `unstack` in Modin, it generates all possible combinations of index levels, when Pandas does some different from that logic



Discussion:
cc @prutskov 
@dchigarev It's a bit strange, because test for this case was added. I'll look on this.
It seems, that #2012 doesn't fix the bug, this reproducer still fails:
```python
if __name__ == "__main__":
    import modin.pandas as pd
    import pandas
    import numpy as np
    from modin.pandas.test.utils import df_equals

    cols = pd.MultiIndex.from_arrays(
        [["c", "b", "a"], ["e", "f", "d"], ["i", "h", "g"]],
        names=["first", "second", "third"],
    )
    ind = pd.MultiIndex.from_arrays(
        [["c", "b", "a"], ["e", "f", "d"], ["i", "h", "g"]],
        names=["first", "second", "third"],
    )
    data = {
        f"{i}": np.arange(len(ind)) for i in range(len(cols))
    }

    md_df = pd.DataFrame(data, index=ind)
    pd_df = pandas.DataFrame(data, index=ind)

    pd_df.columns = cols
    md_df.columns = cols

    df_equals(md_df.unstack([0, 1, 2]), pd_df.unstack([0, 1, 2]))
```

Output:
```
AssertionError: Series are different

Series length are different
[left]:  729, MultiIndex([('c', 'e', 'i', 'c', 'e', 'i'),
            ('c', 'e', 'i', 'c', 'e', 'h'),
            ('c', 'e', 'i', 'c', 'e', 'g'),
            ('c', 'e', 'i', 'c', 'f', 'i'),
            ('c', 'e', 'i', 'c', 'f', 'h'),
            ('c', 'e', 'i', 'c', 'f', 'g'),
            ('c', 'e', 'i', 'c', 'd', 'i'),
            ('c', 'e', 'i', 'c', 'd', 'h'),
            ('c', 'e', 'i', 'c', 'd', 'g'),
            ('c', 'e', 'i', 'b', 'e', 'i'),
            ...
            ('a', 'd', 'g', 'b', 'd', 'g'),
            ('a', 'd', 'g', 'a', 'e', 'i'),
            ('a', 'd', 'g', 'a', 'e', 'h'),
            ('a', 'd', 'g', 'a', 'e', 'g'),
            ('a', 'd', 'g', 'a', 'f', 'i'),
            ('a', 'd', 'g', 'a', 'f', 'h'),
            ('a', 'd', 'g', 'a', 'f', 'g'),
            ('a', 'd', 'g', 'a', 'd', 'i'),
            ('a', 'd', 'g', 'a', 'd', 'h'),
            ('a', 'd', 'g', 'a', 'd', 'g')],
           names=['first', 'second', 'third', 'first', 'second', 'third'], length=729)
[right]: 81, MultiIndex([('c', 'e', 'i', 'a', 'd', 'g'),
            ('c', 'e', 'i', 'a', 'd', 'h'),
            ('c', 'e', 'i', 'a', 'd', 'i'),
            ('c', 'e', 'i', 'a', 'e', 'g'),
            ('c', 'e', 'i', 'a', 'e', 'h'),
            ('c', 'e', 'i', 'a', 'e', 'i'),
            ('c', 'e', 'i', 'a', 'f', 'g'),
            ('c', 'e', 'i', 'a', 'f', 'h'),
            ('c', 'e', 'i', 'a', 'f', 'i'),
            ('c', 'e', 'i', 'b', 'd', 'g'),
            ('c', 'e', 'i', 'b', 'd', 'h'),
            ('c', 'e', 'i', 'b', 'd', 'i'),
            ('c', 'e', 'i', 'b', 'e', 'g'),
            ('c', 'e', 'i', 'b', 'e', 'h'),
            ('c', 'e', 'i', 'b', 'e', 'i'),
            ('c', 'e', 'i', 'b', 'f', 'g'),
            ('c', 'e', 'i', 'b', 'f', 'h'),
            ('c', 'e', 'i', 'b', 'f', 'i'),
            ('c', 'e', 'i', 'c', 'd', 'g'),
            ('c', 'e', 'i', 'c', 'd', 'h'),
            ('c', 'e', 'i', 'c', 'd', 'i'),
            ('c', 'e', 'i', 'c', 'e', 'g'),
            ('c', 'e', 'i', 'c', 'e', 'h'),
            ('c', 'e', 'i', 'c', 'e', 'i'),
            ('c', 'e', 'i', 'c', 'f', 'g'),
            ('c', 'e', 'i', 'c', 'f', 'h'),
            ('c', 'e', 'i', 'c', 'f', 'i'),
            ('b', 'f', 'h', 'a', 'd', 'g'),
            ('b', 'f', 'h', 'a', 'd', 'h'),
            ('b', 'f', 'h', 'a', 'd', 'i'),
            ('b', 'f', 'h', 'a', 'e', 'g'),
            ('b', 'f', 'h', 'a', 'e', 'h'),
            ('b', 'f', 'h', 'a', 'e', 'i'),
            ('b', 'f', 'h', 'a', 'f', 'g'),
            ('b', 'f', 'h', 'a', 'f', 'h'),
            ('b', 'f', 'h', 'a', 'f', 'i'),
            ('b', 'f', 'h', 'b', 'd', 'g'),
            ('b', 'f', 'h', 'b', 'd', 'h'),
            ('b', 'f', 'h', 'b', 'd', 'i'),
            ('b', 'f', 'h', 'b', 'e', 'g'),
            ('b', 'f', 'h', 'b', 'e', 'h'),
            ('b', 'f', 'h', 'b', 'e', 'i'),
            ('b', 'f', 'h', 'b', 'f', 'g'),
            ('b', 'f', 'h', 'b', 'f', 'h'),
            ('b', 'f', 'h', 'b', 'f', 'i'),
            ('b', 'f', 'h', 'c', 'd', 'g'),
            ('b', 'f', 'h', 'c', 'd', 'h'),
            ('b', 'f', 'h', 'c', 'd', 'i'),
            ('b', 'f', 'h', 'c', 'e', 'g'),
            ('b', 'f', 'h', 'c', 'e', 'h'),
            ('b', 'f', 'h', 'c', 'e', 'i'),
            ('b', 'f', 'h', 'c', 'f', 'g'),
            ('b', 'f', 'h', 'c', 'f', 'h'),
            ('b', 'f', 'h', 'c', 'f', 'i'),
            ('a', 'd', 'g', 'a', 'd', 'g'),
            ('a', 'd', 'g', 'a', 'd', 'h'),
            ('a', 'd', 'g', 'a', 'd', 'i'),
            ('a', 'd', 'g', 'a', 'e', 'g'),
            ('a', 'd', 'g', 'a', 'e', 'h'),
            ('a', 'd', 'g', 'a', 'e', 'i'),
            ('a', 'd', 'g', 'a', 'f', 'g'),
            ('a', 'd', 'g', 'a', 'f', 'h'),
            ('a', 'd', 'g', 'a', 'f', 'i'),
            ('a', 'd', 'g', 'b', 'd', 'g'),
            ('a', 'd', 'g', 'b', 'd', 'h'),
            ('a', 'd', 'g', 'b', 'd', 'i'),
            ('a', 'd', 'g', 'b', 'e', 'g'),
            ('a', 'd', 'g', 'b', 'e', 'h'),
            ('a', 'd', 'g', 'b', 'e', 'i'),
            ('a', 'd', 'g', 'b', 'f', 'g'),
            ('a', 'd', 'g', 'b', 'f', 'h'),
            ('a', 'd', 'g', 'b', 'f', 'i'),
            ('a', 'd', 'g', 'c', 'd', 'g'),
            ('a', 'd', 'g', 'c', 'd', 'h'),
            ('a', 'd', 'g', 'c', 'd', 'i'),
            ('a', 'd', 'g', 'c', 'e', 'g'),
            ('a', 'd', 'g', 'c', 'e', 'h'),
            ('a', 'd', 'g', 'c', 'e', 'i'),
            ('a', 'd', 'g', 'c', 'f', 'g'),
            ('a', 'd', 'g', 'c', 'f', 'h'),
            ('a', 'd', 'g', 'c', 'f', 'i')],
           names=['first', 'second', 'third', 'first', 'second', 'third'])

```
#2084 

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -1100,13 +1100,16 @@ def unstack(self, level, fill_value):
         def map_func(df):
             return pandas.DataFrame(df.unstack(level=level, fill_value=fill_value))
 
-        is_correct_len = True
-        if isinstance(self.index, pandas.MultiIndex):
+        def is_tree_like_or_1d(calc_index, valid_index):
+            if not isinstance(calc_index, pandas.MultiIndex):
+                return True
             actual_len = 1
-            for lvl in self.index.levels:
+            for lvl in calc_index.levels:
                 actual_len *= len(lvl)
-            if len(self.index) * len(self.columns) != actual_len * len(self.columns):
-                is_correct_len = False
+            return len(self.index) * len(self.columns) == actual_len * len(valid_index)
+
+        is_tree_like_or_1d_index = is_tree_like_or_1d(self.index, self.columns)
+        is_tree_like_or_1d_cols = is_tree_like_or_1d(self.columns, self.index)
 
         is_all_multi_list = False
         if (
@@ -1114,7 +1117,8 @@ def map_func(df):
             and isinstance(self.columns, pandas.MultiIndex)
             and is_list_like(level)
             and len(level) == self.index.nlevels
-            and is_correct_len
+            and is_tree_like_or_1d_index
+            and is_tree_like_or_1d_cols
         ):
             is_all_multi_list = True
             real_cols_bkp = self.columns
@@ -1148,7 +1152,7 @@ def get_unique_level_values(index):
             )
             return pandas.MultiIndex.from_product([*new_columns, *new_index])
 
-        if is_all_multi_list and is_correct_len:
+        if is_all_multi_list and is_tree_like_or_1d_index and is_tree_like_or_1d_cols:
             result = result.sort_index()
             index_level_values = [lvl for lvl in obj.index.levels]
 
@@ -1158,7 +1162,7 @@ def get_unique_level_values(index):
             return result
 
         if need_reindex:
-            if is_correct_len:
+            if is_tree_like_or_1d_index and is_tree_like_or_1d_cols:
                 is_recompute_index = isinstance(self.index, pandas.MultiIndex)
                 is_recompute_columns = not is_recompute_index and isinstance(
                     self.columns, pandas.MultiIndex
@@ -1166,18 +1170,32 @@ def get_unique_level_values(index):
                 new_index = compute_index(
                     self.index, self.columns, is_recompute_index, is_recompute_columns
                 )
-            else:
-                if isinstance(self.columns, pandas.MultiIndex):
-                    new_index = compute_index(self.index, self.columns)
+            elif is_tree_like_or_1d_index != is_tree_like_or_1d_cols:
+                if isinstance(self.columns, pandas.MultiIndex) or not isinstance(
+                    self.index, pandas.MultiIndex
+                ):
+                    return result
                 else:
+                    index = (
+                        self.index.sortlevel()[0]
+                        if is_tree_like_or_1d_index
+                        and not is_tree_like_or_1d_cols
+                        and isinstance(self.index, pandas.MultiIndex)
+                        else self.index
+                    )
                     index = pandas.MultiIndex.from_tuples(
-                        list(self.index) * len(self.columns)
+                        list(index) * len(self.columns)
                     )
                     columns = self.columns.repeat(len(self.index))
                     index_levels = [
                         index.get_level_values(i) for i in range(index.nlevels)
                     ]
-                    new_index = pandas.MultiIndex.from_arrays([columns] + index_levels)
+                    new_index = pandas.MultiIndex.from_arrays(
+                        [columns] + index_levels,
+                        names=self.columns.names + self.index.names,
+                    )
+            else:
+                return result
             result = result.reindex(0, new_index)
         return result
 


Test patch:
diff --git a/modin/pandas/test/dataframe/test_default.py b/modin/pandas/test/dataframe/test_default.py
--- a/modin/pandas/test/dataframe/test_default.py
+++ b/modin/pandas/test/dataframe/test_default.py
@@ -959,8 +959,7 @@ def test_tz_localize():
 @pytest.mark.parametrize("is_multi_col", [True, False], ids=["col_multi", "col_index"])
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_unstack(data, is_multi_idx, is_multi_col):
-    pandas_df = pandas.DataFrame(data)
-    modin_df = pd.DataFrame(data)
+    modin_df, pandas_df = create_test_dfs(data)
 
     if is_multi_idx:
         index = generate_multiindex(len(pandas_df), nlevels=4, is_tree_like=True)
@@ -974,16 +973,12 @@ def test_unstack(data, is_multi_idx, is_multi_col):
     else:
         columns = pandas_df.columns
 
-    pandas_df.columns = columns
-    pandas_df.index = index
-
-    modin_df.columns = columns
-    modin_df.index = index
+    pandas_df.columns = modin_df.columns = columns
+    pandas_df.index = modin_df.index = index
 
     df_equals(modin_df.unstack(), pandas_df.unstack())
-
+    df_equals(modin_df.unstack(level=1), pandas_df.unstack(level=1))
     if is_multi_idx:
-        df_equals(modin_df.unstack(level=1), pandas_df.unstack(level=1))
         df_equals(modin_df.unstack(level=[0, 1]), pandas_df.unstack(level=[0, 1]))
         df_equals(modin_df.unstack(level=[0, 1, 2]), pandas_df.unstack(level=[0, 1, 2]))
         df_equals(
@@ -992,47 +987,33 @@ def test_unstack(data, is_multi_idx, is_multi_col):
 
 
 @pytest.mark.parametrize(
-    "is_multi_col",
-    [
-        pytest.param(
-            True,
-            marks=pytest.mark.xfail(
-                reason="Unstack fails in case of MultiIndex columns. See #1997 for more details."
-            ),
-        ),
-        pytest.param(
-            False,
-            marks=pytest.mark.xfail(
-                reason="Unstack loses indices names. See issue #2084 for more details."
-            ),
-        ),
-    ],
-    ids=["col_multi", "col_index"],
+    "multi_col", ["col_multi_tree", "col_multi_not_tree", "col_index"]
+)
+@pytest.mark.parametrize(
+    "multi_idx", ["idx_multi_tree", "idx_multi_not_tree", "idx_index"]
 )
-def test_unstack_not_unique(is_multi_col):
-    MAX_NROWS = 34
-    MAX_NCOLS = 36
+def test_unstack_multiindex_types(multi_col, multi_idx):
+    MAX_NROWS = MAX_NCOLS = 36
 
     pandas_df = pandas.DataFrame(test_data["int_data"]).iloc[:MAX_NROWS, :MAX_NCOLS]
     modin_df = pd.DataFrame(test_data["int_data"]).iloc[:MAX_NROWS, :MAX_NCOLS]
 
-    index = generate_multiindex(len(pandas_df), nlevels=3)
-
-    if is_multi_col:
-        columns = generate_multiindex(len(pandas_df.columns), nlevels=2)
-    else:
-        columns = pandas_df.columns
-
-    pandas_df.columns = columns
-    pandas_df.index = index
+    def get_new_index(index, cond):
+        if cond == "col_multi_tree" or cond == "idx_multi_tree":
+            return generate_multiindex(len(index), nlevels=3, is_tree_like=True)
+        elif cond == "col_multi_not_tree" or cond == "idx_multi_not_tree":
+            return generate_multiindex(len(index), nlevels=3)
+        else:
+            return index
 
-    modin_df.columns = columns
-    modin_df.index = index
+    pandas_df.columns = modin_df.columns = get_new_index(pandas_df.columns, multi_col)
+    pandas_df.index = modin_df.index = get_new_index(pandas_df.index, multi_idx)
 
     df_equals(modin_df.unstack(), pandas_df.unstack())
     df_equals(modin_df.unstack(level=1), pandas_df.unstack(level=1))
-    df_equals(modin_df.unstack(level=[0, 1]), pandas_df.unstack(level=[0, 1]))
-    df_equals(modin_df.unstack(level=[0, 1, 2]), pandas_df.unstack(level=[0, 1, 2]))
+    if multi_idx != "idx_index":
+        df_equals(modin_df.unstack(level=[0, 1]), pandas_df.unstack(level=[0, 1]))
+        df_equals(modin_df.unstack(level=[0, 1, 2]), pandas_df.unstack(level=[0, 1, 2]))
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -717,11 +717,13 @@ def generate_level(length, nlevel):
 
         lvl_len = int(lvl_len_d)
         result = pd.MultiIndex.from_product(
-            [generate_level(lvl_len, i) for i in range(nlevels - penalty_level)]
+            [generate_level(lvl_len, i) for i in range(nlevels - penalty_level)],
+            names=[f"level-{i}" for i in range(nlevels - penalty_level)],
         )
         if penalty_level:
             result = pd.MultiIndex.from_tuples(
-                [("base_level", *ml_tuple) for ml_tuple in result]
+                [("base_level", *ml_tuple) for ml_tuple in result],
+                names=[f"level-{i}" for i in range(nlevels)],
             )
         return result.sort_values()
     else: