Repository: modin-project/modin

Issue:
Groupby dictionary aggregation with renaming do not inserts 'by' columns when 'as_index=False'
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Any
- **Modin version** (`modin.__version__`): f98a7b3a741a13ef4a2c25b5a2375475dea4e3bf
- **Python version**: 3.7.5
- **Code we can use to reproduce**:
```python
import pandas
import modin.pandas as pd
from modin.pandas.test.utils import test_data, df_equals, create_test_dfs
import numpy as np
import pandas

md_df, pd_df = create_test_dfs(test_data["int_data"])

by = [md_df.columns[0], md_df.columns[1]]
agg_dict = {
    "max": (md_df.columns[2], "max"),
    "sum": (md_df.columns[3], "sum"),
}

as_index = False
md_res = md_df.groupby(by, as_index=as_index).agg(**agg_dict)
pd_res = pd_df.groupby(by, as_index=as_index).agg(**agg_dict)
df_equals(md_res, pd_res) # AssertionError: shape missmatch

```


<details><summary> Output </summary>


```
Traceback (most recent call last):
  File "../rofl.py", line 18, in <module>
    df_equals(md_res, pd_res)
  File "/localdisk/dchigare/repos/modin_bp/modin/pandas/test/utils.py", line 527, in df_equals
    check_categorical=False,
  File "/localdisk/dchigare/miniconda3/envs/modin_tests/lib/python3.7/site-packages/pandas/_testing.py", line 1562, in assert_frame_equal
    obj, f"{obj} shape mismatch", f"{repr(left.shape)}", f"{repr(right.shape)}",
  File "/localdisk/dchigare/miniconda3/envs/modin_tests/lib/python3.7/site-packages/pandas/_testing.py", line 1036, in raise_assert_detail
    raise AssertionError(msg)
AssertionError: DataFrame are different

DataFrame shape mismatch
[left]:  (251, 2)
[right]: (251, 4)

```


</details>

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
This seems to happen because of that logic of renaming, for the renaming we're picking only columns to aggregate and discarding new inserted 'by' columns:
https://github.com/modin-project/modin/blob/f98a7b3a741a13ef4a2c25b5a2375475dea4e3bf/modin/pandas/groupby.py#L426-L429



Discussion:

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -2483,7 +2483,12 @@ def _groupby_dict_reduce(
                     raise TypeError
 
                 map_fns.append((new_col_name, groupby_reduce_functions[func][0]))
-                reduce_dict[(col, new_col_name)] = groupby_reduce_functions[func][1]
+                reduced_col_name = (
+                    (*col, new_col_name)
+                    if isinstance(col, tuple)
+                    else (col, new_col_name)
+                )
+                reduce_dict[reduced_col_name] = groupby_reduce_functions[func][1]
             map_dict[col] = map_fns
         return GroupbyReduceFunction.register(map_dict, reduce_dict)(
             query_compiler=self,
diff --git a/modin/pandas/groupby.py b/modin/pandas/groupby.py
--- a/modin/pandas/groupby.py
+++ b/modin/pandas/groupby.py
@@ -425,6 +425,20 @@ def try_get_str_func(fn):
         )
 
         if relabeling_required:
+            if not self._as_index:
+                nby_cols = len(result.columns) - len(new_columns)
+                order = np.concatenate([np.arange(nby_cols), order + nby_cols])
+                by_cols = result.columns[:nby_cols]
+                new_columns = pandas.Index(new_columns)
+                if by_cols.nlevels != new_columns.nlevels:
+                    by_cols = by_cols.remove_unused_levels()
+                    empty_levels = [
+                        i
+                        for i, level in enumerate(by_cols.levels)
+                        if len(level) == 1 and level[0] == ""
+                    ]
+                    by_cols = by_cols.droplevel(empty_levels)
+                new_columns = by_cols.append(new_columns)
             result = result.iloc[:, order]
             result.columns = new_columns
         return result


Test patch:
diff --git a/modin/pandas/test/test_groupby.py b/modin/pandas/test/test_groupby.py
--- a/modin/pandas/test/test_groupby.py
+++ b/modin/pandas/test/test_groupby.py
@@ -25,6 +25,7 @@
     test_data,
     test_data_values,
     modin_df_almost_equals_pandas,
+    generate_multiindex,
 )
 
 pd.DEFAULT_NPARTITIONS = 4
@@ -1247,18 +1248,25 @@ def test_shift_freq(groupby_axis, shift_axis):
             ],
             "agg_dict": {
                 "max": (list(test_data["int_data"].keys())[1], max),
-                "min": (list(test_data["int_data"].keys())[-1], min),
+                "min": (list(test_data["int_data"].keys())[-2], min),
             },
         },
+        pytest.param(
+            {
+                "by": [
+                    list(test_data["int_data"].keys())[0],
+                    list(test_data["int_data"].keys())[-1],
+                ],
+                "agg_dict": {
+                    "max": (list(test_data["int_data"].keys())[1], max),
+                    "min": (list(test_data["int_data"].keys())[-1], min),
+                },
+            },
+            marks=pytest.mark.skip("See Modin issue #2542"),
+        ),
     ],
 )
-@pytest.mark.parametrize(
-    "as_index",
-    [
-        True,
-        pytest.param(False, marks=pytest.mark.xfail(reason="See modin issue #2543")),
-    ],
-)
+@pytest.mark.parametrize("as_index", [True, False])
 def test_agg_func_None_rename(by_and_agg_dict, as_index):
     modin_df, pandas_df = create_test_dfs(test_data["int_data"])
 
@@ -1271,6 +1279,44 @@ def test_agg_func_None_rename(by_and_agg_dict, as_index):
     df_equals(modin_result, pandas_result)
 
 
+@pytest.mark.parametrize(
+    "as_index",
+    [
+        True,
+        pytest.param(
+            False,
+            marks=pytest.mark.xfail_backends(
+                ["BaseOnPython"], reason="See Pandas issue #39103"
+            ),
+        ),
+    ],
+)
+@pytest.mark.parametrize("by_length", [1, 3])
+@pytest.mark.parametrize(
+    "agg_fns",
+    [["sum", "min", "max"], ["mean", "quantile"]],
+    ids=["reduction", "aggregation"],
+)
+def test_dict_agg_rename_mi_columns(as_index, by_length, agg_fns):
+    md_df, pd_df = create_test_dfs(test_data["int_data"])
+    mi_columns = generate_multiindex(len(md_df.columns), nlevels=4)
+
+    md_df.columns, pd_df.columns = mi_columns, mi_columns
+
+    by = list(md_df.columns[:by_length])
+    agg_cols = list(md_df.columns[by_length : by_length + 3])
+
+    agg_dict = {
+        f"custom-{i}" + str(agg_fns[i % len(agg_fns)]): (col, agg_fns[i % len(agg_fns)])
+        for i, col in enumerate(agg_cols)
+    }
+
+    md_res = md_df.groupby(by, as_index=as_index).agg(**agg_dict)
+    pd_res = md_df.groupby(by, as_index=as_index).agg(**agg_dict)
+
+    df_equals(md_res, pd_res)
+
+
 @pytest.mark.parametrize(
     "operation",
     [