Repository: modin-project/modin

Issue:
Support grouping on a `modin.pandas.Series` with range-partitioning groupby
<b>This issue is a part of this epic #5926 </b>

This should work with range-partitioning groupby:
```python
import modin.pandas as pd
import modin.config as cfg

cfg.RangePartitioningGroupby.put(True)

sr = pd.Series([1, 1, 2, 2])
df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [10, 20, 10, 20]})

res = df.groupby([sr, "b"]).sum()
```


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/dataframe/dataframe.py b/modin/core/dataframe/pandas/dataframe/dataframe.py
--- a/modin/core/dataframe/pandas/dataframe/dataframe.py
+++ b/modin/core/dataframe/pandas/dataframe/dataframe.py
@@ -18,6 +18,7 @@
 for pandas storage format.
 """
 import datetime
+import re
 from typing import TYPE_CHECKING, Callable, Dict, Hashable, List, Optional, Union
 
 import numpy as np
@@ -2423,7 +2424,14 @@ def combine_and_apply(
         )
 
     def _apply_func_to_range_partitioning(
-        self, key_columns, func, ascending=True, preserve_columns=False, **kwargs
+        self,
+        key_columns,
+        func,
+        ascending=True,
+        preserve_columns=False,
+        data=None,
+        data_key_columns=None,
+        **kwargs,
     ):
         """
         Reshuffle data so it would be range partitioned and then apply the passed function row-wise.
@@ -2438,6 +2446,12 @@ def _apply_func_to_range_partitioning(
             Whether the range should be built in ascending or descending order.
         preserve_columns : bool, default: False
             If the columns cache should be preserved (specify this flag if `func` doesn't change column labels).
+        data : PandasDataframe, optional
+            Dataframe to range-partition along with the `self` frame. If specified, the `func` will recieve
+            a dataframe with an additional MultiIndex level in columns that separates `self` and `data`:
+            ``df["grouper"] # self`` and ``df["data"] # data``.
+        data_key_columns : list of hashables, optional
+            Additional key columns from `data`. Will be combined with `key_columns`.
         **kwargs : dict
             Additional arguments to forward to the range builder function.
 
@@ -2446,21 +2460,60 @@ def _apply_func_to_range_partitioning(
         PandasDataframe
             A new dataframe.
         """
+        if data is not None:
+            # adding an extra MultiIndex level in order to separate `self grouper` from the `data`
+            # after concatenation
+            new_grouper_cols = pandas.MultiIndex.from_tuples(
+                [
+                    ("grouper", *col) if isinstance(col, tuple) else ("grouper", col)
+                    for col in self.columns
+                ]
+            )
+            grouper = self.copy()
+            grouper.columns = new_grouper_cols
+
+            new_data_cols = pandas.MultiIndex.from_tuples(
+                [
+                    ("data", *col) if isinstance(col, tuple) else ("data", col)
+                    for col in data.columns
+                ]
+            )
+            data = data.copy()
+            data.columns = new_data_cols
+
+            grouper = grouper.concat(axis=1, others=[data], how="right", sort=False)
+
+            # since original column names were modified, have to modify 'key_columns' as well
+            key_columns = [
+                ("grouper", *col) if isinstance(col, tuple) else ("grouper", col)
+                for col in key_columns
+            ]
+            if data_key_columns is None:
+                data_key_columns = []
+            else:
+                data_key_columns = [
+                    ("data", *col) if isinstance(col, tuple) else ("data", col)
+                    for col in data_key_columns
+                ]
+            key_columns += data_key_columns
+        else:
+            grouper = self
+
         # If there's only one row partition can simply apply the function row-wise without the need to reshuffle
-        if self._partitions.shape[0] == 1:
-            result = self.apply_full_axis(
+        if grouper._partitions.shape[0] == 1:
+            result = grouper.apply_full_axis(
                 axis=1,
                 func=func,
-                new_columns=self.copy_columns_cache() if preserve_columns else None,
+                new_columns=grouper.copy_columns_cache() if preserve_columns else None,
             )
             if preserve_columns:
-                result._set_axis_lengths_cache(self._column_widths_cache, axis=1)
+                result._set_axis_lengths_cache(grouper._column_widths_cache, axis=1)
             return result
 
         # don't want to inherit over-partitioning so doing this 'min' check
-        ideal_num_new_partitions = min(len(self._partitions), NPartitions.get())
-        m = len(self) / ideal_num_new_partitions
-        sampling_probability = (1 / m) * np.log(ideal_num_new_partitions * len(self))
+        ideal_num_new_partitions = min(len(grouper._partitions), NPartitions.get())
+        m = len(grouper) / ideal_num_new_partitions
+        sampling_probability = (1 / m) * np.log(ideal_num_new_partitions * len(grouper))
         # If this df is overpartitioned, we try to sample each partition with probability
         # greater than 1, which leads to an error. In this case, we can do one of the following
         # two things. If there is only enough rows for one partition, and we have only 1 column
@@ -2471,39 +2524,39 @@ def _apply_func_to_range_partitioning(
         if sampling_probability >= 1:
             from modin.config import MinPartitionSize
 
-            ideal_num_new_partitions = round(len(self) / MinPartitionSize.get())
-            if len(self) < MinPartitionSize.get() or ideal_num_new_partitions < 2:
+            ideal_num_new_partitions = round(len(grouper) / MinPartitionSize.get())
+            if len(grouper) < MinPartitionSize.get() or ideal_num_new_partitions < 2:
                 # If the data is too small, we shouldn't try reshuffling/repartitioning but rather
                 # simply combine all partitions and apply the sorting to the whole dataframe
-                return self.combine_and_apply(func=func)
+                return grouper.combine_and_apply(func=func)
 
-            if ideal_num_new_partitions < len(self._partitions):
-                if len(self._partitions) % ideal_num_new_partitions == 0:
+            if ideal_num_new_partitions < len(grouper._partitions):
+                if len(grouper._partitions) % ideal_num_new_partitions == 0:
                     joining_partitions = np.split(
-                        self._partitions, ideal_num_new_partitions
+                        grouper._partitions, ideal_num_new_partitions
                     )
                 else:
-                    step = round(len(self._partitions) / ideal_num_new_partitions)
+                    step = round(len(grouper._partitions) / ideal_num_new_partitions)
                     joining_partitions = np.split(
-                        self._partitions,
-                        range(step, len(self._partitions), step),
+                        grouper._partitions,
+                        range(step, len(grouper._partitions), step),
                     )
 
                 new_partitions = np.array(
                     [
-                        self._partition_mgr_cls.column_partitions(
+                        grouper._partition_mgr_cls.column_partitions(
                             ptn_grp, full_axis=False
                         )
                         for ptn_grp in joining_partitions
                     ]
                 )
             else:
-                new_partitions = self._partitions
+                new_partitions = grouper._partitions
         else:
-            new_partitions = self._partitions
+            new_partitions = grouper._partitions
 
         shuffling_functions = ShuffleSortFunctions(
-            self,
+            grouper,
             key_columns,
             ascending[0] if is_list_like(ascending) else ascending,
             ideal_num_new_partitions,
@@ -2511,25 +2564,25 @@ def _apply_func_to_range_partitioning(
         )
 
         # here we want to get indices of those partitions that hold the key columns
-        key_indices = self.columns.get_indexer_for(key_columns)
+        key_indices = grouper.columns.get_indexer_for(key_columns)
         partition_indices = np.unique(
-            np.digitize(key_indices, np.cumsum(self.column_widths))
+            np.digitize(key_indices, np.cumsum(grouper.column_widths))
         )
 
-        new_partitions = self._partition_mgr_cls.shuffle_partitions(
+        new_partitions = grouper._partition_mgr_cls.shuffle_partitions(
             new_partitions,
             partition_indices,
             shuffling_functions,
             func,
         )
 
-        result = self.__constructor__(new_partitions)
+        result = grouper.__constructor__(new_partitions)
         if preserve_columns:
-            result.set_columns_cache(self.copy_columns_cache())
+            result.set_columns_cache(grouper.copy_columns_cache())
             # We perform the final steps of the sort on full axis partitions, so we know that the
             # length of each partition is the full length of the dataframe.
-            if self.has_materialized_columns:
-                result._set_axis_lengths_cache([len(self.columns)], axis=1)
+            if grouper.has_materialized_columns:
+                result._set_axis_lengths_cache([len(grouper.columns)], axis=1)
         return result
 
     @lazy_metadata_decorator(apply_axis="both")
@@ -3756,10 +3809,13 @@ def _compute_new_widths():
     def groupby(
         self,
         axis: Union[int, Axis],
-        by: Union[str, List[str]],
+        internal_by: List[str],
+        external_by: List["PandasDataframe"],
+        by_positions: List[int],
         operator: Callable,
         result_schema: Optional[Dict[Hashable, type]] = None,
         align_result_columns: bool = False,
+        series_groupby: bool = False,
         add_missing_cats: bool = False,
         **kwargs: dict,
     ) -> "PandasDataframe":
@@ -3770,8 +3826,22 @@ def groupby(
         ----------
         axis : int or modin.core.dataframe.base.utils.Axis
             The axis to apply the grouping over.
-        by : string or list of strings
-            One or more column labels to use for grouping.
+        internal_by : list of strings
+            One or more column labels from the `self` dataframe to use for grouping.
+        external_by : list of PandasDataframes
+            PandasDataframes to group by (may be specified along with or without `internal_by`).
+        by_positions : list of ints
+            Specifies the order of grouping by `internal_by` and `external_by` columns.
+            Each element in `by_positions` specifies an index from either `external_by` or `internal_by`.
+            Indices for `external_by` are positive and start from 0. Indices for `internal_by` are negative
+            and start from -1 (so in order to convert them to a valid indices one should do ``-idx - 1``).
+            '''
+            by_positions = [0, -1, 1, -2, 2, 3]
+            internal_by = ["col1", "col2"]
+            external_by = [sr1, sr2, sr3, sr4]
+
+            df.groupby([sr1, "col1", sr2, "col2", sr3, sr4])
+            '''.
         operator : callable(pandas.core.groupby.DataFrameGroupBy) -> pandas.DataFrame
             The operation to carry out on each of the groups. The operator is another
             algebraic operator with its own user-defined function parameter, depending
@@ -3782,6 +3852,8 @@ def groupby(
             Whether to manually align columns between all the resulted row partitions.
             This flag is helpful when dealing with UDFs as they can change the partition's shape
             and labeling unpredictably, resulting in an invalid dataframe.
+        series_groupby : bool, default: False
+            Whether to convert a one-column DataFrame to a Series before performing groupby.
         add_missing_cats : bool, default: False
             Whether to add missing categories from `by` columns to the result.
         **kwargs : dict
@@ -3810,14 +3882,56 @@ def groupby(
                 f"Algebra groupby only implemented row-wise. {axis.name} axis groupby not implemented yet!"
             )
 
-        if not isinstance(by, list):
-            by = [by]
-
-        kwargs["observed"] = True
+        has_external_grouper = len(external_by) > 0
         skip_on_aligning_flag = "__skip_me_on_aligning__"
+        duplicated_suffix = "__duplicated_suffix__"
+        duplicated_pattern = r"_[\d]*__duplicated_suffix__"
+        kwargs["observed"] = True
 
         def apply_func(df):  # pragma: no cover
+            if has_external_grouper:
+                external_grouper = df["grouper"]
+                external_grouper = [
+                    # `df.groupby()` can only take a list of Series'es, so splitting
+                    # the df into a list of individual Series'es
+                    external_grouper.iloc[:, i]
+                    for i in range(len(external_grouper.columns))
+                ]
+
+                # renaming 'None' and duplicated names back to their original names
+                for obj in external_grouper:
+                    if not isinstance(obj, pandas.Series):
+                        continue
+                    name = obj.name
+                    if isinstance(name, str):
+                        if name.startswith(MODIN_UNNAMED_SERIES_LABEL):
+                            name = None
+                        elif name.endswith(duplicated_suffix):
+                            name = re.sub(duplicated_pattern, "", name)
+                    elif isinstance(name, tuple):
+                        if name[-1].endswith(duplicated_suffix):
+                            name = (
+                                *name[:-1],
+                                re.sub(duplicated_pattern, "", name[-1]),
+                            )
+                    obj.name = name
+
+                df = df["data"]
+            else:
+                external_grouper = []
+
+            by = []
+            # restoring original order of 'by' columns
+            for idx in by_positions:
+                if idx >= 0:
+                    by.append(external_grouper[idx])
+                else:
+                    by.append(internal_by[-idx - 1])
+
+            if series_groupby:
+                df = df.squeeze(axis=1)
             result = operator(df.groupby(by, **kwargs))
+
             if (
                 align_result_columns
                 and df.empty
@@ -3832,9 +3946,49 @@ def apply_func(df):  # pragma: no cover
                 result.attrs[skip_on_aligning_flag] = True
             return result
 
-        result = self._apply_func_to_range_partitioning(
-            key_columns=by,
+        if has_external_grouper:
+            grouper = (
+                external_by[0]
+                if len(external_by) == 1
+                else external_by[0].concat(
+                    axis=1, others=external_by[1:], how="left", sort=False
+                )
+            )
+
+            new_grouper_cols = []
+            columns_were_changed = False
+            same_columns = {}
+            # duplicated names break range-partitioning mechanism, so renaming them.
+            # original names will be reverted in the actual groupby kernel
+            for col in grouper.columns:
+                suffix = same_columns.get(col)
+                if suffix is None:
+                    same_columns[col] = 0
+                else:
+                    same_columns[col] += 1
+                    col = (
+                        (*col[:-1], f"{col[-1]}_{suffix}{duplicated_suffix}")
+                        if isinstance(col, tuple)
+                        else f"{col}_{suffix}{duplicated_suffix}"
+                    )
+                    columns_were_changed = True
+                new_grouper_cols.append(col)
+
+            if columns_were_changed:
+                grouper.columns = pandas.Index(new_grouper_cols)
+            grouper_key_columns = grouper.columns
+            data = self
+            data_key_columns = internal_by
+        else:
+            grouper = self
+            grouper_key_columns = internal_by
+            data, data_key_columns = None, None
+
+        result = grouper._apply_func_to_range_partitioning(
+            key_columns=grouper_key_columns,
             func=apply_func,
+            data=data,
+            data_key_columns=data_key_columns,
         )
         # no need aligning columns if there's only one row partition
         if add_missing_cats or align_result_columns and result._partitions.shape[0] > 1:
@@ -3869,7 +4023,7 @@ def apply_func(df):  # pragma: no cover
                 else:
                     original_dtypes = None
 
-                def compute_aligned_columns(*dfs, initial_columns=None):
+                def compute_aligned_columns(*dfs, initial_columns=None, by=None):
                     """Take row partitions, filter empty ones, and return joined columns for them."""
                     if align_result_columns:
                         valid_dfs = [
@@ -3908,13 +4062,27 @@ def compute_aligned_columns(*dfs, initial_columns=None):
                         else (None, masks)
                     )
 
+                external_by_cols = [
+                    None if col.startswith(MODIN_UNNAMED_SERIES_LABEL) else col
+                    for obj in external_by
+                    for col in obj.columns
+                ]
+                by = []
+                # restoring original order of 'by' columns
+                for idx in by_positions:
+                    if idx >= 0:
+                        by.append(external_by_cols[idx])
+                    else:
+                        by.append(internal_by[-idx - 1])
+
                 # Passing all partitions to the 'compute_aligned_columns' kernel to get
                 # aligned columns
                 parts = result._partitions.flatten()
                 aligned_columns = parts[0].apply(
                     compute_aligned_columns,
                     *[part._data for part in parts[1:]],
-                    initial_columns=self.columns,
+                    initial_columns=pandas.Index(external_by_cols).append(self.columns),
+                    by=by,
                 )
 
                 def apply_aligned(df, args, partition_idx):
@@ -3979,8 +4147,8 @@ def join_cols(df, *cols):
                 row_lengths=result._row_lengths_cache,
             )
 
-        if not result.has_materialized_index:
-            by_dtypes = ModinDtypes(self._dtypes).lazy_get(by)
+        if not result.has_materialized_index and not has_external_grouper:
+            by_dtypes = ModinDtypes(self._dtypes).lazy_get(internal_by)
             if by_dtypes.is_materialized:
                 new_index = ModinIndex(value=result, axis=0, dtypes=by_dtypes)
                 result.set_index_cache(new_index)
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -3459,9 +3459,9 @@ def _callable_func(self, func, axis, *args, **kwargs):
     # nature. They require certain data to exist on the same partition, and
     # after the shuffle, there should be only a local map required.
 
-    def _groupby_internal_columns(self, by, drop):
+    def _groupby_separate_by(self, by, drop):
         """
-        Extract internal columns from by argument of groupby.
+        Separate internal and external groupers in `by` argument of groupby.
 
         Parameters
         ----------
@@ -3472,31 +3472,52 @@ def _groupby_internal_columns(self, by, drop):
 
         Returns
         -------
-        by : list of BaseQueryCompiler, column or index label, or Grouper
+        external_by : list of BaseQueryCompiler and arrays
+            Values to group by.
         internal_by : list of str
-            List of internal column name to be dropped during groupby.
+            List of column names from `self` to group by.
+        by_positions : list of ints
+            Specifies the order of grouping by `internal_by` and `external_by` columns.
+            Each element in `by_positions` specifies an index from either `external_by` or `internal_by`.
+            Indices for `external_by` are positive and start from 0. Indices for `internal_by` are negative
+            and start from -1 (so in order to convert them to a valid indices one should do ``-idx - 1``)
+            '''
+            by_positions = [0, -1, 1, -2, 2, 3]
+            internal_by = ["col1", "col2"]
+            external_by = [sr1, sr2, sr3, sr4]
+
+            df.groupby([sr1, "col1", sr2, "col2", sr3, sr4])
+            '''.
         """
         if isinstance(by, type(self)):
             if drop:
-                internal_by = by.columns
-                by = [by]
+                internal_by = by.columns.tolist()
+                external_by = []
+                by_positions = [-i - 1 for i in range(len(internal_by))]
             else:
                 internal_by = []
-                by = [by]
+                external_by = [by]
+                by_positions = [i for i in range(len(external_by[0].columns))]
         else:
             if not isinstance(by, list):
                 by = [by] if by is not None else []
             internal_by = []
+            external_by = []
+            external_by_counter = 0
+            by_positions = []
             for o in by:
                 if isinstance(o, pandas.Grouper) and o.key in self.columns:
                     internal_by.append(o.key)
+                    by_positions.append(-len(internal_by))
                 elif hashable(o) and o in self.columns:
                     internal_by.append(o)
-            internal_qc = (
-                [self.getitem_column_array(internal_by)] if len(internal_by) else []
-            )
-            by = internal_qc + by[len(internal_by) :]
-        return by, internal_by
+                    by_positions.append(-len(internal_by))
+                else:
+                    external_by.append(o)
+                    for _ in range(len(o.columns) if isinstance(o, type(self)) else 1):
+                        by_positions.append(external_by_counter)
+                        external_by_counter += 1
+        return external_by, internal_by, by_positions
 
     groupby_all = GroupbyReduceImpl.build_qc_method("all")
     groupby_any = GroupbyReduceImpl.build_qc_method("any")
@@ -3542,7 +3563,7 @@ def groupby_mean(self, by, axis, groupby_kwargs, agg_args, agg_kwargs, drop=Fals
                     + "\nFalling back to a TreeReduce implementation."
                 )
 
-        _, internal_by = self._groupby_internal_columns(by, drop)
+        _, internal_by, _ = self._groupby_separate_by(by, drop)
 
         numeric_only = agg_kwargs.get("numeric_only", False)
         datetime_cols = (
@@ -3760,6 +3781,7 @@ def _groupby_shuffle(
         agg_kwargs,
         drop=False,
         how="axis_wise",
+        series_groupby=False,
     ):
         # Defaulting to pandas in case of an empty frame as we can't process it properly.
         # Higher API level won't pass empty data here unless the frame has delayed
@@ -3770,19 +3792,31 @@ def _groupby_shuffle(
                 by, agg_func, axis, groupby_kwargs, agg_args, agg_kwargs, how, drop
             )
 
-        if isinstance(by, type(self)) and drop:
-            by = by.columns.tolist()
+        if groupby_kwargs.get("level") is not None:
+            raise NotImplementedError(
+                "Grouping on an index level is not yet supported by range-partitioning groupby implementation: "
+                + "https://github.com/modin-project/modin/issues/5926"
+            )
 
-        if not isinstance(by, list):
-            by = [by]
+        if any(
+            isinstance(obj, pandas.Grouper)
+            for obj in (by if isinstance(by, list) else [by])
+        ):
+            raise NotImplementedError(
+                "Grouping on a pandas.Grouper with range-partitioning groupby is not yet supported: "
+                + "https://github.com/modin-project/modin/issues/5926"
+            )
 
-        is_all_labels = all(isinstance(col, (str, tuple)) for col in by)
-        is_all_column_names = (
-            all(col in self.columns for col in by) if is_all_labels else False
-        )
+        external_by, internal_by, by_positions = self._groupby_separate_by(by, drop)
 
-        is_transform = how == "transform" or GroupBy.is_transformation_kernel(agg_func)
+        all_external_are_qcs = all(isinstance(obj, type(self)) for obj in external_by)
+        if not all_external_are_qcs:
+            raise NotImplementedError(
+                "Grouping on an external grouper with range-partitioning groupby is only supported with Series'es: "
+                + "https://github.com/modin-project/modin/issues/5926"
+            )
 
+        is_transform = how == "transform" or GroupBy.is_transformation_kernel(agg_func)
         if is_transform:
             # https://github.com/modin-project/modin/issues/5924
             ErrorMessage.mismatch_with_pandas(
@@ -3790,21 +3824,31 @@ def _groupby_shuffle(
                 message="the order of rows may be shuffled for the result",
             )
 
-        if not is_all_column_names:
-            raise NotImplementedError(
-                "Range-partitioning groupby is only supported when grouping on a column(s) of the same frame. "
-                + "https://github.com/modin-project/modin/issues/5926"
-            )
-
         # This check materializes dtypes for 'by' columns
         if not is_transform and groupby_kwargs.get("observed", False) in (
             False,
             lib.no_default,
         ):
-            if isinstance(self._modin_frame._dtypes, ModinDtypes):
-                by_dtypes = self._modin_frame._dtypes.lazy_get(by).get()
-            else:
-                by_dtypes = self.dtypes[by]
+            # The following 'dtypes' check materializes dtypes for 'by' columns
+            internal_dtypes = pandas.Series()
+            external_dtypes = pandas.Series()
+            if len(internal_by) > 0:
+                internal_dtypes = (
+                    self._modin_frame._dtypes.lazy_get(internal_by).get()
+                    if isinstance(self._modin_frame._dtypes, ModinDtypes)
+                    else self.dtypes[internal_by]
+                )
+            if len(external_by) > 0:
+                dtypes_list = []
+                for obj in external_by:
+                    if not isinstance(obj, type(self)):
+                        # we're only interested in categorical dtypes here, which can only
+                        # appear in modin objects
+                        continue
+                    dtypes_list.append(obj.dtypes)
+                external_dtypes = pandas.concat(dtypes_list)
+
+            by_dtypes = pandas.concat([internal_dtypes, external_dtypes])
             add_missing_cats = any(
                 isinstance(dtype, pandas.CategoricalDtype) for dtype in by_dtypes
             )
@@ -3824,7 +3868,7 @@ def _groupby_shuffle(
                 how == "axis_wise"
             ), f"Only 'axis_wise' aggregation is supported with dictionary functions, got: {how}"
 
-            subset = by + list(agg_func.keys())
+            subset = internal_by + list(agg_func.keys())
             # extracting unique values; no we can't use np.unique here as it would
             # convert a list of tuples to a 2D matrix and so mess up the result
             subset = list(dict.fromkeys(subset))
@@ -3832,7 +3876,9 @@ def _groupby_shuffle(
         else:
             obj = self
 
-        agg_method = GroupByDefault.get_aggregation_method(how)
+        agg_method = (
+            SeriesGroupByDefault if series_groupby else GroupByDefault
+        ).get_aggregation_method(how)
         original_agg_func = agg_func
 
         def agg_func(grp, *args, **kwargs):
@@ -3848,7 +3894,13 @@ def agg_func(grp, *args, **kwargs):
 
         result = obj._modin_frame.groupby(
             axis=axis,
-            by=by,
+            internal_by=internal_by,
+            external_by=[
+                obj._modin_frame if isinstance(obj, type(self)) else obj
+                for obj in external_by
+            ],
+            by_positions=by_positions,
+            series_groupby=series_groupby,
             operator=lambda grp: agg_func(grp, *agg_args, **agg_kwargs),
             # UDFs passed to '.apply()' are allowed to produce results with arbitrary shapes,
             # that's why we have to align the partition's shapes/labeling across different
@@ -3992,6 +4044,7 @@ def groupby_agg(
                     agg_kwargs=agg_kwargs,
                     drop=drop,
                     how=how,
+                    series_groupby=series_groupby,
                 )
             except NotImplementedError as e:
                 # if a user wants to use range-partitioning groupby explicitly, then we should print a visible
@@ -4033,7 +4086,11 @@ def agg_func(grp, *args, **kwargs):
         groupby_kwargs = groupby_kwargs.copy()
 
         as_index = groupby_kwargs.get("as_index", True)
-        by, internal_by = self._groupby_internal_columns(by, drop)
+        external_by, internal_by, _ = self._groupby_separate_by(by, drop)
+        internal_qc = (
+            [self.getitem_column_array(internal_by)] if len(internal_by) else []
+        )
+        by = internal_qc + external_by
 
         broadcastable_by = [o._modin_frame for o in by if isinstance(o, type(self))]
         not_broadcastable_by = [o for o in by if not isinstance(o, type(self))]


Test patch:
diff --git a/modin/pandas/test/test_groupby.py b/modin/pandas/test/test_groupby.py
--- a/modin/pandas/test/test_groupby.py
+++ b/modin/pandas/test/test_groupby.py
@@ -111,6 +111,52 @@
 ]
 
 
+def get_external_groupers(df, columns, drop_from_original_df=False, add_plus_one=False):
+    """
+    Construct ``by`` argument containing external groupers.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame or modin.pandas.DataFrame
+    columns : list[tuple[bool, str]]
+        Columns to group on. If ``True`` do ``df[col]``, otherwise keep the column name.
+        '''
+        >>> columns = [(True, "a"), (False, "b")]
+        >>> get_external_groupers(df, columns)
+        [
+            pandas.Series(..., name="a"),
+            "b"
+        ]
+        '''
+    drop_from_original_df : bool, default: False
+        Whether to drop selected external columns from `df`.
+    add_plus_one : bool, default: False
+        Whether to do ``df[name] + 1`` for external groupers (so they won't be considered as
+        sibling with `df`).
+
+    Returns
+    -------
+    new_df : pandas.DataFrame or modin.pandas.DataFrame
+        If `drop_from_original_df` was True, returns a new dataframe with
+        dropped external columns, otherwise returns `df`.
+    by : list
+        Groupers to pass to `df.groupby(by)`.
+    """
+    new_df = df
+    by = []
+    for lookup, name in columns:
+        if lookup:
+            ser = df[name].copy()
+            if add_plus_one:
+                ser = ser + 1
+            by.append(ser)
+            if drop_from_original_df:
+                new_df = new_df.drop(columns=[name])
+        else:
+            by.append(name)
+    return new_df, by
+
+
 def modin_groupby_equals_pandas(modin_groupby, pandas_groupby):
     eval_general(
         modin_groupby, pandas_groupby, lambda grp: grp.indices, comparator=dict_equals
@@ -1926,26 +1972,68 @@ def test_to_pandas_convertion(kwargs):
     [
         [(False, "a"), (False, "b"), (False, "c")],
         [(False, "a"), (False, "b")],
-        [(True, "a"), (True, "b"), (True, "c")],
+        [(True, "b"), (True, "a"), (True, "c")],
         [(True, "a"), (True, "b")],
-        [(False, "a"), (False, "b"), (True, "c")],
+        [(True, "c"), (False, "a"), (False, "b")],
         [(False, "a"), (True, "c")],
     ],
 )
-def test_mixed_columns(columns):
-    def get_columns(df):
-        return [df[name] if lookup else name for (lookup, name) in columns]
+@pytest.mark.parametrize("drop_from_original_df", [True, False])
+@pytest.mark.parametrize("as_index", [True, False])
+def test_mixed_columns(columns, drop_from_original_df, as_index):
+    data = {
+        "a": [1, 1, 2, 2] * 64,
+        "b": [11, 11, 22, 22] * 64,
+        "c": [111, 111, 222, 222] * 64,
+        "data": [1, 2, 3, 4] * 64,
+    }
 
-    data = {"a": [1, 1, 2], "b": [11, 11, 22], "c": [111, 111, 222]}
+    md_df, pd_df = create_test_dfs(data)
+    md_df, md_by = get_external_groupers(md_df, columns, drop_from_original_df)
+    pd_df, pd_by = get_external_groupers(pd_df, columns, drop_from_original_df)
+
+    md_grp = md_df.groupby(md_by, as_index=as_index)
+    pd_grp = pd_df.groupby(pd_by, as_index=as_index)
+
+    df_equals(md_grp.size(), pd_grp.size())
+    df_equals(md_grp.sum(), pd_grp.sum())
+    df_equals(md_grp.apply(lambda df: df.sum()), pd_grp.apply(lambda df: df.sum()))
+
+
+@pytest.mark.parametrize("as_index", [True, False])
+def test_groupby_external_grouper_duplicated_names(as_index):
+    data = {
+        "a": [1, 1, 2, 2] * 64,
+        "b": [11, 11, 22, 22] * 64,
+        "c": [111, 111, 222, 222] * 64,
+        "data": [1, 2, 3, 4] * 64,
+    }
+
+    md_df, pd_df = create_test_dfs(data)
 
-    df1 = pandas.DataFrame(data)
-    df1 = pandas.concat([df1])
-    ref = df1.groupby(get_columns(df1)).size()
+    md_unnamed_series1, pd_unnamed_series1 = create_test_series([1, 1, 2, 2] * 64)
+    md_unnamed_series2, pd_unnamed_series2 = create_test_series([10, 10, 20, 20] * 64)
 
-    df2 = pd.DataFrame(data)
-    df2 = pd.concat([df2])
-    exp = df2.groupby(get_columns(df2)).size()
-    df_equals(ref, exp)
+    md_grp = md_df.groupby([md_unnamed_series1, md_unnamed_series2], as_index=as_index)
+    pd_grp = pd_df.groupby([pd_unnamed_series1, pd_unnamed_series2], as_index=as_index)
+
+    df_equals(md_grp.sum(), pd_grp.sum())
+
+    md_same_named_series1, pd_same_named_series1 = create_test_series(
+        [1, 1, 2, 2] * 64, name="series_name"
+    )
+    md_same_named_series2, pd_same_named_series2 = create_test_series(
+        [10, 10, 20, 20] * 64, name="series_name"
+    )
+
+    md_grp = md_df.groupby(
+        [md_same_named_series1, md_same_named_series2], as_index=as_index
+    )
+    pd_grp = pd_df.groupby(
+        [pd_same_named_series1, pd_same_named_series2], as_index=as_index
+    )
+
+    df_equals(md_grp.sum(), pd_grp.sum())
 
 
 @pytest.mark.parametrize(
@@ -1962,13 +2050,10 @@ def get_columns(df):
     ],
 )
 def test_internal_by_detection(columns):
-    def get_columns(df):
-        return [(df[name] + 1) if lookup else name for (lookup, name) in columns]
-
     data = {"a": [1, 1, 2], "b": [11, 11, 22], "c": [111, 111, 222]}
 
     md_df = pd.DataFrame(data)
-    by = get_columns(md_df)
+    _, by = get_external_groupers(md_df, columns, add_plus_one=True)
     md_grp = md_df.groupby(by)
 
     ref = frozenset(
@@ -1996,15 +2081,13 @@ def test_mixed_columns_not_from_df(columns, as_index):
     Unlike the previous test, in this case the Series is not just a column from
     the original DataFrame, so you can't use a fasttrack.
     """
-
-    def get_columns(df):
-        return [(df[name] + 1) if lookup else name for (lookup, name) in columns]
-
     data = {"a": [1, 1, 2], "b": [11, 11, 22], "c": [111, 111, 222]}
     groupby_kw = {"as_index": as_index}
 
     md_df, pd_df = create_test_dfs(data)
-    by_md, by_pd = map(get_columns, [md_df, pd_df])
+    (_, by_md), (_, by_pd) = map(
+        lambda df: get_external_groupers(df, columns, add_plus_one=True), [md_df, pd_df]
+    )
 
     pd_grp = pd_df.groupby(by_pd, **groupby_kw)
     md_grp = md_df.groupby(by_md, **groupby_kw)
@@ -2031,16 +2114,13 @@ def get_columns(df):
     ],
 )
 def test_unknown_groupby(columns):
-    def get_columns(df):
-        return [df[name] if lookup else name for (lookup, name) in columns]
-
     data = {"b": [11, 11, 22, 200], "c": [111, 111, 222, 7000]}
     modin_df, pandas_df = pd.DataFrame(data), pandas.DataFrame(data)
 
     with pytest.raises(KeyError):
-        pandas_df.groupby(by=get_columns(pandas_df))
+        pandas_df.groupby(by=get_external_groupers(pandas_df, columns)[1])
     with pytest.raises(KeyError):
-        modin_df.groupby(by=get_columns(modin_df))
+        modin_df.groupby(by=get_external_groupers(modin_df, columns)[1])
 
 
 @pytest.mark.parametrize(
@@ -3250,3 +3330,25 @@ def test_range_groupby_categories(
     # break the order of rows in the result for multi-column groupbys. Placing the sorting-hack for now
     # https://github.com/modin-project/modin/issues/6875
     df_equals(md_res.sort_index(axis=0), pd_res.sort_index(axis=0))
+
+
+@pytest.mark.parametrize("cat_cols", [["a"], ["b"], ["a", "b"]])
+@pytest.mark.parametrize(
+    "columns", [[(False, "a"), (True, "b")], [(True, "a")], [(True, "a"), (True, "b")]]
+)
+def test_range_groupby_categories_external_grouper(columns, cat_cols):
+    data = {
+        "a": [1, 1, 2, 2] * 64,
+        "b": [11, 11, 22, 22] * 64,
+        "c": [111, 111, 222, 222] * 64,
+        "data": [1, 2, 3, 4] * 64,
+    }
+
+    md_df, pd_df = create_test_dfs(data)
+    md_df = md_df.astype({col: "category" for col in cat_cols})
+    pd_df = pd_df.astype({col: "category" for col in cat_cols})
+
+    md_df, md_by = get_external_groupers(md_df, columns, drop_from_original_df=True)
+    pd_df, pd_by = get_external_groupers(pd_df, columns, drop_from_original_df=True)
+
+    eval_general(md_df.groupby(md_by), pd_df.groupby(pd_by), lambda grp: grp.count())
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -50,6 +50,7 @@
     bool_arg_values,
     categories_equals,
     create_test_dfs,
+    create_test_series,
     default_to_pandas_ignore_string,
     df_equals,
     df_equals_with_non_stable_indices,
@@ -227,19 +228,6 @@ def inter_df_math_helper_one_side(
             getattr(modin_df_multi_level, op)(modin_df_multi_level, level=1)
 
 
-def create_test_series(vals, sort=False, **kwargs):
-    if isinstance(vals, dict):
-        modin_series = pd.Series(vals[next(iter(vals.keys()))], **kwargs)
-        pandas_series = pandas.Series(vals[next(iter(vals.keys()))], **kwargs)
-    else:
-        modin_series = pd.Series(vals, **kwargs)
-        pandas_series = pandas.Series(vals, **kwargs)
-    if sort:
-        modin_series = modin_series.sort_values().reset_index(drop=True)
-        pandas_series = pandas_series.sort_values().reset_index(drop=True)
-    return modin_series, pandas_series
-
-
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_to_frame(data):
     modin_series, pandas_series = create_test_series(data)
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -1033,9 +1033,17 @@ def create_test_dfs(*args, **kwargs):
     )
 
 
-def create_test_series(*args, **kwargs):
-    post_fn = kwargs.pop("post_fn", lambda df: df)
-    return map(post_fn, [pd.Series(*args, **kwargs), pandas.Series(*args, **kwargs)])
+def create_test_series(vals, sort=False, **kwargs):
+    if isinstance(vals, dict):
+        modin_series = pd.Series(vals[next(iter(vals.keys()))], **kwargs)
+        pandas_series = pandas.Series(vals[next(iter(vals.keys()))], **kwargs)
+    else:
+        modin_series = pd.Series(vals, **kwargs)
+        pandas_series = pandas.Series(vals, **kwargs)
+    if sort:
+        modin_series = modin_series.sort_values().reset_index(drop=True)
+        pandas_series = pandas_series.sort_values().reset_index(drop=True)
+    return modin_series, pandas_series
 
 
 def generate_dfs():