Repository: modin-project/modin

Issue:
REFACTOR: remove code duplication for `PandasOnRayDataframePartitionManager`
In the current implementation, the following methods of `PandasOnRayDataframePartitionManager` class (child class of `PandasDataframePartitionManager`) are overridden just to apply the `progress_bar_wrapper` decorator:
- map_partitions
- lazy_map_partitions
- map_axis_partitions
- _apply_func_to_list_of_partitions
- apply_func_to_select_indices
- apply_func_to_select_indices_along_full_axis
- apply_func_to_indices_both_axis
- binary_operation

Moving the decorator to the base class will remove the duplication of functions.


Discussion:
@anmyachev I don't think it's possible to move since there's quite a bit of Ray specific code needed for the progress bar.
What we can do is we can replace the pieces like
```python
    @classmethod
    @progress_bar_wrapper
    def lazy_map_partitions(cls, partitions, map_func):
        """
        Apply `map_func` to every partition in `partitions` *lazily*.

        Parameters
        ----------
        partitions : np.ndarray
            A NumPy 2D array of partitions to perform operation on.
        map_func : callable
            Function to apply.

        Returns
        -------
        np.ndarray
            A NumPy array of partitions.
        """
        return super(PandasOnRayDataframePartitionManager, cls).lazy_map_partitions(
            partitions, map_func
        )
```
with (untested, illustrating the idea)
```python
    lazy_map_partitions = progress_bar_wrapper(GenericRayDataframePartitionManager.lazy_map_partitions)
```

Resolving patch:
diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py
--- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py
+++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py
@@ -28,7 +28,6 @@
 )
 from .partition import PandasOnRayDataframePartition
 from modin.core.execution.ray.generic.modin_aqp import call_progress_bar
-from pandas._libs.lib import no_default
 
 
 def progress_bar_wrapper(f):
@@ -131,291 +130,40 @@ def wait_partitions(cls, partitions):
         ]
         ray.wait(blocks, num_returns=len(blocks))
 
-    @classmethod
-    @progress_bar_wrapper
-    def map_partitions(cls, partitions, map_func):
-        """
-        Apply `map_func` to every partition in `partitions`.
-
-        Parameters
-        ----------
-        partitions : np.ndarray
-            A NumPy 2D array of partitions to perform operation on.
-        map_func : callable
-            Function to apply.
-
-        Returns
-        -------
-        np.ndarray
-            A NumPy array of partitions.
-        """
-        return super(PandasOnRayDataframePartitionManager, cls).map_partitions(
-            partitions, map_func
-        )
-
-    @classmethod
-    @progress_bar_wrapper
-    def lazy_map_partitions(cls, partitions, map_func):
-        """
-        Apply `map_func` to every partition in `partitions` *lazily*.
-
-        Parameters
-        ----------
-        partitions : np.ndarray
-            A NumPy 2D array of partitions to perform operation on.
-        map_func : callable
-            Function to apply.
-
-        Returns
-        -------
-        np.ndarray
-            A NumPy array of partitions.
-        """
-        return super(PandasOnRayDataframePartitionManager, cls).lazy_map_partitions(
-            partitions, map_func
-        )
-
-    @classmethod
-    @progress_bar_wrapper
-    def map_axis_partitions(
-        cls,
-        axis,
-        partitions,
-        map_func,
-        keep_partitioning=False,
-        lengths=None,
-        enumerate_partitions=False,
-        **kwargs,
-    ):
-        """
-        Apply `map_func` to every partition in `partitions` along given `axis`.
-
-        Parameters
-        ----------
-        axis : {0, 1}
-            Axis to perform the map across (0 - index, 1 - columns).
-        partitions : np.ndarray
-            A NumPy 2D array of partitions to perform operation on.
-        map_func : callable
-            Function to apply.
-        keep_partitioning : bool, default: False
-            Whether to keep partitioning for Modin Frame.
-            Setting it to True prevents data shuffling between partitions.
-        lengths : list of ints, default: None
-            List of lengths to shuffle the object.
-        enumerate_partitions : bool, default: False
-            Whether or not to pass partition index into `map_func`.
-            Note that `map_func` must be able to accept `partition_idx` kwarg.
-        **kwargs : dict
-            Additional options that could be used by different engines.
-
-        Returns
-        -------
-        np.ndarray
-            A NumPy array of new partitions for Modin Frame.
-
-        Notes
-        -----
-        This method should be used in the case when `map_func` relies on
-        some global information about the axis.
-        """
-        return super(PandasOnRayDataframePartitionManager, cls).map_axis_partitions(
-            axis,
-            partitions,
-            map_func,
-            keep_partitioning,
-            lengths,
-            enumerate_partitions,
-            **kwargs,
-        )
-
-    @classmethod
-    @progress_bar_wrapper
-    def _apply_func_to_list_of_partitions(cls, func, partitions, **kwargs):
-        """
-        Apply a `func` to a list of remote `partitions`.
-
-        Parameters
-        ----------
-        func : callable
-            The func to apply.
-        partitions : np.ndarray
-            The partitions to which the `func` will apply.
-        **kwargs : dict
-            Keyword arguments.
-
-        Returns
-        -------
-        list
-            A list of ``RayFramePartition`` objects.
-
-        Notes
-        -----
-        This preprocesses the `func` first before applying it to the partitions.
-        """
-        return super(
-            PandasOnRayDataframePartitionManager, cls
-        )._apply_func_to_list_of_partitions(func, partitions, **kwargs)
-
-    @classmethod
-    @progress_bar_wrapper
-    def apply_func_to_select_indices(
-        cls, axis, partitions, func, indices, keep_remaining=False
-    ):
-        """
-        Apply a `func` to select `indices` of `partitions`.
-
-        Parameters
-        ----------
-        axis : {0, 1}
-            Axis to apply the `func` over.
-        partitions : np.ndarray
-            The partitions to which the `func` will apply.
-        func : callable
-            The function to apply to these indices of partitions.
-        indices : dict
-            The indices to apply the function to.
-        keep_remaining : bool, default: False
-            Whether or not to keep the other partitions. Some operations
-            may want to drop the remaining partitions and keep
-            only the results.
-
-        Returns
-        -------
-        np.ndarray
-            A NumPy array with partitions.
-
-        Notes
-        -----
-        Your internal function must take a kwarg `internal_indices` for
-        this to work correctly. This prevents information leakage of the
-        internal index to the external representation.
-        """
-        return super(
-            PandasOnRayDataframePartitionManager, cls
-        ).apply_func_to_select_indices(
-            axis, partitions, func, indices, keep_remaining=keep_remaining
-        )
-
-    @classmethod
-    @progress_bar_wrapper
-    def apply_func_to_select_indices_along_full_axis(
-        cls, axis, partitions, func, indices, keep_remaining=False
-    ):
-        """
-        Apply a `func` to a select subset of full columns/rows.
-
-        Parameters
-        ----------
-        axis : {0, 1}
-            The axis to apply the `func` over.
-        partitions : np.ndarray
-            The partitions to which the `func` will apply.
-        func : callable
-            The function to apply.
-        indices : list-like
-            The global indices to apply the func to.
-        keep_remaining : bool, default: False
-            Whether or not to keep the other partitions.
-            Some operations may want to drop the remaining partitions and
-            keep only the results.
 
-        Returns
-        -------
-        np.ndarray
-            A NumPy array with partitions.
-
-        Notes
-        -----
-        This should be used when you need to apply a function that relies
-        on some global information for the entire column/row, but only need
-        to apply a function to a subset.
-        For your func to operate directly on the indices provided,
-        it must use `internal_indices` as a keyword argument.
-        """
-        return super(
-            PandasOnRayDataframePartitionManager, cls
-        ).apply_func_to_select_indices_along_full_axis(
-            axis, partitions, func, indices, keep_remaining
-        )
-
-    @classmethod
-    @progress_bar_wrapper
-    def apply_func_to_indices_both_axis(
-        cls,
-        partitions,
-        func,
-        row_partitions_list,
-        col_partitions_list,
-        item_to_distribute=no_default,
-        row_lengths=None,
-        col_widths=None,
-    ):
-        """
-        Apply a function along both axes.
-
-        Parameters
-        ----------
-        partitions : np.ndarray
-            The partitions to which the `func` will apply.
-        func : callable
-            The function to apply.
-        row_partitions_list : list
-            List of row partitions.
-        col_partitions_list : list
-            List of column partitions.
-        item_to_distribute : np.ndarray or scalar, default: no_default
-            The item to split up so it can be applied over both axes.
-        row_lengths : list of ints, optional
-            Lengths of partitions for every row. If not specified this information
-            is extracted from partitions itself.
-        col_widths : list of ints, optional
-            Widths of partitions for every column. If not specified this information
-            is extracted from partitions itself.
-
-        Returns
-        -------
-        np.ndarray
-            A NumPy array with partitions.
-
-        Notes
-        -----
-        For your func to operate directly on the indices provided,
-        it must use ``row_internal_indices`` and ``col_internal_indices`` as keyword
-        arguments.
-        """
-        return super(
-            PandasOnRayDataframePartitionManager, cls
-        ).apply_func_to_indices_both_axis(
-            partitions,
-            func,
-            row_partitions_list,
-            col_partitions_list,
-            item_to_distribute,
-            row_lengths,
-            col_widths,
-        )
-
-    @classmethod
-    @progress_bar_wrapper
-    def binary_operation(cls, left, func, right):
-        """
-        Apply a function that requires partitions of two ``PandasOnRayDataframe`` objects.
+def _make_wrapped_method(name: str):
+    """
+    Define new attribute that should work with progress bar.
 
-        Parameters
-        ----------
-        left : np.ndarray
-            The partitions of left ``PandasOnRayDataframe``.
-        func : callable
-            The function to apply.
-        right : np.ndarray
-            The partitions of right ``PandasOnRayDataframe``.
+    Parameters
+    ----------
+    name : str
+        Name of `GenericRayDataframePartitionManager` attribute that should be reused.
 
-        Returns
-        -------
-        np.ndarray
-            A NumPy array with new partitions.
-        """
-        return super(PandasOnRayDataframePartitionManager, cls).binary_operation(
-            left, func, right
-        )
+    Notes
+    -----
+    - `classmethod` decorator shouldn't be applied twice, so we refer to `__func__` attribute.
+    - New attribute is defined for `PandasOnRayDataframePartitionManager`.
+    """
+    setattr(
+        PandasOnRayDataframePartitionManager,
+        name,
+        classmethod(
+            progress_bar_wrapper(
+                getattr(GenericRayDataframePartitionManager, name).__func__
+            )
+        ),
+    )
+
+
+for method in (
+    "map_partitions",
+    "lazy_map_partitions",
+    "map_axis_partitions",
+    "_apply_func_to_list_of_partitions",
+    "apply_func_to_select_indices",
+    "apply_func_to_select_indices_along_full_axis",
+    "apply_func_to_indices_both_axis",
+    "binary_operation",
+):
+    _make_wrapped_method(method)


Test patch: