Repository: modin-project/modin

Issue:
Add a way to pass additional parameters to the engines for `map_axis_partitions` function
Through the way, we should be able to pass additional parameters to the engines, for example, `max_retries` for Ray engine, which will help get rid of problems in the implementation of `to_csv` function. See details in #3659.


Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/partitioning/partition_manager.py b/modin/core/dataframe/pandas/partitioning/partition_manager.py
--- a/modin/core/dataframe/pandas/partitioning/partition_manager.py
+++ b/modin/core/dataframe/pandas/partitioning/partition_manager.py
@@ -378,6 +378,7 @@ def broadcast_axis_partitions(
         apply_indices=None,
         enumerate_partitions=False,
         lengths=None,
+        **kwargs,
     ):
         """
         Broadcast the `right` partitions to `left` and apply `apply_func` along full `axis`.
@@ -402,6 +403,8 @@ def broadcast_axis_partitions(
             Note that `apply_func` must be able to accept `partition_idx` kwarg.
         lengths : list of ints, default: None
             The list of lengths to shuffle the object.
+        **kwargs : dict
+            Additional options that could be used by different engines.
 
         Returns
         -------
@@ -441,6 +444,7 @@ def broadcast_axis_partitions(
                     preprocessed_map_func,
                     **kw,
                     **({"partition_idx": idx} if enumerate_partitions else {}),
+                    **kwargs,
                 )
                 for idx, i in enumerate(apply_indices)
             ]
@@ -511,6 +515,7 @@ def map_axis_partitions(
         keep_partitioning=False,
         lengths=None,
         enumerate_partitions=False,
+        **kwargs,
     ):
         """
         Apply `map_func` to every partition in `partitions` along given `axis`.
@@ -531,6 +536,8 @@ def map_axis_partitions(
         enumerate_partitions : bool, default: False
             Whether or not to pass partition index into `map_func`.
             Note that `map_func` must be able to accept `partition_idx` kwarg.
+        **kwargs : dict
+            Additional options that could be used by different engines.
 
         Returns
         -------
@@ -550,6 +557,7 @@ def map_axis_partitions(
             right=None,
             lengths=lengths,
             enumerate_partitions=enumerate_partitions,
+            **kwargs,
         )
 
     @classmethod
diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py
--- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py
+++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py
@@ -241,6 +241,7 @@ def map_axis_partitions(
         keep_partitioning=False,
         lengths=None,
         enumerate_partitions=False,
+        **kwargs,
     ):
         """
         Apply `map_func` to every partition in `partitions` along given `axis`.
@@ -261,6 +262,8 @@ def map_axis_partitions(
         enumerate_partitions : bool, default: False
             Whether or not to pass partition index into `map_func`.
             Note that `map_func` must be able to accept `partition_idx` kwarg.
+        **kwargs : dict
+            Additional options that could be used by different engines.
 
         Returns
         -------
@@ -273,7 +276,13 @@ def map_axis_partitions(
         some global information about the axis.
         """
         return super(PandasOnRayDataframePartitionManager, cls).map_axis_partitions(
-            axis, partitions, map_func, keep_partitioning, lengths, enumerate_partitions
+            axis,
+            partitions,
+            map_func,
+            keep_partitioning,
+            lengths,
+            enumerate_partitions,
+            **kwargs,
         )
 
     @classmethod


Test patch: