Repository: modin-project/modin

Issue:
BUG: Applying dictionary function to columns raises TypeError: dict_apply_builder() got an unexpected keyword argument 'internal_indices'
Looks like it is in a worker process

```
2022-08-15 13:56:11,155	ERROR worker.py:94 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::_apply_func() (pid=54258, ip=127.0.0.1)
  At least one of the input arguments for this task could not be computed:
ray.exceptions.RayTaskError: ray::deploy_ray_func() (pid=54301, ip=127.0.0.1)
  File "[...]/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/virtual_partition.py", line 548, in deploy_ray_func
    result = func(*args, **kwargs)
  File "[...]/modin/core/dataframe/pandas/partitioning/axis_partition.py", line 136, in deploy_axis_func
    result = func(dataframe, *kwargs.pop("args", ()), **kwargs)
TypeError: dict_apply_builder() got an unexpected keyword argument 'internal_indices'
```

reproducer at verison bd326f1c4175102489f08d271a53cf374bd9125e (edited in by @mvashishtha ):

```python
import modin.pandas as pd

df = pd.DataFrame([1])
print(df.apply({0: (lambda x: x)}))
```


Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -2700,15 +2700,24 @@ def _dict_func(self, func, axis, *args, **kwargs):
         if "axis" not in kwargs:
             kwargs["axis"] = axis
 
-        def dict_apply_builder(df, func_dict={}):  # pragma: no cover
+        func = {k: wrap_udf_function(v) if callable(v) else v for k, v in func.items()}
+
+        def dict_apply_builder(df, internal_indices=[]):  # pragma: no cover
             # Sometimes `apply` can return a `Series`, but we require that internally
             # all objects are `DataFrame`s.
-            return pandas.DataFrame(df.apply(func_dict, *args, **kwargs))
+            # It looks like it doesn't need to use `internal_indices` option internally
+            # for the case since `apply` use labels from dictionary keys in `func` variable.
+            return pandas.DataFrame(df.apply(func, *args, **kwargs))
 
-        func = {k: wrap_udf_function(v) if callable(v) else v for k, v in func.items()}
+        labels = list(func.keys())
         return self.__constructor__(
             self._modin_frame.apply_full_axis_select_indices(
-                axis, dict_apply_builder, func, keep_remaining=False
+                axis,
+                dict_apply_builder,
+                labels,
+                new_index=labels if axis == 1 else None,
+                new_columns=labels if axis == 0 else None,
+                keep_remaining=False,
             )
         )
 


Test patch:
diff --git a/modin/pandas/test/dataframe/test_udf.py b/modin/pandas/test/dataframe/test_udf.py
--- a/modin/pandas/test/dataframe/test_udf.py
+++ b/modin/pandas/test/dataframe/test_udf.py
@@ -40,7 +40,7 @@
     arg_keys,
     default_to_pandas_ignore_string,
 )
-from modin.config import NPartitions, StorageFormat
+from modin.config import NPartitions
 from modin.test.test_utils import warns_that_defaulting_to_pandas
 from modin.utils import get_current_execution
 
@@ -116,13 +116,6 @@ def test_aggregate_error_checking():
         modin_df.aggregate("NOT_EXISTS")
 
 
-@pytest.mark.xfail(
-    StorageFormat.get() == "Pandas",
-    reason="DataFrame.apply(dict) raises an exception because of a bug in its"
-    + "implementation for pandas storage format, this prevents us from catching the desired"
-    + "exception. You can track this bug at:"
-    + "https://github.com/modin-project/modin/issues/3221",
-)
 @pytest.mark.parametrize(
     "func",
     agg_func_values + agg_func_except_values,
@@ -245,6 +238,51 @@ def test_apply_udf(data, func):
     )
 
 
+def test_apply_dict_4828():
+    data = [[2, 4], [1, 3]]
+    modin_df1, pandas_df1 = create_test_dfs(data)
+    eval_general(
+        modin_df1,
+        pandas_df1,
+        lambda df: df.apply({0: (lambda x: x**2)}),
+    )
+    eval_general(
+        modin_df1,
+        pandas_df1,
+        lambda df: df.apply({0: (lambda x: x**2)}, axis=1),
+    )
+
+    # several partitions along axis 0
+    modin_df2, pandas_df2 = create_test_dfs(data, index=[2, 3])
+    modin_df3 = pd.concat([modin_df1, modin_df2], axis=0)
+    pandas_df3 = pandas.concat([pandas_df1, pandas_df2], axis=0)
+    eval_general(
+        modin_df3,
+        pandas_df3,
+        lambda df: df.apply({0: (lambda x: x**2)}),
+    )
+    eval_general(
+        modin_df3,
+        pandas_df3,
+        lambda df: df.apply({0: (lambda x: x**2)}, axis=1),
+    )
+
+    # several partitions along axis 1
+    modin_df4, pandas_df4 = create_test_dfs(data, columns=[2, 3])
+    modin_df5 = pd.concat([modin_df1, modin_df4], axis=1)
+    pandas_df5 = pandas.concat([pandas_df1, pandas_df4], axis=1)
+    eval_general(
+        modin_df5,
+        pandas_df5,
+        lambda df: df.apply({0: (lambda x: x**2)}),
+    )
+    eval_general(
+        modin_df5,
+        pandas_df5,
+        lambda df: df.apply({0: (lambda x: x**2)}, axis=1),
+    )
+
+
 def test_apply_modin_func_4635():
     data = [1]
     modin_df, pandas_df = create_test_dfs(data)