Repository: modin-project/modin

Issue:
df.apply throws error
<!--
General questions should be asked on the mailing list modin-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information


- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Win10 Enterprise
- **Modin installed from (source or binary)**: pip install modin[dask]
- **Modin version**: 0.6.3
- **Python version**: 3.7.3
Jupyter notebook 7.4.0
- **Exact command to reproduce**: `df.apply(extract_data_per_user, axis=1, args=(data_per_user,))`

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
I'm using modin.pandas in a jupyter notebook (provided by anaconda), just as I would use regular pandas. However, when running
`df.apply(extract_data_per_user, axis=1, args=(data_per_user,))`
I get a 
`TypeError: apply() got multiple values for argument 'axis'` error. This same code works just fine in pandas, which is why I opened this issue. 

`extract_data_per_user(row, data_per_user)` looks at the value of a specific cel in a row and modifies values in the `data_per_user` dict based on what it reads.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->

Here's the stack trace:
```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-69-936ab004d2a8> in <module>
      2     df = pd.read_csv(file_path)
      3     data_per_user = generate_data_per_user_dicts()
----> 4     df.apply(extract_data_per_user, axis=1, args=(data_per_user,))
      5 
      6     file_name = file_path.split("/")[-1].strip(".csv")

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\pandas\dataframe.py in apply(self, func, axis, broadcast, raw, reduce, result_type, convert_dtype, args, **kwds)
    279             convert_dtype=convert_dtype,
    280             args=args,
--> 281             **kwds
    282         )
    283         if not isinstance(query_compiler, type(self._query_compiler)):

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\pandas\base.py in apply(self, func, axis, broadcast, raw, reduce, result_type, convert_dtype, args, **kwds)
    561         elif not callable(func) and not is_list_like(func):
    562             raise TypeError("{} object is not callable".format(type(func)))
--> 563         query_compiler = self._query_compiler.apply(func, axis, *args, **kwds)
    564         return query_compiler
    565 

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\backends\pandas\query_compiler.py in apply(self, func, axis, *args, **kwargs)
   1034         """
   1035         if callable(func):
-> 1036             return self._callable_func(func, axis, *args, **kwargs)
   1037         elif isinstance(func, dict):
   1038             return self._dict_func(func, axis, *args, **kwargs)

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\backends\pandas\query_compiler.py in _callable_func(self, func, axis, *args, **kwargs)
   1111         else:
   1112             new_modin_frame = self._modin_frame._apply_full_axis(
-> 1113                 axis, lambda df: df.apply(func, axis=axis, *args, **kwargs)
   1114             )
   1115         return self.__constructor__(new_modin_frame)

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\engines\base\frame\data.py in _apply_full_axis(self, axis, func, new_index, new_columns, dtypes)
    847         if new_columns is None:
    848             new_columns = self._frame_mgr_cls.get_indices(
--> 849                 1, new_partitions, lambda df: df.columns
    850             )
    851         if new_index is None:

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\engines\dask\pandas_on_dask_futures\frame\partition_manager.py in get_indices(cls, axis, partitions, index_func)
     53                 else []
     54             )
---> 55         new_idx = client.gather(new_idx)
     56         return new_idx[0].append(new_idx[1:]) if len(new_idx) else new_idx

~\AppData\Local\Continuum\anaconda3\lib\site-packages\distributed\client.py in gather(self, futures, errors, direct, asynchronous)
   1871                 direct=direct,
   1872                 local_worker=local_worker,
-> 1873                 asynchronous=asynchronous,
   1874             )
   1875 

~\AppData\Local\Continuum\anaconda3\lib\site-packages\distributed\client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
    766         else:
    767             return sync(
--> 768                 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
    769             )
    770 

~\AppData\Local\Continuum\anaconda3\lib\site-packages\distributed\utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
    332     if error[0]:
    333         typ, exc, tb = error[0]
--> 334         raise exc.with_traceback(tb)
    335     else:
    336         return result[0]

~\AppData\Local\Continuum\anaconda3\lib\site-packages\distributed\utils.py in f()
    316             if callback_timeout is not None:
    317                 future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
--> 318             result[0] = yield future
    319         except Exception as exc:
    320             error[0] = sys.exc_info()

~\AppData\Local\Continuum\anaconda3\lib\site-packages\tornado\gen.py in run(self)
    727 
    728                     try:
--> 729                         value = future.result()
    730                     except Exception:
    731                         exc_info = sys.exc_info()

~\AppData\Local\Continuum\anaconda3\lib\site-packages\distributed\client.py in _gather(self, futures, errors, direct, local_worker)
   1727                             exc = CancelledError(key)
   1728                         else:
-> 1729                             raise exception.with_traceback(traceback)
   1730                         raise exc
   1731                     if errors == "skip":

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\engines\base\frame\axis_partition.py in deploy_axis_func()
    187 
    188         dataframe = pandas.concat(list(partitions), axis=axis, copy=False)
--> 189         result = func(dataframe, **kwargs)
    190         if isinstance(result, pandas.Series):
    191             if num_splits == 1:

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\engines\base\frame\data.py in _map_reduce_func()
    652 
    653         def _map_reduce_func(df):
--> 654             series_result = func(df)
    655             if axis == 0 and isinstance(series_result, pandas.Series):
    656                 # In the case of axis=0, we need to keep the shape of the data

~\AppData\Local\Continuum\anaconda3\lib\site-packages\modin\backends\pandas\query_compiler.py in <lambda>()
   1111         else:
   1112             new_modin_frame = self._modin_frame._apply_full_axis(
-> 1113                 axis, lambda df: df.apply(func, axis=axis, *args, **kwargs)
   1114             )
   1115         return self.__constructor__(new_modin_frame)

TypeError: apply() got multiple values for argument 'axis'
```


Discussion:
Hi @JeroenStravers1, thanks for posting!

I was able to reproduce this, oddly it only occurs when `args` is set for the `apply` method. 

I will get this fixed, but there is another issue here related to updating the dictionary. Accessing and updating variables from inside of an `apply` would not work out of the box (right now) for a few reasons:

- Since this is all happening in parallel, each row would have to assume the same input dictionary instead of inheriting the result of the previous row. This is because the workers do not directly communicate their `apply` outputs to each other.
- Updating a dictionary inplace will only update it for the workers because they are separate processes. The updates would get lost when the worker was finished computing the `apply`.

In the future, to handle this case, we will have to inspect the function code being applied for any of these types of updates (plus other behaviors) and use an Actor (Dask and Ray both support Actors) to support updates from different workers. Then we will have to propagate the changes back to the dictionary in the main Python process. This strategy would work for single or multi-node and allow updates like this, but it will be a big undertaking because inspecting the code for specific behaviors will be very difficult and likely slowish at first. That particular problem isn't likely to be solved for a few months at least, because it will require some design documentation and a lot of testing.
Hey @devin-petersohn, thanks for your reply! I'm glad you were able to reproduce the issue.

No worries btw, this issue isn't blocking me or anything. I was playing around with Modin because I think it looks promising, and wanted to contribute a little ;)

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -561,7 +561,7 @@ def apply(
                 )
         elif not callable(func) and not is_list_like(func):
             raise TypeError("{} object is not callable".format(type(func)))
-        query_compiler = self._query_compiler.apply(func, axis, *args, **kwds)
+        query_compiler = self._query_compiler.apply(func, axis, args=args, **kwds)
         return query_compiler
 
     def as_blocks(self, copy=True):


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -1070,6 +1070,26 @@ def test_apply(self, request, data, func, axis):
             modin_result = modin_df.apply(func, axis)
             df_equals(modin_result, pandas_result)
 
+    @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+    @pytest.mark.parametrize("axis", axis_values, ids=axis_keys)
+    def test_apply_args(self, data, axis):
+        modin_df = pd.DataFrame(data)
+        pandas_df = pandas.DataFrame(data)
+
+        def apply_func(series, y):
+            try:
+                return series + y
+            except TypeError:
+                return series.map(str) + str(y)
+
+        modin_result = modin_df.apply(apply_func, axis=axis, args=(1,))
+        pandas_result = pandas_df.apply(apply_func, axis=axis, args=(1,))
+        df_equals(modin_result, pandas_result)
+
+        modin_result = modin_df.apply(apply_func, axis=axis, args=("_A",))
+        pandas_result = pandas_df.apply(apply_func, axis=axis, args=("_A",))
+        df_equals(modin_result, pandas_result)
+
     def test_apply_metadata(self):
         def add(a, b, c):
             return a + b + c