Repository: modin-project/modin

Issue:
ETL part of LoanPd benchmark on Modin Cloud
Source will be added bit later


Discussion:
I unexpectedly ran into the problem:
`NoCredentialsError: Unable to locate credentials`
It seems that problem in version of boto3. Need pin to working version.
Currently master last changes work well with credentials in fresh environment.
Current problem:
```python
with remote:
    print(data.shape)
    print(data)
    print(data["loan_condition"])
```
```python
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-44-334eadba54bc> in <module>
      1 with remote:
      2     print(data.shape)
----> 3     print(data["loan_condition"])

/localdisk/amyachev/modin/modin/experimental/cloud/rpyc_proxy.py in method(_self, __method_name__, *_args, **_kw)
    370                                     __method_name__
    371                                 ] = remote = remote_cls.__getattr__(__method_name__)
--> 372                             return remote(_self.__remote_end__, *_args, **_kw)
    373 
    374                         method.__name__ = name

/localdisk/amyachev/miniconda3/envs/modin/lib/python3.8/site-packages/rpyc/core/netref.py in __call__(_self, *args, **kwargs)
    251         def __call__(_self, *args, **kwargs):
    252             kwargs = tuple(kwargs.items())
--> 253             return syncreq(_self, consts.HANDLE_CALL, args, kwargs)
    254         __call__.__doc__ = doc
    255         return __call__

/localdisk/amyachev/miniconda3/envs/modin/lib/python3.8/site-packages/rpyc/core/netref.py in syncreq(proxy, handler, *args)
     74     """
     75     conn = object.__getattribute__(proxy, "____conn__")
---> 76     return conn.sync_request(handler, proxy, *args)
     77 
     78 

/localdisk/amyachev/modin/modin/experimental/cloud/rpyc_proxy.py in sync_request(self, handler, *args)
    154                 return result
    155 
--> 156         return super().sync_request(handler, *args)
    157 
    158     def async_request(self, handler, *args, **kw):

/localdisk/amyachev/miniconda3/envs/modin/lib/python3.8/site-packages/rpyc/core/protocol.py in sync_request(self, handler, *args)
    467         """
    468         timeout = self._config["sync_request_timeout"]
--> 469         return self.async_request(handler, *args, timeout=timeout).value
    470 
    471     def _async_request(self, handler, args=(), callback=(lambda a, b: None)):  # serving

/localdisk/amyachev/miniconda3/envs/modin/lib/python3.8/site-packages/rpyc/core/async_.py in value(self)
    100         self.wait()
    101         if self._is_exc:
--> 102             raise self._obj
    103         else:
    104             return self._obj

AttributeError:  Can't get attribute 'loan_condition' on <module '__main__' from '/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/ray/workers/default_worker.py'>

========= Remote Traceback (1) =========
Traceback (most recent call last):
  File "/home/ubuntu/modin/modin/engines/ray/pandas_on_ray/frame/partition.py", line 44, in get
    return ray.get(self.oid)
  File "/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/ray/worker.py", line 1538, in get
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(AttributeError): ray::modin.engines.ray.pandas_on_ray.frame.partition.deploy_ray_func() (pid=3089, ip=172.31.36.90)
  File "python/ray/_raylet.pyx", line 479, in ray._raylet.execute_task
  File "/home/ubuntu/modin/modin/engines/ray/pandas_on_ray/frame/partition.py", line 212, in deploy_ray_func
    func = deserialize(func)
  File "/home/ubuntu/modin/modin/engines/ray/pandas_on_ray/frame/partition.py", line 207, in deserialize
    return ray.get(obj)
  File "/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/ray/serialization.py", line 308, in deserialize_objects
    self._deserialize_object(data, metadata, object_ref))
  File "/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/ray/serialization.py", line 247, in _deserialize_object
    return self._deserialize_msgpack_data(data, metadata)
  File "/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/ray/serialization.py", line 228, in _deserialize_msgpack_data
    python_objects = self._deserialize_pickle5_data(pickle5_data)
  File "/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/ray/serialization.py", line 216, in _deserialize_pickle5_data
    obj = pickle.loads(in_band)
AttributeError: Can't get attribute 'loan_condition' on <module '__main__' from '/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/ray/workers/default_worker.py'>

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/rpyc/core/protocol.py", line 320, in _dispatch_request
    res = self._HANDLERS[handler](self, *args)
  File "/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/rpyc/core/protocol.py", line 593, in _handle_call
    return obj(*args, **dict(kwargs))
  File "/home/ubuntu/modin/modin/pandas/base.py", line 3324, in __str__
    return repr(self)
  File "/home/ubuntu/modin/modin/pandas/series.py", line 306, in __repr__
    temp_df = self._build_repr_df(num_rows, num_cols)
  File "/home/ubuntu/modin/modin/pandas/base.py", line 108, in _build_repr_df
    return self.iloc[indexer]._query_compiler.to_pandas()
  File "/home/ubuntu/modin/modin/backends/pandas/query_compiler.py", line 195, in to_pandas
    return self._modin_frame.to_pandas()
  File "/home/ubuntu/modin/modin/engines/base/frame/data.py", line 1805, in to_pandas
    df = self._frame_mgr_cls.to_pandas(self._partitions)
  File "/home/ubuntu/modin/modin/engines/base/frame/partition_manager.py", line 339, in to_pandas
    retrieved_objects = [[obj.to_pandas() for obj in part] for part in partitions]
  File "/home/ubuntu/modin/modin/engines/base/frame/partition_manager.py", line 339, in <listcomp>
    retrieved_objects = [[obj.to_pandas() for obj in part] for part in partitions]
  File "/home/ubuntu/modin/modin/engines/base/frame/partition_manager.py", line 339, in <listcomp>
    retrieved_objects = [[obj.to_pandas() for obj in part] for part in partitions]
  File "/home/ubuntu/modin/modin/engines/ray/pandas_on_ray/frame/partition.py", line 92, in to_pandas
    dataframe = self.get()
  File "/home/ubuntu/modin/modin/engines/ray/pandas_on_ray/frame/partition.py", line 46, in get
    handle_ray_task_error(e)
  File "/home/ubuntu/modin/modin/engines/ray/utils.py", line 30, in handle_ray_task_error
    raise att_err
  File "/home/ubuntu/modin/modin/engines/ray/utils.py", line 25, in handle_ray_task_error
    raise getattr(builtins, s.split(":")[0])("".join(s.split(":")[1:]))
AttributeError:  Can't get attribute 'loan_condition' on <module '__main__' from '/home/ubuntu/miniconda/envs/modin/lib/python3.8/site-packages/ray/workers/default_worker.py'>
```
Most of it is ported, to continue working requires the ability to install additional packages in a remote context.

Created #2089
Need to make some changes to modin.
Faced with the error(`pip install "ray==0.8.7"`):
```python
>>> import ray
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "C:\Users\amyachev\AppData\Local\Continuum\miniconda3\envs\modin\lib\site-packages\ray\__init__.py", line 54, in <module>
    CDLL(so_path, ctypes.RTLD_GLOBAL)
  File "C:\Users\amyachev\AppData\Local\Continuum\miniconda3\envs\modin\lib\ctypes\__init__.py", line 364, in __init__
    self._handle = _dlopen(self._name, mode)
OSError: [WinError 126] The specified module could not be found
```

Resolving patch:
diff --git a/modin/experimental/cloud/rpyc_proxy.py b/modin/experimental/cloud/rpyc_proxy.py
--- a/modin/experimental/cloud/rpyc_proxy.py
+++ b/modin/experimental/cloud/rpyc_proxy.py
@@ -618,7 +618,17 @@ def dtypes(self):
 
     DeliveringDataFrame = _deliveringWrapper(
         DataFrame,
-        ["groupby", "agg", "aggregate", "__getitem__", "astype", "drop", "merge"],
+        [
+            "groupby",
+            "agg",
+            "aggregate",
+            "__getitem__",
+            "astype",
+            "drop",
+            "merge",
+            "apply",
+            "applymap",
+        ],
         DataFrameOverrides,
         "DataFrame",
     )
@@ -659,4 +669,4 @@ def make_series_wrapper(Series):
     are overridded here, so what it mostly does is it produces a wrapper class
     inherited from normal Series but wrapping all access to remote end transparently.
     """
-    return _deliveringWrapper(Series, target_name="Series")
+    return _deliveringWrapper(Series, ["apply"], target_name="Series")


Test patch: