Repository: modin-project/modin

Issue:
Set Dask `client.submit` in partitions to `pure=False`

### Describe the problem
<!-- Describe the problem clearly here. -->
We use `lambda` functions in many cases, which all share the same name. If run sequentially, Dask will cache the results of `gt` and return them on a `ge` because they share the `labmda` name. The `key` created by Dask is a function of the function name and parameters, causing some issues with the `lambda` functions we use internally.

At first, we can use `pure=False`, but there may be some opportunity later to name the `lambda` functions we're sending to the workers so we can still take advantage of Dask's task result caching.

### Source code / logs

There are a few places this needs to be changed:

https://github.com/modin-project/modin/blob/898889c8ac8804b7eed4cb3d733ee0d1e29686a8/modin/engines/dask/pandas_on_dask_futures/frame/partition.py#L67

https://github.com/modin-project/modin/blob/898889c8ac8804b7eed4cb3d733ee0d1e29686a8/modin/engines/dask/pandas_on_dask_futures/frame/axis_partition.py#L26

https://github.com/modin-project/modin/blob/898889c8ac8804b7eed4cb3d733ee0d1e29686a8/modin/engines/dask/pandas_on_dask_futures/frame/axis_partition.py#L39

https://github.com/modin-project/modin/blob/898889c8ac8804b7eed4cb3d733ee0d1e29686a8/modin/engines/dask/pandas_on_dask_futures/frame/axis_partition.py#L46

https://github.com/modin-project/modin/blob/898889c8ac8804b7eed4cb3d733ee0d1e29686a8/modin/engines/dask/pandas_on_dask_futures/frame/axis_partition.py#L59



Discussion:

Resolving patch:
diff --git a/modin/engines/dask/pandas_on_dask_futures/frame/axis_partition.py b/modin/engines/dask/pandas_on_dask_futures/frame/axis_partition.py
--- a/modin/engines/dask/pandas_on_dask_futures/frame/axis_partition.py
+++ b/modin/engines/dask/pandas_on_dask_futures/frame/axis_partition.py
@@ -30,13 +30,17 @@ def deploy_axis_func(
             num_splits,
             kwargs,
             maintain_partitioning,
-            *partitions
+            *partitions,
+            pure=False,
         )
         if num_splits == 1:
             return axis_result
         # We have to do this to split it back up. It is already split, but we need to
         # get futures for each.
-        return [client.submit(lambda l: l[i], axis_result) for i in range(num_splits)]
+        return [
+            client.submit(lambda l: l[i], axis_result, pure=False)
+            for i in range(num_splits)
+        ]
 
     @classmethod
     def deploy_func_between_two_axis_partitions(
@@ -50,13 +54,17 @@ def deploy_func_between_two_axis_partitions(
             num_splits,
             len_of_left,
             kwargs,
-            *partitions
+            *partitions,
+            pure=False,
         )
         if num_splits == 1:
             return axis_result
         # We have to do this to split it back up. It is already split, but we need to
         # get futures for each.
-        return [client.submit(lambda l: l[i], axis_result) for i in range(num_splits)]
+        return [
+            client.submit(lambda l: l[i], axis_result, pure=False)
+            for i in range(num_splits)
+        ]
 
 
 class PandasOnDaskFrameColumnPartition(PandasOnDaskFrameAxisPartition):
diff --git a/modin/engines/dask/pandas_on_dask_futures/frame/partition.py b/modin/engines/dask/pandas_on_dask_futures/frame/partition.py
--- a/modin/engines/dask/pandas_on_dask_futures/frame/partition.py
+++ b/modin/engines/dask/pandas_on_dask_futures/frame/partition.py
@@ -64,7 +64,9 @@ def apply(self, func, **kwargs):
              applied to it.
         """
         call_queue = self.call_queue + [[func, kwargs]]
-        future = self.client.submit(apply_list_of_funcs, call_queue, self.future)
+        future = self.client.submit(
+            apply_list_of_funcs, call_queue, self.future, pure=False
+        )
         return PandasOnDaskFramePartition(future)
 
     def add_to_apply_calls(self, func, **kwargs):
@@ -132,7 +134,7 @@ def put(cls, obj):
             A `RemotePartitions` object.
         """
         client = _get_global_client()
-        return cls(client.scatter(obj))
+        return cls(client.scatter(obj, hash=False))
 
     @classmethod
     def preprocess_func(cls, func):


Test patch:
diff --git a/modin/experimental/pandas/test/test_io_exp.py b/modin/experimental/pandas/test/test_io_exp.py
--- a/modin/experimental/pandas/test/test_io_exp.py
+++ b/modin/experimental/pandas/test/test_io_exp.py
@@ -8,6 +8,10 @@
 )
 
 
+@pytest.mark.skipif(
+    os.environ.get("MODIN_ENGINE", "Ray").title() == "Dask",
+    reason="Dask does not have experimental API",
+)
 def test_from_sql_distributed(make_sql_connection):  # noqa: F811
     if os.environ.get("MODIN_ENGINE", "") == "Ray":
         filename = "test_from_sql_distributed.db"
@@ -27,6 +31,10 @@ def test_from_sql_distributed(make_sql_connection):  # noqa: F811
         assert modin_df_equals_pandas(modin_df_from_table, pandas_df)
 
 
+@pytest.mark.skipif(
+    os.environ.get("MODIN_ENGINE", "Ray").title() == "Dask",
+    reason="Dask does not have experimental API",
+)
 def test_from_sql_defaults(make_sql_connection):  # noqa: F811
     filename = "test_from_sql_distributed.db"
     table = "test_from_sql_distributed"
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -2672,6 +2672,7 @@ def test_var(data, skipna, ddof):
         df_equals(modin_result, pandas_result)
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Windows memory issue #960")
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_view(data):
     modin_series, pandas_series = create_test_series(data)