Repository: modin-project/modin

Issue:
PERF: internal `_launch_tasks` function takes too long while `read_csv` is executing
We can reduce the amount of data transferred to workers.

Profile (on Ray engine):
```bash
   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
      400    0.050    0.000   18.654    0.047 .../modin/modin/core/io/text/text_file_dispatcher.py:599(_launch_tasks)
```


Discussion:

Resolving patch:
diff --git a/modin/core/execution/dask/common/engine_wrapper.py b/modin/core/execution/dask/common/engine_wrapper.py
--- a/modin/core/execution/dask/common/engine_wrapper.py
+++ b/modin/core/execution/dask/common/engine_wrapper.py
@@ -13,6 +13,8 @@
 
 """Module houses class responsible for execution of remote operations."""
 
+from collections import UserDict
+
 from distributed.client import default_client
 
 
@@ -121,5 +123,13 @@ def put(cls, data, **kwargs):
         -------
         List, dict, iterator, or queue of futures matching the type of input.
         """
+        if isinstance(data, dict):
+            # there is a bug that looks similar to https://github.com/dask/distributed/issues/3965;
+            # to avoid this we could change behaviour for serialization:
+            # <Future: finished, type: collections.UserDict, key: UserDict-b8a15c164319c1d32fd28481125de455>
+            # vs
+            # {'sep': <Future: finished, type: pandas._libs.lib._NoDefault, key: sep>, \
+            #  'delimiter': <Future: finished, type: NoneType, key: delimiter> ...
+            data = UserDict(data)
         client = default_client()
         return client.scatter(data, **kwargs)
diff --git a/modin/core/io/text/text_file_dispatcher.py b/modin/core/io/text/text_file_dispatcher.py
--- a/modin/core/io/text/text_file_dispatcher.py
+++ b/modin/core/io/text/text_file_dispatcher.py
@@ -591,8 +591,19 @@ def _define_metadata(
 
         return column_widths, num_splits
 
+    _parse_func = None
+
+    @classmethod
+    def preprocess_func(cls):  # noqa: RT01
+        """Prepare a function for transmission to remote workers."""
+        if cls._parse_func is None:
+            cls._parse_func = cls.put(cls.parse)
+        return cls._parse_func
+
     @classmethod
-    def _launch_tasks(cls, splits: list, **partition_kwargs) -> Tuple[list, list, list]:
+    def _launch_tasks(
+        cls, splits: list, *partition_args, **partition_kwargs
+    ) -> Tuple[list, list, list]:
         """
         Launch tasks to read partitions.
 
@@ -601,6 +612,8 @@ def _launch_tasks(cls, splits: list, **partition_kwargs) -> Tuple[list, list, li
         splits : list
             List of tuples with partitions data, which defines
             parser task (start/end read bytes and etc.).
+        *partition_args : tuple
+            Positional arguments to be passed to the parser function.
         **partition_kwargs : dict
             `kwargs` that should be passed to the parser function.
 
@@ -616,10 +629,13 @@ def _launch_tasks(cls, splits: list, **partition_kwargs) -> Tuple[list, list, li
         partition_ids = [None] * len(splits)
         index_ids = [None] * len(splits)
         dtypes_ids = [None] * len(splits)
+        # this is done mostly for performance; see PR#5678 for details
+        func = cls.preprocess_func()
         for idx, (start, end) in enumerate(splits):
             partition_kwargs.update({"start": start, "end": end})
             *partition_ids[idx], index_ids[idx], dtypes_ids[idx] = cls.deploy(
-                func=cls.parse,
+                func=func,
+                f_args=partition_args,
                 f_kwargs=partition_kwargs,
                 num_returns=partition_kwargs.get("num_splits") + 2,
             )
@@ -1024,8 +1040,8 @@ def _read(cls, filepath_or_buffer, **kwargs):
         if not use_modin_impl:
             return cls.single_worker_read(
                 filepath_or_buffer,
+                kwargs,
                 reason=fallback_reason,
-                **kwargs,
             )
 
         is_quoting = kwargs["quoting"] != QUOTE_NONE
@@ -1100,8 +1116,6 @@ def _read(cls, filepath_or_buffer, **kwargs):
         # kwargs that will be passed to the workers
         partition_kwargs = dict(
             kwargs,
-            fname=filepath_or_buffer_md,
-            num_splits=num_splits,
             header_size=0 if use_inferred_column_names else header_size,
             names=column_names if use_inferred_column_names else names,
             header="infer" if use_inferred_column_names else header,
@@ -1110,8 +1124,14 @@ def _read(cls, filepath_or_buffer, **kwargs):
             nrows=None,
             compression=compression_infered,
         )
+        # this is done mostly for performance; see PR#5678 for details
+        filepath_or_buffer_md_ref = cls.put(filepath_or_buffer_md)
+        kwargs_ref = cls.put(partition_kwargs)
         partition_ids, index_ids, dtypes_ids = cls._launch_tasks(
-            splits, **partition_kwargs
+            splits,
+            filepath_or_buffer_md_ref,
+            kwargs_ref,
+            num_splits=num_splits,
         )
 
         new_query_compiler = cls._get_new_qc(
diff --git a/modin/core/storage_formats/pandas/parsers.py b/modin/core/storage_formats/pandas/parsers.py
--- a/modin/core/storage_formats/pandas/parsers.py
+++ b/modin/core/storage_formats/pandas/parsers.py
@@ -86,6 +86,13 @@
 _doc_parse_parameters_common = """fname : str or path object
     Name of the file or path to read."""
 
+_doc_common_read_kwargs = """common_read_kwargs : dict
+    Common keyword parameters for read functions.
+"""
+_doc_parse_parameters_common2 = "\n".join(
+    (_doc_parse_parameters_common, _doc_common_read_kwargs)
+)
+
 
 def _split_result_for_readers(axis, num_splits, df):  # pragma: no cover
     """
@@ -265,7 +272,7 @@ def get_dtypes(cls, dtypes_ids, columns):
         return frame_dtypes
 
     @classmethod
-    def single_worker_read(cls, fname, *, reason: str, **kwargs):
+    def single_worker_read(cls, fname, *args, reason: str, **kwargs):
         """
         Perform reading by single worker (default-to-pandas implementation).
 
@@ -273,6 +280,8 @@ def single_worker_read(cls, fname, *, reason: str, **kwargs):
         ----------
         fname : str, path object or file-like object
             Name of the file or file-like object to read.
+        *args : tuple
+            Positional arguments to be passed into `read_*` function.
         reason : str
             Message describing the reason for falling back to pandas.
         **kwargs : dict
@@ -289,7 +298,7 @@ def single_worker_read(cls, fname, *, reason: str, **kwargs):
         """
         ErrorMessage.default_to_pandas(reason=reason)
         # Use default args for everything
-        pandas_frame = cls.parse(fname, **kwargs)
+        pandas_frame = cls.parse(fname, *args, **kwargs)
         if isinstance(pandas_frame, pandas.io.parsers.TextFileReader):
             pd_read = pandas_frame.read
             pandas_frame.read = (
@@ -311,10 +320,14 @@ def single_worker_read(cls, fname, *, reason: str, **kwargs):
 @doc(_doc_pandas_parser_class, data_type="CSV files")
 class PandasCSVParser(PandasParser):
     @staticmethod
-    @doc(_doc_parse_func, parameters=_doc_parse_parameters_common)
-    def parse(fname, **kwargs):
-        kwargs["callback"] = PandasCSVParser.read_callback
-        return PandasParser.generic_parse(fname, **kwargs)
+    @doc(_doc_parse_func, parameters=_doc_parse_parameters_common2)
+    def parse(fname, common_read_kwargs, **kwargs):
+        return PandasParser.generic_parse(
+            fname,
+            callback=PandasCSVParser.read_callback,
+            **common_read_kwargs,
+            **kwargs,
+        )
 
     @staticmethod
     def read_callback(*args, **kwargs):
@@ -430,10 +443,14 @@ def parse(fname, **kwargs):
 @doc(_doc_pandas_parser_class, data_type="tables with fixed-width formatted lines")
 class PandasFWFParser(PandasParser):
     @staticmethod
-    @doc(_doc_parse_func, parameters=_doc_parse_parameters_common)
-    def parse(fname, **kwargs):
-        kwargs["callback"] = PandasFWFParser.read_callback
-        return PandasParser.generic_parse(fname, **kwargs)
+    @doc(_doc_parse_func, parameters=_doc_parse_parameters_common2)
+    def parse(fname, common_read_kwargs, **kwargs):
+        return PandasParser.generic_parse(
+            fname,
+            callback=PandasFWFParser.read_callback,
+            **common_read_kwargs,
+            **kwargs,
+        )
 
     @staticmethod
     def read_callback(*args, **kwargs):


Test patch: