Repository: modin-project/modin

Issue:
Presence of skiprows and nrows parameters make read_csv fall back to single threaded
<!--
General questions should be asked on the mailing list modin-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
Ubuntu 19.10
- **Modin installed from (source or binary)**:
Source
- **Modin version**:
0.7.0+3.g7528bf3
- **Python version**:
Python 3.7.5
- **Exact command to reproduce**:

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

When `skiprows` is defined as a list or `nrows` parameter is present, `read_csv` implementation gives a warning that it falls back to single threaded implementation and works slowly.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->

```python
#import pandas as pd
import ray
ray.init(huge_pages=True, plasma_directory="/mnt/hugepages")
import modin.pandas as pd

dtypes = {
    'object_id': 'int32',
    'mjd': 'float32',
    'passband': 'int32',
    'flux': 'float32',
    'flux_err': 'float32',
    'detected': 'int32'
}

PATH='/localdisk/benchmark_datasets/plasticc'
GPU_MEMORY = 16
TEST_ROWS = 453653104
OVERHEAD = 1.2
SKIP_ROWS = int((1 - GPU_MEMORY/(32.0*OVERHEAD))*TEST_ROWS)

test = pd.read_csv('%s/test_set_100000.csv'%PATH,
        skiprows=range(1, 1+SKIP_ROWS),
        dtype=dtypes)
```
[test_set_100000.csv.zip](https://github.com/modin-project/modin/files/4172070/test_set_100000.csv.zip)



Discussion:

Resolving patch:
diff --git a/modin/engines/base/io/text/csv_dispatcher.py b/modin/engines/base/io/text/csv_dispatcher.py
--- a/modin/engines/base/io/text/csv_dispatcher.py
+++ b/modin/engines/base/io/text/csv_dispatcher.py
@@ -18,11 +18,13 @@
     ColumnNamesTypes,
 )
 import pandas
+from pandas.core.dtypes.common import is_list_like
 from csv import QUOTE_NONE, Dialect
 import sys
 from typing import Union, Sequence, Callable, Dict, Tuple
 from pandas._typing import FilePathOrBuffer
 import pandas._libs.lib as lib
+import numpy as np
 
 from modin.config import NPartitions
 
@@ -55,6 +57,13 @@ def _read(cls, filepath_or_buffer, **kwargs):
         -------
         new_query_compiler : BaseQueryCompiler
             Query compiler with imported data for further processing.
+
+        Notes
+        -----
+        `skiprows` is handled diferently based on the parameter type because of
+        performance reasons. If `skiprows` is integer - rows will be skipped during
+        data file partitioning and wouldn't be actually read. If `skiprows` is array
+        or callable - full data file will be read and only then rows will be dropped.
         """
         filepath_or_buffer_md = (
             cls.get_path(filepath_or_buffer)
@@ -64,17 +73,29 @@ def _read(cls, filepath_or_buffer, **kwargs):
         compression_infered = cls.infer_compression(
             filepath_or_buffer, kwargs.get("compression")
         )
-        use_modin_impl = cls._read_csv_check_support(
-            filepath_or_buffer, kwargs, compression_infered
-        )
-        if not use_modin_impl:
-            return cls.single_worker_read(filepath_or_buffer, **kwargs)
-
         # Getting frequently used read_csv kwargs
         names = kwargs.get("names", lib.no_default)
         index_col = kwargs.get("index_col", None)
         encoding = kwargs.get("encoding", None)
         skiprows = kwargs.get("skiprows", None)
+        header = kwargs.get("header", "infer")
+        # Define header size for further skipping (Header can be skipped because header
+        # information will be obtained further from empty_df, so no need to handle it
+        # by workers)
+        header_size = cls._define_header_size(
+            header,
+            names,
+        )
+        skiprows_md, pre_reading = cls._manage_skiprows_parameter(skiprows, header_size)
+        should_handle_skiprows = skiprows_md is not None and not isinstance(
+            skiprows_md, int
+        )
+
+        use_modin_impl = cls._read_csv_check_support(
+            filepath_or_buffer, kwargs, compression_infered
+        )
+        if not use_modin_impl:
+            return cls.single_worker_read(filepath_or_buffer, **kwargs)
 
         is_quoting = kwargs.get("quoting", "") != QUOTE_NONE
         quotechar = kwargs.get("quotechar", '"').encode(
@@ -90,13 +111,9 @@ def _read(cls, filepath_or_buffer, **kwargs):
         # information will be obtained further from empty_df, so no need to handle it
         # by workers)
         header_size = cls._define_header_size(
-            kwargs.get("header", "infer"),
+            header,
             names,
         )
-        # Since skiprows can be only integer here (non-integer skiprows trigger fallback
-        # to pandas implementation for now) we can process header_size and skiprows
-        # simultaneously
-        skiprows = skiprows + header_size if skiprows else header_size
 
         pd_df_metadata = pandas.read_csv(
             filepath_or_buffer,
@@ -112,6 +129,7 @@ def _read(cls, filepath_or_buffer, **kwargs):
             num_splits=num_splits,
             header_size=header_size if not pass_names else 0,
             names=names if not pass_names else column_names,
+            header=header if not pass_names else "infer",
             skipfooter=0,
             skiprows=None,
             nrows=None,
@@ -122,10 +140,12 @@ def _read(cls, filepath_or_buffer, **kwargs):
             splits = cls.partitioned_file(
                 f,
                 num_partitions=NPartitions.get(),
-                nrows=kwargs.get("nrows", None),
-                skiprows=skiprows,
+                nrows=kwargs.get("nrows", None) if not should_handle_skiprows else None,
+                skiprows=skiprows_md if not should_handle_skiprows else None,
                 quotechar=quotechar,
                 is_quoting=is_quoting,
+                header_size=header_size,
+                pre_reading=pre_reading,
             )
 
         partition_ids, index_ids, dtypes_ids = cls._launch_tasks(
@@ -140,9 +160,12 @@ def _read(cls, filepath_or_buffer, **kwargs):
             index_name=pd_df_metadata.index.name,
             column_widths=column_widths,
             column_names=column_names,
+            skiprows_md=skiprows_md if should_handle_skiprows else None,
+            header_size=header_size,
             squeeze=kwargs.get("squeeze", False),
             skipfooter=kwargs.get("skipfooter", None),
             parse_dates=kwargs.get("parse_dates", False),
+            nrows=kwargs.get("nrows", None) if should_handle_skiprows else None,
         )
         return new_query_compiler
 
@@ -190,10 +213,6 @@ def _read_csv_check_support(
         if read_csv_kwargs.get("chunksize") is not None:
             return False
 
-        skiprows = read_csv_kwargs.get("skiprows")
-        if skiprows is not None and not isinstance(skiprows, int):
-            return False
-
         return True
 
     @classmethod
@@ -203,7 +222,7 @@ def _define_index(
         index_name: str,
     ) -> Tuple[IndexColType, list]:
         """
-        Compute the resulting DataFrame index and index lengths for each or partitions.
+        Compute the resulting DataFrame index and index lengths for each of partitions.
 
         Parameters
         ----------
@@ -241,6 +260,8 @@ def _get_new_qc(
         index_name: str,
         column_widths: list,
         column_names: ColumnNamesTypes,
+        skiprows_md: Union[Sequence, callable, None] = None,
+        header_size: int = None,
         **kwargs,
     ):
         """
@@ -263,6 +284,10 @@ def _get_new_qc(
             Number of columns in each partition.
         column_names : ColumnNamesTypes
             Array with columns names.
+        skiprows_md : array-like or callable, optional
+            Specifies rows to skip.
+        header_size : int, default: 0
+            Number of rows, that occupied by header.
         **kwargs : dict
             Parameters of `read_csv` function needed for postprocessing.
 
@@ -300,9 +325,84 @@ def _get_new_qc(
             new_query_compiler = new_query_compiler.drop(
                 new_query_compiler.index[-skipfooter:]
             )
+        if skiprows_md is not None:
+            # skip rows that passed as array or callable
+            nrows = kwargs.get("nrows", None)
+            index_range = pandas.RangeIndex(len(new_query_compiler.index))
+            if is_list_like(skiprows_md):
+                new_query_compiler = new_query_compiler.view(
+                    index=index_range.delete(skiprows_md - header_size)
+                )
+            elif callable(skiprows_md):
+                mod_index = skiprows_md(index_range + header_size)
+                mod_index = (
+                    mod_index
+                    if isinstance(mod_index, np.ndarray)
+                    else mod_index.to_numpy("bool")
+                )
+                view_idx = index_range[~mod_index]
+                new_query_compiler = new_query_compiler.view(index=view_idx)
+            else:
+                raise TypeError(
+                    f"Not acceptable type of `skiprows` parameter: {type(skiprows_md)}"
+                )
+
+            if not isinstance(new_query_compiler.index, pandas.MultiIndex):
+                new_query_compiler = new_query_compiler.reset_index(drop=True)
+
+            if nrows:
+                new_query_compiler = new_query_compiler.view(
+                    pandas.RangeIndex(len(new_query_compiler.index))[:nrows]
+                )
         if kwargs.get("squeeze", False) and len(new_query_compiler.columns) == 1:
             return new_query_compiler[new_query_compiler.columns[0]]
         if index_col is None:
             new_query_compiler._modin_frame.synchronize_labels(axis=0)
 
         return new_query_compiler
+
+    @classmethod
+    def _manage_skiprows_parameter(
+        cls,
+        skiprows: Union[int, Sequence[int], Callable, None] = None,
+        header_size: int = 0,
+    ) -> Tuple[Union[int, Sequence, Callable], bool, int]:
+        """
+        Manage read_csv `skiprows` parameter.
+
+        Change `skiprows` parameter in the way Modin could more optimally
+        process it. If `skiprows` is an array, this array will be sorted and
+        then, if array is uniformly distributed, `skiprows` will be "squashed"
+        into integer value and `pre_reading` parameter will be set if needed
+        (in this case fastpath can be done).
+
+        Parameters
+        ----------
+        skiprows : int, array or callable, optional
+                Original skiprows parameter of read_csv function.
+        header_size : int, default: 0
+                Number of rows that are used by header.
+
+        Returns
+        -------
+        skiprows : int, array or callable
+                Updated skiprows parameter.
+        pre_reading : int
+                The number of rows that should be read before data file
+                splitting for further reading (the number of rows for
+                the first partition).
+        """
+        pre_reading = 0
+        uniform_skiprows = False
+        if is_list_like(skiprows):
+            skiprows = np.sort(skiprows)
+            if np.all(np.diff(skiprows) == 1):
+                uniform_skiprows = True
+
+        if uniform_skiprows:
+            pre_reading = max(0, skiprows[0] - header_size)
+            skiprows = len(skiprows)
+        elif skiprows is None:
+            skiprows = 0
+
+        return skiprows, pre_reading
diff --git a/modin/engines/base/io/text/text_file_dispatcher.py b/modin/engines/base/io/text/text_file_dispatcher.py
--- a/modin/engines/base/io/text/text_file_dispatcher.py
+++ b/modin/engines/base/io/text/text_file_dispatcher.py
@@ -198,6 +198,8 @@ def partitioned_file(
         skiprows: int = None,
         quotechar: bytes = b'"',
         is_quoting: bool = True,
+        header_size: int = 0,
+        pre_reading: int = 0,
     ):
         """
         Compute chunk sizes in bytes for every partition.
@@ -211,33 +213,59 @@ def partitioned_file(
             If not specified grabs the value from `modin.config.NPartitions.get()`.
         nrows : int, optional
             Number of rows of file to read.
-        skiprows : array or callable, optional
+        skiprows : int, optional
             Specifies rows to skip.
         quotechar : bytes, default: b'"'
             Indicate quote in a file.
         is_quoting : bool, default: True
             Whether or not to consider quotes.
+        header_size : int, default: 0
+            Number of rows, that occupied by header.
+        pre_reading : int, default: 0
+            Number of rows between header and skipped rows, that should be read.
 
         Returns
         -------
         list
-            An array, where each element of array is a tuple of two ints:
-            beginning and the end offsets of the current chunk.
+            List with the next elements:
+                int : partition start read byte
+                int : partition end read byte
         """
+        read_rows_counter = 0
+        outside_quotes = True
+
         if num_partitions is None:
-            num_partitions = NPartitions.get()
+            num_partitions = NPartitions.get() - 1 if pre_reading else NPartitions.get()
 
         rows_skipper = cls.rows_skipper_builder(f, quotechar, is_quoting=is_quoting)
         result = []
 
         file_size = cls.file_size(f)
 
+        rows_skipper(header_size)
+
+        if pre_reading:
+            pre_reading_start = f.tell()
+            outside_quotes, read_rows = cls._read_rows(
+                f,
+                nrows=pre_reading,
+                quotechar=quotechar,
+                is_quoting=is_quoting,
+                outside_quotes=outside_quotes,
+            )
+            read_rows_counter += read_rows
+
+            result.append((pre_reading_start, f.tell()))
+
+            # add outside_quotes
+            if is_quoting and not outside_quotes:
+                warnings.warn("File has mismatched quotes")
+
         rows_skipper(skiprows)
 
         start = f.tell()
 
         if nrows:
-            read_rows_counter = 0
             partition_size = max(1, num_partitions, nrows // num_partitions)
             while f.tell() < file_size and read_rows_counter < nrows:
                 if read_rows_counter + partition_size > nrows:


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -16,6 +16,7 @@
 import pandas
 from pandas.errors import ParserWarning
 import pandas._libs.lib as lib
+from pandas.core.dtypes.common import is_list_like
 from collections import OrderedDict
 from modin.config import TestDatasetSize, Engine, Backend, IsExperimental
 from modin.utils import to_pandas
@@ -339,55 +340,115 @@ def test_read_csv_parsing_1(
             skipfooter=skipfooter,
         )
 
-    @pytest.mark.parametrize("true_values", [["Yes"], ["Yes", "true"], None])
-    @pytest.mark.parametrize("false_values", [["No"], ["No", "false"], None])
-    @pytest.mark.parametrize("skiprows", [2, lambda x: x % 2])
-    @pytest.mark.parametrize("skipfooter", [0, 10])
+    @pytest.mark.parametrize("header", ["infer", None, 0])
+    @pytest.mark.parametrize(
+        "skiprows",
+        [
+            2,
+            lambda x: x % 2,
+            lambda x: x > 25,
+            lambda x: x > 128,
+            np.arange(10, 50),
+            np.arange(10, 50, 2),
+        ],
+    )
     @pytest.mark.parametrize("nrows", [35, None])
-    @pytest.mark.parametrize("names", [["c1", "c2", "c3", "c4"], None])
+    @pytest.mark.parametrize(
+        "names",
+        [
+            [f"c{col_number}" for col_number in range(4)],
+            [f"c{col_number}" for col_number in range(6)],
+            None,
+        ],
+    )
+    @pytest.mark.parametrize("encoding", ["latin1", "windows-1251", None])
     def test_read_csv_parsing_2(
         self,
-        true_values,
-        false_values,
+        make_csv_file,
+        request,
+        header,
         skiprows,
-        skipfooter,
         nrows,
         names,
+        encoding,
     ):
-        xfail_case_1 = (
-            (false_values or true_values)
-            and Engine.get() != "Python"
-            and Backend.get() != "Omnisci"
-        )
-        xfail_case_2 = (
+        xfail_case = (
             Backend.get() == "Omnisci"
+            and header is not None
             and isinstance(skiprows, int)
             and names is None
-            and skipfooter == 0
             and nrows is None
-            and not true_values
-            and not false_values
         )
-        if xfail_case_1:
-            pytest.xfail("modin and pandas dataframes differs - issue #2446")
-        if xfail_case_2:
+        if xfail_case:
             pytest.xfail(
                 "read_csv fails because of duplicated columns names - issue #3080"
             )
+        if request.config.getoption(
+            "--simulate-cloud"
+        ).lower() != "off" and is_list_like(skiprows):
+            pytest.xfail(
+                reason="The reason of tests fail in `cloud` mode is unknown for now - issue #2340"
+            )
+        if encoding:
+            unique_filename = get_unique_filename()
+            make_csv_file(
+                filename=unique_filename,
+                encoding=encoding,
+            )
+        kwargs = {
+            "filepath_or_buffer": unique_filename
+            if encoding
+            else pytest.csvs_names["test_read_csv_regular"],
+            "header": header,
+            "skiprows": skiprows,
+            "nrows": nrows,
+            "names": names,
+            "encoding": encoding,
+        }
 
+        if Engine.get() != "Python":
+            df = pandas.read_csv(**dict(kwargs, nrows=1))
+            # in that case first partition will contain str
+            if df[df.columns[0]][df.index[0]] in ["c1", "col1", "c3", "col3"]:
+                pytest.xfail("read_csv incorrect output with float data - issue #2634")
         eval_io(
             fn_name="read_csv",
             check_exception_type=None,  # issue #2320
             raising_exceptions=None,
             check_kwargs_callable=not callable(skiprows),
             # read_csv kwargs
+            **kwargs,
+        )
+
+    @pytest.mark.parametrize("true_values", [["Yes"], ["Yes", "true"], None])
+    @pytest.mark.parametrize("false_values", [["No"], ["No", "false"], None])
+    @pytest.mark.parametrize("skipfooter", [0, 10])
+    @pytest.mark.parametrize("nrows", [35, None])
+    def test_read_csv_parsing_3(
+        self,
+        true_values,
+        false_values,
+        skipfooter,
+        nrows,
+    ):
+        xfail_case = (
+            (false_values or true_values)
+            and Engine.get() != "Python"
+            and Backend.get() != "Omnisci"
+        )
+        if xfail_case:
+            pytest.xfail("modin and pandas dataframes differs - issue #2446")
+
+        eval_io(
+            fn_name="read_csv",
+            check_exception_type=None,  # issue #2320
+            raising_exceptions=None,
+            # read_csv kwargs
             filepath_or_buffer=pytest.csvs_names["test_read_csv_yes_no"],
             true_values=true_values,
             false_values=false_values,
-            skiprows=skiprows,
             skipfooter=skipfooter,
             nrows=nrows,
-            names=names,
         )
 
     def test_read_csv_skipinitialspace(self):
@@ -891,12 +952,6 @@ def test_read_csv_default_to_pandas(self):
                 StringIO(open(pytest.csvs_names["test_read_csv_regular"], "r").read())
             )
 
-        with pytest.warns(UserWarning):
-            pd.read_csv(
-                pytest.csvs_names["test_read_csv_regular"],
-                skiprows=lambda x: x in [0, 2],
-            )
-
     @pytest.mark.xfail(
         condition="config.getoption('--simulate-cloud').lower() != 'off'",
         reason="The reason of tests fail in `cloud` mode is unknown for now - issue #2340",