Repository: modin-project/modin

Issue:
read_csv skiprows parameter bug (possible)
Here is the code
```python
# import pandas as pd
import modin.pandas as pd
from distributed import Client

z = 1
print(f"{z }")

if __name__ == '__main__':
    if pd.__name__ == 'modin.pandas':
        client = Client(n_workers=3)

    us_columns_dtype = {
        'date/time': 'string',
        'sku': 'string',
        'description': 'string',
        'type': 'string',
        'settlement id': 'string',
        'order id': 'string',
        'quantity': 'Int32',
        'product sales': "float64",
        'fulfillment': str,
    }

    skiprows = 0
    file_path = r"D:\_\OD\DR_US_2021Jun6-2022Jun5CustomUnifiedTransaction.csv"
    separator = ','
    en_cod_ = 'utf-8'
    columns_dtype = us_columns_dtype.copy()

    readed_into_df = pd.read_csv(
        str(file_path),
        sep=separator,
        encoding=en_cod_,
        thousands=",",
        skiprows=skiprows,
        on_bad_lines="skip",
        usecols=columns_dtype.keys(),
        dtype=columns_dtype,
    )
```
Usage the _skiprows_ parameter cause the next error listing
```

1
1
1
1
2022-06-07 14:45:15,373 - distributed.worker - WARNING - Compute Failed
Key:       parse-41bece4b074ebd6bc2c6f8a110fd29b1
Function:  parse
args:      ()
kwargs:    {'sep': ',', 'delimiter': None, 'header': 'infer', 'names': Index(['date/time', 'settlement id', 'type', 'order id', 'sku', 'description',
       'quantity', 'fulfillment', 'product sales'],
      dtype='object'), 'index_col': None, 'usecols': dict_keys(['date/time', 'sku', 'description', 'type', 'settlement id', 'order id', 'quantity', 'product sales', 
'fulfillment']), 'prefix': <no_default>, 'mangle_dupe_cols': True, 'dtype': {'date/time': 'string', 'sku': 'string', 'description': 'string', 'type': 'string', 'settlement id': 'string', 'order id': 'string', 'quantity': 'Int32', 'product sales': 'float64', 'fulfillment': <class 'str'>}, 'engine': None, 'converters': None, 'true_values': None, 'false_values': None, 'skipinitialspace': False, 'skiprows': None, 'nrows': None, 'na_values': None, 'keep_default_na': True, 'na_filter': True, 'verbose': False, 'skip_blank_lines': True, 'parse_dates': False, 'infer_datetime_format': False, 'keep_date_col': False, 'date_parser': None, 'dayfirst'
Exception: 'ValueError("could not convert string to float: \'Standard Orders\'")'

Traceback (most recent call last):
  File "d:\OD\OneDrive\Projects\Chud_Amaz\Soft_in_dev\moduled_way_OOP\modin_test_bug_report.py", line 30, in <module>
    readed_into_df = pd.read_csv(
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\pandas\io.py", line 135, in read_csv
    return _read(**kwargs)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\pandas\io.py", line 60, in _read
    pd_obj = FactoryDispatcher.read_csv(**kwargs)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\data_management\factories\dispatcher.py", line 178, in read_csv
    return cls.__factory._read_csv(**kwargs)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\data_management\factories\factories.py", line 206, in _read_csv
    return cls.io_cls.read_csv(**kwargs)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\engines\base\io\file_dispatcher.py", line 68, in read
    query_compiler = cls._read(*args, **kwargs)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\engines\base\io\text\csv_dispatcher.py", line 157, in _read
    new_query_compiler = cls._get_new_qc(
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\engines\base\io\text\csv_dispatcher.py", line 300, in _get_new_qc
    new_index, row_lengths = cls._define_index(index_ids, index_name)2022-06-07 14:45:15,440 - distributed.worker - WARNING - Compute Failed
Key:       parse-12a66d09f9162c480b1d2f6c748fc0ec
Function:  parse
args:      ()
kwargs:    {'sep': ',', 'delimiter': None, 'header': 'infer', 'names': Index(['date/time', 'settlement id', 'type', 'order id', 'sku', 'description',
       'quantity', 'fulfillment', 'product sales'],
      dtype='object'), 'index_col': None, 'usecols': dict_keys(['date/time', 'sku', 'description', 'type', 'settlement id', 'order id', 'quantity', 'product sales', 
'fulfillment']), 'prefix': <no_default>, 'mangle_dupe_cols': True, 'dtype': {'date/time': 'string', 'sku': 'string', 'description': 'string', 'type': 'string', 'settlement id': 'string', 'order id': 'string', 'quantity': 'Int32', 'product sales': 'float64', 'fulfillment': <class 'str'>}, 'engine': None, 'converters': None, 'true_values': None, 'false_values': None, 'skipinitialspace': False, 'skiprows': None, 'nrows': None, 'na_values': None, 'keep_default_na': True, 'na_filter': True, 'verbose': False, 'skip_blank_lines': True, 'parse_dates': False, 'infer_datetime_format': False, 'keep_date_col': False, 'date_parser': None, 'dayfirst'
Exception: 'ValueError("could not convert string to float: \'Standard Orders\'")'

2022-06-07 14:45:15,424 - distributed.worker - WARNING - Compute Failed
Key:       parse-9d26a0a1270fdd144b14b9d766dcb368
Function:  parse
args:      ()
kwargs:    {'sep': ',', 'delimiter': None, 'header': 'infer', 'names': Index(['date/time', 'settlement id', 'type', 'order id', 'sku', 'description',
       'quantity', 'fulfillment', 'product sales'],
      dtype='object'), 'index_col': None, 'usecols': dict_keys(['date/time', 'sku', 'description', 'type', 'settlement id', 'order id', 'quantity', 'product sales', 
'fulfillment']), 'prefix': <no_default>, 'mangle_dupe_cols': True, 'dtype': {'date/time': 'string', 'sku': 'string', 'description': 'string', 'type': 'string', 'settlement id': 'string', 'order id': 'string', 'quantity': 'Int32', 'product sales': 'float64', 'fulfillment': <class 'str'>}, 'engine': None, 'converters': None, 'true_values': None, 'false_values': None, 'skipinitialspace': False, 'skiprows': None, 'nrows': None, 'na_values': None, 'keep_default_na': True, 'na_filter': True, 'verbose': False, 'skip_blank_lines': True, 'parse_dates': False, 'infer_datetime_format': False, 'keep_date_col': False, 'date_parser': None, 'dayfirst'
Exception: 'ValueError("could not convert string to float: \'Standard Orders\'")'


  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\engines\base\io\text\csv_dispatcher.py", line 243, in _define_index
    index_objs = cls.materialize(index_ids)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\engines\dask\task_wrapper.py", line 64, in materialize
    return client.gather(future)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\distributed\client.py", line 2178, in gather
    return self.sync(
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\distributed\utils.py", line 318, in sync
    return sync(
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\distributed\utils.py", line 385, in sync
    raise exc.with_traceback(tb)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\distributed\utils.py", line 358, in f
    result = yield future
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\tornado\gen.py", line 762, in run
    value = future.result()
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\distributed\client.py", line 2041, in _gather
    raise exception.with_traceback(traceback)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\modin\backends\pandas\parsers.py", line 272, in parse
    pandas_df = pandas.read_csv(BytesIO(to_read), **kwargs)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\pandas\util\_decorators.py", line 311, in wrapper
    return func(*args, **kwargs)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\pandas\io\parsers\readers.py", line 586, in read_csv
    return _read(filepath_or_buffer, kwds)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\pandas\io\parsers\readers.py", line 488, in _read
    return parser.read(nrows)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\pandas\io\parsers\readers.py", line 1047, in read
    index, columns, col_dict = self._engine.read(nrows)
  File "D:\Games\Anaconda\envs\AnazModin\lib\site-packages\pandas\io\parsers\c_parser_wrapper.py", line 223, in read
    chunks = self._reader.read_low_memory(nrows)
  File "pandas\_libs\parsers.pyx", line 801, in pandas._libs.parsers.TextReader.read_low_memory
  File "pandas\_libs\parsers.pyx", line 880, in pandas._libs.parsers.TextReader._read_rows
  File "pandas\_libs\parsers.pyx", line 1026, in pandas._libs.parsers.TextReader._convert_column_data
  File "pandas\_libs\parsers.pyx", line 1101, in pandas._libs.parsers.TextReader._convert_tokens
ValueError: could not convert string to float: 'Standard Orders'

```
If I only commen row with  **_skiprows=skiprows,_** all works. The same code with pure pandas works properly

**Environment**:

- Dask version: IDK.  
- >>> pd.__pandas_version__
'1.3.4'
- Python version: 3.9.12
- Operating System: Windows10
- Install method:  conda 

It seams patameter **_skiprows=skiprows,_** processing wrong way. BC in pure pandas all works.    
But some files works on properly...

I have tryed read my file without parameter  `dtype=columns_dtype`  and the code was finished without any messages....

I just was trying transfer my working **pandas** project to **modin[dask]** so please help me to avoid this issue.

### So in short.
**_read_csv()_**   works with parameter `skiprows`  or with parameter `dtype`. Using this parameters together produce crash.

-----
This issue is a copy feom my **_Dask_**  [issue](https://github.com/dask/distributed/issues/6522). They have recomended me to repost my qiestion here bc they think:

> 

> It looks like you are running into an issue with modin.pandas. afaik, modin is re-implementing the pandas API so you're likely hitting a bug in their implementation.

And really  -  I had used `import dask.dataframe as pd` and  error disappeared. So it looks like real modin arguments passing error.  Can anybody help?   Or I have to switch from `modin` to `dask`?



[DR_US_2021Jun6-2022Jun5CustomUnifiedTransaction.csv](https://github.com/dask/distributed/files/8852859/DR_US_2021Jun6-2022Jun5CustomUnifiedTransaction.csv)




Discussion:
Hello @VasilijKolomiets! Thank you for submitting the issue!

I downsized a bit your reproducer.

```python
# import pandas as pd
import modin.pandas as pd

import modin.config as cfg
cfg.Engine.put("dask")


if __name__ == '__main__':
    us_columns_dtype = {
        'product sales': str,  # to see what's trying to read in real
        # 'product sales': "float64",
    }

    skiprows = 0
    file_path = "DR_US_2021Jun6-2022Jun5CustomUnifiedTransaction.csv"
    separator = ','
    en_cod_ = 'utf-8'
    columns_dtype = us_columns_dtype.copy()

    readed_into_df = pd.read_csv(
        str(file_path),
        sep=separator,
        encoding=en_cod_,
        thousands=",",
        skiprows=skiprows,
        on_bad_lines="skip",
        usecols=columns_dtype.keys(),
        dtype=columns_dtype,
    )
    print(readed_into_df)
```
<details><summary>Outputs: </summary>

Output for `modin.pandas`:

```bash
(modin) prutskov@prutskovPC:~/projects/modin$ python test2.py
UserWarning: Dask execution environment not yet initialized. Initializing...
To remove this warning, run the following python code before doing dataframe operations:

    from distributed import Client

    client = Client()

                     product sales
0       Jun 6, 2021 1:29:07 AM PDT
1       Jun 6, 2021 1:29:07 AM PDT
2       Jun 6, 2021 1:29:43 AM PDT
3       Jun 6, 2021 1:43:17 AM PDT
4       Jun 6, 2021 2:14:52 AM PDT
...                            ...
31753  Jun 5, 2022 10:38:01 PM PDT
31754  Jun 5, 2022 11:04:58 PM PDT
31755  Jun 5, 2022 11:42:56 PM PDT
31756  Jun 5, 2022 11:44:37 PM PDT
31757  Jun 5, 2022 11:54:52 PM PDT

[31758 rows x 1 columns]
```

Output for `pandas`:

```bash
(modin) prutskov@prutskovPC:~/projects/modin$ python test2.py
      product sales
0             29.49
1             29.49
2            149.99
3             39.99
4             58.98
...             ...
31753         33.99
31754             0
31755         29.99
31756             0
31757         59.99

[31758 rows x 1 columns]
```

</details>

Commenting of `skiprows` parameter makes output similar to pandas. The issue should be more investigated.

The additional one problem is discovered in case `skiprows=[]`:

<details><summary>Outputs 2: </summary>

Output for `modin.pandas`:

```bash
(modin) prutskov@prutskovPC:~/projects/modin$ python test2.py
(modin) prutskov@prutskovPC:~/projects/modin$ python test2.py
UserWarning: Dask execution environment not yet initialized. Initializing...
To remove this warning, run the following python code before doing dataframe operations:

    from distributed import Client

    client = Client()

Traceback (most recent call last):
  File "test2.py", line 20, in <module>
    readed_into_df = pd.read_csv(
  File "/home/prutskov/projects/modin/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/home/prutskov/projects/modin/modin/pandas/io.py", line 140, in read_csv
    return _read(**kwargs)
  File "/home/prutskov/projects/modin/modin/pandas/io.py", line 61, in _read
    pd_obj = FactoryDispatcher.read_csv(**kwargs)
  File "/home/prutskov/projects/modin/modin/core/execution/dispatching/factories/dispatcher.py", line 185, in read_csv
    return cls.__factory._read_csv(**kwargs)
  File "/home/prutskov/projects/modin/modin/core/execution/dispatching/factories/factories.py", line 217, in _read_csv
    return cls.io_cls.read_csv(**kwargs)
  File "/home/prutskov/projects/modin/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/home/prutskov/projects/modin/modin/core/io/file_dispatcher.py", line 153, in read
    query_compiler = cls._read(*args, **kwargs)
  File "/home/prutskov/projects/modin/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/home/prutskov/projects/modin/modin/core/io/text/text_file_dispatcher.py", line 1004, in _read
    ) = cls._manage_skiprows_parameter(skiprows, header_size)
  File "/home/prutskov/projects/modin/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/home/prutskov/projects/modin/modin/core/io/text/text_file_dispatcher.py", line 800, in _manage_skiprows_parameter
    skiprows_md[0] - header_size if skiprows_md[0] > header_size else 0
IndexError: index 0 is out of bounds for axis 0 with size 0
```

Output for `pandas`:

```bash
(modin) prutskov@prutskovPC:~/projects/modin$ python test2.py
      product sales
0             29.49
1             29.49
2            149.99
3             39.99
4             58.98
...             ...
31753         33.99
31754             0
31755         29.99
31756             0
31757         59.99

[31758 rows x 1 columns]
```

</details>

The root cause of the second issue: https://github.com/modin-project/modin/blob/dcee13d57ebf9a006460deedb734c15791acae7a/modin/core/io/text/text_file_dispatcher.py#L795-L803

The both issues are reproduced on both Ray/Dask engines. We will start working on this!


Resolving patch:
diff --git a/modin/core/io/text/text_file_dispatcher.py b/modin/core/io/text/text_file_dispatcher.py
--- a/modin/core/io/text/text_file_dispatcher.py
+++ b/modin/core/io/text/text_file_dispatcher.py
@@ -792,7 +792,7 @@ def _manage_skiprows_parameter(
         pre_reading = skiprows_partitioning = skiprows_md = 0
         if isinstance(skiprows, int):
             skiprows_partitioning = skiprows
-        elif is_list_like(skiprows):
+        elif is_list_like(skiprows) and len(skiprows) > 0:
             skiprows_md = np.sort(skiprows)
             if np.all(np.diff(skiprows_md) == 1):
                 # `skiprows` is uniformly distributed array.
@@ -1018,10 +1018,8 @@ def _read(cls, filepath_or_buffer, **kwargs):
             )
 
         is_quoting = kwargs["quoting"] != QUOTE_NONE
-        # In these cases we should pass additional metadata
-        # to the workers to match pandas output
-        pass_names = names in [None, lib.no_default] and (
-            skiprows is not None or kwargs["skipfooter"] != 0
+        use_inferred_column_names = cls._uses_inferred_column_names(
+            names, skiprows, kwargs.get("skipfooter", 0), kwargs.get("usecols", None)
         )
 
         pd_df_metadata = cls.read_callback(
@@ -1036,9 +1034,9 @@ def _read(cls, filepath_or_buffer, **kwargs):
             kwargs,
             fname=filepath_or_buffer_md,
             num_splits=num_splits,
-            header_size=header_size if not pass_names else 0,
-            names=names if not pass_names else column_names,
-            header=header if not pass_names else "infer",
+            header_size=0 if use_inferred_column_names else header_size,
+            names=column_names if use_inferred_column_names else names,
+            header="infer" if use_inferred_column_names else header,
             skipfooter=0,
             skiprows=None,
             nrows=None,
@@ -1115,3 +1113,53 @@ def _get_skip_mask(cls, rows_index: pandas.Index, skiprows: Callable):
             mask = rows_index.map(skiprows)
 
         return mask
+
+    @staticmethod
+    @logger_decorator(
+        "PANDAS-API", "TextFileDispatcher._uses_inferred_column_names", "INFO"
+    )
+    def _uses_inferred_column_names(names, skiprows, skipfooter, usecols):
+        """
+        Tell whether need to use inferred column names in workers or not.
+
+        1) ``False`` is returned in 2 cases and means next:
+            1.a) `names` parameter was provided from the API layer. In this case parameter
+            `names` must be provided as `names` parameter for ``read_csv`` in the workers.
+            1.b) `names` parameter wasn't provided from the API layer. In this case column names
+            inference must happen in each partition.
+        2) ``True`` is returned in case when inferred column names from pre-reading stage must be
+            provided as `names` parameter for ``read_csv`` in the workers.
+
+        In case `names` was provided, the other parameters aren't checked. Otherwise, inferred column
+        names should be used in a case of not full data reading which is defined by `skipfooter` parameter,
+        when need to skip lines at the bottom of file or by `skiprows` parameter, when need to skip lines at
+        the top of file (but if `usecols` was provided, column names inference must happen in the workers).
+
+        Parameters
+        ----------
+        names : array-like
+            List of column names to use.
+        skiprows : list-like, int or callable
+            Line numbers to skip (0-indexed) or number of lines to skip (int) at
+            the start of the file. If callable, the callable function will be
+            evaluated against the row indices, returning ``True`` if the row should
+            be skipped and ``False`` otherwise.
+        skipfooter : int
+            Number of lines at bottom of the file to skip.
+        usecols : list-like or callable
+            Subset of the columns.
+
+        Returns
+        -------
+        bool
+            Whether to use inferred column names in ``read_csv`` of the workers or not.
+        """
+        if names not in [None, lib.no_default]:
+            return False
+        if skipfooter != 0:
+            return True
+        if isinstance(skiprows, int) and skiprows == 0:
+            return False
+        if is_list_like(skiprows):
+            return usecols is None
+        return skiprows is not None


Test patch:
diff --git a/modin/pandas/test/data/issue_4543.csv b/modin/pandas/test/data/issue_4543.csv
new file mode 100644
--- /dev/null
+++ b/modin/pandas/test/data/issue_4543.csv
@@ -0,0 +1,4 @@
+str_data,float_data,country
+fanta,3.14,usa
+cocacola,9.8,france
+sprite,89.2,china
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -1000,6 +1000,18 @@ def test_read_csv_newlines_in_quotes(self, nrows, skiprows):
             cast_to_str=StorageFormat.get() != "Omnisci",
         )
 
+    @pytest.mark.parametrize("skiprows", [None, 0, [], [1, 2], np.arange(0, 2)])
+    def test_read_csv_skiprows_with_usecols(self, skiprows):
+        usecols = {"float_data": "float64"}
+        eval_io(
+            fn_name="read_csv",
+            # read_csv kwargs
+            filepath_or_buffer="modin/pandas/test/data/issue_4543.csv",
+            skiprows=skiprows,
+            usecols=usecols.keys(),
+            dtype=usecols,
+        )
+
     def test_read_csv_sep_none(self):
         eval_io(
             fn_name="read_csv",