Repository: modin-project/modin

Issue:
`read_csv` fails with some callable `skiprows` parameters
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Any
- **Modin version**: c2b399a0d69baa519dab67acd715061e93986b22
- **Python version**: 3.8.12
- **Code we can use to reproduce**:
```
import os
import modin.pandas as pd
from modin.config import NPartitions

NPartitions.put(4)

test_filename = "test.csv"
data = """col1,col2
1,2
3,4
5,6
7,8
"""

try :
    with open(test_filename, "w") as f:
        f.write(data)

    df = pd.read_csv(test_filename, skiprows=lambda x: x in [2,3])

finally:
    os.remove(test_filename)
```
<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->
```
Traceback (most recent call last):
  File "test.py", line 1189, in <module>
    df = pd.read_csv(test_filename, skiprows=lambda x: x in [2,3])
  File "/modin/modin/pandas/io.py", line 131, in read_csv
    return _read(**kwargs)
  File "/modin/modin/pandas/io.py", line 59, in _read
    pd_obj = FactoryDispatcher.read_csv(**kwargs)
  File "/modin/modin/data_management/factories/dispatcher.py", line 178, in read_csv
    return cls.__factory._read_csv(**kwargs)
  File "/modin/modin/data_management/factories/factories.py", line 206, in _read_csv
    return cls.io_cls.read_csv(**kwargs)
  File "/modin/modin/engines/base/io/file_dispatcher.py", line 68, in read
    query_compiler = cls._read(*args, **kwargs)
  File "/modin/modin/engines/base/io/text/csv_dispatcher.py", line 151, in _read
    new_query_compiler = cls._get_new_qc(
  File "/modin/modin/engines/base/io/text/csv_dispatcher.py", line 333, in _get_new_qc
    mod_index = skiprows_md(index_range)
  File "/modin/modin/engines/base/io/text/csv_dispatcher.py", line 459, in skiprows_func
    return skiprows(x + header_size)
  File "test.py", line 1189, in <lambda>
    df = pd.read_csv(test_filename, skiprows=lambda x: x in [2,3])
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
```



Discussion:

Resolving patch:
diff --git a/modin/core/io/text/text_file_dispatcher.py b/modin/core/io/text/text_file_dispatcher.py
--- a/modin/core/io/text/text_file_dispatcher.py
+++ b/modin/core/io/text/text_file_dispatcher.py
@@ -889,10 +889,10 @@ def _get_new_qc(
                     index=index_range.delete(skiprows_md)
                 )
             elif callable(skiprows_md):
-                mod_index = skiprows_md(index_range)
-                if not isinstance(mod_index, np.ndarray):
-                    mod_index = mod_index.to_numpy("bool")
-                view_idx = index_range[~mod_index]
+                skip_mask = cls._get_skip_mask(index_range, skiprows_md)
+                if not isinstance(skip_mask, np.ndarray):
+                    skip_mask = skip_mask.to_numpy("bool")
+                view_idx = index_range[~skip_mask]
                 new_query_compiler = new_query_compiler.view(index=view_idx)
             else:
                 raise TypeError(
@@ -1037,3 +1037,33 @@ def _read(cls, filepath_or_buffer: FilePathOrBuffer, **kwargs):
             nrows=kwargs["nrows"] if should_handle_skiprows else None,
         )
         return new_query_compiler
+
+    @classmethod
+    def _get_skip_mask(cls, rows_index: pandas.Index, skiprows: Callable):
+        """
+        Get mask of skipped by callable `skiprows` rows.
+
+        Parameters
+        ----------
+        rows_index : pandas.Index
+            Rows index to get mask for.
+        skiprows : Callable
+            Callable to check whether row index should be skipped.
+
+        Returns
+        -------
+        pandas.Index
+        """
+        try:
+            # direct `skiprows` call is more efficient than using of
+            # map method, but in some cases it can work incorrectly, e.g.
+            # when `skiprows` contains `in` operator
+            mask = skiprows(rows_index)
+            assert is_list_like(mask)
+        except (ValueError, TypeError, AssertionError):
+            # ValueError can be raised if `skiprows` callable contains membership operator
+            # TypeError is raised if `skiprows` callable contains bitwise operator
+            # AssertionError is raised if unexpected behavior was detected
+            mask = rows_index.map(skiprows)
+
+        return mask


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -1132,6 +1132,30 @@ def test_read_csv_empty_frame(self):
             index_col="col1",
         )
 
+    @pytest.mark.parametrize(
+        "skiprows",
+        [
+            lambda x: x > 20,
+            lambda x: True,
+            lambda x: x in [10, 20],
+            pytest.param(
+                lambda x: x << 10,
+                marks=pytest.mark.skipif(
+                    condition="config.getoption('--simulate-cloud').lower() != 'off'",
+                    reason="The reason of tests fail in `cloud` mode is unknown for now - issue #2340",
+                ),
+            ),
+        ],
+    )
+    def test_read_csv_skiprows_corner_cases(self, skiprows):
+        eval_io(
+            fn_name="read_csv",
+            check_kwargs_callable=not callable(skiprows),
+            # read_csv kwargs
+            filepath_or_buffer=pytest.csvs_names["test_read_csv_regular"],
+            skiprows=skiprows,
+        )
+
 
 class TestTable:
     def test_read_table(self, make_csv_file):