Repository: modin-project/modin

Issue:
Cannot use function for usecols argument in read_csv
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Windows 10 and Ubuntu 18.04.3 LTS
- **Modin version** (`modin.__version__`): 0.8.1.1
- **Python version**: 3.6.9
- **Code we can use to reproduce**: https://colab.research.google.com/drive/1yylpw4TYqnGGUpgb4H3t7do7qnyua9w1?usp=sharing


### Describe the problem
Using a function  for the usecols argument with the read_csv function throws an exception.
This is because of the list call with as argument the function here: usecols = all_cols.get_indexer_for(list(usecols_md[0]))
See also the log.

### Source code / logs
**Source code:**   https://colab.research.google.com/drive/1yylpw4TYqnGGUpgb4H3t7do7qnyua9w1?usp=sharing

<details><summary>Traceback</summary>

```
TypeError                                 Traceback (most recent call last)

<ipython-input-4-6eaf150bb793> in <module>()
----> 1 df = pd.read_csv('/tmp/tmp_csv.csv',usecols=column_selector)
      2 df.head()

5 frames

/usr/local/lib/python3.6/dist-packages/modin/pandas/io.py in parser_func(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)
    110         if kwargs.get("sep", sep) is False:
    111             kwargs["sep"] = "\t"
--> 112         return _read(**kwargs)
    113 
    114     return parser_func

/usr/local/lib/python3.6/dist-packages/modin/pandas/io.py in _read(**kwargs)
    125     from modin.data_management.factories.dispatcher import EngineDispatcher
    126 
--> 127     pd_obj = EngineDispatcher.read_csv(**kwargs)
    128     # This happens when `read_csv` returns a TextFileReader object for iterating through
    129     if isinstance(pd_obj, pandas.io.parsers.TextFileReader):

/usr/local/lib/python3.6/dist-packages/modin/data_management/factories/dispatcher.py in read_csv(cls, **kwargs)
    111     @classmethod
    112     def read_csv(cls, **kwargs):
--> 113         return cls.__engine._read_csv(**kwargs)
    114 
    115     @classmethod

/usr/local/lib/python3.6/dist-packages/modin/data_management/factories/factories.py in _read_csv(cls, **kwargs)
     85     @classmethod
     86     def _read_csv(cls, **kwargs):
---> 87         return cls.io_cls.read_csv(**kwargs)
     88 
     89     @classmethod

/usr/local/lib/python3.6/dist-packages/modin/engines/base/io/file_reader.py in read(cls, *args, **kwargs)
     27     @classmethod
     28     def read(cls, *args, **kwargs):
---> 29         query_compiler = cls._read(*args, **kwargs)
     30         # TODO (devin-petersohn): Make this section more general for non-pandas kernel
     31         # implementations.

/usr/local/lib/python3.6/dist-packages/modin/engines/base/io/text/csv_reader.py in _read(cls, filepath_or_buffer, **kwargs)
     82                 **dict(kwargs, nrows=0, skipfooter=0),
     83             ).columns
---> 84             usecols = all_cols.get_indexer_for(list(usecols_md[0]))
     85         parse_dates = kwargs.pop("parse_dates", False)
     86         partition_kwargs = dict(

TypeError: 'function' object is not iterable
```

</details>




Discussion:
Hi @hannesdm sorry for the delay on responding!

We will get this worked into the next release, thanks for reporting with the reproducible example, extremely helpful!

Resolving patch:
diff --git a/modin/engines/base/io/text/csv_dispatcher.py b/modin/engines/base/io/text/csv_dispatcher.py
--- a/modin/engines/base/io/text/csv_dispatcher.py
+++ b/modin/engines/base/io/text/csv_dispatcher.py
@@ -13,7 +13,6 @@
 
 from modin.engines.base.io.text.text_file_dispatcher import TextFileDispatcher
 from modin.data_management.utils import compute_chunksize
-from pandas.io.parsers import _validate_usecols_arg
 import pandas
 import csv
 import sys
@@ -90,14 +89,6 @@ def _read(cls, filepath_or_buffer, **kwargs):
         column_names = empty_pd_df.columns
         skipfooter = kwargs.get("skipfooter", None)
         skiprows = kwargs.pop("skiprows", None)
-        usecols_md = _validate_usecols_arg(usecols)
-        if usecols is not None and usecols_md[1] != "integer":
-            del kwargs["usecols"]
-            all_cols = pandas.read_csv(
-                cls.file_open(filepath_or_buffer, "rb"),
-                **dict(kwargs, nrows=0, skipfooter=0),
-            ).columns
-            usecols = all_cols.get_indexer_for(list(usecols_md[0]))
         parse_dates = kwargs.pop("parse_dates", False)
         partition_kwargs = dict(
             kwargs,
@@ -106,7 +97,6 @@ def _read(cls, filepath_or_buffer, **kwargs):
             skipfooter=0,
             skiprows=None,
             parse_dates=parse_dates,
-            usecols=usecols,
         )
         encoding = kwargs.get("encoding", None)
         quotechar = kwargs.get("quotechar", '"').encode(


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -292,6 +292,13 @@ def test_read_csv_col_handling(
             skip_blank_lines=skip_blank_lines,
         )
 
+    @pytest.mark.parametrize("usecols", [lambda col_name: col_name in ["a", "b", "e"]])
+    def test_from_csv_with_callable_usecols(self, usecols):
+        fname = "modin/pandas/test/data/test_usecols.csv"
+        pandas_df = pandas.read_csv(fname, usecols=usecols)
+        modin_df = pd.read_csv(fname, usecols=usecols)
+        df_equals(modin_df, pandas_df)
+
     # General Parsing Configuration
     @pytest.mark.parametrize("dtype", [None, True])
     @pytest.mark.parametrize("engine", [None, "python", "c"])