Repository: mars-project/mars

Issue:
[BUG] DataFrame['field'] failed with `ValueError: invalid entry in coordinates array`
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

DataFrame['field'] failed with `ValueError: invalid entry in coordinates array`.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```python
import mars.dataframe as md

names = ["utdid ", "id","appid", "server_time", "os_version", "brand","resolution", "province", "city","ex_dt", "flag" ,
         "cmt_utdid","cate1", "cate2", "content", "cmtid","article_id", "title ", "item_type",
         "word_vec_content", "word_vec_title", "cmt_dt", 
         "read_user_f","cmt_user_f"]
usecols = ["province", "city","flag" ,
         "cate1", "cate2", "content","item_type",
         "word_vec_content", 
         "read_user_f","cmt_user_f"]
dtypes = {'flag':int}

data = md.read_csv('./data/train_data/test.csv', sep="\u0001", names=names, usecols = usecols, dtype=dtypes)
d = data.execute()
d['flag'].execute()
```

Error

```
In [20]: d.flag.execute()                                                       
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-20-68cd215e82a2> in <module>
----> 1 d.flag.execute()

~/Workspace/mars/mars/core.py in execute(self, session, **kw)
    641 
    642         if wait:
--> 643             return run()
    644         else:
    645             thread_executor = ThreadPoolExecutor(1)

~/Workspace/mars/mars/core.py in run()
    637 
    638         def run():
--> 639             self.data.execute(session, **kw)
    640             return self
    641 

~/Workspace/mars/mars/core.py in execute(self, session, **kw)
    377 
    378         if wait:
--> 379             return run()
    380         else:
    381             # leverage ThreadPoolExecutor to submit task,

~/Workspace/mars/mars/core.py in run()
    372         def run():
    373             # no more fetch, thus just fire run
--> 374             session.run(self, **kw)
    375             # return Tileable or ExecutableTuple itself
    376             return self

~/Workspace/mars/mars/session.py in run(self, *tileables, **kw)
    497         tileables = tuple(mt.tensor(t) if not isinstance(t, (Entity, Base)) else t
    498                           for t in tileables)
--> 499         result = self._sess.run(*tileables, **kw)
    500 
    501         for t in tileables:

~/Workspace/mars/mars/session.py in run(self, *tileables, **kw)
    106             # set number of running cores
    107             self.context.set_ncores(kw['n_parallel'])
--> 108             res = self._executor.execute_tileables(tileables, **kw)
    109             return res
    110 

~/Workspace/mars/mars/utils.py in _inner(*args, **kwargs)
    449         def _inner(*args, **kwargs):
    450             with self:
--> 451                 return func(*args, **kwargs)
    452 
    453         return _inner

~/Workspace/mars/mars/executor.py in execute_tileables(self, tileables, fetch, n_parallel, n_thread, print_progress, mock, compose, name)
    859                 # build chunk graph, tile will be done during building
    860                 chunk_graph = chunk_graph_builder.build(
--> 861                     tileables, tileable_graph=tileable_graph)
    862                 tileable_graph = chunk_graph_builder.prev_tileable_graph
    863                 temp_result_keys = set(result_keys)

~/Workspace/mars/mars/utils.py in _inner(*args, **kwargs)
    449         def _inner(*args, **kwargs):
    450             with self:
--> 451                 return func(*args, **kwargs)
    452 
    453         return _inner

~/Workspace/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    346 
    347         chunk_graph = super().build(
--> 348             tileables, tileable_graph=tileable_graph)
    349         self._iterative_chunk_graphs.append(chunk_graph)
    350         if len(self._interrupted_ops) == 0:

~/Workspace/mars/mars/utils.py in _inner(*args, **kwargs)
    449         def _inner(*args, **kwargs):
    450             with self:
--> 451                 return func(*args, **kwargs)
    452 
    453         return _inner

~/Workspace/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    260                     # for further execution
    261                     partial_tiled_chunks = \
--> 262                         self._on_tile_failure(tileable_data.op, exc_info)
    263                     if partial_tiled_chunks is not None and \
    264                             len(partial_tiled_chunks) > 0:

~/Workspace/mars/mars/tiles.py in inner(op, exc_info)
    299                     on_tile_failure(op, exc_info)
    300                 else:
--> 301                     raise exc_info[1].with_traceback(exc_info[2]) from None
    302         return inner
    303 

~/Workspace/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    240                 continue
    241             try:
--> 242                 tiled = self._tile(tileable_data, tileable_graph)
    243                 tiled_op.add(tileable_data.op)
    244                 for t, td in zip(tileable_data.op.outputs, tiled):

~/Workspace/mars/mars/tiles.py in _tile(self, tileable_data, tileable_graph)
    335         if any(inp.op in self._interrupted_ops for inp in tileable_data.inputs):
    336             raise TilesError('Tile fail due to failure of inputs')
--> 337         return super()._tile(tileable_data, tileable_graph)
    338 
    339     @enter_mode(build=True, kernel=True)

~/Workspace/mars/mars/tiles.py in _tile(self, tileable_data, tileable_graph)
    199                 t._nsplits = o.nsplits
    200         elif on_tile is None:
--> 201             tds[0]._inplace_tile()
    202         else:
    203             tds = on_tile(tileable_data.op.outputs, tds)

~/Workspace/mars/mars/core.py in _inplace_tile(self)
    166 
    167     def _inplace_tile(self):
--> 168         return handler.inplace_tile(self)
    169 
    170     def __getattr__(self, attr):

~/Workspace/mars/mars/tiles.py in inplace_tile(self, to_tile)
    134         if not to_tile.is_coarse():
    135             return to_tile
--> 136         dispatched = self.dispatch(to_tile.op)
    137         self._assign_to([d.data for d in dispatched], to_tile.op.outputs)
    138         return to_tile

~/Workspace/mars/mars/utils.py in _inner(*args, **kwargs)
    449         def _inner(*args, **kwargs):
    450             with self:
--> 451                 return func(*args, **kwargs)
    452 
    453         return _inner

~/Workspace/mars/mars/tiles.py in dispatch(self, op)
    117         else:
    118             try:
--> 119                 tiled = op_cls.tile(op)
    120             except NotImplementedError as ex:
    121                 cause = ex

~/Workspace/mars/mars/dataframe/indexing/getitem.py in tile(cls, op)
    269     def tile(cls, op):
    270         if op.col_names is not None:
--> 271             return cls.tile_with_columns(op)
    272         else:
    273             return cls.tile_with_mask(op)

~/Workspace/mars/mars/dataframe/indexing/getitem.py in tile_with_columns(cls, op)
    340             dtype = in_df.dtypes[col_names]
    341             for i in range(in_df.chunk_shape[0]):
--> 342                 c = in_df.cix[(i, column_index)]
    343                 op = DataFrameIndex(col_names=col_names)
    344                 out_chunks.append(op.new_chunk([c], shape=(c.shape[0],), index=(i,), dtype=dtype,

~/Workspace/mars/mars/core.py in __getitem__(self, item)
    714             indexes = tuple(zip(*itertools.product(*slices)))
    715 
--> 716             flat_index = np.ravel_multi_index(indexes, self._tileable.chunk_shape)
    717             if singleton:
    718                 return self._tileable._chunks[flat_index[0]]

<__array_function__ internals> in ravel_multi_index(*args, **kwargs)

ValueError: invalid entry in coordinates array
```

[test.csv.zip](https://github.com/mars-project/mars/files/5615097/test.csv.zip)




Discussion:

Resolving patch:
diff --git a/mars/dataframe/datasource/read_csv.py b/mars/dataframe/datasource/read_csv.py
--- a/mars/dataframe/datasource/read_csv.py
+++ b/mars/dataframe/datasource/read_csv.py
@@ -628,6 +628,15 @@ def read_csv(path, names=None, sep=',', index_col=None, compression=None, header
             b = f.read(head_end - head_start)
         mini_df = pd.read_csv(BytesIO(b), sep=sep, index_col=index_col, dtype=dtype,
                               names=names, header=header)
+        if names is None:
+            names = list(mini_df.columns)
+        else:
+            # if names specified, header should be None
+            header = None
+        if usecols:
+            usecols = usecols if isinstance(usecols, list) else [usecols]
+            col_index = sorted(mini_df.columns.get_indexer(usecols))
+            mini_df = mini_df.iloc[:, col_index]
 
     if isinstance(mini_df.index, pd.RangeIndex):
         index_value = parse_index(pd.RangeIndex(-1))
@@ -636,7 +645,6 @@ def read_csv(path, names=None, sep=',', index_col=None, compression=None, header
     columns_value = parse_index(mini_df.columns, store_data=True)
     if index_col and not isinstance(index_col, int):
         index_col = list(mini_df.columns).index(index_col)
-    names = list(mini_df.columns)
     op = DataFrameReadCSV(path=path, names=names, sep=sep, header=header, index_col=index_col,
                           usecols=usecols, compression=compression, gpu=gpu,
                           incremental_index=incremental_index, use_arrow_dtype=use_arrow_dtype,


Test patch:
diff --git a/mars/dataframe/datasource/tests/test_datasource_execution.py b/mars/dataframe/datasource/tests/test_datasource_execution.py
--- a/mars/dataframe/datasource/tests/test_datasource_execution.py
+++ b/mars/dataframe/datasource/tests/test_datasource_execution.py
@@ -324,6 +324,33 @@ def testReadCSVExecution(self):
             mdf = self.executor.execute_dataframe(md.read_csv(file_path, index_col=0, nrows=1), concat=True)[0]
             pd.testing.assert_frame_equal(df[:1], mdf)
 
+        # test names and usecols
+        with tempfile.TemporaryDirectory() as tempdir:
+            file_path = os.path.join(tempdir, 'test.csv')
+            df = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.int64),
+                              columns=['a', 'b', 'c'])
+            df.to_csv(file_path, index=False)
+
+            mdf = self.executor.execute_dataframe(md.read_csv(file_path,
+                                                              usecols=['c', 'b']), concat=True)[0]
+            pd.testing.assert_frame_equal(
+                pd.read_csv(file_path, usecols=['c', 'b']), mdf)
+
+            mdf = self.executor.execute_dataframe(md.read_csv(file_path, names=['a', 'b', 'c'],
+                                                              usecols=['c', 'b']), concat=True)[0]
+            pd.testing.assert_frame_equal(
+                pd.read_csv(file_path, names=['a', 'b', 'c'], usecols=['c', 'b']), mdf)
+
+            mdf = self.executor.execute_dataframe(md.read_csv(file_path, names=['a', 'b', 'c'],
+                                                              usecols=['a', 'c']), concat=True)[0]
+            pd.testing.assert_frame_equal(
+                pd.read_csv(file_path, names=['a', 'b', 'c'], usecols=['a', 'c']), mdf)
+
+            mdf = self.executor.execute_dataframe(
+                md.read_csv(file_path, usecols=['a', 'c']), concat=True)[0]
+            pd.testing.assert_frame_equal(
+                pd.read_csv(file_path, usecols=['a', 'c']), mdf)
+
         # test sep
         with tempfile.TemporaryDirectory() as tempdir:
             file_path = os.path.join(tempdir, 'test.csv')