Repository: mars-project/mars

Issue:
[BUG] Creating DataFrame failed if columns provided
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Creating DataFrame failed if columns provided.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
In [1]: import mars.dataframe as md                                             

In [2]: import mars.tensor as mt                                                

In [3]: df = md.DataFrame(mt.random.rand(10, 3), columns=list('abc'))           
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-3-8015891094de> in <module>
----> 1 df = md.DataFrame(mt.random.rand(10, 3), columns=list('abc'))

~/Workspace/mars/mars/dataframe/initializer.py in __init__(self, data, index, columns, dtype, copy, chunk_size, gpu, sparse)
     29                  chunk_size=None, gpu=None, sparse=None):
     30         if isinstance(data, TENSOR_TYPE):
---> 31             df = from_tensor(data, index=index, columns=columns, gpu=gpu, sparse=sparse)
     32         elif isinstance(data, DATAFRAME_TYPE):
     33             df = data

~/Workspace/mars/mars/dataframe/datasource/from_tensor.py in from_tensor(tensor, index, columns, gpu, sparse)
    135     gpu = tensor.op.gpu if gpu is None else gpu
    136     op = DataFrameFromTensor(dtypes=pd.Series([tensor.dtype] * col_num), gpu=gpu, sparse=sparse)
--> 137     return op(tensor, index, columns)

~/Workspace/mars/mars/dataframe/datasource/from_tensor.py in __call__(self, input_tensor, index, columns)
     51     def __call__(self, input_tensor, index, columns):
     52         if index is not None or columns is not None:
---> 53             if input_tensor.shape != (len(index), len(columns)):
     54                 raise ValueError(
     55                     '({0},{1}) should have the same shape with tensor: {2}'.format(index, columns, input_tensor.shape))

TypeError: object of type 'NoneType' has no len()
```



Discussion:
I could take a look at this issue, should be a minor bug.
> I could take a look at this issue, should be a minor bug.

Great! 

Better add cases for specify only `columns` or `index` when test `from_tensor`.

Resolving patch:
diff --git a/mars/dataframe/core.py b/mars/dataframe/core.py
--- a/mars/dataframe/core.py
+++ b/mars/dataframe/core.py
@@ -253,7 +253,7 @@ def min_max(self):
     def has_value(self):
         if isinstance(self._index_value, self.RangeIndex):
             return True
-        elif getattr(self, '_data', None) is not None:
+        elif getattr(self._index_value, '_data', None) is not None:
             return True
         return False
 
diff --git a/mars/dataframe/datasource/from_tensor.py b/mars/dataframe/datasource/from_tensor.py
--- a/mars/dataframe/datasource/from_tensor.py
+++ b/mars/dataframe/datasource/from_tensor.py
@@ -17,6 +17,7 @@
 import numpy as np
 import pandas as pd
 
+from ...core import Base
 from ...serialize import KeyField, SeriesField
 from ... import opcodes as OperandDef
 from ..operands import DataFrameOperand, DataFrameOperandMixin, ObjectType
@@ -49,26 +50,36 @@ def _set_inputs(self, inputs):
         self._input = inputs[0]
 
     def __call__(self, input_tensor, index, columns):
-        if index is not None or columns is not None:
-            if input_tensor.shape != (len(index), len(columns)):
-                raise ValueError(
-                    '({0},{1}) should have the same shape with tensor: {2}'.format(index, columns, input_tensor.shape))
-        if input_tensor.ndim == 1:
-            # convert to 1-d DataFrame
-            index_value = parse_index(pd.RangeIndex(start=0, stop=input_tensor.shape[0]))
-            columns_value = parse_index(pd.RangeIndex(start=0, stop=1))
-        elif input_tensor.ndim != 2:
+        if input_tensor.ndim != 1 and input_tensor.ndim != 2:
             raise ValueError('Must pass 1-d or 2-d input')
-        else:
-            # convert to DataFrame
-            index_value = parse_index(pd.RangeIndex(start=0, stop=input_tensor.shape[0]))
-            columns_value = parse_index(pd.RangeIndex(start=0, stop=input_tensor.shape[1]))
 
-        # overwrite index_value and columns_value if user has set them
         if index is not None:
+            if input_tensor.shape[0] != len(index):
+                raise ValueError(
+                    'index {0} should have the same shape with tensor: {1}'.format(index, input_tensor.shape[0]))
+            if not isinstance(index, pd.Index):
+                if isinstance(index, Base):
+                    raise NotImplementedError('The index value cannot be a tileable')
+                index = pd.Index(index)
             index_value = parse_index(index, store_data=True)
+        else:
+            index_value = parse_index(pd.RangeIndex(start=0, stop=input_tensor.shape[0]))
+
         if columns is not None:
+            if input_tensor.shape[1] != len(columns):
+                raise ValueError(
+                    'columns {0} should have the same shape with tensor: {1}'.format(columns, input_tensor.shape[1]))
+            if not isinstance(columns, pd.Index):
+                if isinstance(index, Base):
+                    raise NotImplementedError('The index value cannot be a tileable')
+                columns = pd.Index(columns)
             columns_value = parse_index(columns, store_data=True)
+        else:
+            if input_tensor.ndim == 1:
+                # convert to 1-d DataFrame
+                columns_value = parse_index(pd.RangeIndex(start=0, stop=1), store_data=True)
+            else:
+                columns_value = parse_index(pd.RangeIndex(start=0, stop=input_tensor.shape[1]), store_data=True)
 
         return self.new_dataframe([input_tensor], input_tensor.shape, dtypes=self.dtypes,
                                   index_value=index_value,
@@ -99,7 +110,7 @@ def tile(cls, op):
                                             store_data=True)
 
             index_stop = cum_size[0][i]
-            if out_df.index_value is not None:
+            if out_df.index_value.has_value():
                 index_value = parse_index(out_df.index_value.to_pandas()[index_stop - in_chunk.shape[0]:index_stop],
                                           store_data=True)
             else:


Test patch:
diff --git a/mars/dataframe/datasource/tests/test_datasource.py b/mars/dataframe/datasource/tests/test_datasource.py
--- a/mars/dataframe/datasource/tests/test_datasource.py
+++ b/mars/dataframe/datasource/tests/test_datasource.py
@@ -302,6 +302,22 @@ def testFromTensor(self):
         with self.assertRaises(TypeError):
             from_tensor(scalar)
 
+        # from tensor with given index
+        df = from_tensor(tensor, index=np.arange(0, 20, 2))
+        df.tiles()
+        pd.testing.assert_index_equal(df.chunks[0].index_value.to_pandas(), pd.Index(np.arange(0, 10, 2)))
+        pd.testing.assert_index_equal(df.chunks[1].index_value.to_pandas(), pd.Index(np.arange(0, 10, 2)))
+        pd.testing.assert_index_equal(df.chunks[2].index_value.to_pandas(), pd.Index(np.arange(10, 20, 2)))
+        pd.testing.assert_index_equal(df.chunks[3].index_value.to_pandas(), pd.Index(np.arange(10, 20, 2)))
+
+        # from tensor with given columns
+        df = from_tensor(tensor, columns=list('abcdefghij'))
+        df.tiles()
+        pd.testing.assert_index_equal(df.chunks[0].columns.to_pandas(), pd.Index(['a', 'b', 'c', 'd', 'e']))
+        pd.testing.assert_index_equal(df.chunks[1].columns.to_pandas(), pd.Index(['f', 'g', 'h', 'i', 'j']))
+        pd.testing.assert_index_equal(df.chunks[2].columns.to_pandas(), pd.Index(['a', 'b', 'c', 'd', 'e']))
+        pd.testing.assert_index_equal(df.chunks[3].columns.to_pandas(), pd.Index(['f', 'g', 'h', 'i', 'j']))
+
     def testFromRecords(self):
         dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')])
 
diff --git a/mars/dataframe/datasource/tests/test_datasource_execution.py b/mars/dataframe/datasource/tests/test_datasource_execution.py
--- a/mars/dataframe/datasource/tests/test_datasource_execution.py
+++ b/mars/dataframe/datasource/tests/test_datasource_execution.py
@@ -86,6 +86,22 @@ def testFromTensorExecution(self):
         pdf_expected = pd.DataFrame(self.executor.execute_tensor(tensor4, concat=True)[0])
         pd.testing.assert_frame_equal(pdf_expected, result4)
 
+        # from tensor with given index
+        tensor5 = mt.ones((10, 10), chunk_size=3)
+        df5 = from_tensor(tensor5, index=np.arange(0, 20, 2))
+        result5 = self.executor.execute_dataframe(df5, concat=True)[0]
+        pdf_expected = pd.DataFrame(self.executor.execute_tensor(tensor5, concat=True)[0],
+                                    index=np.arange(0, 20, 2))
+        pd.testing.assert_frame_equal(pdf_expected, result5)
+
+        # from tensor with given columns
+        tensor6 = mt.ones((10, 10), chunk_size=3)
+        df6 = from_tensor(tensor6, columns=list('abcdefghij'))
+        result6 = self.executor.execute_dataframe(df6, concat=True)[0]
+        pdf_expected = pd.DataFrame(self.executor.execute_tensor(tensor6, concat=True)[0],
+                                    columns=list('abcdefghij'))
+        pd.testing.assert_frame_equal(pdf_expected, result6)
+
     def testFromRecordsExecution(self):
         dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')])