Repository: mars-project/mars

Issue:
[BUG] Creating DataFrame from dict does not support scalar value
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
Creating DataFrame from dict does not support scalar value.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
In [7]: md.DataFrame({'a': '1', 'b': md.Series([1, 2, 3])}).execute()           
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-7-ec6db392e00f> in <module>
----> 1 md.DataFrame({'a': '1', 'b': md.Series([1, 2, 3])}).execute()

~/Workspace/mars/mars/dataframe/initializer.py in __init__(self, data, index, columns, dtype, copy, chunk_size, gpu, sparse)
     46             df = dataframe_from_1d_tensors(
     47                 tensors, index=index, columns=columns,
---> 48                 gpu=gpu, sparse=sparse)
     49         else:
     50             pdf = pd.DataFrame(data, index=index, columns=columns, dtype=dtype, copy=copy)

~/Workspace/mars/mars/dataframe/datasource/from_tensor.py in dataframe_from_1d_tensors(tensors, index, columns, gpu, sparse)
    328     op = DataFrameFromTensor(dtypes=dtypes, from_1d_tensors=True,
    329                              gpu=gpu, sparse=sparse)
--> 330     return op(tensors, index, columns)
    331 
    332 

~/Workspace/mars/mars/dataframe/datasource/from_tensor.py in __call__(self, input_tensor, index, columns)
     73     def __call__(self, input_tensor, index, columns):
     74         if self._from_1d_tensors:
---> 75             return self._call_input_1d_tensors(input_tensor, index, columns)
     76         else:
     77             return self._call_input_tensor(input_tensor, index, columns)

~/Workspace/mars/mars/dataframe/datasource/from_tensor.py in _call_input_1d_tensors(self, input_1d_tensors, index, columns)
     99     def _call_input_1d_tensors(self, input_1d_tensors, index, columns):
    100         if len({t.shape for t in input_1d_tensors}) != 1:
--> 101             raise ValueError('input 1-d tensors should have same shape')
    102 
    103         inputs = list(input_1d_tensors)

ValueError: input 1-d tensors should have same shape
```



Discussion:

Resolving patch:
diff --git a/mars/dataframe/datasource/from_tensor.py b/mars/dataframe/datasource/from_tensor.py
--- a/mars/dataframe/datasource/from_tensor.py
+++ b/mars/dataframe/datasource/from_tensor.py
@@ -21,7 +21,7 @@
 
 from ... import opcodes as OperandDef
 from ...core import Base, Entity
-from ...serialize import KeyField, SeriesField, DataTypeField, BoolField
+from ...serialize import KeyField, SeriesField, DataTypeField, AnyField
 from ...tensor.datasource import tensor as astensor
 from ...tensor.utils import unify_chunks
 from ...tiles import TilesError
@@ -37,15 +37,13 @@ class DataFrameFromTensor(DataFrameOperand, DataFrameOperandMixin):
     """
     _op_type_ = OperandDef.DATAFRAME_FROM_TENSOR
 
-    _input = KeyField('input')
+    _input = AnyField('input')
     _index = KeyField('index')
     _dtypes = SeriesField('dtypes')
-    _from_1d_tensors = BoolField('from_1d_tensors')
 
-    def __init__(self, index=None, dtypes=None, from_1d_tensors=None,
-                 gpu=None, sparse=None, **kw):
-        super().__init__(_index=index, _dtypes=dtypes, _from_1d_tensors=from_1d_tensors,
-                         _gpu=gpu, _sparse=sparse, _object_type=ObjectType.dataframe, **kw)
+    def __init__(self, input_=None, index=None, dtypes=None, gpu=None, sparse=None, **kw):
+        super().__init__(_input=input_, _index=index, _dtypes=dtypes, _gpu=gpu,
+                         _sparse=sparse, _object_type=ObjectType.dataframe, **kw)
 
     @property
     def dtypes(self):
@@ -59,20 +57,28 @@ def input(self):
     def index(self):
         return self._index
 
-    @property
-    def from_1d_tensors(self):
-        return self._from_1d_tensors
-
     def _set_inputs(self, inputs):
         super()._set_inputs(inputs)
-        if not self._from_1d_tensors:
-            self._input = inputs[0]
+        inputs_iter = iter(self._inputs)
+        if self._input is not None:
+            if not isinstance(self._input, dict):
+                self._input = next(inputs_iter)
+            else:
+                # check each value for input
+                new_input = OrderedDict()
+                for k, v in self._input.items():
+                    if isinstance(v, (Base, Entity)):
+                        new_input[k] = next(inputs_iter)
+                    else:
+                        new_input[k] = v
+                self._input = new_input
+
         if self._index is not None:
-            self._index = inputs[-1]
+            self._index = next(inputs_iter)
 
     def __call__(self, input_tensor, index, columns):
-        if self._from_1d_tensors:
-            return self._call_input_1d_tensors(input_tensor, index, columns)
+        if isinstance(input_tensor, dict):
+            return self._call_input_1d_tileables(input_tensor, index, columns)
         else:
             return self._call_input_tensor(input_tensor, index, columns)
 
@@ -96,34 +102,43 @@ def _process_index(self, index, inputs):
             index_value = parse_index(index, store_data=True)
         return index_value
 
-    def _call_input_1d_tensors(self, input_1d_tensors, index, columns):
-        if len({t.shape for t in input_1d_tensors}) != 1:
-            raise ValueError('input 1-d tensors should have same shape')
+    def _call_input_1d_tileables(self, input_1d_tileables, index, columns):
+        tileables = []
+        shape = None
+        for tileable in input_1d_tileables.values():
+            tileable_shape = astensor(tileable).shape
+            if len(tileable_shape) > 0:
+                if shape is None:
+                    shape = tileable_shape
+                elif shape != tileable_shape:
+                    raise ValueError('input 1-d tensors should have same shape')
+
+            if isinstance(tileable, (Base, Entity)):
+                tileables.append(tileable)
 
-        inputs = list(input_1d_tensors)
         if index is not None:
-            if input_1d_tensors[0].shape[0] != len(index):
+            if tileables[0].shape[0] != len(index):
                 raise ValueError(
                     'index {} should have the same shape with tensor: {}'.format(
-                        index, input_1d_tensors[0].shape[0]))
-            index_value = self._process_index(index, inputs)
+                        index, input_1d_tileables[0].shape[0]))
+            index_value = self._process_index(index, tileables)
         else:
-            index_value = parse_index(pd.RangeIndex(0, input_1d_tensors[0].shape[0]))
+            index_value = parse_index(pd.RangeIndex(0, tileables[0].shape[0]))
 
         if columns is not None:
-            if len(input_1d_tensors) != len(columns):
+            if len(input_1d_tileables) != len(columns):
                 raise ValueError(
-                    'columns {0} should have size {1}'.format(columns, len(input_1d_tensors)))
+                    'columns {0} should have size {1}'.format(columns, len(input_1d_tileables)))
             if not isinstance(columns, pd.Index):
                 if isinstance(columns, Base):
                     raise NotImplementedError('The columns value cannot be a tileable')
                 columns = pd.Index(columns)
             columns_value = parse_index(columns, store_data=True)
         else:
-            columns_value = parse_index(pd.RangeIndex(0, len(input_1d_tensors)), store_data=True)
+            columns_value = parse_index(pd.RangeIndex(0, len(input_1d_tileables)), store_data=True)
 
-        shape = (input_1d_tensors[0].shape[0], len(input_1d_tensors))
-        return self.new_dataframe(inputs, shape, dtypes=self.dtypes,
+        shape = (shape[0], len(input_1d_tileables))
+        return self.new_dataframe(tileables, shape, dtypes=self.dtypes,
                                   index_value=index_value, columns_value=columns_value)
 
     def _call_input_tensor(self, input_tensor, index, columns):
@@ -171,13 +186,13 @@ def tile(cls, op):
         # make sure all tensor have known chunk shapes
         check_chunks_unknown_shape(op.inputs, TilesError)
 
-        if op.from_1d_tensors:
-            return cls._tile_input_1d_tensors(op)
+        if isinstance(op.input, dict):
+            return cls._tile_input_1d_tileables(op)
         else:
             return cls._tile_input_tensor(op)
 
     @classmethod
-    def _tile_input_1d_tensors(cls, op):
+    def _tile_input_1d_tileables(cls, op):
         out_df = op.outputs[0]
         in_tensors = op.inputs
         in_tensors = unify_chunks(*in_tensors)
@@ -187,6 +202,19 @@ def _tile_input_1d_tensors(cls, op):
         out_chunks = []
         for i in range(in_tensors[0].chunk_shape[0]):
             chunk_op = op.copy().reset_key()
+            new_input = OrderedDict()
+            for k, v in op.input.items():
+                if not isinstance(v, (Base, Entity)):
+                    try:
+                        new_input[k] = v[cum_sizes[i]: cum_sizes[i + 1]]
+                    except TypeError:
+                        # scalar
+                        new_input[k] = v
+                else:
+                    # do not need to do slice,
+                    # will be done in set_inputs
+                    new_input[k] = v
+            chunk_op._input = new_input
             columns_value = out_df.columns_value
             dtypes = out_df.dtypes
             chunk_index = (i, 0)
@@ -286,16 +314,17 @@ def _tile_input_tensor(cls, op):
     def execute(cls, ctx, op):
         chunk = op.outputs[0]
 
-        if op.from_1d_tensors:
+        if isinstance(op.input, dict):
             d = OrderedDict()
-            tensors = [ctx[inp.key] for inp in op.inputs]
+            for k, v in op.input.items():
+                if hasattr(v, 'key'):
+                    d[k] = ctx[v.key]
+                else:
+                    d[k] = v
             if op.index is not None:
-                tensors_data, index_data = tensors[:-1], tensors[-1]
+                index_data = ctx[op.index.key]
             else:
-                tensors_data = tensors
                 index_data = chunk.index_value.to_pandas()
-            for name, data_1d in zip(chunk.dtypes.index, tensors_data):
-                d[name] = data_1d
             ctx[chunk.key] = pd.DataFrame(d, index=index_data,
                                           columns=chunk.columns_value.to_pandas())
         else:
@@ -317,17 +346,21 @@ def dataframe_from_tensor(tensor, index=None, columns=None, gpu=None, sparse=Fal
     except IndexError:
         col_num = 1
     gpu = tensor.op.gpu if gpu is None else gpu
-    op = DataFrameFromTensor(dtypes=pd.Series([tensor.dtype] * col_num, index=columns),
+    op = DataFrameFromTensor(input_=tensor,
+                             dtypes=pd.Series([tensor.dtype] * col_num, index=columns),
                              gpu=gpu, sparse=sparse)
     return op(tensor, index, columns)
 
 
-def dataframe_from_1d_tensors(tensors, index=None, columns=None, gpu=None, sparse=False):
-    gpu = tensors[0].op.gpu if gpu is None else gpu
-    dtypes = pd.Series([t.dtype for t in tensors], index=columns)
-    op = DataFrameFromTensor(dtypes=dtypes, from_1d_tensors=True,
+def dataframe_from_1d_tileables(d, index=None, columns=None, gpu=None, sparse=False):
+    tileables = list(d.values())
+    columns = list(d.keys()) if columns is None else columns
+    gpu = next(t.op.gpu for t in tileables if hasattr(t, 'op')) if gpu is None else gpu
+    dtypes = pd.Series([t.dtype if hasattr(t, 'dtype') else pd.Series(t).dtype
+                        for t in tileables], index=columns)
+    op = DataFrameFromTensor(input_=d, dtypes=dtypes,
                              gpu=gpu, sparse=sparse)
-    return op(tensors, index, columns)
+    return op(d, index, columns)
 
 
 class SeriesFromTensor(DataFrameOperand, DataFrameOperandMixin):
diff --git a/mars/dataframe/initializer.py b/mars/dataframe/initializer.py
--- a/mars/dataframe/initializer.py
+++ b/mars/dataframe/initializer.py
@@ -16,12 +16,11 @@
 
 from ..core import Base, Entity
 from ..tensor.core import TENSOR_TYPE
-from ..tensor import tensor as astensor
 from .core import DATAFRAME_TYPE, SERIES_TYPE, DataFrame as _Frame, Series as _Series
 from .datasource.dataframe import from_pandas as from_pandas_df
 from .datasource.series import from_pandas as from_pandas_series
 from .datasource.from_tensor import dataframe_from_tensor, series_from_tensor, \
-    dataframe_from_1d_tensors
+    dataframe_from_1d_tileables
 
 
 class DataFrame(_Frame):
@@ -39,13 +38,8 @@ def __init__(self, data=None, index=None, columns=None, dtype=None, copy=False,
                 df = data
         elif isinstance(data, dict) and any(isinstance(v, (Base, Entity)) for v in data.values()):
             # data is a dict and some value is tensor
-            columns = list(data.keys()) if columns is None else columns
-            tensors = []
-            for c in columns:
-                tensors.append(astensor(data[c]))
-            df = dataframe_from_1d_tensors(
-                tensors, index=index, columns=columns,
-                gpu=gpu, sparse=sparse)
+            df = dataframe_from_1d_tileables(
+                data, index=index, columns=columns, gpu=gpu, sparse=sparse)
         else:
             pdf = pd.DataFrame(data, index=index, columns=columns, dtype=dtype, copy=copy)
             df = from_pandas_df(pdf, chunk_size=chunk_size, gpu=gpu, sparse=sparse)
diff --git a/mars/executor.py b/mars/executor.py
--- a/mars/executor.py
+++ b/mars/executor.py
@@ -537,7 +537,8 @@ def _submit_operand_to_execute(self):
             # error happens, ignore
             return
 
-        to_submit_op = self._queue.pop(0)
+        with self._lock:
+            to_submit_op = self._queue.pop(0)
         assert to_submit_op.key not in self._submitted_op_keys
         self._submitted_op_keys.add(to_submit_op.key)
 


Test patch:
diff --git a/mars/dataframe/datasource/tests/test_datasource.py b/mars/dataframe/datasource/tests/test_datasource.py
--- a/mars/dataframe/datasource/tests/test_datasource.py
+++ b/mars/dataframe/datasource/tests/test_datasource.py
@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from weakref import ReferenceType
 import os
 import tempfile
 import shutil
+from collections import OrderedDict
+from weakref import ReferenceType
 
 import numpy as np
 import pandas as pd
@@ -29,7 +30,7 @@
 from mars.dataframe.datasource.series import from_pandas as from_pandas_series
 from mars.dataframe.datasource.index import from_pandas as from_pandas_index, from_tileable
 from mars.dataframe.datasource.from_tensor import dataframe_from_tensor, \
-    series_from_tensor, dataframe_from_1d_tensors
+    series_from_tensor, dataframe_from_1d_tileables
 from mars.dataframe.datasource.from_records import from_records
 from mars.dataframe.datasource.read_csv import read_csv, DataFrameReadCSV
 from mars.dataframe.datasource.read_sql_table import read_sql_table, DataFrameReadSQLTable
@@ -414,8 +415,9 @@ def testFromTensor(self):
         pd.testing.assert_index_equal(series.chunks[2].index_value.to_pandas(), pd.RangeIndex(8, 10))
         self.assertEqual(series.chunks[2].name, 'a')
 
-        df = dataframe_from_1d_tensors([mt.tensor(np.random.rand(4)),
-                                        mt.tensor(np.random.rand(4))])
+        d = OrderedDict([(0, mt.tensor(np.random.rand(4))),
+                         (1, mt.tensor(np.random.rand(4)))])
+        df = dataframe_from_1d_tileables(d)
         pd.testing.assert_index_equal(df.columns_value.to_pandas(), pd.RangeIndex(2))
 
         df = df.tiles()
@@ -448,20 +450,20 @@ def testFromTensor(self):
             dataframe_from_tensor(mt.tensor(np.random.rand(3, 2)),
                                   index=mt.tensor(np.random.rand(3, 2)))
 
-        # 1-d tensors should have same shapen
+        # 1-d tensors should have same shape
         with self.assertRaises(ValueError):
-            dataframe_from_1d_tensors([mt.tensor(np.random.rand(3)),
-                                       mt.tensor(np.random.rand(2))])
+            dataframe_from_1d_tileables(OrderedDict([(0, mt.tensor(np.random.rand(3))),
+                                                     (1, mt.tensor(np.random.rand(2)))]))
 
         # index has wrong shape
         with self.assertRaises(ValueError):
-            dataframe_from_1d_tensors([mt.tensor(np.random.rand(3))],
-                                      index=mt.tensor(np.random.rand(2)))
+            dataframe_from_1d_tileables({0: mt.tensor(np.random.rand(3))},
+                                        index=mt.tensor(np.random.rand(2)))
 
         # columns have wrong shape
         with self.assertRaises(ValueError):
-            dataframe_from_1d_tensors([mt.tensor(np.random.rand(3))],
-                                      columns=['a', 'b'])
+            dataframe_from_1d_tileables({0: mt.tensor(np.random.rand(3))},
+                                        columns=['a', 'b'])
 
         # index should be 1-d
         with self.assertRaises(ValueError):
diff --git a/mars/dataframe/datasource/tests/test_datasource_execution.py b/mars/dataframe/datasource/tests/test_datasource_execution.py
--- a/mars/dataframe/datasource/tests/test_datasource_execution.py
+++ b/mars/dataframe/datasource/tests/test_datasource_execution.py
@@ -28,7 +28,7 @@
 from mars.dataframe.datasource.dataframe import from_pandas as from_pandas_df
 from mars.dataframe.datasource.series import from_pandas as from_pandas_series
 from mars.dataframe.datasource.index import from_pandas as from_pandas_index, from_tileable
-from mars.dataframe.datasource.from_tensor import dataframe_from_tensor, dataframe_from_1d_tensors
+from mars.dataframe.datasource.from_tensor import dataframe_from_tensor, dataframe_from_1d_tileables
 from mars.dataframe.datasource.from_records import from_records
 
 
@@ -204,8 +204,12 @@ def testFromTensorExecution(self):
         # from 1d tensors
         raws8 = [('a', np.random.rand(8)), ('b', np.random.randint(10, size=8)),
                  ('c', [''.join(np.random.choice(list(printable), size=6)) for _ in range(8)])]
-        tensors8 = [mt.tensor(r[1], chunk_size=3) for r in raws8]
-        df8 = dataframe_from_1d_tensors(tensors8, columns=[r[0] for r in raws8])
+        tensors8 = OrderedDict((r[0], mt.tensor(r[1], chunk_size=3)) for r in raws8)
+        raws8.append(('d', 1))
+        raws8.append(('e', pd.date_range('2020-1-1', periods=8)))
+        tensors8['d'] = 1
+        tensors8['e'] = raws8[-1][1]
+        df8 = dataframe_from_1d_tileables(tensors8, columns=[r[0] for r in raws8])
         result = self.executor.execute_dataframe(df8, concat=True)[0]
         pdf_expected = pd.DataFrame(OrderedDict(raws8))
         pd.testing.assert_frame_equal(result, pdf_expected)
@@ -213,15 +217,15 @@ def testFromTensorExecution(self):
         # from 1d tensors and specify index with a tensor
         index_raw9 = np.random.rand(8)
         index9 = mt.tensor(index_raw9, chunk_size=4)
-        df9 = dataframe_from_1d_tensors(tensors8, columns=[r[0] for r in raws8],
-                                        index=index9)
+        df9 = dataframe_from_1d_tileables(tensors8, columns=[r[0] for r in raws8],
+                                          index=index9)
         result = self.executor.execute_dataframe(df9, concat=True)[0]
         pdf_expected = pd.DataFrame(OrderedDict(raws8), index=index_raw9)
         pd.testing.assert_frame_equal(result, pdf_expected)
 
         # from 1d tensors and specify index
-        df11 = dataframe_from_1d_tensors(tensors8, columns=[r[0] for r in raws8],
-                                         index=md.date_range('2020-1-1', periods=8))
+        df11 = dataframe_from_1d_tileables(tensors8, columns=[r[0] for r in raws8],
+                                           index=md.date_range('2020-1-1', periods=8))
         result = self.executor.execute_dataframe(df11, concat=True)[0]
         pdf_expected = pd.DataFrame(OrderedDict(raws8),
                                     index=pd.date_range('2020-1-1', periods=8))