Repository: mars-project/mars

Issue:
[BUG] `mt.transpose` failed when input tensor has unknown chunk shape
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

`mt.transpose` failed when input tensor has unknown chunk shape.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```python
import mars.dataframe as md
import mars.tensor as mt

df = md.DataFrame(mt.random.rand(50, 10, chunk_size=10),
                  columns=[f's{i}' for i in range(10)])
df = df[df['s0'] < 0]

t = df.to_tensor()
t.dot(t.T).execute()
```

Error:

```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()
    648             try:
--> 649                 self._set_tuple(value, field_obj, tp=field.type, weak_ref=field.weak_ref)
    650             except TypeError:

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_tuple()
    364                     it_obj = Value()
--> 365                     self._set_value(val, it_obj, tp=tp.type if tp is not None else tp)
    366                     res.append(it_obj)

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_value()
    559         else:
--> 560             cls._set_typed_value(value, obj, tp, weak_ref=weak_ref)
    561 

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_typed_value()
    443             value_field = PRIMITIVE_TYPE_TO_VALUE_FIELD[tp]
--> 444             setattr(obj, value_field, value)
    445         elif tp is ValueType.slice:

TypeError: 10.0 has type float, but expected one of: int, long

The above exception was the direct cause of the following exception:

TypeError                                 Traceback (most recent call last)
/home/admin/work/public-mars-0.5.1.zip/mars/scheduler/graph.py in _execute_graph()

/home/admin/work/public-mars-0.5.1.zip/mars/utils.py in _wrapped()

/home/admin/work/public-mars-0.5.1.zip/mars/scheduler/graph.py in create_operand_actors()

/home/admin/work/public-mars-0.5.1.zip/mars/utils.py in _wrapped()

/home/admin/work/public-mars-0.5.1.zip/mars/scheduler/graph.py in get_executable_operand_dag()

/home/admin/work/public-mars-0.5.1.zip/mars/utils.py in serialize_graph()

~/Workspace/mars/mars/graph.pyx in mars.graph.DirectedGraph.to_pb()
    420         return graph
--> 421 
    422     def to_pb(self, pb_obj=None, data_serial_type=None, pickle_protocol=None):

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.to_pb()
    686                                              pickle_protocol=pickle_protocol)
--> 687         return self.serialize(provider, obj=obj)
    688 

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.serialize()
    669     def serialize(self, Provider provider, obj=None):
--> 670         return provider.serialize_model(self, obj=obj)
    671 

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Provider.serialize_model()
    797         for name, field in model_instance._FIELDS.items():
--> 798             field.serialize(self, model_instance, obj)
    799 

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()
    154 
--> 155     cpdef serialize(self, Provider provider, model_instance, obj):
    156         return provider.serialize_field(self, model_instance, obj)

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()
    155     cpdef serialize(self, Provider provider, model_instance, obj):
--> 156         return provider.serialize_field(self, model_instance, obj)
    157 

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()
    630                     if val is not None:
--> 631                         self._serial_reference_value(tag, field.type.type.model, val, it_obj)
    632                     elif isinstance(it_obj, Value):

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._serial_reference_value()
    572                 field_obj = value.cls(self)()
--> 573                 value.serialize(self, obj=field_obj)
    574                 value_pb.type_id = value.__serializable_index__

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.serialize()
    669     def serialize(self, Provider provider, obj=None):
--> 670         return provider.serialize_model(self, obj=obj)
    671 

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Provider.serialize_model()
    797         for name, field in model_instance._FIELDS.items():
--> 798             field.serialize(self, model_instance, obj)
    799 

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()
    154 
--> 155     cpdef serialize(self, Provider provider, model_instance, obj):
    156         return provider.serialize_field(self, model_instance, obj)

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()
    155     cpdef serialize(self, Provider provider, model_instance, obj):
--> 156         return provider.serialize_field(self, model_instance, obj)
    157 

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()
    651                 exc_info = sys.exc_info()
--> 652                 raise TypeError(f'Failed to set field `{tag}` for {model_instance} with '
    653                                 f'value {value}, reason: {exc_info[1]}') \

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()
    648             try:
--> 649                 self._set_tuple(value, field_obj, tp=field.type, weak_ref=field.weak_ref)
    650             except TypeError:

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_tuple()
    364                     it_obj = Value()
--> 365                     self._set_value(val, it_obj, tp=tp.type if tp is not None else tp)
    366                     res.append(it_obj)

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_value()
    559         else:
--> 560             cls._set_typed_value(value, obj, tp, weak_ref=weak_ref)
    561 

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_typed_value()
    443             value_field = PRIMITIVE_TYPE_TO_VALUE_FIELD[tp]
--> 444             setattr(obj, value_field, value)
    445         elif tp is ValueType.slice:

TypeError: Failed to set field `shape` for Chunk <op=TensorTranspose, key=6fd87469bf886695950bdfb3164a7ce7> with value (10.0, -1), reason: 10.0 has type float, but expected one of: int, long
```



Discussion:

Resolving patch:
diff --git a/mars/learn/neighbors/_faiss.py b/mars/learn/neighbors/_faiss.py
--- a/mars/learn/neighbors/_faiss.py
+++ b/mars/learn/neighbors/_faiss.py
@@ -272,11 +272,12 @@ def _execute_one_chunk(cls, ctx, op):
 
     @classmethod
     def _execute_map(cls, ctx, op):
-        (data,), device_id, _ = as_same_device(
+        (data,), device_id, xp = as_same_device(
             [ctx[op.inputs[0].key]], device=op.device, ret_extra=True)
         index = ctx[op.inputs[1].key] if len(op.inputs) == 2 else None
 
         with device(device_id):
+            data = xp.ascontiguousarray(data)
             if index is not None:
                 # fetch the trained index
                 trained_index = _load_index(ctx, op, index, device_id)
diff --git a/mars/tensor/base/transpose.py b/mars/tensor/base/transpose.py
--- a/mars/tensor/base/transpose.py
+++ b/mars/tensor/base/transpose.py
@@ -47,7 +47,8 @@ def axes(self):
         return getattr(self, '_axes', None)
 
     def __call__(self, a):
-        shape = _reorder(a.shape, self._axes)
+        shape = tuple(s if np.isnan(s) else int(s)
+                      for s in _reorder(a.shape, self._axes))
         if self._axes == list(reversed(range(a.ndim))):
             # order reversed
             tensor_order = reverse_order(a.order)
@@ -74,7 +75,8 @@ def tile(cls, op):
         out_chunks = []
         for c in op.inputs[0].chunks:
             chunk_op = op.copy().reset_key()
-            chunk_shape = _reorder(c.shape, op.axes)
+            chunk_shape = tuple(s if np.isnan(s) else int(s)
+                                for s in _reorder(c.shape, op.axes))
             chunk_idx = _reorder(c.index, op.axes)
             out_chunk = chunk_op.new_chunk([c], shape=chunk_shape,
                                            index=chunk_idx, order=tensor.order)


Test patch:
diff --git a/mars/tensor/base/tests/test_base_execute.py b/mars/tensor/base/tests/test_base_execute.py
--- a/mars/tensor/base/tests/test_base_execute.py
+++ b/mars/tensor/base/tests/test_base_execute.py
@@ -18,6 +18,7 @@
 import scipy.sparse as sps
 import pandas as pd
 
+from mars import dataframe as md
 from mars import tensor as mt
 from mars.tensor.base import copyto, transpose, moveaxis, broadcast_to, broadcast_arrays, where, \
     expand_dims, rollaxis, atleast_1d, atleast_2d, atleast_3d, argwhere, array_split, split, \
@@ -142,6 +143,15 @@ def testTransposeExecution(self):
         self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS'])
         self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS'])
 
+        df = md.DataFrame(mt.random.rand(10, 5, chunk_size=5))
+        df = df[df[0] < 1]
+        # generate tensor with unknown shape
+        t = df.to_tensor()
+        t2 = transpose(t)
+
+        res = self.executor.execute_tensor(t2, concat=True)[0]
+        self.assertEqual(res.shape, (5, 10))
+
     def testSwapaxesExecution(self):
         raw = np.random.random((11, 8, 5))
         arr = swapaxes(raw, 2, 0)