Repository: mars-project/mars

Issue:
[BUG] Serializers cannot serialize pandas.Timestamp correctly
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Serializers cannot serialize pandas.Timestamp correctly.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
In [112]: import numpy as np                                                    

In [113]: import pandas as pd                                                   

In [114]: import mars.dataframe as md                                           

In [115]: pdf = pd.DataFrame(np.random.rand(3, 3), index=pd.date_range('2020-1-1
     ...: ', periods=3))                                                        

In [116]: df = md.DataFrame(pdf)                                                

In [117]: df.to_pb()                                                            
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_value()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_untyped_value()

TypeError: Unknown type to serialize: <class 'pandas._libs.tslibs.timestamps.Timestamp'>

The above exception was the direct cause of the following exception:

TypeError                                 Traceback (most recent call last)
<ipython-input-117-6fec40d3e7d0> in <module>
----> 1 df.to_pb()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.to_pb()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.serialize()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Provider.serialize_model()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._serial_reference_value()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.serialize()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Provider.serialize_model()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._serial_reference_value()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.serialize()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Provider.serialize_model()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.serialize()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider.serialize_field()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_value()

~/Workspace/mars/mars/serialize/pbserializer.pyx in mars.serialize.pbserializer.ProtobufSerializeProvider._set_untyped_value()

TypeError: Failed to set field `max_val` for <mars.dataframe.core.IndexValue.DatetimeIndex object at 0xa1f15def0>, reason: Unknown type to serialize: <class 'pandas._libs.tslibs.timestamps.Timestamp'>
```




Discussion:

Resolving patch:
diff --git a/mars/dataframe/core.py b/mars/dataframe/core.py
--- a/mars/dataframe/core.py
+++ b/mars/dataframe/core.py
@@ -25,7 +25,7 @@
     is_eager_mode, build_mode
 from ..core import ChunkData, Chunk, TileableEntity, TileableData, HasShapeTileableData
 from ..serialize import Serializable, ValueType, ProviderType, DataTypeField, AnyField, \
-    SeriesField, BoolField, Int64Field, Int32Field, StringField, ListField, SliceField, \
+    SeriesField, BoolField, Int32Field, StringField, ListField, SliceField, \
     TupleField, OneOfField, ReferenceField, NDArrayField
 
 
@@ -126,7 +126,7 @@ class DatetimeIndex(IndexBase):
         _data = NDArrayField('data')
         _freq = AnyField('freq')
         _start = AnyField('start')
-        _periods = Int64Field('periods')
+        _periods = AnyField('periods')
         _end = AnyField('end')
         _closed = AnyField('closed')
         _tz = AnyField('tz')
@@ -140,7 +140,7 @@ class TimedeltaIndex(IndexBase):
         _unit = AnyField('unit')
         _freq = AnyField('freq')
         _start = AnyField('start')
-        _periods = Int64Field('periods')
+        _periods = AnyField('periods')
         _end = AnyField('end')
         _closed = AnyField('closed')
 
@@ -149,7 +149,7 @@ class PeriodIndex(IndexBase):
         _data = NDArrayField('data')
         _freq = AnyField('freq')
         _start = AnyField('start')
-        _periods = Int64Field('periods')
+        _periods = AnyField('periods')
         _end = AnyField('end')
         _year = AnyField('year')
         _month = AnyField('month')


Test patch:
diff --git a/mars/dataframe/datasource/tests/test_datasource.py b/mars/dataframe/datasource/tests/test_datasource.py
--- a/mars/dataframe/datasource/tests/test_datasource.py
+++ b/mars/dataframe/datasource/tests/test_datasource.py
@@ -72,7 +72,8 @@ def testChunkSerialize(self):
 
     def testDataFrameGraphSerialize(self):
         df = from_pandas_df(pd.DataFrame(np.random.rand(10, 10),
-                                         columns=[np.random.bytes(10) for _ in range(10)]))
+                                         columns=pd.timedelta_range(start='1 day', periods=10),
+                                         index=pd.date_range('2020-1-1', periods=10)))
         graph = df.build_graph(tiled=False)
 
         pb = graph.to_pb()