Repository: mars-project/mars

Issue:
[BUG] Type erasion bug of structured numpy arrays
**Describe the bug**

The `dtype` is incorrectly erased when evaluating structured arrays. 

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version: 3.7
2. The version of Mars you use: master
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```python
In [1]: import mars.tensor as mt
In [2]: import numpy as np
In [3]: x = mt.array([(1,2),(3,4)], dtype=np.dtype([('h1', 'int32'), ('h2', 'double')]))
In [4]: from mars.deploy.local.core import new_cluster
In [5]: cluster = new_cluster(scheduler_n_process=2, worker_n_process=2,
   ...:                          shared_memory='20M')
In [6]: from mars.session import new_session
In [7]: sess = new_session(cluster.endpoint).as_default()
In [8]: x.execute()
Out[8]:
array([[b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF0\x3F',
        b'\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x40'],
       [b'\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x08\x40',
        b'\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x40']],
      dtype='|V12')
```

As you can see in the above code that the `dtype` is incorrectly interpreted.

**Expected behavior**

```python
array([[(1, 1.), (2, 2.)],
       [(3, 3.), (4, 4.)]], dtype=[('h1', '<i4'), ('h2', '<f8')])
```

**Additional context**

We have submitted a pr to arrow (apache/arrow#4953) and a temporary fix (#585) in our test case.



Discussion:

Resolving patch:
diff --git a/mars/compat.py b/mars/compat.py
--- a/mars/compat.py
+++ b/mars/compat.py
@@ -423,8 +423,89 @@ def _exitfunc(cls):
                     gc.enable()
 
 
+def apply_pyarrow_serialization_patch(serialization_context):
+    '''
+    Fix the bug about dtype serialization in pyarrow (pyarrow#4953).
+
+    From the JIRA of arrow, the fixed version of this bug is 1.0, so we only apply
+    the patch for pyarrow less than version 1.0.
+    '''
+    from distutils.version import LooseVersion
+    import numpy as np
+    import pyarrow
+
+    try:
+        # This function is available after numpy-0.16.0.
+        # See also: https://github.com/numpy/numpy/blob/master/numpy/lib/format.py
+        from numpy.lib.format import descr_to_dtype
+    except ImportError:
+        def descr_to_dtype(descr):
+            '''
+            descr may be stored as dtype.descr, which is a list of
+            (name, format, [shape]) tuples where format may be a str or a tuple.
+            Offsets are not explicitly saved, rather empty fields with
+            name, format == '', '|Vn' are added as padding.
+            This function reverses the process, eliminating the empty padding fields.
+            '''
+            if isinstance(descr, str):
+                # No padding removal needed
+                return np.dtype(descr)
+            elif isinstance(descr, tuple):
+                # subtype, will always have a shape descr[1]
+                dt = descr_to_dtype(descr[0])
+                return np.dtype((dt, descr[1]))
+            fields = []
+            offset = 0
+            for field in descr:
+                if len(field) == 2:
+                    name, descr_str = field
+                    dt = descr_to_dtype(descr_str)
+                else:
+                    name, descr_str, shape = field
+                    dt = np.dtype((descr_to_dtype(descr_str), shape))
+
+                # Ignore padding bytes, which will be void bytes with '' as name
+                # Once support for blank names is removed, only "if name == ''" needed)
+                is_pad = (name == '' and dt.type is np.void and dt.names is None)
+                if not is_pad:
+                    fields.append((name, dt, offset))
+
+                offset += dt.itemsize
+
+            names, formats, offsets = zip(*fields)
+            # names may be (title, names) tuples
+            nametups = (n  if isinstance(n, tuple) else (None, n) for n in names)
+            titles, names = zip(*nametups)
+            return np.dtype({'names': names, 'formats': formats, 'titles': titles,
+                                'offsets': offsets, 'itemsize': offset})
+
+    def _serialize_numpy_array_list(obj):
+        if obj.dtype.str != '|O':
+            # Make the array c_contiguous if necessary so that we can call change
+            # the view.
+            if not obj.flags.c_contiguous:
+                obj = np.ascontiguousarray(obj)
+            return obj.view('uint8'), np.lib.format.dtype_to_descr(obj.dtype)
+        else:
+            return obj.tolist(), np.lib.format.dtype_to_descr(obj.dtype)
+
+    def _deserialize_numpy_array_list(data):
+        if data[1] != '|O':
+            assert data[0].dtype == np.uint8
+            return data[0].view(descr_to_dtype(data[1]))
+        else:
+            return np.array(data[0], dtype=np.dtype(data[1]))
+
+    if LooseVersion(pyarrow.__version__) < LooseVersion('1.0'):
+        serialization_context.register_type(
+            np.ndarray, 'np.array',
+            custom_serializer=_serialize_numpy_array_list,
+            custom_deserializer=_deserialize_numpy_array_list)
+
+
 __all__ = ['PY27', 'sys', 'builtins', 'long_type', 'OrderedDict', 'dictconfig', 'suppress', 'reduce', 'reload_module',
            'Queue', 'PriorityQueue', 'Empty', 'urlretrieve', 'pickle', 'urlencode', 'urlparse', 'unquote',
            'quote', 'quote_plus', 'parse_qsl', 'Enum', 'ConfigParser', 'decimal', 'Decimal', 'DECIMAL_TYPES',
            'FixedOffset', 'utc', 'finalize', 'functools32', 'zip_longest', 'OrderedDict3', 'BrokenPipeError',
-           'TimeoutError', 'ConnectionResetError', 'ConnectionRefusedError', 'izip', 'accumulate']
+           'TimeoutError', 'ConnectionResetError', 'ConnectionRefusedError', 'izip', 'accumulate',
+           'apply_pyarrow_serialization_patch']
diff --git a/mars/serialize/dataserializer.py b/mars/serialize/dataserializer.py
--- a/mars/serialize/dataserializer.py
+++ b/mars/serialize/dataserializer.py
@@ -315,9 +315,11 @@ def _deserialize_sparse_csr_list(data):
 def mars_serialize_context():
     global _serialize_context
     if _serialize_context is None:
+        from ..compat import apply_pyarrow_serialization_patch
         ctx = pyarrow.default_serialization_context()
         ctx.register_type(SparseNDArray, 'mars.SparseNDArray',
                           custom_serializer=_serialize_sparse_csr_list,
                           custom_deserializer=_deserialize_sparse_csr_list)
+        apply_pyarrow_serialization_patch(ctx)
         _serialize_context = ctx
     return _serialize_context


Test patch:
diff --git a/mars/serialize/tests/test_serialize.py b/mars/serialize/tests/test_serialize.py
--- a/mars/serialize/tests/test_serialize.py
+++ b/mars/serialize/tests/test_serialize.py
@@ -527,6 +527,13 @@ def testDataSerialize(self):
                 assert_array_equal(array, dataserializer.loads(dataserializer.dumps(
                     array, compress=dataserializer.CompressType.GZIP)))
 
+            # test structured arrays.
+            rec_dtype = np.dtype([('a', 'int64'), ('b', 'double'), ('c', '<U8')])
+            array = np.ones((100,), dtype=rec_dtype)
+            array_loaded = dataserializer.loads(dataserializer.dumps(array))
+            self.assertEqual(array.dtype, array_loaded.dtype)
+            assert_array_equal(array, array_loaded)
+
             fn = os.path.join(tempfile.gettempdir(), 'test_dump_file_%d.bin' % id(self))
             try:
                 array = np.random.rand(1000, 100).T  # test non c-contiguous
diff --git a/mars/tests/test_mutable.py b/mars/tests/test_mutable.py
--- a/mars/tests/test_mutable.py
+++ b/mars/tests/test_mutable.py
@@ -263,7 +263,7 @@ def testWithGivenSession(session):
 
             # check dtype and value
             self.assertEqual(np.dtype(arr.dtype), expected.dtype)
-            np.testing.assert_array_equal(session.fetch(arr).view(dtype=rec_type), expected)
+            np.testing.assert_array_equal(session.fetch(arr), expected)
 
         with new_session().as_default() as session:
             testWithGivenSession(session)
@@ -401,7 +401,7 @@ def testWithGivenSession(session):
             expected = np.full((4, 5), fill_value=123.456, dtype=dtype)
             expected[1:4, 2] = (1, 2.)
             expected[2:4] = np.arange(10).reshape(2, 5)
-            np.testing.assert_array_equal(session.fetch(arr2).view(dtype=dtype), expected)
+            np.testing.assert_array_equal(session.fetch(arr2), expected)
 
         with new_session().as_default() as session:
             testWithGivenSession(session)