Repository: mars-project/mars

Issue:
[BUG] worker chunkstore error cause failure
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

```

Traceback (most recent call last):
  File "/home/admin/work/_public-mars-0.1.1.zip/mars/scheduler/operand.py", line 461, in _rejecter
    six.reraise(*exc)
  File "/home/admin/work/_public-mars-0.1.1.zip/mars/lib/six.py", line 702, in reraise
    raise value.with_traceback(tb)
  File "/home/admin/work/_public-mars-0.1.1.zip/mars/promise.py", line 92, in _wrapped
    result = func(*args, **kwargs)
  File "/home/admin/work/_public-mars-0.1.1.zip/mars/utils.py", line 288, in _wrapped
    return func(*args, **kwargs)
  File "/home/admin/work/_public-mars-0.1.1.zip/mars/worker/calc.py", line 72, in _try_put_chunk
    ref = self._chunk_store.put(session_id, chunk_key, _calc_result_cache[chunk_key][1])
  File "/home/admin/work/_public-mars-0.1.1.zip/mars/worker/chunkstore.py", line 184, in put
    value = DataTuple(*value)
TypeError: tuple() takes at most 1 argument (2 given)
```

**To Reproduce**

Can reproduce for branch 0.1 .


Discussion:

Resolving patch:
diff --git a/mars/serialize/dataserializer.py b/mars/serialize/dataserializer.py
--- a/mars/serialize/dataserializer.py
+++ b/mars/serialize/dataserializer.py
@@ -91,9 +91,6 @@ def decompress(data):  # pragma: no cover
 SERIAL_VERSION = 0
 
 
-DATA_FLAG_DENSE = 0
-DATA_FLAG_CSR = 1
-
 COMPRESS_FLAG_NONE = 0
 COMPRESS_FLAG_LZ4 = 1
 COMPRESS_FLAG_GZIP = 2
@@ -311,10 +308,6 @@ def peek_serialized_size(file):
         file.seek(0)
 
 
-class DataTuple(tuple):
-    pass
-
-
 def _serialize_numpy_array_list(obj):
     if obj.dtype.str != '|O':
         # Make the array c_contiguous if necessary so that we can call change
@@ -366,37 +359,6 @@ def _deserialize_sparse_csr_list(data):
     return SparseNDArray(target_csr, shape=shape)
 
 
-def _serialize_data_tuple(obj):
-    data_meta = []
-    outputs = [None]
-    for item in obj:
-        if isinstance(item, np.ndarray):
-            serialized_items = _serialize_numpy_array_list(item)
-            outputs.extend(serialized_items)
-            data_meta.append(DATA_FLAG_DENSE)
-            data_meta.append(len(serialized_items))
-        else:
-            serialized_items = _serialize_sparse_csr_list(item)
-            outputs.extend(serialized_items)
-            data_meta.append(DATA_FLAG_CSR)
-            data_meta.append(len(serialized_items))
-    outputs[0] = struct.pack('<' + 'B' * len(data_meta), *data_meta)
-    return tuple(outputs)
-
-
-def _deserialize_data_tuple(data):
-    data_meta = struct.unpack('<' + 'B' * len(data[0]), data[0])
-    pos = 1
-    data_parts = []
-    for flag, size in zip(data_meta[0::2], data_meta[1::2]):
-        if flag == DATA_FLAG_DENSE:
-            data_parts.append(_deserialize_numpy_array_list(data[pos:pos + size]))
-        elif flag == DATA_FLAG_CSR:
-            data_parts.append(_deserialize_sparse_csr_list(data[pos:pos + size]))
-        pos += size
-    return DataTuple(data_parts)
-
-
 _serialize_context = None
 
 
@@ -407,8 +369,5 @@ def mars_serialize_context():
         ctx.register_type(SparseNDArray, 'mars.SparseNDArray',
                           custom_serializer=_serialize_sparse_csr_list,
                           custom_deserializer=_deserialize_sparse_csr_list)
-        ctx.register_type(DataTuple, 'mars.DataTuple',
-                          custom_serializer=_serialize_data_tuple,
-                          custom_deserializer=_deserialize_data_tuple)
         _serialize_context = ctx
     return _serialize_context
diff --git a/mars/worker/chunkstore.py b/mars/worker/chunkstore.py
--- a/mars/worker/chunkstore.py
+++ b/mars/worker/chunkstore.py
@@ -115,12 +115,17 @@ def create(self, session_id, chunk_key, size):
         obj_id = self._new_object_id(session_id, chunk_key)
 
         try:
+            self._plasma_client.evict(size)
             buffer = self._plasma_client.create(obj_id, size)
             return buffer
         except PlasmaStoreFull:
             exc_type = PlasmaStoreFull
+            self._mapper_ref.delete(session_id, chunk_key)
             logger.warning('Chunk %s(%d) failed to store to plasma due to StoreFullError',
                            chunk_key, size)
+        except:  # noqa: E722  # pragma: no cover
+            self._mapper_ref.delete(session_id, chunk_key)
+            raise
 
         if exc_type is PlasmaStoreFull:
             raise StoreFull
@@ -131,7 +136,7 @@ def seal(self, session_id, chunk_key):
         try:
             self._plasma_client.seal(obj_id)
         except PlasmaObjectNonexistent:
-            raise KeyError('(%r, %r)' % (session_id, chunk_key))
+            raise KeyError((session_id, chunk_key))
 
     def get(self, session_id, chunk_key):
         """
@@ -142,7 +147,7 @@ def get(self, session_id, chunk_key):
         obj_id = self._get_object_id(session_id, chunk_key)
         obj = self._plasma_client.get(obj_id, serialization_context=self._serialize_context, timeout_ms=10)
         if obj is ObjectNotAvailable:
-            raise KeyError('(%r, %r)' % (session_id, chunk_key))
+            raise KeyError((session_id, chunk_key))
         return obj
 
     def get_buffer(self, session_id, chunk_key):
@@ -152,7 +157,7 @@ def get_buffer(self, session_id, chunk_key):
         obj_id = self._get_object_id(session_id, chunk_key)
         [buf] = self._plasma_client.get_buffers([obj_id], timeout_ms=10)
         if buf is None:
-            raise KeyError('(%r, %r)' % (session_id, chunk_key))
+            raise KeyError((session_id, chunk_key))
         return buf
 
     def get_actual_size(self, session_id, chunk_key):
@@ -163,6 +168,8 @@ def get_actual_size(self, session_id, chunk_key):
         try:
             obj_id = self._get_object_id(session_id, chunk_key)
             [buf] = self._plasma_client.get_buffers([obj_id], timeout_ms=10)
+            if buf is None:
+                raise KeyError((session_id, chunk_key))
             size = buf.size
         finally:
             del buf
@@ -177,11 +184,8 @@ def put(self, session_id, chunk_key, value):
         """
         import pyarrow
         from pyarrow.lib import PlasmaStoreFull
-        from ..serialize.dataserializer import DataTuple
 
         data_size = calc_data_size(value)
-        if isinstance(value, tuple):
-            value = DataTuple(*value)
 
         try:
             obj_id = self._new_object_id(session_id, chunk_key)
@@ -212,6 +216,9 @@ def put(self, session_id, chunk_key, value):
             logger.warning('Chunk %s(%d) failed to store to plasma due to StoreFullError',
                            chunk_key, data_size)
             exc = PlasmaStoreFull
+        except:  # noqa: E722  # pragma: no cover
+            self._mapper_ref.delete(session_id, chunk_key)
+            raise
 
         if exc is PlasmaStoreFull:
             raise StoreFull


Test patch:
diff --git a/mars/serialize/tests/test_serialize.py b/mars/serialize/tests/test_serialize.py
--- a/mars/serialize/tests/test_serialize.py
+++ b/mars/serialize/tests/test_serialize.py
@@ -363,7 +363,7 @@ def testArrowSerialize(self):
         except ImportError:
             sps = None
 
-        from mars.serialize.dataserializer import DataTuple, mars_serialize_context
+        from mars.serialize.dataserializer import mars_serialize_context
         context = mars_serialize_context()
 
         if np:
@@ -378,7 +378,7 @@ def testArrowSerialize(self):
         if np and sps:
             array = np.random.rand(1000, 100)
             mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr'))
-            tp = DataTuple((array, mat))
+            tp = (array, mat)
             des_tp = pyarrow.deserialize(pyarrow.serialize(tp, context).to_buffer(), context)
             assert_array_equal(tp[0], des_tp[0])
             self.assertTrue((tp[1].spmatrix != des_tp[1].spmatrix).nnz == 0)