Repository: numpy/numpy

Issue:
pickling fails on some masked arrays (Trac #1342)
_Original ticket http://projects.scipy.org/numpy/ticket/1342 on 2009-12-19 by @pierregm, assigned to @pierregm._

The current pickling process fails sometimes

> > > base = ma.array([{'a':1}])
> > > target = cPickle.loads(cPickle.dumps(base))

.../numpy/ma/core.pyc in **setstate**(self, state)
   5408         """
   5409         (_, shp, typ, isf, raw, msk, flv) = state
-> 5410         ndarray.__setstate__(self, (shp, typ, isf, raw))
   5411         self._mask.__setstate__((shp, make_mask_descr(typ), isf, msk))
   5412         self.fill_value = flv

Using 

> > > self._data.tolist(),
> > > instead of
> > > self._data.tostring(cf)
> > > in MaskedArray.__getstate__ solves that particular issue, but breaks the other pickle. 

There's a need for a whole new approach for pickling masked arrays



Discussion:
This bug report is still valid.

And it is still so today! The problem comes from the fact that this pickling approach re-uses `ndarray` methods for `__setstate__` but not for `__reduce__`. Note that `ndarray` does not implement `__getstate__` at all, so there is likely no use for it in masked arrays either.


Resolving patch:
diff --git a/numpy/ma/core.py b/numpy/ma/core.py
--- a/numpy/ma/core.py
+++ b/numpy/ma/core.py
@@ -5860,17 +5860,8 @@ def __getstate__(self):
 
         """
         cf = 'CF'[self.flags.fnc]
-        state = (1,
-                 self.shape,
-                 self.dtype,
-                 self.flags.fnc,
-                 self._data.tobytes(cf),
-                 # self._data.tolist(),
-                 getmaskarray(self).tobytes(cf),
-                 # getmaskarray(self).tolist(),
-                 self._fill_value,
-                 )
-        return state
+        data_state = super(MaskedArray, self).__reduce__()[2]
+        return data_state + (getmaskarray(self).tobytes(cf), self._fill_value)
 
     def __setstate__(self, state):
         """Restore the internal state of the masked array, for


Test patch:
diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py
--- a/numpy/ma/tests/test_core.py
+++ b/numpy/ma/tests/test_core.py
@@ -476,13 +476,24 @@ def test_str_repr(self):
 
     def test_pickling(self):
         # Tests pickling
-        a = arange(10)
-        a[::3] = masked
-        a.fill_value = 999
-        a_pickled = pickle.loads(a.dumps())
-        assert_equal(a_pickled._mask, a._mask)
-        assert_equal(a_pickled._data, a._data)
-        assert_equal(a_pickled.fill_value, 999)
+        for dtype in (int, float, str, object):
+            a = arange(10).astype(dtype)
+            a.fill_value = 999
+
+            masks = ([0, 0, 0, 1, 0, 1, 0, 1, 0, 1],  # partially masked
+                     True,                            # Fully masked
+                     False)                           # Fully unmasked
+
+            for mask in masks:
+                a.mask = mask
+                a_pickled = pickle.loads(a.dumps())
+                assert_equal(a_pickled._mask, a._mask)
+                assert_equal(a_pickled._data, a._data)
+                if dtype in (object, int):
+                    assert_equal(a_pickled.fill_value, 999)
+                else:
+                    assert_equal(a_pickled.fill_value, dtype(999))
+                assert_array_equal(a_pickled.mask, mask)
 
     def test_pickling_subbaseclass(self):
         # Test pickling w/ a subclass of ndarray