Repository: ipython/ipython

Issue:
IPython parallel function failed to pickle large array of object
I started 2 engines with `default` profile in the clusters page of IPython notebook:

```
from IPython import parallel
client = parallel.Client()
view = client.load_balanced_view()
print view.client.ids
```

Then ran a simple parallel function on a numpy array with `dtype=object`:

```
x = np.empty([700,6], dtype=object)

@view.parallel(block=False, chunksize=175)
def vector_func(x):
    return x.shape
res = vector_func(x)
res.wait_interactive(interval=5)
print res.get()
```

And got the following error:

```
C:\Anaconda\lib\site-packages\IPython\utils\pickleutil.pyc in get_object(self, g)
    209         if self.pickled:
    210             # no shape, we just pickled it
--> 211             return pickle.loads(data)
    212         else:
    213             return frombuffer(data, dtype=self.dtype).reshape(self.shape)
TypeError: must be string, not zmq.backend.cython.message.Frame
```

If I reduce `chunksize`, and thus the size of array to be pickled, the code will run successfully:

```
@view.parallel(block=False, chunksize=174) # Reduce the chunksize to anything <175
```

Finally, if the array is of numerical dtype, it will also work without any problem:

```
x = np.empty([700,6], dtype=float)
```

I've reproduced the same error on two Windows 7 (64-bit) desktops running Python 2.7.7, Anaconda 2.0.1 (64-bit), IPython 2.1.0.

I guess the problem is associate with [this issue](https://github.com/ipython/ipython/issues/4863/)?



Discussion:
It's because pickle.loads doesn't accept  buffers on Python 2. This works fine on Python 3. Fix incoming.


Resolving patch:
diff --git a/IPython/kernel/zmq/serialize.py b/IPython/kernel/zmq/serialize.py
--- a/IPython/kernel/zmq/serialize.py
+++ b/IPython/kernel/zmq/serialize.py
@@ -11,14 +11,14 @@
     import pickle
 
 # IPython imports
-from IPython.utils import py3compat
+from IPython.utils.py3compat import PY3, buffer_to_bytes_py2
 from IPython.utils.data import flatten
 from IPython.utils.pickleutil import (
     can, uncan, can_sequence, uncan_sequence, CannedObject,
     istype, sequence_types, PICKLE_PROTOCOL,
 )
 
-if py3compat.PY3:
+if PY3:
     buffer = memoryview
 
 #-----------------------------------------------------------------------------
@@ -105,10 +105,7 @@ def deserialize_object(buffers, g=None):
     (newobj, bufs) : unpacked object, and the list of remaining unused buffers.
     """
     bufs = list(buffers)
-    pobj = bufs.pop(0)
-    if not isinstance(pobj, bytes):
-        # a zmq message
-        pobj = bytes(pobj)
+    pobj = buffer_to_bytes_py2(bufs.pop(0))
     canned = pickle.loads(pobj)
     if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
         for c in canned:
@@ -161,11 +158,10 @@ def unpack_apply_message(bufs, g=None, copy=True):
     Returns: original f,args,kwargs"""
     bufs = list(bufs) # allow us to pop
     assert len(bufs) >= 2, "not enough buffers!"
-    if not copy:
-        for i in range(2):
-            bufs[i] = bufs[i].bytes
-    f = uncan(pickle.loads(bufs.pop(0)), g)
-    info = pickle.loads(bufs.pop(0))
+    pf = buffer_to_bytes_py2(bufs.pop(0))
+    f = uncan(pickle.loads(pf), g)
+    pinfo = buffer_to_bytes_py2(bufs.pop(0))
+    info = pickle.loads(pinfo)
     arg_bufs, kwarg_bufs = bufs[:info['narg_bufs']], bufs[info['narg_bufs']:]
     
     args = []
diff --git a/IPython/utils/pickleutil.py b/IPython/utils/pickleutil.py
--- a/IPython/utils/pickleutil.py
+++ b/IPython/utils/pickleutil.py
@@ -17,7 +17,7 @@
 from . import codeutil  # This registers a hook when it's imported
 from . import py3compat
 from .importstring import import_item
-from .py3compat import string_types, iteritems
+from .py3compat import string_types, iteritems, buffer_to_bytes_py2
 
 from IPython.config import Application
 from IPython.utils.log import get_logger
@@ -260,8 +260,8 @@ def get_object(self, g=None):
         from numpy import frombuffer
         data = self.buffers[0]
         if self.pickled:
-            # no shape, we just pickled it
-            return pickle.loads(data)
+            # we just pickled it
+            return pickle.loads(buffer_to_bytes_py2(data))
         else:
             return frombuffer(data, dtype=self.dtype).reshape(self.shape)
 
diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py
--- a/IPython/utils/py3compat.py
+++ b/IPython/utils/py3compat.py
@@ -30,6 +30,12 @@ def cast_bytes(s, encoding=None):
         return encode(s, encoding)
     return s
 
+def buffer_to_bytes(buf):
+    """Cast a buffer object to bytes"""
+    if not isinstance(buf, bytes):
+        buf = bytes(buf)
+    return buf
+
 def _modify_str_or_docstring(str_change_func):
     @functools.wraps(str_change_func)
     def wrapper(func_or_str):
@@ -86,6 +92,7 @@ def input(prompt=''):
     bytes_to_str = decode
     cast_bytes_py2 = no_code
     cast_unicode_py2 = no_code
+    buffer_to_bytes_py2 = no_code
     
     string_types = (str,)
     unicode_type = str
@@ -151,6 +158,7 @@ def input(prompt=''):
     bytes_to_str = no_code
     cast_bytes_py2 = cast_bytes
     cast_unicode_py2 = cast_unicode
+    buffer_to_bytes_py2 = buffer_to_bytes
     
     string_types = (str, unicode)
     unicode_type = unicode


Test patch: