Repository: dask/distributed

Issue:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte
**What happened**: crash with msgpack 0.6.2 – the `requirements.txt` file specifies minimum 0.6, so, it should work. Maybe it's too low, IDK.

**What you expected to happen**: no crash

**Minimal Complete Verifiable Example**:

```python
from dask.distributed import Client, LocalCluster
import dask.array as da
cluster = LocalCluster(
    n_workers=1,
    threads_per_worker=1,
    processes=True,
    diagnostics_port=None,
)
client = Client(cluster)
x = da.random.random((300, 300), chunks=(10, 10))
y = x + x.T
z = (y.mean(axis=1) / y.shape[0]).std()
print(z.compute())
```

causing: `UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte`

```
distributed.core - ERROR - 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte                                   
Traceback (most recent call last):                                                                                                  
  File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 589, in handle_stream                                     
    handler(**merge(extra, msg))                                                                                                    
  File "/usr/local/lib/python3.8/dist-packages/distributed/scheduler.py", line 2286, in update_graph_hlg                            
    dsk, dependencies, annotations = highlevelgraph_unpack(hlg)                                                                     
  File "/usr/local/lib/python3.8/dist-packages/distributed/protocol/highlevelgraph.py", line 125, in highlevelgraph_unpack          
    hlg = msgpack.loads(                                                                                                            
  File "msgpack/_unpacker.pyx", line 201, in msgpack._cmsgpack.unpackb                                                              
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte                                          
distributed.core - ERROR - Exception while handling op register-client                                                              
Traceback (most recent call last):                                                                                                  
  File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 517, in handle_comm                                       
    result = await result                                                                                                           
  File "/usr/local/lib/python3.8/dist-packages/distributed/scheduler.py", line 3149, in add_client                                  
    await self.handle_stream(comm=comm, extra={"client": client})                                                                   
  File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 589, in handle_stream                                     
    handler(**merge(extra, msg))                                                                                                    
  File "/usr/local/lib/python3.8/dist-packages/distributed/scheduler.py", line 2286, in update_graph_hlg                            
    dsk, dependencies, annotations = highlevelgraph_unpack(hlg)                                                                     
  File "/usr/local/lib/python3.8/dist-packages/distributed/protocol/highlevelgraph.py", line 125, in highlevelgraph_unpack          
    hlg = msgpack.loads(                                                                                                            
  File "msgpack/_unpacker.pyx", line 201, in msgpack._cmsgpack.unpackb                                                              
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte                                          
tornado.application - ERROR - Exception in callback functools.partial(<function TCPServer._handle_connection.<locals>.<lambda> at 0x
7fa2a45254c0>, <Task finished name='Task-21' coro=<BaseTCPListener._handle_stream() done, defined at /usr/local/lib/python3.8/dist-p
ackages/distributed/comm/tcp.py:459> exception=UnicodeDecodeError('utf-8', b"\x80\x05\x95=\x00\x00\x00\x00\x00\x00\x00\x8c9('var-agg
regate-_sqrt-42c4ccf3de0d30d6231d8723571f3516',)\x94.", 0, 1, 'invalid start byte')>)                                               
Traceback (most recent call last):                                                                                                  
  File "/usr/local/lib/python3.8/dist-packages/tornado/ioloop.py", line 743, in _run_callback                                       
    ret = callback()                                                                                                                
  File "/usr/local/lib/python3.8/dist-packages/tornado/tcpserver.py", line 327, in <lambda>                                         
    gen.convert_yielded(future), lambda f: f.result()                                                                               
  File "/usr/local/lib/python3.8/dist-packages/distributed/comm/tcp.py", line 476, in _handle_stream                                
    await self.comm_handler(comm)                                                                                                   
  File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 517, in handle_comm                                       
    result = await result                                                                                                           
  File "/usr/local/lib/python3.8/dist-packages/distributed/scheduler.py", line 3149, in add_client                                  
    await self.handle_stream(comm=comm, extra={"client": client})                                                                   
  File "/usr/local/lib/python3.8/dist-packages/distributed/core.py", line 589, in handle_stream                                     
    handler(**merge(extra, msg))                                                                                                    
  File "/usr/local/lib/python3.8/dist-packages/distributed/scheduler.py", line 2286, in update_graph_hlg                            
    dsk, dependencies, annotations = highlevelgraph_unpack(hlg)                                                                     
  File "/usr/local/lib/python3.8/dist-packages/distributed/protocol/highlevelgraph.py", line 125, in highlevelgraph_unpack          
    hlg = msgpack.loads(                                                                                                            
  File "msgpack/_unpacker.pyx", line 201, in msgpack._cmsgpack.unpackb                                                              
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte                                          
Traceback (most recent call last):                                                                                                  
  File "<stdin>", line 1, in <module>                                                                                               
  File "/usr/local/lib/python3.8/dist-packages/dask/base.py", line 279, in compute                                                  
    (result,) = compute(self, traverse=False, **kwargs)                                                                             
  File "/usr/local/lib/python3.8/dist-packages/dask/base.py", line 567, in compute                                                  
    results = schedule(dsk, keys, **kwargs)                                                                                         
  File "/usr/local/lib/python3.8/dist-packages/distributed/client.py", line 2676, in get                                            
    results = self.gather(packed, asynchronous=asynchronous, direct=direct)                                                         
  File "/usr/local/lib/python3.8/dist-packages/distributed/client.py", line 1985, in gather                                         
    return self.sync(                                                                                                               
  File "/usr/local/lib/python3.8/dist-packages/distributed/client.py", line 831, in sync                                            
    return sync(                                                                                                                    
  File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 340, in sync                                             
    raise exc.with_traceback(tb)                                                                                                    
  File "/usr/local/lib/python3.8/dist-packages/distributed/utils.py", line 324, in f                                                
    result[0] = yield future                                                                                                        
  File "/usr/local/lib/python3.8/dist-packages/tornado/gen.py", line 735, in run                                                    
    value = future.result()                                                                                                         
  File "/usr/local/lib/python3.8/dist-packages/distributed/client.py", line 1851, in _gather                                        
    raise exc                                                                                                                       
concurrent.futures._base.CancelledError: ('_sqrt-42c4ccf3de0d30d6231d8723571f3516',)                                                
```

**Anything else we need to know?**:

**Environment**:

- Dask version:  (and distributed): are at `2020.12.0`
- `msgpack.version: (0, 6, 2)`
- Python version:  3.8.5
- Operating System: Ubuntu 20.04 Linux in a Docker environment
- Install method (conda, pip, source): pip



Discussion:
well, updated msgpack to `msgpack-1.0.2-cp38-cp38-manylinux1_x86_64.whl` and now it works.

So, you  either need tighter  `requirements.txt` or some tweaks for backwards compatibility. It might be worth checking which version of msgpack works, to figure out the change that makes distributed work again.
I am seeing the same thing and also confirm that bumping `msgpack` works. Just bumping in case this got missed during the holidays; pinging @jrbourbeau just because your name is on the current `msgpack` version blame 🙂.
Thanks @haraldschilly for reporting and providing a nice reproducible example! Looking into this now
I've proposed a fix over in https://github.com/dask/distributed/pull/4407. @bnaul @haraldschilly if either of you are able to confirm this fixes things for you that would be appreciated -- though no obligation 

Resolving patch:
diff --git a/distributed/client.py b/distributed/client.py
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2585,11 +2585,11 @@ def _graph_to_futures(
             if not isinstance(dsk, HighLevelGraph):
                 dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
 
-            dsk = highlevelgraph_pack(dsk, self, keyset)
-
             if isinstance(retries, Number) and retries > 0:
                 retries = {k: retries for k in dsk}
 
+            dsk = highlevelgraph_pack(dsk, self, keyset)
+
             # Create futures before sending graph (helps avoid contention)
             futures = {key: Future(key, self, inform=False) for key in keyset}
             self._send_to_scheduler(
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
--- a/distributed/protocol/highlevelgraph.py
+++ b/distributed/protocol/highlevelgraph.py
@@ -1,4 +1,3 @@
-import msgpack
 from tlz import valmap
 
 from dask.core import keys_in_tasks
@@ -10,12 +9,8 @@
 
 from ..utils import CancelledError
 
-from .utils import msgpack_opts
-from .serialize import (
-    import_allowed_module,
-    msgpack_encode_default,
-    msgpack_decode_default,
-)
+from .core import dumps_msgpack, loads_msgpack
+from .serialize import import_allowed_module
 
 
 def _materialized_layer_pack(
@@ -93,8 +88,8 @@ def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
 
     Returns
     -------
-    data: bytes
-        Packed high level graph serialized by msgpack
+    data: list of header and payload
+        Packed high level graph serialized by dumps_msgpack
     """
     layers = []
 
@@ -126,8 +121,7 @@ def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
                 ),
             }
         )
-
-    return msgpack.dumps({"layers": layers}, default=msgpack_encode_default)
+    return dumps_msgpack({"layers": layers})
 
 
 def _materialized_layer_unpack(state, dsk, dependencies, annotations):
@@ -151,8 +145,8 @@ def highlevelgraph_unpack(dumped_hlg):
 
     Parameters
     ----------
-    dumped_hlg: bytes
-        Packed high level graph serialized by msgpack
+    dumped_hlg: list of header and payload
+        Packed high level graph serialized by dumps_msgpack
 
     Returns
     -------
@@ -164,10 +158,7 @@ def highlevelgraph_unpack(dumped_hlg):
         Annotations for `dsk`
     """
 
-    # Notice, we set `use_list=False`, which makes msgpack convert lists to tuples
-    hlg = msgpack.loads(
-        dumped_hlg, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts
-    )
+    hlg = loads_msgpack(*dumped_hlg)
 
     dsk = {}
     deps = {}


Test patch: