Repository: dask/distributed

Issue:
Failed to deserialize when using packages from anaconda defaults
Apologies if this is a bug I should file with conda or somewhere else but this seemed like a good place to start.

I have the following environment.yaml

```yaml
name: sb2018
dependencies:
- python>=3.6
- jupyter
- boto3
- requests
- pandas
- path.py
- dask
- fastparquet
- python-snappy
- bokeh
- numba >=0.37
- pip:
  - tldextract
  - urlextract
```

If I specify no channels so the packages all come from defaults, I end up with the following error when running `read_parquet`:

```
distributed.protocol.core - CRITICAL - Failed to deserialize
Traceback (most recent call last):
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/core.py", line 96, in loads
    msg = loads_msgpack(small_header, small_payload)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/core.py", line 168, in loads_msgpack
    payload = decompress(payload)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/compression.py", line 50, in _fixed_snappy_decompress
    return snappy.decompress(data)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/snappy.py", line 91, in uncompress
    return _uncompress(data)
TypeError: argument 1 must be read-only bytes-like object, not bytearray
distributed.scheduler - ERROR - argument 1 must be read-only bytes-like object, not bytearray
Traceback (most recent call last):
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/scheduler.py", line 1897, in handle_client
    msgs = yield comm.read()
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/tornado/gen.py", line 1099, in run
    value = future.result()
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/tornado/gen.py", line 1107, in run
    yielded = self.gen.throw(*exc_info)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/comm/tcp.py", line 202, in read
    msg = yield from_frames(frames, deserialize=self.deserialize)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/tornado/gen.py", line 1099, in run
    value = future.result()
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/tornado/gen.py", line 315, in wrapper
    yielded = next(result)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/comm/utils.py", line 75, in from_frames
    res = _from_frames()
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/comm/utils.py", line 61, in _from_frames
    return protocol.loads(frames, deserialize=deserialize)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/core.py", line 96, in loads
    msg = loads_msgpack(small_header, small_payload)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/core.py", line 168, in loads_msgpack
    payload = decompress(payload)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/compression.py", line 50, in _fixed_snappy_decompress
    return snappy.decompress(data)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/snappy.py", line 91, in uncompress
    return _uncompress(data)
TypeError: argument 1 must be read-only bytes-like object, not bytearray
distributed.utils - ERROR - argument 1 must be read-only bytes-like object, not bytearray
Traceback (most recent call last):
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/utils.py", line 623, in log_errors
    yield
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/client.py", line 904, in _handle_report
    six.reraise(*clean_exception(**msg))
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/six.py", line 692, in reraise
    raise value.with_traceback(tb)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/comm/tcp.py", line 202, in read
    msg = yield from_frames(frames, deserialize=self.deserialize)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/tornado/gen.py", line 1099, in run
    value = future.result()
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/tornado/gen.py", line 315, in wrapper
    yielded = next(result)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/comm/utils.py", line 75, in from_frames
    res = _from_frames()
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/comm/utils.py", line 61, in _from_frames
    return protocol.loads(frames, deserialize=deserialize)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/core.py", line 96, in loads
    msg = loads_msgpack(small_header, small_payload)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/core.py", line 168, in loads_msgpack
    payload = decompress(payload)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/distributed/protocol/compression.py", line 50, in _fixed_snappy_decompress
    return snappy.decompress(data)
  File "/home/bird/miniconda3/envs/sb2018/lib/python3.6/site-packages/snappy.py", line 91, in uncompress
    return _uncompress(data)
TypeError: argument 1 must be read-only bytes-like object, not bytearray
```

If I specify the channel as conda-forge in my environment.yaml then all packages come from conda-forge and everything's okay.

export env with defaults: 

```yaml
name: sb2018
channels:
  - defaults
  - conda-forge
dependencies:
  - asn1crypto=0.24.0=py36_0
  - attrs=17.4.0=py36_0
  - bleach=2.1.3=py36_0
  - bokeh=0.12.14=py36_0
  - boto3=1.5.32=py36_0
  - botocore=1.8.46=py36_0
  - ca-certificates=2017.08.26=h1d4fec5_0
  - certifi=2018.1.18=py36_0
  - cffi=1.11.5=py36h9745a5d_0
  - chardet=3.0.4=py36h0f667ec_1
  - click=6.7=py36h5253387_0
  - cloudpickle=0.5.2=py36_1
  - cryptography=2.1.4=py36hd09be54_0
  - dask=0.17.1=py36_0
  - dask-core=0.17.1=py36_0
  - dbus=1.12.2=hc3f9b76_1
  - decorator=4.2.1=py36_0
  - distributed=1.21.2=py36_0
  - docutils=0.14=py36hb0f60f5_0
  - entrypoints=0.2.3=py36h1aec115_2
  - expat=2.2.5=he0dffb1_0
  - fastparquet=0.1.4=py36h14c3975_0
  - fontconfig=2.12.6=h49f89f6_0
  - freetype=2.8=hab7d2ae_1
  - glib=2.53.6=h5d9569c_2
  - gmp=6.1.2=h6c8ec71_1
  - gst-plugins-base=1.12.4=h33fb286_0
  - gstreamer=1.12.4=hb53b477_0
  - heapdict=1.0.0=py36_2
  - html5lib=1.0.1=py36h2f9c1c0_0
  - icu=58.2=h9c2bf20_1
  - idna=2.6=py36h82fb2a8_1
  - intel-openmp=2018.0.0=hc7b2577_8
  - ipykernel=4.8.2=py36_0
  - ipython=6.2.1=py36h88c514a_1
  - ipython_genutils=0.2.0=py36hb52b0d5_0
  - ipywidgets=7.1.2=py36_0
  - jedi=0.11.1=py36_0
  - jinja2=2.10=py36ha16c418_0
  - jmespath=0.9.3=py36hd3948f9_0
  - jpeg=9b=h024ee3a_2
  - jsonschema=2.6.0=py36h006f8b5_0
  - jupyter=1.0.0=py36_4
  - jupyter_client=5.2.2=py36_0
  - jupyter_console=5.2.0=py36he59e554_1
  - jupyter_core=4.4.0=py36h7c827e3_0
  - libedit=3.1=heed3624_0
  - libffi=3.2.1=hd88cf55_4
  - libgcc-ng=7.2.0=hdf63c60_3
  - libgfortran-ng=7.2.0=hdf63c60_3
  - libpng=1.6.34=hb9fc6fc_0
  - libsodium=1.0.15=hf101ebd_0
  - libstdcxx-ng=7.2.0=hdf63c60_3
  - libxcb=1.12=hcd93eb1_4
  - libxml2=2.9.7=h26e45fe_0
  - llvmlite=0.22.0=py36ha27ea49_0
  - locket=0.2.0=py36h787c0ad_1
  - markupsafe=1.0=py36hd9260cd_1
  - mistune=0.8.3=py36_0
  - mkl=2018.0.1=h19d6760_4
  - msgpack-python=0.5.5=py36h6bb024c_0
  - nbconvert=5.3.1=py36hb41ffb7_0
  - nbformat=4.4.0=py36h31c9010_0
  - ncurses=6.0=h9df7e31_2
  - notebook=5.4.0=py36_0
  - numba=0.37.0=np114py36h5fcf951_0
  - numpy=1.14.1=py36ha266831_2
  - openssl=1.0.2n=hb7f436b_0
  - packaging=17.1=py36_0
  - pandas=0.22.0=py36hf484d3e_0
  - pandoc=1.19.2.1=hea2e7c5_1
  - pandocfilters=1.4.2=py36ha6701b7_1
  - parso=0.1.1=py36h35f843b_0
  - partd=0.3.8=py36h36fd896_0
  - path.py=11.0=py36_0
  - pcre=8.41=hc27e229_1
  - pexpect=4.4.0=py36_0
  - pickleshare=0.7.4=py36h63277f8_0
  - pip=9.0.1=py36_5
  - pluggy=0.6.0=py36hb689045_0
  - prompt_toolkit=1.0.15=py36h17d85b1_0
  - psutil=5.4.3=py36h14c3975_0
  - ptyprocess=0.5.2=py36h69acd42_0
  - py=1.5.2=py36h29bf505_0
  - pycparser=2.18=py36hf9f622e_1
  - pygments=2.2.0=py36h0d3125c_0
  - pyopenssl=17.5.0=py36h20ba746_0
  - pyparsing=2.2.0=py36hee85983_1
  - pyqt=5.6.0=py36h0386399_5
  - pysocks=1.6.8=py36_0
  - pytest=3.4.2=py36_0
  - python=3.6.4=hc3d631a_1
  - python-dateutil=2.6.1=py36h88d3b88_1
  - python-snappy=0.5.1=py36_0
  - pytz=2018.3=py36_0
  - pyyaml=3.12=py36hafb9ca4_1
  - pyzmq=17.0.0=py36h14c3975_0
  - qt=5.6.2=hd25b39d_14
  - qtconsole=4.3.1=py36h8f73b5b_0
  - readline=7.0=ha6073c6_4
  - requests=2.18.4=py36he2e5f8d_1
  - s3transfer=0.1.13=py36_0
  - send2trash=1.5.0=py36_0
  - setuptools=38.5.1=py36_0
  - simplegeneric=0.8.1=py36_2
  - sip=4.18.1=py36h51ed4ed_2
  - six=1.11.0=py36h372c433_1
  - snappy=1.1.7=hbae5bb6_3
  - sortedcontainers=1.5.9=py36_0
  - sqlite=3.22.0=h1bed415_0
  - tblib=1.3.2=py36h34cf8b6_0
  - terminado=0.8.1=py36_1
  - testpath=0.3.1=py36h8cadb63_0
  - thrift=0.11.0=py36hf484d3e_0
  - tk=8.6.7=hc745277_3
  - toolz=0.9.0=py36_0
  - tornado=5.0=py36_0
  - traitlets=4.3.2=py36h674d592_0
  - urllib3=1.22=py36hbe7ace6_0
  - wcwidth=0.1.7=py36hdf4376a_0
  - webencodings=0.5.1=py36h800622e_1
  - wheel=0.30.0=py36hfd4bba0_1
  - widgetsnbextension=3.1.4=py36_0
  - xz=5.2.3=h55aa19d_2
  - yaml=0.1.7=had09818_2
  - zeromq=4.2.3=h439df22_3
  - zict=0.1.3=py36h3a3bf81_0
  - zlib=1.2.11=ha838bed_2
  - pip:
    - requests-file==1.4.3
    - tldextract==2.2.0
    - uritools==2.1.0
    - urlextract==0.8.3
prefix: /home/bird/miniconda3/envs/sb2018
```

export env with conda-forge:

```yaml
name: sb2018
channels:
  - defaults
  - conda-forge
dependencies:
  - asn1crypto=0.24.0=py36_0
  - attrs=17.4.0=py_0
  - bleach=2.1.3=py_0
  - bokeh=0.12.14=py36_0
  - boto3=1.6.5=py_0
  - botocore=1.9.6=py_0
  - ca-certificates=2018.1.18=0
  - certifi=2018.1.18=py36_0
  - cffi=1.11.5=py36_0
  - chardet=3.0.4=py36_0
  - click=6.7=py_1
  - cloudpickle=0.5.2=py_0
  - cryptography=2.1.4=py36_0
  - cytoolz=0.9.0.1=py36_0
  - dask=0.17.1=py_2
  - dask-core=0.17.1=py_0
  - dbus=1.10.22=0
  - decorator=4.2.1=py36_0
  - distributed=1.21.3=py36_0
  - docutils=0.14=py36_0
  - entrypoints=0.2.3=py36_1
  - expat=2.2.5=0
  - fastparquet=0.1.4=py36_0
  - fontconfig=2.12.6=0
  - freetype=2.8.1=0
  - gettext=0.19.8.1=0
  - glib=2.55.0=0
  - gmp=6.1.2=0
  - gst-plugins-base=1.8.0=0
  - gstreamer=1.8.0=1
  - heapdict=1.0.0=py36_0
  - html5lib=1.0.1=py_0
  - icu=58.2=0
  - idna=2.6=py36_1
  - ipykernel=4.8.2=py36_0
  - ipython=6.2.1=py36_1
  - ipython_genutils=0.2.0=py36_0
  - ipywidgets=7.1.2=py36_0
  - jedi=0.11.1=py36_0
  - jinja2=2.10=py36_0
  - jmespath=0.9.3=py36_0
  - jpeg=9b=2
  - jsonschema=2.6.0=py36_1
  - jupyter=1.0.0=py36_0
  - jupyter_client=5.2.2=py36_0
  - jupyter_console=5.2.0=py36_0
  - jupyter_core=4.4.0=py_0
  - libffi=3.2.1=3
  - libiconv=1.15=0
  - libpng=1.6.34=0
  - libsodium=1.0.15=1
  - libxcb=1.12=1
  - libxml2=2.9.7=0
  - locket=0.2.0=py36_1
  - markupsafe=1.0=py36_0
  - mistune=0.8.3=py_0
  - msgpack-python=0.5.5=py36_0
  - nbconvert=5.3.1=py_1
  - nbformat=4.4.0=py36_0
  - ncurses=5.9=10
  - notebook=5.4.0=py36_0
  - openssl=1.0.2n=0
  - packaging=17.1=py_0
  - pandas=0.22.0=py36_0
  - pandoc=2.1.2=0
  - pandocfilters=1.4.1=py36_0
  - parso=0.1.1=py_0
  - partd=0.3.8=py36_0
  - path.py=10.3.1=py36_0
  - pcre=8.39=0
  - pexpect=4.4.0=py36_0
  - pickleshare=0.7.4=py36_0
  - pip=9.0.1=py36_1
  - pluggy=0.6.0=py_0
  - prompt_toolkit=1.0.15=py36_0
  - psutil=5.4.3=py36_0
  - ptyprocess=0.5.2=py36_0
  - py=1.5.2=py_0
  - pycparser=2.18=py36_0
  - pygments=2.2.0=py36_0
  - pyopenssl=17.5.0=py36_0
  - pyparsing=2.2.0=py36_0
  - pyqt=5.6.0=py36_4
  - pysocks=1.6.8=py36_1
  - pytest=3.4.2=py36_0
  - python=3.6.4=0
  - python-dateutil=2.6.1=py36_0
  - python-snappy=0.5.2=py36_0
  - pytz=2018.3=py_0
  - pyyaml=3.12=py36_1
  - pyzmq=17.0.0=py36_3
  - qt=5.6.2=7
  - qtconsole=4.3.1=py36_0
  - readline=7.0=0
  - requests=2.18.4=py36_1
  - s3transfer=0.1.13=py36_0
  - send2trash=1.5.0=py_0
  - setuptools=38.5.2=py36_0
  - simplegeneric=0.8.1=py36_0
  - sip=4.18=py36_1
  - six=1.11.0=py36_1
  - snappy=1.1.7=1
  - sortedcontainers=1.5.7=py36_0
  - sqlite=3.20.1=2
  - tblib=1.3.2=py36_0
  - terminado=0.8.1=py36_0
  - testpath=0.3.1=py36_0
  - thrift=0.11.0=py36_0
  - tk=8.6.7=0
  - toolz=0.9.0=py_0
  - tornado=4.5.3=py36_0
  - traitlets=4.3.2=py36_0
  - urllib3=1.22=py36_0
  - wcwidth=0.1.7=py36_0
  - webencodings=0.5=py36_0
  - wheel=0.30.0=py36_2
  - widgetsnbextension=3.1.4=py36_0
  - xorg-libxau=1.0.8=3
  - xorg-libxdmcp=1.1.2=3
  - xz=5.2.3=0
  - yaml=0.1.7=0
  - zeromq=4.2.3=2
  - zict=0.1.3=py_0
  - zlib=1.2.11=0
  - intel-openmp=2018.0.0=hc7b2577_8
  - libgcc-ng=7.2.0=hdf63c60_3
  - libgfortran-ng=7.2.0=hdf63c60_3
  - libstdcxx-ng=7.2.0=hdf63c60_3
  - llvmlite=0.22.0=py36ha27ea49_0
  - mkl=2018.0.1=h19d6760_4
  - numba=0.37.0=np114py36h5fcf951_0
  - numpy=1.14.1=py36ha266831_2
  - pip:
    - requests-file==1.4.3
    - tldextract==2.2.0
    - uritools==2.1.0
    - urlextract==0.8.3
prefix: /home/bird/miniconda3/envs/sb2018

```


Discussion:
I doubt this is the slight difference in snappy versions - fastparquet works fine with the older one in anaconda, and this is not happening in the parquet code path. 

At a guess, this may have to do with the newer no-copy buffers being used by tornado (note the versions in the package lists), and you happen to be a rare case whose default compression is snappy (which you have as a dependency of parquet). You could try installing blosc of lz4 - I don't actually know how the default is chosen.
Thanks @martindurant. Feel free to close, I just wanted to document.
I'm not sure I understand your workaround though, can you explain in more detail what I would do / change in my code?

Yup, that's a valuable bug report.  Thanks @birdsarah .  For now I recommend downgrading to `tornado=4.5`.  The recent 5.0 release has raised a number of small issues like this around the ecosystem.  

I'll take a look at this.
Alternatively also install `lz4`

Resolving patch:
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -47,6 +47,8 @@ def _fixed_snappy_decompress(data):
         # snappy.decompress() doesn't accept memoryviews
         if isinstance(data, memoryview):
             data = data.tobytes()
+        elif isinstance(data, bytearray):
+            data = bytes(data)
         return snappy.decompress(data)
 
     compressions['snappy'] = {'compress': snappy.compress,
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3769,7 +3769,7 @@ def transition(self, key, finish, *args, **kwargs):
             if start == finish:
                 return {}
 
-            if self.plugins and finish == 'forgotten':
+            if self.plugins:
                 dependents = set(ts.dependents)
                 dependencies = set(ts.dependencies)
 


Test patch:
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -8,9 +8,11 @@
 from tornado import gen
 import pytest
 
+import dask
 from distributed.compatibility import finalize
 from distributed.core import (pingpong, Server, rpc, connect, send_recv,
                                coerce_to_address, ConnectionPool)
+from distributed.protocol.compression import compressions
 
 from distributed.metrics import time
 from distributed.protocol import to_serialize
@@ -56,6 +58,10 @@ def echo_serialize(comm, x):
     return {'result': to_serialize(x)}
 
 
+def echo_no_serialize(comm, x):
+    return {'result': x}
+
+
 def test_server(loop):
     """
     Simple Server test.
@@ -572,3 +578,23 @@ def test_tick_logging(s, a, b):
         assert 'Scheduler' in text or 'Worker' in text
     finally:
         core.tick_maximum_delay = old
+
+
+@pytest.mark.parametrize('compression', list(compressions))
+@pytest.mark.parametrize('serialize', [echo_serialize, echo_no_serialize])
+def test_compression(compression, serialize, loop):
+    with dask.set_options(compression=compression):
+
+        @gen.coroutine
+        def f():
+            server = Server({'echo': serialize})
+            server.listen('tcp://')
+
+            with rpc(server.address) as r:
+                data = b'1' * 1000000
+                result = yield r.echo(x=to_serialize(data))
+                assert result == {'result': data}
+
+            server.stop()
+
+        loop.run_sync(f)