Repository: dask/distributed

Issue:
Serialization problem with arrow Tables?
I think the below code should work (but it doesn't) is this a bug or am I missing something?

Setup code
```python 
import pandas as pd
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq

# create parquet files
filenames = []
for n in range(10):
    df = n*pd.DataFrame(np.ones((10, 5)))
    tbl = pa.Table.from_pandas(df)
    filename = f'part{n}.pq'
    pq.write_table(tbl, filename, version='2.0')
    filenames.append(filename)
```
Demo of problem:
```python
def read_table(filename):
    import pyarrow.parquet as pq
    tbl = pq.read_table(filename)
    return tbl


from distributed import LocalCluster, Client

cluster = LocalCluster(2, threads_per_worker=1)
client = Client(cluster.scheduler.address)
futs = client.map(read_table, filenames)
tbl = client.submit(pa.concat_tables, futs).result()
```
Exception:
```python-traceback
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-67-74f644c68e47> in <module>
      1 futs = client.map(read_table, filenames)
----> 2 tbl = client.submit(pa.concat_tables, futs).result()

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in result(self, timeout)
    191         # shorten error traceback
    192         result = self.client.sync(self._result, callback_timeout=timeout,
--> 193                                   raiseit=False)
    194         if self.status == 'error':
    195             six.reraise(*result)

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in sync(self, func, *args, **kwargs)
    673             return future
    674         else:
--> 675             return sync(self.loop, func, *args, **kwargs)
    676 
    677     def __repr__(self):

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\utils.py in sync(loop, func, *args, **kwargs)
    275             e.wait(10)
    276     if error[0]:
--> 277         six.reraise(*error[0])
    278     else:
    279         return result[0]

C:\Miniconda3\envs\py-dev\lib\site-packages\six.py in reraise(tp, value, tb)
    691             if value.__traceback__ is not tb:
    692                 raise value.with_traceback(tb)
--> 693             raise value
    694         finally:
    695             value = None

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\utils.py in f()
    260             if timeout is not None:
    261                 future = gen.with_timeout(timedelta(seconds=timeout), future)
--> 262             result[0] = yield future
    263         except Exception as exc:
    264             error[0] = sys.exc_info()

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1139                     if exc_info is not None:
   1140                         try:
-> 1141                             yielded = self.gen.throw(*exc_info)
   1142                         finally:
   1143                             # Break up a reference to itself

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in _result(self, raiseit)
    216                 raise gen.Return(exception)
    217         else:
--> 218             result = yield self.client._gather([self])
    219             raise gen.Return(result[0])
    220 

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1139                     if exc_info is not None:
   1140                         try:
-> 1141                             yielded = self.gen.throw(*exc_info)
   1142                         finally:
   1143                             # Break up a reference to itself

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in _gather(self, futures, errors, direct, local_worker)
   1520                 else:
   1521                     self._gather_future = future
-> 1522                 response = yield future
   1523 
   1524             if response['status'] == 'error':

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1139                     if exc_info is not None:
   1140                         try:
-> 1141                             yielded = self.gen.throw(*exc_info)
   1142                         finally:
   1143                             # Break up a reference to itself

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in _gather_remote(self, direct, local_worker)
   1569 
   1570             else:  # ask scheduler to gather data for us
-> 1571                 response = yield self.scheduler.gather(keys=keys)
   1572         finally:
   1573             self._gather_semaphore.release()

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1139                     if exc_info is not None:
   1140                         try:
-> 1141                             yielded = self.gen.throw(*exc_info)
   1142                         finally:
   1143                             # Break up a reference to itself

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\core.py in send_recv_from_rpc(**kwargs)
    578             try:
    579                 comm = yield self.live_comm()
--> 580                 result = yield send_recv(comm=comm, op=key, **kwargs)
    581             except (RPCClosed, CommClosedError) as e:
    582                 raise e.__class__("%s: while trying to call remote method %r"

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
   1145                             exc_info = None
   1146                     else:
-> 1147                         yielded = self.gen.send(value)
   1148 
   1149                     if stack_context._state.contexts is not orig_stack_contexts:

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\core.py in send_recv(comm, reply, serializers, deserializers, **kwargs)
    470     if isinstance(response, dict) and response.get('status') == 'uncaught-error':
    471         if comm.deserialize:
--> 472             six.reraise(*clean_exception(**response))
    473         else:
    474             raise Exception(response['text'])

C:\Miniconda3\envs\py-dev\lib\site-packages\six.py in reraise(tp, value, tb)
    690                 value = tp()
    691             if value.__traceback__ is not tb:
--> 692                 raise value.with_traceback(tb)
    693             raise value
    694         finally:

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\core.py in handle_comm()
    344                         if type(result) is gen.Future:
    345                             self._ongoing_coroutines.add(result)
--> 346                             result = yield result
    347                     except (CommClosedError, CancelledError) as e:
    348                         if self.status == 'running':

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
   1139                     if exc_info is not None:
   1140                         try:
-> 1141                             yielded = self.gen.throw(*exc_info)
   1142                         finally:
   1143                             # Break up a reference to itself

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\scheduler.py in gather()
   2315 
   2316         data, missing_keys, missing_workers = yield gather_from_workers(
-> 2317             who_has, rpc=self.rpc, close=False, serializers=serializers)
   2318         if not missing_keys:
   2319             result = {'status': 'OK', 'data': data}

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
   1139                     if exc_info is not None:
   1140                         try:
-> 1141                             yielded = self.gen.throw(*exc_info)
   1142                         finally:
   1143                             # Break up a reference to itself

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\utils_comm.py in gather_from_workers()
     65             for worker, c in coroutines.items():
     66                 try:
---> 67                     r = yield c
     68                 except EnvironmentError:
     69                     missing_workers.add(worker)

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
   1139                     if exc_info is not None:
   1140                         try:
-> 1141                             yielded = self.gen.throw(*exc_info)
   1142                         finally:
   1143                             # Break up a reference to itself

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\worker.py in get_data_from_worker()
   2669                                    deserializers=deserializers,
   2670                                    op='get_data', keys=keys, who=who,
-> 2671                                    max_connections=max_connections)
   2672         try:
   2673             status = response['status']

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
   1131 
   1132                     try:
-> 1133                         value = future.result()
   1134                     except Exception:
   1135                         self.had_exception = True

C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
   1145                             exc_info = None
   1146                     else:
-> 1147                         yielded = self.gen.send(value)
   1148 
   1149                     if stack_context._state.contexts is not orig_stack_contexts:

C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\core.py in send_recv()
    472             six.reraise(*clean_exception(**response))
    473         else:
--> 474             raise Exception(response['text'])
    475     raise gen.Return(response)
    476 

Exception: Object <pyarrow.lib.Buffer object at 0x000002A6C3FF03E8> is neither a bytes object nor has an encode method
```

I'll try updating and reproducing but won't get to that for a while so any confirmation appreciated...
```
print(pa.__version__)
print(dask.__version__)
print(distributed.__version__)

0.12.0
1.1.1
1.25.3
```


Discussion:
You might want to try updating Dask.  That version is decently old.

On Mon, Aug 19, 2019 at 9:32 PM Dave Hirschfeld <notifications@github.com>
wrote:

> I think the below code should work (but it doesn't) is this a bug or am I
> missing something?
>
> Setup code
>
> import pandas as pdimport numpy as npimport pyarrow as paimport pyarrow.parquet as pq
> # create parquet files
> filenames = []for n in range(10):
>     df = n*pd.DataFrame(np.ones((10, 5)))
>     tbl = pa.Table.from_pandas(df)
>     filename = f'part{n}.pq'
>     pq.write_table(tbl, filename, version='2.0')
>     filenames.append(filename)
>
> Demo of problem:
>
> def read_table(filename):
>     import pyarrow.parquet as pq
>     tbl = pq.read_table(filename)
>     return tbl
>
> from distributed import LocalCluster, Client
>
> cluster = LocalCluster(2, threads_per_worker=1)
> client = Client(cluster.scheduler.address)
> futs = client.map(read_table, filenames)
> tbl = client.submit(pa.concat_tables, futs).result()
>
> Exception:
>
> ---------------------------------------------------------------------------
> Exception                                 Traceback (most recent call last)
> <ipython-input-67-74f644c68e47> in <module>
>       1 futs = client.map(read_table, filenames)
> ----> 2 tbl = client.submit(pa.concat_tables, futs).result()
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in result(self, timeout)
>     191         # shorten error traceback
>     192         result = self.client.sync(self._result, callback_timeout=timeout,
> --> 193                                   raiseit=False)
>     194         if self.status == 'error':
>     195             six.reraise(*result)
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in sync(self, func, *args, **kwargs)
>     673             return future
>     674         else:
> --> 675             return sync(self.loop, func, *args, **kwargs)
>     676
>     677     def __repr__(self):
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\utils.py in sync(loop, func, *args, **kwargs)
>     275             e.wait(10)
>     276     if error[0]:
> --> 277         six.reraise(*error[0])
>     278     else:
>     279         return result[0]
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\six.py in reraise(tp, value, tb)
>     691             if value.__traceback__ is not tb:
>     692                 raise value.with_traceback(tb)
> --> 693             raise value
>     694         finally:
>     695             value = None
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\utils.py in f()
>     260             if timeout is not None:
>     261                 future = gen.with_timeout(timedelta(seconds=timeout), future)
> --> 262             result[0] = yield future
>     263         except Exception as exc:
>     264             error[0] = sys.exc_info()
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1139                     if exc_info is not None:
>    1140                         try:
> -> 1141                             yielded = self.gen.throw(*exc_info)
>    1142                         finally:
>    1143                             # Break up a reference to itself
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in _result(self, raiseit)
>     216                 raise gen.Return(exception)
>     217         else:
> --> 218             result = yield self.client._gather([self])
>     219             raise gen.Return(result[0])
>     220
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1139                     if exc_info is not None:
>    1140                         try:
> -> 1141                             yielded = self.gen.throw(*exc_info)
>    1142                         finally:
>    1143                             # Break up a reference to itself
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in _gather(self, futures, errors, direct, local_worker)
>    1520                 else:
>    1521                     self._gather_future = future
> -> 1522                 response = yield future
>    1523
>    1524             if response['status'] == 'error':
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1139                     if exc_info is not None:
>    1140                         try:
> -> 1141                             yielded = self.gen.throw(*exc_info)
>    1142                         finally:
>    1143                             # Break up a reference to itself
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\client.py in _gather_remote(self, direct, local_worker)
>    1569
>    1570             else:  # ask scheduler to gather data for us
> -> 1571                 response = yield self.scheduler.gather(keys=keys)
>    1572         finally:
>    1573             self._gather_semaphore.release()
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1139                     if exc_info is not None:
>    1140                         try:
> -> 1141                             yielded = self.gen.throw(*exc_info)
>    1142                         finally:
>    1143                             # Break up a reference to itself
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\core.py in send_recv_from_rpc(**kwargs)
>     578             try:
>     579                 comm = yield self.live_comm()
> --> 580                 result = yield send_recv(comm=comm, op=key, **kwargs)
>     581             except (RPCClosed, CommClosedError) as e:
>     582                 raise e.__class__("%s: while trying to call remote method %r"
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run(self)
>    1145                             exc_info = None
>    1146                     else:
> -> 1147                         yielded = self.gen.send(value)
>    1148
>    1149                     if stack_context._state.contexts is not orig_stack_contexts:
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\core.py in send_recv(comm, reply, serializers, deserializers, **kwargs)
>     470     if isinstance(response, dict) and response.get('status') == 'uncaught-error':
>     471         if comm.deserialize:
> --> 472             six.reraise(*clean_exception(**response))
>     473         else:
>     474             raise Exception(response['text'])
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\six.py in reraise(tp, value, tb)
>     690                 value = tp()
>     691             if value.__traceback__ is not tb:
> --> 692                 raise value.with_traceback(tb)
>     693             raise value
>     694         finally:
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\core.py in handle_comm()
>     344                         if type(result) is gen.Future:
>     345                             self._ongoing_coroutines.add(result)
> --> 346                             result = yield result
>     347                     except (CommClosedError, CancelledError) as e:
>     348                         if self.status == 'running':
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
>    1139                     if exc_info is not None:
>    1140                         try:
> -> 1141                             yielded = self.gen.throw(*exc_info)
>    1142                         finally:
>    1143                             # Break up a reference to itself
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\scheduler.py in gather()
>    2315
>    2316         data, missing_keys, missing_workers = yield gather_from_workers(
> -> 2317             who_has, rpc=self.rpc, close=False, serializers=serializers)
>    2318         if not missing_keys:
>    2319             result = {'status': 'OK', 'data': data}
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
>    1139                     if exc_info is not None:
>    1140                         try:
> -> 1141                             yielded = self.gen.throw(*exc_info)
>    1142                         finally:
>    1143                             # Break up a reference to itself
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\utils_comm.py in gather_from_workers()
>      65             for worker, c in coroutines.items():
>      66                 try:
> ---> 67                     r = yield c
>      68                 except EnvironmentError:
>      69                     missing_workers.add(worker)
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
>    1139                     if exc_info is not None:
>    1140                         try:
> -> 1141                             yielded = self.gen.throw(*exc_info)
>    1142                         finally:
>    1143                             # Break up a reference to itself
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\worker.py in get_data_from_worker()
>    2669                                    deserializers=deserializers,
>    2670                                    op='get_data', keys=keys, who=who,
> -> 2671                                    max_connections=max_connections)
>    2672         try:
>    2673             status = response['status']
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
>    1131
>    1132                     try:
> -> 1133                         value = future.result()
>    1134                     except Exception:
>    1135                         self.had_exception = True
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\tornado\gen.py in run()
>    1145                             exc_info = None
>    1146                     else:
> -> 1147                         yielded = self.gen.send(value)
>    1148
>    1149                     if stack_context._state.contexts is not orig_stack_contexts:
>
> C:\Miniconda3\envs\py-dev\lib\site-packages\distributed\core.py in send_recv()
>     472             six.reraise(*clean_exception(**response))
>     473         else:
> --> 474             raise Exception(response['text'])
>     475     raise gen.Return(response)
>     476
> Exception: Object <pyarrow.lib.Buffer object at 0x000002A6C3FF03E8> is neither a bytes object nor has an encode method
>
> I'll try updating and reproducing but won't get to that for a while so any
> confirmation appreciated...
>
> print(pa.__version__)
> print(dask.__version__)
> print(distributed.__version__)
>
> 0.12.0
> 1.1.1
> 1.25.3
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/2966?email_source=notifications&email_token=AACKZTHDVPHWRXW6HIF4BNDQFNXUHA5CNFSM4INN4F72YY3PNVWWK3TUL52HS4DFUVEXG43VMWVGG33NNVSW45C7NFSM4HGEPKRA>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AACKZTBACJS7UYSG6QQAYBDQFNXUHANCNFSM4INN4F7Q>
> .
>

Same result on the latest versions
```python
In [8]: print(pa.__version__)
   ...: print(dask.__version__)
   ...: print(distributed.__version__)
0.14.1
2.3.0
2.3.0
```
```python-traceback
Traceback (most recent call last):

  File "<ipython-input-9-74f644c68e47>", line 2, in <module>
    tbl = client.submit(pa.concat_tables, futs).result()

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\client.py", line 220, in result
    result = self.client.sync(self._result, callback_timeout=timeout, raiseit=False)

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\client.py", line 753, in sync
    self.loop, func, *args, callback_timeout=callback_timeout, **kwargs

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\utils.py", line 328, in sync
    six.reraise(*error[0])

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\six.py", line 693, in reraise
    raise value

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\utils.py", line 313, in f
    result[0] = yield future

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\tornado\gen.py", line 729, in run
    value = future.result()

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\client.py", line 243, in _result
    result = await self.client._gather([self])

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\client.py", line 1697, in _gather
    response = await future

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\client.py", line 1749, in _gather_remote
    response = await self.scheduler.gather(keys=keys)

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\core.py", line 750, in send_recv_from_rpc
    result = await send_recv(comm=comm, op=key, **kwargs)

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\core.py", line 559, in send_recv
    six.reraise(*clean_exception(**response))

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\six.py", line 692, in reraise
    raise value.with_traceback(tb)

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\core.py", line 416, in handle_comm
    result = await result

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\scheduler.py", line 2510, in gather
    who_has, rpc=self.rpc, close=False, serializers=serializers

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\utils_comm.py", line 75, in gather_from_workers
    r = await c

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\worker.py", line 3084, in get_data_from_worker
    max_connections=max_connections,

  File "C:\Users\dhirschfeld\envs\dev\lib\site-packages\distributed\core.py", line 561, in send_recv
    raise Exception(response["text"])

Exception: Object <pyarrow.lib.Buffer object at 0x0000022D1A25F1B8> is neither a bytes object nor has an encode method
```
Here are some, perhaps more informative tracebacks

```python-traceback
distributed.protocol.core - CRITICAL - Failed to Serialize
Traceback (most recent call last):
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/core.py", line 58, in dumps
    compression, frames = zip(*map(maybe_compress, frames))
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/compression.py", line 166, in maybe_compress
    sample = byte_sample(payload, sample_size, nsamples)
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/compression.py", line 135, in byte_sample
    return b"".join(map(ensure_bytes, parts))
  File "/Users/mrocklin/workspace/distributed/distributed/utils.py", line 922, in ensure_bytes
    raise TypeError("Object %s is neither a bytes object nor has an encode method" % s)
TypeError: Object <pyarrow.lib.Buffer object at 0x10ae09670> is neither a bytes object nor has an encode method
distributed.comm.utils - ERROR - Object <pyarrow.lib.Buffer object at 0x10ae09670> is neither a bytes object nor has an encode method
Traceback (most recent call last):
  File "/Users/mrocklin/workspace/distributed/distributed/comm/utils.py", line 29, in _to_frames
    msg, serializers=serializers, on_error=on_error, context=context
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/core.py", line 58, in dumps
    compression, frames = zip(*map(maybe_compress, frames))
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/compression.py", line 166, in maybe_compress
    sample = byte_sample(payload, sample_size, nsamples)
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/compression.py", line 135, in byte_sample
    return b"".join(map(ensure_bytes, parts))
  File "/Users/mrocklin/workspace/distributed/distributed/utils.py", line 922, in ensure_bytes
    raise TypeError("Object %s is neither a bytes object nor has an encode method" % s)
TypeError: Object <pyarrow.lib.Buffer object at 0x10ae09670> is neither a bytes object nor has an encode method
distributed.core - ERROR - Object <pyarrow.lib.Buffer object at 0x10ae09670> is neither a bytes object nor has an encode method
Traceback (most recent call last):
  File "/Users/mrocklin/workspace/distributed/distributed/core.py", line 416, in handle_comm
    result = await result
  File "/Users/mrocklin/workspace/distributed/distributed/worker.py", line 1185, in get_data
    compressed = await comm.write(msg, serializers=serializers)
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/site-packages/tornado/gen.py", line 742, in run
    yielded = self.gen.throw(*exc_info)  # type: ignore
  File "/Users/mrocklin/workspace/distributed/distributed/comm/tcp.py", line 235, in write
    context={"sender": self._local_addr, "recipient": self._peer_addr},
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
    value = future.result()
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/site-packages/tornado/gen.py", line 742, in run
    yielded = self.gen.throw(*exc_info)  # type: ignore
  File "/Users/mrocklin/workspace/distributed/distributed/comm/utils.py", line 37, in to_frames
    res = yield offload(_to_frames)
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
    value = future.result()
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/site-packages/tornado/gen.py", line 742, in run
    yielded = self.gen.throw(*exc_info)  # type: ignore
  File "/Users/mrocklin/workspace/distributed/distributed/utils.py", line 1489, in offload
    return (yield _offload_executor.submit(fn, *args, **kwargs))
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
    value = future.result()
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/concurrent/futures/_base.py", line 428, in result
    return self.__get_result()
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/Users/mrocklin/miniconda/envs/dev/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/Users/mrocklin/workspace/distributed/distributed/comm/utils.py", line 29, in _to_frames
    msg, serializers=serializers, on_error=on_error, context=context
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/core.py", line 58, in dumps
    compression, frames = zip(*map(maybe_compress, frames))
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/compression.py", line 166, in maybe_compress
    sample = byte_sample(payload, sample_size, nsamples)
  File "/Users/mrocklin/workspace/distributed/distributed/protocol/compression.py", line 135, in byte_sample
    return b"".join(map(ensure_bytes, parts))
  File "/Users/mrocklin/workspace/distributed/distributed/utils.py", line 922, in ensure_bytes
    raise TypeError("Object %s is neither a bytes object nor has an encode method" % s)
TypeError: Object <pyarrow.lib.Buffer object at 0x10ae09670> is neither a bytes object nor has an encode method
distributed.core - ERROR - Object <pyarrow.lib.Buffer object at 0x10ae09670> is neither a bytes object nor has an encode method
```

I ran git blame on the arrow serialization code to see who we could ask about this problem.  Your name came up :)  Any thoughts?  Any interest in debugging this problem?
I'm not sure what changed, but

```diff
diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py
index 1f2b4e83..15f3be44 100644
--- a/distributed/protocol/arrow.py
+++ b/distributed/protocol/arrow.py
@@ -34,7 +34,7 @@ def serialize_table(tbl):
     writer = pyarrow.RecordBatchStreamWriter(sink, tbl.schema)
     writer.write_table(tbl)
     writer.close()
-    buf = sink.getvalue()
+    buf = memoryview(sink.getvalue())
     header = {}
     frames = [buf]
     return header, frames
```

is sufficient to fix. Working on a test.

Resolving patch:
diff --git a/distributed/utils.py b/distributed/utils.py
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -903,22 +903,43 @@ def tmpfile(extension=""):
 
 
 def ensure_bytes(s):
-    """ Turn string or bytes to bytes
+    """Attempt to turn `s` into bytes.
+
+    Parameters
+    ----------
+    s : Any
+        The object to be converted. Will correctly handled
+
+        * str
+        * bytes
+        * objects implementing the buffer protocol (memoryview, ndarray, etc.)
+
+    Returns
+    -------
+    b : bytes
+
+    Raises
+    ------
+    TypeError
+        When `s` cannot be converted
+
+    Examples
+    --------
 
     >>> ensure_bytes('123')
     b'123'
     >>> ensure_bytes(b'123')
     b'123'
     """
-    if isinstance(s, bytes):
-        return s
-    if isinstance(s, memoryview):
-        return s.tobytes()
-    if isinstance(s, bytearray):  # noqa: F821
-        return bytes(s)
     if hasattr(s, "encode"):
         return s.encode()
-    raise TypeError("Object %s is neither a bytes object nor has an encode method" % s)
+    else:
+        try:
+            return bytes(s)
+        except Exception as e:
+            raise TypeError(
+                "Object %s is neither a bytes object nor has an encode method" % s
+            ) from e
 
 
 def divide_n_among_bins(n, bins):


Test patch:
diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py
--- a/distributed/protocol/tests/test_arrow.py
+++ b/distributed/protocol/tests/test_arrow.py
@@ -3,9 +3,9 @@
 
 pa = pytest.importorskip("pyarrow")
 
+import distributed
 from distributed.utils_test import gen_cluster
-from distributed.protocol import deserialize, serialize
-
+from distributed.protocol import deserialize, serialize, to_serialize
 
 df = pd.DataFrame({"A": list("abc"), "B": [1, 2, 3]})
 tbl = pa.Table.from_pandas(df, preserve_index=False)
@@ -35,3 +35,12 @@ def run_test(client, scheduler, worker1, worker2):
         assert obj.equals(result)
 
     run_test()
+
+
+def test_dumps_compression():
+    # https://github.com/dask/distributed/issues/2966
+    # large enough to trigger compression
+    t = pa.Table.from_pandas(pd.DataFrame({"A": [1] * 10000}))
+    msg = {"op": "update", "data": to_serialize(t)}
+    result = distributed.protocol.loads(distributed.protocol.dumps(msg))
+    assert result["data"].equals(t)
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,3 +1,4 @@
+import array
 import datetime
 from functools import partial
 import io
@@ -275,13 +276,25 @@ def f():
 
 
 def test_ensure_bytes():
-    data = [b"1", "1", memoryview(b"1"), bytearray(b"1")]
+    data = [b"1", "1", memoryview(b"1"), bytearray(b"1"), array.array("b", [49])]
     for d in data:
         result = ensure_bytes(d)
         assert isinstance(result, bytes)
         assert result == b"1"
 
 
+def test_ensure_bytes_ndarray():
+    result = ensure_bytes(np.arange(12))
+    assert isinstance(result, bytes)
+
+
+def test_ensure_bytes_pyarrow_buffer():
+    pa = pytest.importorskip("pyarrow")
+    buf = pa.py_buffer(b"123")
+    result = ensure_bytes(buf)
+    assert isinstance(result, bytes)
+
+
 def test_nbytes():
     def check(obj, expected):
         assert nbytes(obj) == expected