Repository: dask/distributed

Issue:
Inconsistent versions of pandas can result in pickling errors
This isn't really a bug per se, but the most recent version of pandas (1.3.0, published a couple of days ago) changes the pickling protocol for dataframes. Distributed clusters with different versions of pandas can run into trouble serializing dataframes.

A minimal example:

First, in an environment running `pandas=1.2.x`
```python
import distributed
cluster = distributed.Cluster("tcp://127.0.0.1:44444")
```
Then, in an otherwise identical environment running `pandas=1.3.0`:
```python
import distributed
import pandas

client = distributed.Client("tcp://127.0.0.1:44444")
df = pandas.DataFrame({"a": [1,2,3], "b": [4,5,6]})
client.scatter(df)
```

produces the following error:
<details>

```python-traceback
In [10]: client.scatter(df)
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
<ipython-input-10-379fe3935c01> in <module>
----> 1 client.scatter(df)

~/miniconda3/envs/coiled/lib/python3.8/site-packages/distributed/client.py in scatter(self, data, workers, broadcast, direct, hash, timeout, asynchronous)
   2222         else:
   2223             local_worker = None
-> 2224         return self.sync(
   2225             self._scatter,
   2226             data,

~/miniconda3/envs/coiled/lib/python3.8/site-packages/distributed/client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
    860             return future
    861         else:
--> 862             return sync(
    863                 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
    864             )

~/miniconda3/envs/coiled/lib/python3.8/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
    336     if error[0]:
    337         typ, exc, tb = error[0]
--> 338         raise exc.with_traceback(tb)
    339     else:
    340         return result[0]

~/miniconda3/envs/coiled/lib/python3.8/site-packages/distributed/utils.py in f()
    319             if callback_timeout is not None:
    320                 future = asyncio.wait_for(future, callback_timeout)
--> 321             result[0] = yield future
    322         except Exception:
    323             error[0] = sys.exc_info()

~/miniconda3/envs/coiled/lib/python3.8/site-packages/tornado/gen.py in run(self)
    760 
    761                     try:
--> 762                         value = future.result()
    763                     except Exception:
    764                         exc_info = sys.exc_info()

~/miniconda3/envs/coiled/lib/python3.8/site-packages/distributed/client.py in _scatter(self, data, workers, broadcast, direct, local_worker, timeout, hash)
   2113                 )
   2114             else:
-> 2115                 await self.scheduler.scatter(
   2116                     data=data2,
   2117                     workers=workers,

~/miniconda3/envs/coiled/lib/python3.8/site-packages/distributed/core.py in send_recv_from_rpc(**kwargs)
    862             name, comm.name = comm.name, "ConnectionPool." + key
    863             try:
--> 864                 result = await send_recv(comm=comm, op=key, **kwargs)
    865             finally:
    866                 self.pool.reuse(self.addr, comm)

~/miniconda3/envs/coiled/lib/python3.8/site-packages/distributed/core.py in send_recv(comm, reply, serializers, deserializers, **kwargs)
    661         if comm.deserialize:
    662             typ, exc, tb = clean_exception(**response)
--> 663             raise exc.with_traceback(tb)
    664         else:
    665             raise Exception(response["text"])

~/dask/distributed/distributed/core.py in handle_comm()
    500                             result = asyncio.ensure_future(result)
    501                             self._ongoing_coroutines.add(result)
--> 502                             result = await result
    503                     except (CommClosedError, CancelledError) as e:
    504                         if self.status == Status.running:

~/dask/distributed/distributed/scheduler.py in scatter()
   5370         assert isinstance(data, dict)
   5371 
-> 5372         keys, who_has, nbytes = await scatter_to_workers(
   5373             nthreads, data, rpc=self.rpc, report=False
   5374         )

~/dask/distributed/distributed/utils_comm.py in scatter_to_workers()
    143     rpcs = {addr: rpc(addr) for addr in d}
    144     try:
--> 145         out = await All(
    146             [
    147                 rpcs[address].update_data(

~/dask/distributed/distributed/utils.py in All()
    208     while not tasks.done():
    209         try:
--> 210             result = await tasks.next()
    211         except Exception:
    212 

~/dask/distributed/distributed/core.py in send_recv_from_rpc()
    864             name, comm.name = comm.name, "ConnectionPool." + key
    865             try:
--> 866                 result = await send_recv(comm=comm, op=key, **kwargs)
    867             finally:
    868                 self.pool.reuse(self.addr, comm)

~/dask/distributed/distributed/core.py in send_recv()
    667             raise exc.with_traceback(tb)
    668         else:
--> 669             raise Exception(response["text"])
    670     return response
    671 

Exception: Can't get attribute 'new_block' on <module 'pandas.core.internals.blocks' from '/home/ian/miniconda3/envs/dask/lib/python3.8/site-packages/pandas/core/internals/blocks.py'>
```

</details>


I don't actually think there is much to be done about this, except to warn users if different versions of pandas are detected on the client/scheduler/workers. I see that is not currently in the list of default packages that are checked, and that it was removed in #2627 (though I don't know the reason). So, my proposed fix for this is just to add pandas back in to the `get_versions()` check, unless there is a compelling reason not to?

I suppose I am also opening this to flag that others seeing issues with mysterious failures to use dataframes might want to look into their pandas version :)


Discussion:

Resolving patch:
diff --git a/distributed/versions.py b/distributed/versions.py
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -18,6 +18,7 @@
 
 optional_packages = [
     ("numpy", lambda p: p.__version__),
+    ("pandas", lambda p: p.__version__),
     ("lz4", lambda p: p.__version__),
     ("blosc", lambda p: p.__version__),
 ]


Test patch: