Repository: dask/distributed

Issue:
Dataframe shuffle: obscure failure if pyarrow is not installed
If pandas is installed but pyarrow isn't, p2p dataframe shuffle fails with a very obscure

`RuntimeError: P2P shuffling e3b0df60fbfc8044356bd67bb2dc2df4 failed during transfer phase`

This is reproduced e.g. by  `test_index_merge_p2p`.

You can get the use case with a conda env that looks as follows:
```yaml
name: dask-distributed
channels:
  - conda-forge
dependencies:
  - python=3.11
  - pandas
  - pip
  - pip:
      - dask
      - distributed
```

# Expected behaviour
The user should get a clean `ModuleNotFoundError`.

CC @hendrikmakait @graingert 


Discussion:

Resolving patch:
diff --git a/distributed/shuffle/_merge.py b/distributed/shuffle/_merge.py
--- a/distributed/shuffle/_merge.py
+++ b/distributed/shuffle/_merge.py
@@ -8,6 +8,7 @@
 from dask.highlevelgraph import HighLevelGraph
 from dask.layers import Layer
 
+from distributed.shuffle._arrow import check_minimal_arrow_version
 from distributed.shuffle._core import ShuffleId, barrier_key, get_worker_plugin
 from distributed.shuffle._shuffle import shuffle_barrier, shuffle_transfer
 
@@ -212,6 +213,7 @@ def __init__(
         parts_out: Sequence | None = None,
         annotations: dict | None = None,
     ) -> None:
+        check_minimal_arrow_version()
         self.name = name
         self.name_input_left = name_input_left
         self.meta_input_left = meta_input_left


Test patch:
diff --git a/distributed/shuffle/tests/test_merge.py b/distributed/shuffle/tests/test_merge.py
--- a/distributed/shuffle/tests/test_merge.py
+++ b/distributed/shuffle/tests/test_merge.py
@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+import contextlib
+from unittest import mock
+
 import pytest
 
 from distributed.shuffle._merge import hash_join
@@ -13,6 +16,11 @@
 from dask.dataframe.utils import assert_eq
 from dask.utils_test import hlg_layer_topological
 
+try:
+    import pyarrow as pa
+except ImportError:
+    pa = None
+
 pytestmark = pytest.mark.ci1
 
 
@@ -42,6 +50,24 @@ def list_eq(aa, bb):
     dd._compat.assert_numpy_array_equal(av, bv)
 
 
+@gen_cluster(client=True)
+async def test_minimal_version(c, s, a, b):
+    no_pyarrow_ctx = (
+        mock.patch.dict("sys.modules", {"pyarrow": None})
+        if pa is not None
+        else contextlib.nullcontext()
+    )
+    with no_pyarrow_ctx:
+        A = pd.DataFrame({"x": [1, 2, 3, 4, 5, 6], "y": [1, 1, 2, 2, 3, 4]})
+        a = dd.repartition(A, [0, 4, 5])
+
+        B = pd.DataFrame({"y": [1, 3, 4, 4, 5, 6], "z": [6, 5, 4, 3, 2, 1]})
+        b = dd.repartition(B, [0, 2, 5])
+
+        with pytest.raises(ModuleNotFoundError, match="requires pyarrow"):
+            await c.compute(dd.merge(a, b, left_on="x", right_on="z", shuffle="p2p"))
+
+
 @pytest.mark.parametrize("how", ["inner", "left", "right", "outer"])
 @gen_cluster(client=True)
 async def test_basic_merge(c, s, a, b, how, lose_annotations):
diff --git a/distributed/shuffle/tests/test_shuffle.py b/distributed/shuffle/tests/test_shuffle.py
--- a/distributed/shuffle/tests/test_shuffle.py
+++ b/distributed/shuffle/tests/test_shuffle.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import itertools
 import logging
 import os
@@ -116,20 +117,22 @@ async def check_scheduler_cleanup(
     assert not plugin.heartbeats
 
 
-@pytest.mark.skipif(
-    pa is not None,
-    reason="We don't have a CI job that is installing a very old pyarrow version",
-)
 @gen_cluster(client=True)
 async def test_minimal_version(c, s, a, b):
-    df = dask.datasets.timeseries(
-        start="2000-01-01",
-        end="2000-01-10",
-        dtypes={"x": float, "y": float},
-        freq="10 s",
+    no_pyarrow_ctx = (
+        mock.patch.dict("sys.modules", {"pyarrow": None})
+        if pa is not None
+        else contextlib.nullcontext()
     )
-    with pytest.raises(RuntimeError, match="requires pyarrow"):
-        await c.compute(dd.shuffle.shuffle(df, "x", shuffle="p2p"))
+    with no_pyarrow_ctx:
+        df = dask.datasets.timeseries(
+            start="2000-01-01",
+            end="2000-01-10",
+            dtypes={"x": float, "y": float},
+            freq="10 s",
+        )
+        with pytest.raises(ModuleNotFoundError, match="requires pyarrow"):
+            await c.compute(dd.shuffle.shuffle(df, "x", shuffle="p2p"))
 
 
 @pytest.mark.gpu