Repository: dask/distributed

Issue:
Flaky `test_workspace_concurrency_intense`
For example, see [this CI build](https://github.com/dask/distributed/pull/4763/checks?check_run_id=2459092140)

<details>
<summary>Full traceback:</summary>

```python
______________________ test_workspace_concurrency_intense ______________________

tmpdir = local('/private/var/folders/24/8k48jl6d249_n_qfxwsl6xvm0000gn/T/pytest-of-runner/pytest-0/test_workspace_concurrency_int10')

    @pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
    @pytest.mark.slow
    def test_workspace_concurrency_intense(tmpdir):
>       n_created, n_purged = _test_workspace_concurrency(tmpdir, 8.0, 16)

distributed/tests/test_diskutils.py:287: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

tmpdir = local('/private/var/folders/24/8k48jl6d249_n_qfxwsl6xvm0000gn/T/pytest-of-runner/pytest-0/test_workspace_concurrency_int10')
timeout = 8.0, max_procs = 16

    def _test_workspace_concurrency(tmpdir, timeout, max_procs):
        """
        WorkSpace concurrency test.  We merely check that no exception or
        deadlock happens.
        """
        base_dir = str(tmpdir)
    
        err_q = mp_context.Queue()
        purged_q = mp_context.Queue()
        stop_evt = mp_context.Event()
        ws = WorkSpace(base_dir)
        # Make sure purging only happens in the child processes
        ws._purge_leftovers = lambda: None
    
        # Run a bunch of child processes that will try to purge concurrently
        NPROCS = 2 if sys.platform == "win32" else max_procs
        processes = [
            mp_context.Process(
                target=_workspace_concurrency, args=(base_dir, purged_q, err_q, stop_evt)
            )
            for i in range(NPROCS)
        ]
        for p in processes:
            p.start()
    
        n_created = 0
        n_purged = 0
        try:
            t1 = time()
            while time() - t1 < timeout:
                # Add a bunch of locks, and simulate forgetting them.
                # The concurrent processes should try to purge them.
                for i in range(50):
                    d = ws.new_work_dir(prefix="workspace-concurrency-")
                    d._finalizer.detach()
                    n_created += 1
                sleep(1e-2)
        finally:
            stop_evt.set()
            for p in processes:
                p.join()
    
        # Any errors?
        try:
            err = err_q.get_nowait()
        except queue.Empty:
            pass
        else:
            raise err
    
        try:
            while True:
                n_purged += purged_q.get_nowait()
        except queue.Empty:
            pass
        # We attempted to purge most directories at some point
>       assert n_purged >= 0.5 * n_created > 0
E       assert 0 >= (0.5 * 2100)

distributed/tests/test_diskutils.py:272: AssertionError
```

</details>


Discussion:
Findings from stress testing:
1. This is a macos-specific issue.
2. It also affects ``test_workspace_concurrency``, but less frequently
3. When it fails, pytest-rerunfailures is useless, as it fails 10 reruns in a row
4. Failures are very heavily clustered in time, which strongly hints at something being wrong in the CI boxes:
   - stress test on July 4: 0 out of 60 failures
   - stress test on July 6: 0 out of 60 failures
   - stress test on July 8: 15 out of 60 failures

#5022 marks these two tests as XFAIL on MacOS.

Resolving patch:
diff --git a/distributed/utils.py b/distributed/utils.py
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -46,7 +46,7 @@
 from dask.utils import parse_timedelta as _parse_timedelta
 from dask.widgets import get_template
 
-from .compatibility import PYPY, WINDOWS
+from .compatibility import WINDOWS
 from .metrics import time
 
 try:
@@ -67,11 +67,9 @@
 
 
 def _initialize_mp_context():
-    if WINDOWS or PYPY:
-        return multiprocessing
-    else:
-        method = dask.config.get("distributed.worker.multiprocessing-method")
-        ctx = multiprocessing.get_context(method)
+    method = dask.config.get("distributed.worker.multiprocessing-method")
+    ctx = multiprocessing.get_context(method)
+    if method == "forkserver":
         # Makes the test suite much faster
         preload = ["distributed"]
         if "pkg_resources" in sys.modules:
@@ -87,7 +85,8 @@ def _initialize_mp_context():
             else:
                 preload.append(pkg)
         ctx.set_forkserver_preload(preload)
-        return ctx
+
+    return ctx
 
 
 mp_context = _initialize_mp_context()


Test patch:
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -12,6 +12,7 @@
 
 import dask
 
+from distributed.compatibility import WINDOWS
 from distributed.diskutils import WorkSpace
 from distributed.metrics import time
 from distributed.utils import mp_context
@@ -214,8 +215,7 @@ def _workspace_concurrency(base_dir, purged_q, err_q, stop_evt, barrier):
 
 
 @pytest.mark.slow
-@pytest.mark.parametrize("timeout,max_procs", [(5.0, 6), (10.0, 16)])
-def test_workspace_concurrency(tmpdir, timeout, max_procs):
+def test_workspace_concurrency(tmpdir):
     """WorkSpace concurrency test. We merely check that no exception or
     deadlock happens.
     """
@@ -228,6 +228,10 @@ def test_workspace_concurrency(tmpdir, timeout, max_procs):
     # Make sure purging only happens in the child processes
     ws._purge_leftovers = lambda: None
 
+    # Windows (or at least Windows GitHub CI) has been observed to be exceptionally
+    # slow. Don't stress it too much.
+    max_procs = 2 if WINDOWS else 16
+
     # Run a bunch of child processes that will try to purge concurrently
     barrier = mp_context.Barrier(parties=max_procs + 1)
     processes = [
@@ -244,14 +248,16 @@ def test_workspace_concurrency(tmpdir, timeout, max_procs):
     n_purged = 0
     t1 = time()
     try:
-        while time() - t1 < timeout:
-            # Add a bunch of locks, and simulate forgetting them.
+        # On Linux, you will typically end with n_created > 10.000
+        # On Windows, it can take 60 seconds to create 50 locks!
+        while time() - t1 < 10:
+            # Add a bunch of locks and simulate forgetting them.
             # The concurrent processes should try to purge them.
-            for i in range(50):
+            for _ in range(100):
                 d = ws.new_work_dir(prefix="workspace-concurrency-")
                 d._finalizer.detach()
                 n_created += 1
-            sleep(0.01)
+
     finally:
         stop_evt.set()
         for p in processes:
@@ -270,6 +276,6 @@ def test_workspace_concurrency(tmpdir, timeout, max_procs):
             n_purged += purged_q.get_nowait()
     except queue.Empty:
         pass
-    assert n_created >= 100
+
     # We attempted to purge most directories at some point
     assert n_purged >= 0.5 * n_created > 0