Repository: dask/distributed

Issue:
Flaky `test_worker_start_exception`
The interesting thing here is that the `ValueError: broken` only seems to get printed twice when `Error in Nanny killing Worker subprocess` occurs. Maybe something weird in asyncio is happening, possibly causing the exception to pop out again in the wrong place?

But `AttributeError: 'NoneType' object has no attribute 'close'` points to an actual bug. I wonder if we fix that, if the problem will go away?

```
_________________________ test_worker_start_exception __________________________

s = <Scheduler 'tcp://127.0.0.1:57170', workers: 0, cores: 0, tasks: 0>

    @gen_cluster(nthreads=[])
    async def test_worker_start_exception(s):
        nanny = Nanny(s.address, worker_class=BrokenWorker)
        with captured_logger(logger="distributed.nanny", level=logging.WARNING) as logs:
            with raises_with_cause(
                RuntimeError,
                "Nanny failed to start",
                RuntimeError,
                "BrokenWorker failed to start",
            ):
                async with nanny:
                    pass
        assert nanny.status == Status.failed
        # ^ NOTE: `Nanny.close` sets it to `closed`, then `Server.start._close_on_failure` sets it to `failed`
        assert nanny.process is None
        assert "Restarting worker" not in logs.getvalue()
        # Avoid excessive spewing. (It's also printed once extra within the subprocess, which is okay.)
>       assert logs.getvalue().count("ValueError: broken") == 1, logs.getvalue()
E       AssertionError: Failed to start process
E         Traceback (most recent call last):
2022-08-26 06:52:07,459 - distributed.utils_perf - WARNING - full garbage collections took 40% CPU time recently (threshold: 10%)
E           File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
2022-08-26 06:52:08,508 - distributed.utils_perf - WARNING - full garbage collections took 39% CPU time recently (threshold: 10%)
E             await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
2022-08-26 06:52:09,543 - distributed.utils_perf - WARNING - full garbage collections took 37% CPU time recently (threshold: 10%)
E           File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
E             return await fut
E           File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
E             raise ValueError("broken")
E         ValueError: broken
E         
E         The above exception was the direct cause of the following exception:
E         
E         Traceback (most recent call last):
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate
E             result = await self.process.start()
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start
E             msg = await self._wait_until_connected(uid)
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected
E             raise msg["exception"]
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
E             await worker
E           File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
E             raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
E         RuntimeError: BrokenWorker failed to start.
E         Error in Nanny killing Worker subprocess
E         Traceback (most recent call last):
E           File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
E             await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
E           File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
E             return await fut
E           File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
E             raise ValueError("broken")
E         ValueError: broken
E         
E         The above exception was the direct cause of the following exception:
E         
E         Traceback (most recent call last):
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate
E             result = await self.process.start()
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start
E             msg = await self._wait_until_connected(uid)
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected
E             raise msg["exception"]
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
E             await worker
E           File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
E             raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
E         RuntimeError: BrokenWorker failed to start.
E         
E         During handling of the above exception, another exception occurred:
E         
E         Traceback (most recent call last):
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 595, in close
E             await self.kill(timeout=timeout)
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 386, in kill
E             await self.process.kill(timeout=0.8 * (deadline - time()))
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 793, in kill
E             self.child_stop_q.close()
E         AttributeError: 'NoneType' object has no attribute 'close'
E         
E       assert 2 == 1
E        +  where 2 = <built-in method count of str object at 0x7fba2e3f3a00>('ValueError: broken')
E        +    where <built-in method count of str object at 0x7fba2e3f3a00> = 'Failed to start process\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start\n    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)\n  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for\n    return await fut\n  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe\n    raise ValueError("broken")\nValueError: broken\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate\n    result = await self.process.start()\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start\n    msg = await self._wait_until_connected(uid)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected\n    raise msg["exception"]\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run\n    await worker\n  File "/Users/runner/work/distribute...art()\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start\n    msg = await self._wait_until_connected(uid)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected\n    raise msg["exception"]\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run\n    await worker\n  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start\n    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc\nRuntimeError: BrokenWorker failed to start.\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 595, in close\n    await self.kill(timeout=timeout)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 386, in kill\n    await self.process.kill(timeout=0.8 * (deadline - time()))\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 793, in kill\n    self.child_stop_q.close()\nAttributeError: \'NoneType\' object has no attribute \'close\'\n'.count
E        +      where 'Failed to start process\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start\n    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)\n  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for\n    return await fut\n  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe\n    raise ValueError("broken")\nValueError: broken\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate\n    result = await self.process.start()\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start\n    msg = await self._wait_until_connected(uid)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected\n    raise msg["exception"]\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run\n    await worker\n  File "/Users/runner/work/distribute...art()\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start\n    msg = await self._wait_until_connected(uid)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected\n    raise msg["exception"]\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run\n    await worker\n  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start\n    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc\nRuntimeError: BrokenWorker failed to start.\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 595, in close\n    await self.kill(timeout=timeout)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 386, in kill\n    await self.process.kill(timeout=0.8 * (deadline - time()))\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 793, in kill\n    self.child_stop_q.close()\nAttributeError: \'NoneType\' object has no attribute \'close\'\n' = <built-in method getvalue of _io.StringIO object at 0x141232f80>()
E        +        where <built-in method getvalue of _io.StringIO object at 0x141232f80> = <_io.StringIO object at 0x141232f80>.getvalue

distributed/tests/test_nanny.py:520: AssertionError
----------------------------- Captured stdout call -----------------------------
Dumped cluster state to test_cluster_dump/test_worker_start_exception.yaml
----------------------------- Captured stderr call -----------------------------
2022-08-26 06:41:19,345 - distributed.scheduler - INFO - State start
2022-08-26 06:41:19,348 - distributed.scheduler - INFO -   Scheduler at:     tcp://127.0.0.1:57170
2022-08-26 06:41:19,348 - distributed.scheduler - INFO -   dashboard at:           127.0.0.1:57169
2022-08-26 06:41:22,017 - distributed.worker - INFO - Stopping worker
2022-08-26 06:41:22,018 - distributed.worker - INFO - Closed worker has not yet started: Status.init
2022-08-26 06:41:22,020 - distributed.nanny - ERROR - Failed to start worker
Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
    return await fut
  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
    raise ValueError("broken")
ValueError: broken

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
    await worker
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
RuntimeError: BrokenWorker failed to start.
2022-08-26 06:41:22,148 - distributed.nanny - ERROR - Failed to start process
Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
    return await fut
  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
    raise ValueError("broken")
ValueError: broken

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate
    result = await self.process.start()
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start
    msg = await self._wait_until_connected(uid)
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected
    raise msg["exception"]
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
    await worker
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
RuntimeError: BrokenWorker failed to start.
2022-08-26 06:41:22,164 - distributed.nanny - ERROR - Error in Nanny killing Worker subprocess
Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
    return await fut
  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
    raise ValueError("broken")
ValueError: broken

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate
    result = await self.process.start()
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start
    msg = await self._wait_until_connected(uid)
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected
    raise msg["exception"]
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
    await worker
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
RuntimeError: BrokenWorker failed to start.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 595, in close
    await self.kill(timeout=timeout)
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 386, in kill
    await self.process.kill(timeout=0.8 * (deadline - time()))
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 793, in kill
    self.child_stop_q.close()
AttributeError: 'NoneType' object has no attribute 'close'
2022-08-26 06:41:22,203 - distributed.scheduler - INFO - Scheduler closing...
2022-08-26 06:41:22,204 - distributed.scheduler - INFO - Scheduler closing all comms
```

https://github.com/dask/distributed/runs/8030741091?check_suite_focus=true#step:11:1301
Flaky `test_worker_start_exception`
The interesting thing here is that the `ValueError: broken` only seems to get printed twice when `Error in Nanny killing Worker subprocess` occurs. Maybe something weird in asyncio is happening, possibly causing the exception to pop out again in the wrong place?

But `AttributeError: 'NoneType' object has no attribute 'close'` points to an actual bug. I wonder if we fix that, if the problem will go away?

```
_________________________ test_worker_start_exception __________________________

s = <Scheduler 'tcp://127.0.0.1:57170', workers: 0, cores: 0, tasks: 0>

    @gen_cluster(nthreads=[])
    async def test_worker_start_exception(s):
        nanny = Nanny(s.address, worker_class=BrokenWorker)
        with captured_logger(logger="distributed.nanny", level=logging.WARNING) as logs:
            with raises_with_cause(
                RuntimeError,
                "Nanny failed to start",
                RuntimeError,
                "BrokenWorker failed to start",
            ):
                async with nanny:
                    pass
        assert nanny.status == Status.failed
        # ^ NOTE: `Nanny.close` sets it to `closed`, then `Server.start._close_on_failure` sets it to `failed`
        assert nanny.process is None
        assert "Restarting worker" not in logs.getvalue()
        # Avoid excessive spewing. (It's also printed once extra within the subprocess, which is okay.)
>       assert logs.getvalue().count("ValueError: broken") == 1, logs.getvalue()
E       AssertionError: Failed to start process
E         Traceback (most recent call last):
2022-08-26 06:52:07,459 - distributed.utils_perf - WARNING - full garbage collections took 40% CPU time recently (threshold: 10%)
E           File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
2022-08-26 06:52:08,508 - distributed.utils_perf - WARNING - full garbage collections took 39% CPU time recently (threshold: 10%)
E             await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
2022-08-26 06:52:09,543 - distributed.utils_perf - WARNING - full garbage collections took 37% CPU time recently (threshold: 10%)
E           File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
E             return await fut
E           File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
E             raise ValueError("broken")
E         ValueError: broken
E         
E         The above exception was the direct cause of the following exception:
E         
E         Traceback (most recent call last):
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate
E             result = await self.process.start()
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start
E             msg = await self._wait_until_connected(uid)
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected
E             raise msg["exception"]
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
E             await worker
E           File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
E             raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
E         RuntimeError: BrokenWorker failed to start.
E         Error in Nanny killing Worker subprocess
E         Traceback (most recent call last):
E           File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
E             await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
E           File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
E             return await fut
E           File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
E             raise ValueError("broken")
E         ValueError: broken
E         
E         The above exception was the direct cause of the following exception:
E         
E         Traceback (most recent call last):
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate
E             result = await self.process.start()
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start
E             msg = await self._wait_until_connected(uid)
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected
E             raise msg["exception"]
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
E             await worker
E           File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
E             raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
E         RuntimeError: BrokenWorker failed to start.
E         
E         During handling of the above exception, another exception occurred:
E         
E         Traceback (most recent call last):
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 595, in close
E             await self.kill(timeout=timeout)
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 386, in kill
E             await self.process.kill(timeout=0.8 * (deadline - time()))
E           File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 793, in kill
E             self.child_stop_q.close()
E         AttributeError: 'NoneType' object has no attribute 'close'
E         
E       assert 2 == 1
E        +  where 2 = <built-in method count of str object at 0x7fba2e3f3a00>('ValueError: broken')
E        +    where <built-in method count of str object at 0x7fba2e3f3a00> = 'Failed to start process\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start\n    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)\n  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for\n    return await fut\n  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe\n    raise ValueError("broken")\nValueError: broken\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate\n    result = await self.process.start()\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start\n    msg = await self._wait_until_connected(uid)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected\n    raise msg["exception"]\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run\n    await worker\n  File "/Users/runner/work/distribute...art()\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start\n    msg = await self._wait_until_connected(uid)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected\n    raise msg["exception"]\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run\n    await worker\n  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start\n    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc\nRuntimeError: BrokenWorker failed to start.\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 595, in close\n    await self.kill(timeout=timeout)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 386, in kill\n    await self.process.kill(timeout=0.8 * (deadline - time()))\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 793, in kill\n    self.child_stop_q.close()\nAttributeError: \'NoneType\' object has no attribute \'close\'\n'.count
E        +      where 'Failed to start process\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start\n    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)\n  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for\n    return await fut\n  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe\n    raise ValueError("broken")\nValueError: broken\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate\n    result = await self.process.start()\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start\n    msg = await self._wait_until_connected(uid)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected\n    raise msg["exception"]\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run\n    await worker\n  File "/Users/runner/work/distribute...art()\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start\n    msg = await self._wait_until_connected(uid)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected\n    raise msg["exception"]\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run\n    await worker\n  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start\n    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc\nRuntimeError: BrokenWorker failed to start.\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 595, in close\n    await self.kill(timeout=timeout)\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 386, in kill\n    await self.process.kill(timeout=0.8 * (deadline - time()))\n  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 793, in kill\n    self.child_stop_q.close()\nAttributeError: \'NoneType\' object has no attribute \'close\'\n' = <built-in method getvalue of _io.StringIO object at 0x141232f80>()
E        +        where <built-in method getvalue of _io.StringIO object at 0x141232f80> = <_io.StringIO object at 0x141232f80>.getvalue

distributed/tests/test_nanny.py:520: AssertionError
----------------------------- Captured stdout call -----------------------------
Dumped cluster state to test_cluster_dump/test_worker_start_exception.yaml
----------------------------- Captured stderr call -----------------------------
2022-08-26 06:41:19,345 - distributed.scheduler - INFO - State start
2022-08-26 06:41:19,348 - distributed.scheduler - INFO -   Scheduler at:     tcp://127.0.0.1:57170
2022-08-26 06:41:19,348 - distributed.scheduler - INFO -   dashboard at:           127.0.0.1:57169
2022-08-26 06:41:22,017 - distributed.worker - INFO - Stopping worker
2022-08-26 06:41:22,018 - distributed.worker - INFO - Closed worker has not yet started: Status.init
2022-08-26 06:41:22,020 - distributed.nanny - ERROR - Failed to start worker
Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
    return await fut
  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
    raise ValueError("broken")
ValueError: broken

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
    await worker
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
RuntimeError: BrokenWorker failed to start.
2022-08-26 06:41:22,148 - distributed.nanny - ERROR - Failed to start process
Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
    return await fut
  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
    raise ValueError("broken")
ValueError: broken

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate
    result = await self.process.start()
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start
    msg = await self._wait_until_connected(uid)
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected
    raise msg["exception"]
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
    await worker
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
RuntimeError: BrokenWorker failed to start.
2022-08-26 06:41:22,164 - distributed.nanny - ERROR - Error in Nanny killing Worker subprocess
Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 481, in start
    await asyncio.wait_for(self.start_unsafe(), timeout=timeout)
  File "/Users/runner/miniconda3/envs/dask-distributed/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
    return await fut
  File "/Users/runner/work/distributed/distributed/distributed/tests/test_nanny.py", line 500, in start_unsafe
    raise ValueError("broken")
ValueError: broken

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 438, in instantiate
    result = await self.process.start()
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 695, in start
    msg = await self._wait_until_connected(uid)
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 823, in _wait_until_connected
    raise msg["exception"]
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 892, in run
    await worker
  File "/Users/runner/work/distributed/distributed/distributed/core.py", line 489, in start
    raise RuntimeError(f"{type(self).__name__} failed to start.") from exc
RuntimeError: BrokenWorker failed to start.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 595, in close
    await self.kill(timeout=timeout)
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 386, in kill
    await self.process.kill(timeout=0.8 * (deadline - time()))
  File "/Users/runner/work/distributed/distributed/distributed/nanny.py", line 793, in kill
    self.child_stop_q.close()
AttributeError: 'NoneType' object has no attribute 'close'
2022-08-26 06:41:22,203 - distributed.scheduler - INFO - Scheduler closing...
2022-08-26 06:41:22,204 - distributed.scheduler - INFO - Scheduler closing all comms
```

https://github.com/dask/distributed/runs/8030741091?check_suite_focus=true#step:11:1301


Discussion:

Resolving patch:
diff --git a/distributed/nanny.py b/distributed/nanny.py
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -780,9 +780,11 @@ async def kill(self, timeout: float = 2, executor_wait: bool = True) -> None:
         logger.info("Nanny asking worker to close")
 
         process = self.process
-        assert self.process
+        assert process
+        queue = self.child_stop_q
+        assert queue
         wait_timeout = timeout * 0.8
-        self.child_stop_q.put(
+        queue.put(
             {
                 "op": "stop",
                 "timeout": wait_timeout,
@@ -790,19 +792,25 @@ async def kill(self, timeout: float = 2, executor_wait: bool = True) -> None:
             }
         )
         await asyncio.sleep(0)  # otherwise we get broken pipe errors
-        self.child_stop_q.close()
+        queue.close()
+        del queue
 
         try:
-            await process.join(wait_timeout)
-            return
-        except asyncio.TimeoutError:
-            pass
+            try:
+                await process.join(wait_timeout)
+                return
+            except asyncio.TimeoutError:
+                pass
 
-        logger.warning(
-            f"Worker process still alive after {wait_timeout} seconds, killing"
-        )
-        await process.kill()
-        await process.join(max(0, deadline - time()))
+            logger.warning(
+                f"Worker process still alive after {wait_timeout} seconds, killing"
+            )
+            await process.kill()
+            await process.join(max(0, deadline - time()))
+        except ValueError as e:
+            if "invalid operation on closed AsyncProcess" in str(e):
+                return
+            raise
 
     async def _wait_until_connected(self, uid):
         while True:


Test patch: