Repository: PrefectHQ/prefect

Issue:
Temporary network connectivity problems cause Worker to stop working
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar issue and didn't find it.
- [X] I searched the Prefect documentation for this issue.
- [X] I checked that this issue is related to Prefect and not one of its dependencies.

### Bug summary

Network issues seem to get the Worker into a state where it continues to send heartbeats, but stops running flows, resulting in flows getting stuck in "pending" state.

### Reproduction

```python3
Run a worker and wait until it gets "stuck"
```


### Error

```python3
$ k logs prefect-worker-stable-78d664b779-snmln
/usr/local/lib/python3.11/site-packages/prefect/experimental/cli/worker.py:81: ExperimentalFeature: The workers feature is experimental. The interface or behavior may change without warning, we recommend pinning versions to prevent unexpected changes. To disable warnings for this group of experiments, disable PREFECT_EXPERIMENTAL_WARN_WORKERS.
  async with worker_cls(
Worker 'KubernetesWorker 3ee16ee0-5575-4d11-975b-a644c66b2362' started!

Failed the last 3 attempts.  Please check your environment and configuration.
Examples of recent errors:

Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/h2/connection.py", line 224, in 
process_input
    func, target_state = self._transitions[(self.state, input_)]
                         ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^
KeyError: (<ConnectionState.CLOSED: 3>, <ConnectionInputs.SEND_HEADERS: 0>)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/httpcore/_async/http2.py", line 
106, in handle_async_request
    await self._send_request_headers(request=request, stream_id=stream_id)
  File "/usr/local/lib/python3.11/site-packages/httpcore/_async/http2.py", line 
203, in _send_request_headers
    self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
  File "/usr/local/lib/python3.11/site-packages/h2/connection.py", line 766, in 
send_headers
    self.state_machine.process_input(ConnectionInputs.SEND_HEADERS)
  File "/usr/local/lib/python3.11/site-packages/h2/connection.py", line 228, in 
process_input
    raise ProtocolError(
h2.exceptions.ProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in 
state ConnectionState.CLOSED

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/httpx/_transports/default.py", 
line 60, in map_httpcore_exceptions
    yield
  File "/usr/local/lib/python3.11/site-packages/httpx/_transports/default.py", 
line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File 
"/usr/local/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", 
line 253, in handle_async_request
    raise exc
  File 
"/usr/local/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", 
line 237, in handle_async_request
    response = await connection.handle_async_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpcore/_async/connection.py", 
line 90, in handle_async_request
    return await self._connection.handle_async_request(request)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpcore/_async/http2.py", line 
142, in handle_async_request
    raise LocalProtocolError(exc)  # pragma: nocover
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
httpcore.LocalProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in 
state ConnectionState.CLOSED

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/prefect/utilities/services.py", 
line 46, in critical_service_loop
    await workload()
  File 
"/usr/local/lib/python3.11/site-packages/prefect/experimental/workers/base.py", 
line 372, in get_and_submit_flow_runs
    runs_response = await self._get_scheduled_flow_runs()
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File 
"/usr/local/lib/python3.11/site-packages/prefect/experimental/workers/base.py", 
line 440, in _get_scheduled_flow_runs
    await self._client.get_scheduled_flow_runs_for_work_pool(
  File 
"/usr/local/lib/python3.11/site-packages/prefect/client/orchestration.py", line 
2269, in get_scheduled_flow_runs_for_work_pool
    response = await self._client.post(
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1848, in
post
    return await self.request(
           ^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1533, in
request
    return await self.send(request, auth=auth, 
follow_redirects=follow_redirects)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^
  File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 
243, in send
    response = await self._send_with_retry(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/prefect/client/base.py", line 
189, in _send_with_retry
    response = await request()
               ^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1620, in
send
    response = await self._send_handling_auth(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1648, in
_send_handling_auth
    response = await self._send_handling_redirects(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1685, in
_send_handling_redirects
    response = await self._send_single_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_client.py", line 1722, in
_send_single_request
    response = await transport.handle_async_request(request)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/httpx/_transports/default.py", 
line 352, in handle_async_request
    with map_httpcore_exceptions():
  File "/usr/local/lib/python3.11/contextlib.py", line 155, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.11/site-packages/httpx/_transports/default.py", 
line 77, in map_httpcore_exceptions
    raise mapped_exc(message) from exc
httpx.LocalProtocolError: Invalid input ConnectionInputs.SEND_HEADERS in state 
ConnectionState.CLOSED
```


### Versions

```Text
Image:         prefecthq/prefect-dev:main-python3.11-kubernetes
    Image ID:      docker.io/prefecthq/prefect-dev@sha256:0b2bd51e56b1a61b8570f5973192270a7afbf49bf808c9e125168f048c25cdb0
```


### Additional context

_No response_


Discussion:
Possibly related to #7442
Let's focus on the failure mode of our "critical loop service" utility here that results in a hang instead of a bad exit code. I've seen reports of that elsewhere. If we exit with a bad code, the service can be restarted by K8s but we should never hang like this.

Resolving patch:
diff --git a/src/prefect/utilities/services.py b/src/prefect/utilities/services.py
--- a/src/prefect/utilities/services.py
+++ b/src/prefect/utilities/services.py
@@ -64,8 +64,6 @@ async def critical_service_loop(
                 failures.append((exc, sys.exc_info()[-1]))
             else:
                 raise
-        except KeyboardInterrupt:
-            return
 
         # Decide whether to exit now based on recent history.
         #
@@ -89,7 +87,7 @@ async def critical_service_loop(
             # We've failed enough times to be sure something is wrong, the writing is
             # on the wall.  Let's explain what we've seen and exit.
             printer(
-                f"\nFailed the last {consecutive} attempts.  "
+                f"\nFailed the last {consecutive} attempts. "
                 "Please check your environment and configuration."
             )
 
@@ -102,7 +100,8 @@ async def critical_service_loop(
             for exception, traceback in failures_by_type:
                 printer("".join(format_exception(None, exception, traceback)))
                 printer()
-            return
+
+            raise RuntimeError("Service exceeded error threshold.")
 
         if run_once:
             return


Test patch:
diff --git a/tests/utilities/test_services.py b/tests/utilities/test_services.py
--- a/tests/utilities/test_services.py
+++ b/tests/utilities/test_services.py
@@ -7,6 +7,10 @@
 from prefect.utilities.services import critical_service_loop
 
 
+class UncapturedException(BaseException):
+    pass
+
+
 async def test_critical_service_loop_operates_normally():
     workload = AsyncMock(
         side_effect=[
@@ -15,15 +19,25 @@ async def test_critical_service_loop_operates_normally():
             None,
             None,
             None,
-            KeyboardInterrupt,
+            UncapturedException,
         ]
     )
 
-    await critical_service_loop(workload, 0.0)
+    with pytest.raises(UncapturedException):
+        await critical_service_loop(workload, 0.0)
 
     assert workload.await_count == 6
 
 
+async def test_critical_service_loop_does_not_capture_keyboard_interrupt():
+    workload = AsyncMock(side_effect=KeyboardInterrupt)
+
+    with pytest.raises(KeyboardInterrupt):
+        await critical_service_loop(workload, 0.0)
+
+    assert workload.await_count == 1
+
+
 async def test_tolerates_single_intermittent_error():
     workload = AsyncMock(
         side_effect=[
@@ -32,11 +46,12 @@ async def test_tolerates_single_intermittent_error():
             None,
             None,
             None,
-            KeyboardInterrupt,
+            UncapturedException,
         ]
     )
 
-    await critical_service_loop(workload, 0.0)
+    with pytest.raises(UncapturedException):
+        await critical_service_loop(workload, 0.0)
 
     assert workload.await_count == 6
 
@@ -49,11 +64,12 @@ async def test_tolerates_two_consecutive_errors():
             httpx.TimeoutException("oofta"),
             None,
             None,
-            KeyboardInterrupt,
+            UncapturedException,
         ]
     )
 
-    await critical_service_loop(workload, 0.0)
+    with pytest.raises(UncapturedException):
+        await critical_service_loop(workload, 0.0)
 
     assert workload.await_count == 6
 
@@ -66,11 +82,12 @@ async def test_tolerates_majority_errors():
             httpx.TimeoutException("oofta"),
             httpx.TimeoutException("boo"),
             None,
-            KeyboardInterrupt,
+            UncapturedException,
         ]
     )
 
-    await critical_service_loop(workload, 0.0)
+    with pytest.raises(UncapturedException):
+        await critical_service_loop(workload, 0.0)
 
     assert workload.await_count == 6
 
@@ -84,11 +101,12 @@ async def test_quits_after_3_consecutive_errors(capsys: pytest.CaptureFixture):
             httpx.TimeoutException("boo"),
             httpx.ConnectError("woops"),
             None,
-            KeyboardInterrupt,
+            None,
         ]
     )
 
-    await critical_service_loop(workload, 0.0, consecutive=3)
+    with pytest.raises(RuntimeError, match="Service exceeded error threshold"):
+        await critical_service_loop(workload, 0.0, consecutive=3)
 
     assert workload.await_count == 4
     result = capsys.readouterr()
@@ -106,13 +124,16 @@ async def test_consistent_sleeps_between_loops(monkeypatch):
             None,
             None,
             None,
-            KeyboardInterrupt,
+            UncapturedException,
         ]
     )
     sleeper = AsyncMock()
 
     monkeypatch.setattr("prefect.utilities.services.anyio.sleep", sleeper)
-    await critical_service_loop(workload, 0.0)
+
+    with pytest.raises(UncapturedException):
+        await critical_service_loop(workload, 0.0)
+
     assert workload.await_count == 6
 
     sleep_times = [call.args[0] for call in sleeper.await_args_list]
@@ -133,13 +154,16 @@ async def test_jittered_sleeps_between_loops(monkeypatch):
             None,
             None,
             None,
-            KeyboardInterrupt,
+            UncapturedException,
         ]
     )
     sleeper = AsyncMock()
 
     monkeypatch.setattr("prefect.utilities.services.anyio.sleep", sleeper)
-    await critical_service_loop(workload, 42, jitter_range=0.3)
+
+    with pytest.raises(UncapturedException):
+        await critical_service_loop(workload, 42, jitter_range=0.3)
+
     assert workload.await_count == 6
 
     sleep_times = [call.args[0] for call in sleeper.await_args_list]