Repository: PrefectHQ/prefect

Issue:
Agent does not report cancellation of process until after grace period expires
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar request and didn't find it.
- [X] I searched the Prefect documentation for this feature.

### Prefect Version

2.x

### Describe the current behavior

When the agent kills a process, it waits a full 30s grace period to update the state to "CANCELLED"
```
16:29:34.550 | INFO    | prefect.agent - Found 1 flow runs awaiting cancellation.
16:29:34.593 | INFO    | prefect.agent - Killing process Michaels-MacBook-Pro.local:77948 for flow run '9fd80913-57f2-4530-9c4e-5b601fba6cb8'...
16:29:36.495 | ERROR   | prefect.infrastructure.process - Process 'ruby-gorilla' exited with status code: -15; This indicates that the process exited due to a SIGTERM signal. Typically, this is caused by manual cancellation.
16:30:04.630 | INFO    | prefect.agent - Cancelled flow run '9fd80913-57f2-4530-9c4e-5b601fba6cb8'!
```



### Describe the proposed behavior

Ideally, the agent would be able to monitor that the SIGTERM resulted in an exited process and report without waiting the full grace period.

### Example Use

n/a

### Additional context

_No response_


Discussion:

Resolving patch:
diff --git a/src/prefect/infrastructure/process.py b/src/prefect/infrastructure/process.py
--- a/src/prefect/infrastructure/process.py
+++ b/src/prefect/infrastructure/process.py
@@ -168,12 +168,26 @@ async def kill(self, infrastructure_pid: str, grace_seconds: int = 30):
                     f"Unable to kill process {pid!r}: The process was not found."
                 )
 
-            await anyio.sleep(grace_seconds)
+            # Throttle how often we check if the process is still alive to keep
+            # from making too many system calls in a short period of time.
+            check_interval = max(grace_seconds / 10, 1)
+
+            with anyio.move_on_after(grace_seconds):
+                while True:
+                    await anyio.sleep(check_interval)
+
+                    # Detect if the process is still alive. If not do an early
+                    # return as the process respected the SIGTERM from above.
+                    try:
+                        os.kill(pid, 0)
+                    except ProcessLookupError:
+                        return
 
             try:
                 os.kill(pid, signal.SIGKILL)
             except ProcessLookupError:
-                # The process likely exited due to the SIGTERM above.
+                # We shouldn't ever end up here, but it's possible that the
+                # process ended right after the check above.
                 return
 
     def preview(self):


Test patch:
diff --git a/tests/infrastructure/test_process.py b/tests/infrastructure/test_process.py
--- a/tests/infrastructure/test_process.py
+++ b/tests/infrastructure/test_process.py
@@ -201,7 +201,7 @@ async def test_process_kill_mismatching_hostname(monkeypatch):
     process = Process(command=["noop"])
 
     with pytest.raises(InfrastructureNotAvailable):
-        await process.kill(infrastructure_pid=infrastructure_pid, grace_seconds=15)
+        await process.kill(infrastructure_pid=infrastructure_pid)
 
     os_kill.assert_not_called()
 
@@ -215,7 +215,7 @@ async def test_process_kill_no_matching_pid(monkeypatch):
     process = Process(command=["noop"])
 
     with pytest.raises(InfrastructureNotFound):
-        await process.kill(infrastructure_pid=infrastructure_pid, grace_seconds=15)
+        await process.kill(infrastructure_pid=infrastructure_pid)
 
 
 @pytest.mark.skipif(
@@ -224,12 +224,37 @@ async def test_process_kill_no_matching_pid(monkeypatch):
 )
 async def test_process_kill_sends_sigterm_then_sigkill(monkeypatch):
     os_kill = MagicMock()
+    monkeypatch.setattr("os.kill", os_kill)
+
+    infrastructure_pid = f"{socket.gethostname()}:12345"
+    grace_seconds = 2
+
+    process = Process(command=["noop"])
+    await process.kill(
+        infrastructure_pid=infrastructure_pid, grace_seconds=grace_seconds
+    )
+
+    os_kill.assert_has_calls(
+        [
+            call(12345, signal.SIGTERM),
+            call(12345, 0),
+            call(12345, signal.SIGKILL),
+        ]
+    )
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="SIGTERM/SIGKILL are only used in non-Windows environments",
+)
+async def test_process_kill_early_return(monkeypatch):
+    os_kill = MagicMock(side_effect=[None, ProcessLookupError])
     anyio_sleep = AsyncMock()
     monkeypatch.setattr("os.kill", os_kill)
     monkeypatch.setattr("prefect.infrastructure.process.anyio.sleep", anyio_sleep)
 
     infrastructure_pid = f"{socket.gethostname()}:12345"
-    grace_seconds = 15
+    grace_seconds = 30
 
     process = Process(command=["noop"])
     await process.kill(
@@ -239,11 +264,11 @@ async def test_process_kill_sends_sigterm_then_sigkill(monkeypatch):
     os_kill.assert_has_calls(
         [
             call(12345, signal.SIGTERM),
-            call(12345, signal.SIGKILL),
+            call(12345, 0),
         ]
     )
 
-    anyio_sleep.assert_called_once_with(grace_seconds)
+    anyio_sleep.assert_called_once_with(3)
 
 
 @pytest.mark.skipif(