Repository: PrefectHQ/prefect

Issue:
Flow run submission failure by a Prefect Agent should result in `Crashed` state
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar issue and didn't find it.
- [X] I searched the Prefect documentation for this issue.
- [X] I checked that this issue is related to Prefect and not one of its dependencies.

### Bug summary

When an agent or worker fails to submit a flow run, a `Failed` state is set on the flow run.

when a flow run never has a chance to fail within the defined bounds of a workflow, it should be represented by a `Crashed` state

For example (like [this](https://prefect-community.slack.com/archives/CL09KU1K7/p1683746127477319) one), an `on_failure` hook would never be able to run if a k8s pod never spun up for a flow run

### Reproduction

```python3
introduce an infrastructure-level failure between an agent/worker picking up a flow run and successful submission
```


### Error

_No response_

### Versions

```Text
2.10.x
```


### Additional context

_No response_


Discussion:

Resolving patch:
diff --git a/src/prefect/agent.py b/src/prefect/agent.py
--- a/src/prefect/agent.py
+++ b/src/prefect/agent.py
@@ -506,7 +506,9 @@ async def _submit_run_and_capture_errors(
                 )
                 # Mark the task as started to prevent agent crash
                 task_status.started(exc)
-                await self._propose_failed_state(flow_run, exc)
+                await self._propose_crashed_state(
+                    flow_run, "Flow run could not be submitted to infrastructure"
+                )
             else:
                 self.logger.exception(
                     f"An error occured while monitoring flow run '{flow_run.id}'. "
diff --git a/src/prefect/workers/base.py b/src/prefect/workers/base.py
--- a/src/prefect/workers/base.py
+++ b/src/prefect/workers/base.py
@@ -837,7 +837,9 @@ async def _submit_run_and_capture_errors(
                 )
                 # Mark the task as started to prevent agent crash
                 task_status.started(exc)
-                await self._propose_failed_state(flow_run, exc)
+                await self._propose_crashed_state(
+                    flow_run, "Flow run could not be submitted to infrastructure"
+                )
             else:
                 run_logger.exception(
                     f"An error occurred while monitoring flow run '{flow_run.id}'. "


Test patch:
diff --git a/tests/agent/test_agent_run_submission.py b/tests/agent/test_agent_run_submission.py
--- a/tests/agent/test_agent_run_submission.py
+++ b/tests/agent/test_agent_run_submission.py
@@ -707,7 +707,7 @@ async def test_agent_fails_flow_if_get_infrastructure_fails(
         with pytest.raises(FailedRun, match="Submission failed. ValueError: Bad!"):
             await state.result()
 
-    async def test_agent_fails_flow_if_infrastructure_submission_fails(
+    async def test_agent_crashes_flow_if_infrastructure_submission_fails(
         self, prefect_client, deployment, mock_infrastructure_run
     ):
         infra_doc_id = deployment.infrastructure_document_id
@@ -745,8 +745,10 @@ def raise_value_error():
         ), "The concurrency slot should be released"
 
         state = (await prefect_client.read_flow_run(flow_run.id)).state
-        assert state.is_failed()
-        with pytest.raises(FailedRun, match="Submission failed. ValueError: Hello!"):
+        assert state.is_crashed()
+        with pytest.raises(
+            CrashedRun, match="Flow run could not be submitted to infrastructure"
+        ):
             await state.result()
 
     async def test_agent_does_not_fail_flow_if_infrastructure_watch_fails(
diff --git a/tests/fixtures/database.py b/tests/fixtures/database.py
--- a/tests/fixtures/database.py
+++ b/tests/fixtures/database.py
@@ -827,6 +827,34 @@ def hello(name: str):
     return deployment
 
 
+@pytest.fixture
+async def worker_deployment_infra_wq1(
+    session, flow, flow_function, work_queue_1, infrastructure_document_id
+):
+    def hello(name: str):
+        pass
+
+    deployment = await models.deployments.create_deployment(
+        session=session,
+        deployment=schemas.core.Deployment(
+            name="My Deployment 2",
+            tags=["test"],
+            flow_id=flow.id,
+            schedule=schemas.schedules.IntervalSchedule(
+                interval=pendulum.duration(days=1).as_timedelta(),
+                anchor_date=pendulum.datetime(2020, 1, 1),
+            ),
+            path="./subdir",
+            entrypoint="/file.py:flow",
+            parameter_openapi_schema=parameter_schema(hello),
+            work_queue_id=work_queue_1.id,
+            infrastructure_document_id=infrastructure_document_id,
+        ),
+    )
+    await session.commit()
+    return deployment
+
+
 @pytest.fixture
 async def worker_deployment_wq_2(
     session,
diff --git a/tests/workers/test_base_worker.py b/tests/workers/test_base_worker.py
--- a/tests/workers/test_base_worker.py
+++ b/tests/workers/test_base_worker.py
@@ -17,6 +17,7 @@
     InfrastructureNotAvailable,
     InfrastructureNotFound,
     ObjectNotFound,
+    CrashedRun,
 )
 from prefect.flows import flow
 from prefect.server import models
@@ -1749,6 +1750,40 @@ async def test_get_flow_run_logger(
         }
 
 
+class TestInfrastructureIntegration:
+    async def test_agent_crashes_flow_if_infrastructure_submission_fails(
+        self,
+        prefect_client: PrefectClient,
+        worker_deployment_infra_wq1,
+        work_pool,
+        monkeypatch,
+    ):
+        flow_run = await prefect_client.create_flow_run_from_deployment(
+            worker_deployment_infra_wq1.id,
+            state=Scheduled(scheduled_time=pendulum.now("utc")),
+        )
+        await prefect_client.read_flow(worker_deployment_infra_wq1.flow_id)
+
+        def raise_value_error():
+            raise ValueError("Hello!")
+
+        mock_run = MagicMock()
+        mock_run.run = raise_value_error
+
+        async with WorkerTestImpl(work_pool_name=work_pool.name) as worker:
+            worker._work_pool = work_pool
+            monkeypatch.setattr(worker, "run", mock_run)
+            monkeypatch.setattr(worker, "run", mock_run)
+            await worker.get_and_submit_flow_runs()
+
+        state = (await prefect_client.read_flow_run(flow_run.id)).state
+        assert state.is_crashed()
+        with pytest.raises(
+            CrashedRun, match="Flow run could not be submitted to infrastructure"
+        ):
+            await state.result()
+
+
 async def test_worker_set_last_polled_time(
     work_pool,
 ):