Repository: PrefectHQ/prefect

Issue:
Subflow timeout is counting before actually started (upstream tasks)
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar issue and didn't find it.
- [X] I searched the Prefect documentation for this issue.
- [X] I checked that this issue is related to Prefect and not one of its dependencies.

### Bug summary

Subflow timeout seems to count down immediately, even though the subflow isn't started yet due upstream tasks. I would expect the count down only starts after the subflow begins execution. 

Otherwise you have to add the of the parent flow to the timeout of the subflow (if that is even known). 

### Reproduction

```python3
from prefect import flow, task, get_run_logger
from datetime import datetime
from time import sleep

@task
def print_now_task():
    get_run_logger().info(datetime.now().isoformat())

@task
def sleep_task(n):
    sleep(n)


@flow(timeout_seconds=1)
def sub_flow():
    print_now_task()
    sleep_task(10)
    print_now_task()


@flow
def parent_flow():
    print_now_task()

    workers = sleep_task.map(list(range(10)))
    sub_flow(wait_for=[workers])

    print_now_task()

if __name__ == "__main__":
    parent_flow()
```


### Error

```python3
14:19:36.331 | ERROR   | Flow run 'outrageous-moth' - Encountered exception during execution:
Traceback (most recent call last):
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/engine.py", line 624, in orchestrate_flow_run
    result = await run_sync(flow_call)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 132, in run_sync_in_interruptible_worker_thread
    tg.start_soon(
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 660, in __aexit__
    raise ExceptionGroup(exceptions)
anyio._backends._asyncio.ExceptionGroup: 2 exceptions were raised in the task group:
----------------------------
Traceback (most recent call last):
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/to_thread.py", line 31, in run_sync
    return await get_asynclib().run_sync_in_worker_thread(
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 900, in run_sync_in_worker_thread
    await checkpoint()
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 515, in checkpoint
    await sleep(0)
  File "/Users/joell/.pyenv/versions/3.9.13/lib/python3.9/asyncio/tasks.py", line 641, in sleep
    await __sleep0()
  File "/Users/joell/.pyenv/versions/3.9.13/lib/python3.9/asyncio/tasks.py", line 630, in __sleep0
    yield
asyncio.exceptions.CancelledError
----------------------------
Traceback (most recent call last):
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 122, in send_interrupt_to_thread
    await event.wait()
  File "/Users/joell/.pyenv/versions/3.9.13/lib/python3.9/asyncio/locks.py", line 226, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 127, in send_interrupt_to_thread
    raise_async_exception_in_thread(thread, anyio.get_cancelled_exc_class())
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 79, in raise_async_exception_in_thread
    ctypes.c_long(thread.ident), ctypes.py_object(exc_type)
AttributeError: 'NoneType' object has no attribute 'ident'


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/engine.py", line 624, in orchestrate_flow_run
    result = await run_sync(flow_call)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 69, in run_sync_in_worker_thread
    return await anyio.to_thread.run_sync(call, cancellable=True)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/to_thread.py", line 31, in run_sync
    return await get_asynclib().run_sync_in_worker_thread(
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
    return await future
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 867, in run
    result = context.run(func, *args)
  File "/Users/joell/dev/me/feniks-prefect/src/flows/timeout_flow.py", line 26, in parent_flow
    sub_flow(wait_for=[workers])
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/flows.py", line 447, in __call__
    return enter_flow_run_engine_from_flow_call(
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/engine.py", line 166, in enter_flow_run_engine_from_flow_call
    return run_async_from_worker_thread(begin_run)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 152, in run_async_from_worker_thread
    return anyio.from_thread.run(call)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/from_thread.py", line 49, in run
    return asynclib.run_async_from_thread(func, *args)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 970, in run_async_from_thread
    return f.result()
  File "/Users/joell/.pyenv/versions/3.9.13/lib/python3.9/concurrent/futures/_base.py", line 446, in result
    return self.__get_result()
  File "/Users/joell/.pyenv/versions/3.9.13/lib/python3.9/concurrent/futures/_base.py", line 391, in __get_result
    raise self._exception
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/client/utilities.py", line 47, in with_injected_client
    return await fn(*args, **kwargs)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/engine.py", line 533, in create_and_begin_subflow_run
    return await terminal_state.result(fetch=True)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/states.py", line 77, in _get_state_result
    raise await get_state_exception(state)
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/prefect/engine.py", line 626, in orchestrate_flow_run
    waited_for_task_runs = await wait_for_task_runs_and_report_crashes(
  File "/Users/joell/dev/me/feniks-prefect/.venv/lib/python3.9/site-packages/anyio/_core/_tasks.py", line 118, in __exit__
    raise TimeoutError
TimeoutError
```
```


### Versions

```Text
14:31:11.496 | DEBUG   | prefect.client - Connecting to API at https://prefect.feniks.dev/api/
Version:             2.7.1
API version:         0.8.3
Python version:      3.9.13
Git commit:          33f7de50
Built:               Thu, Dec 8, 2022 4:09 PM
OS/Arch:             darwin/arm64
Profile:             joell-dev
Server type:         hosted
```
```


### Additional context

_No response_


Discussion:
Huh weird, this was added in https://github.com/PrefectHQ/prefect/pull/7343 and we're definitely waiting for the upstreams at 

https://github.com/PrefectHQ/prefect/blob/5fb17cec3c716d4603ee91c87862a1db6842b794/src/prefect/engine.py#L435-L436

then starting the timeout context way later at 

https://github.com/PrefectHQ/prefect/blob/5fb17cec3c716d4603ee91c87862a1db6842b794/src/prefect/engine.py#L607
I can, however, reproduce this with your example.

Resolving patch:
diff --git a/src/prefect/engine.py b/src/prefect/engine.py
--- a/src/prefect/engine.py
+++ b/src/prefect/engine.py
@@ -574,11 +574,6 @@ async def orchestrate_flow_run(
     """
     logger = flow_run_logger(flow_run, flow)
 
-    timeout_context = (
-        anyio.fail_after(flow.timeout_seconds)
-        if flow.timeout_seconds
-        else nullcontext()
-    )
     flow_run_context = None
 
     try:
@@ -604,6 +599,10 @@ async def orchestrate_flow_run(
         # Update the flow run to the latest data
         flow_run = await client.read_flow_run(flow_run.id)
         try:
+            timeout_context = anyio.fail_after(
+                flow.timeout_seconds if flow.timeout_seconds else None
+            )
+
             with timeout_context as timeout_scope:
                 with partial_flow_run_context.finalize(
                     flow=flow,


Test patch:
diff --git a/tests/test_flows.py b/tests/test_flows.py
--- a/tests/test_flows.py
+++ b/tests/test_flows.py
@@ -991,6 +991,38 @@ def my_flow():
         assert not canary_file.exists()
         assert runtime < 5, f"The engine returns without waiting; took {runtime}s"
 
+    async def test_subflow_timeout_waits_until_execution_starts(self, tmp_path):
+        """
+        Subflow with a timeout shouldn't start their timeout before the subflow is started.
+        Fixes: https://github.com/PrefectHQ/prefect/issues/7903.
+        """
+
+        canary_file = tmp_path / "canary"
+
+        @flow(timeout_seconds=1)
+        def downstream_flow():
+            canary_file.touch()
+
+        @task
+        def sleep_task(n):
+            time.sleep(n)
+
+        @flow
+        async def my_flow():
+            upstream_sleepers = sleep_task.map(list(range(3)))
+            downstream_flow(wait_for=upstream_sleepers)
+
+        t0 = anyio.current_time()
+        state = await my_flow._run()
+        t1 = anyio.current_time()
+
+        assert state.is_completed()
+
+        # Validate the sleep tasks have ran.
+        # Note: t1 - t0 can be less than exactly 3 (i.e., around 2.9). By comparing with 2.7 we have some leeway.
+        assert t1 - t0 >= 2.7
+        assert canary_file.exists()  # Validate subflow has ran
+
 
 class ParameterTestModel(pydantic.BaseModel):
     data: int