Repository: PrefectHQ/prefect

Issue:
Port `flow_run/lateness` endpoint from cloud to open source.
### First check

- [X] I am a contributor to the Prefect codebase

### Description

In cloud we have a `flow_run/lateness` endpoint that returns the average lateness for a set of flow runs, we should port that to open source so that the upcoming workspace dashboard work can take advantage of it.

### Impact

_No response_

### Additional context

_No response_


Discussion:

Resolving patch:
diff --git a/src/prefect/server/api/flow_runs.py b/src/prefect/server/api/flow_runs.py
--- a/src/prefect/server/api/flow_runs.py
+++ b/src/prefect/server/api/flow_runs.py
@@ -3,10 +3,12 @@
 """
 
 import datetime
-from typing import List
+from typing import List, Optional
 from uuid import UUID
 
+
 import pendulum
+import sqlalchemy as sa
 from fastapi import Body, Depends, HTTPException, Path, Response, status
 from fastapi.responses import ORJSONResponse
 
@@ -110,6 +112,66 @@ async def count_flow_runs(
         )
 
 
+@router.post("/lateness")
+async def average_flow_run_lateness(
+    flows: Optional[schemas.filters.FlowFilter] = None,
+    flow_runs: Optional[schemas.filters.FlowRunFilter] = None,
+    task_runs: Optional[schemas.filters.TaskRunFilter] = None,
+    deployments: Optional[schemas.filters.DeploymentFilter] = None,
+    work_pools: Optional[schemas.filters.WorkPoolFilter] = None,
+    work_pool_queues: Optional[schemas.filters.WorkQueueFilter] = None,
+    db: PrefectDBInterface = Depends(provide_database_interface),
+) -> Optional[float]:
+    """
+    Query for average flow-run lateness in seconds.
+    """
+    async with db.session_context() as session:
+        if db.dialect.name == "sqlite":
+            # Since we want an _average_ of the lateness we're unable to use
+            # the existing FlowRun.expected_start_time_delta property as it
+            # returns a timedelta and SQLite is unable to properly deal with it
+            # and always returns 1970.0 as the average. This copies the same
+            # logic but ensures that it returns the number of seconds instead
+            # so it's compatible with SQLite.
+            base_query = sa.case(
+                (
+                    db.FlowRun.start_time > db.FlowRun.expected_start_time,
+                    sa.func.strftime("%s", db.FlowRun.start_time)
+                    - sa.func.strftime("%s", db.FlowRun.expected_start_time),
+                ),
+                (
+                    db.FlowRun.start_time.is_(None)
+                    & db.FlowRun.state_type.notin_(schemas.states.TERMINAL_STATES)
+                    & (db.FlowRun.expected_start_time < sa.func.datetime("now")),
+                    sa.func.strftime("%s", sa.func.datetime("now"))
+                    - sa.func.strftime("%s", db.FlowRun.expected_start_time),
+                ),
+                else_=0,
+            )
+        else:
+            base_query = db.FlowRun.estimated_start_time_delta
+
+        query = await models.flow_runs._apply_flow_run_filters(
+            sa.select(sa.func.avg(base_query)),
+            flow_filter=flows,
+            flow_run_filter=flow_runs,
+            task_run_filter=task_runs,
+            deployment_filter=deployments,
+            work_pool_filter=work_pools,
+            work_queue_filter=work_pool_queues,
+        )
+        result = await session.execute(query)
+
+        avg_lateness = result.scalar()
+
+        if avg_lateness is None:
+            return None
+        elif isinstance(avg_lateness, datetime.timedelta):
+            return avg_lateness.total_seconds()
+        else:
+            return avg_lateness
+
+
 @router.post("/history")
 async def flow_run_history(
     history_start: DateTimeTZ = Body(..., description="The history's start time."),
diff --git a/src/prefect/server/database/interface.py b/src/prefect/server/database/interface.py
--- a/src/prefect/server/database/interface.py
+++ b/src/prefect/server/database/interface.py
@@ -8,6 +8,7 @@
 from prefect.server.database.configurations import BaseDatabaseConfiguration
 from prefect.server.database.orm_models import BaseORMConfiguration
 from prefect.server.database.query_components import BaseQueryComponents
+from prefect.server.utilities.database import get_dialect
 from prefect.utilities.asyncutils import run_sync_in_worker_thread
 
 
@@ -119,6 +120,10 @@ async def session_context(
             else:
                 yield session
 
+    @property
+    def dialect(self) -> sa.engine.Dialect:
+        return get_dialect(self.database_config.connection_url)
+
     @property
     def Base(self):
         """Base class for orm models"""


Test patch:
diff --git a/tests/server/api/test_flow_runs.py b/tests/server/api/test_flow_runs.py
--- a/tests/server/api/test_flow_runs.py
+++ b/tests/server/api/test_flow_runs.py
@@ -1063,3 +1063,73 @@ async def test_history_interval_must_be_one_second_or_larger(self, client):
         )
         assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY
         assert b"History interval must not be less than 1 second" in response.content
+
+
+class TestFlowRunLateness:
+    @pytest.fixture
+    def url(self) -> str:
+        return "/flow_runs/lateness"
+
+    @pytest.fixture()
+    async def late_flow_runs(self, session, flow):
+        flow_runs = []
+        for i in range(5):
+            one_minute_ago = pendulum.now().subtract(minutes=1)
+            flow_run = await models.flow_runs.create_flow_run(
+                session=session,
+                flow_run=schemas.core.FlowRun(
+                    flow_id=flow.id,
+                    state=schemas.states.Scheduled(scheduled_time=one_minute_ago),
+                ),
+            )
+            await models.flow_runs.set_flow_run_state(
+                session=session,
+                flow_run_id=flow_run.id,
+                state=schemas.states.Running(timestamp=one_minute_ago.add(seconds=i)),
+            )
+            flow_runs.append(flow_run)
+
+            await session.commit()
+
+        return flow_runs
+
+    async def test_average_lateness_no_flow_runs(self, url: str, client):
+        response = await client.post(url)
+        assert response.status_code == 200
+
+        # If no flow runs match the filter, the average lateness is null.
+        assert response.content == b"null"
+
+    async def test_average_lateness(
+        self,
+        url: str,
+        client,
+        late_flow_runs,
+    ):
+        response = await client.post(url)
+        assert response.status_code == 200
+
+        # The flow runs in `late_flow_runs` are created in a loop and the
+        # lateness is the iteration count of the loop. There are 5 flow runs,
+        # so avg(0 + 1 + 2 + 3 + 4) == 2.0
+        assert response.content == b"2.0"
+
+    async def test_supports_filters(
+        self,
+        url: str,
+        client,
+        late_flow_runs,
+    ):
+        flow_run_ids = [flow_run.id for flow_run in late_flow_runs[-2:]]
+        flow_run_filter = schemas.filters.FlowRunFilter(
+            id=schemas.filters.FlowRunFilterId(any_=flow_run_ids)
+        )
+        response = await client.post(
+            url, json={"flow_runs": flow_run_filter.dict(json_compatible=True)}
+        )
+        assert response.status_code == 200
+
+        # The flow runs in `late_flow_runs` are created in a loop and the
+        # lateness is the iteration count of the loop. We're only looking at
+        # the last two flow runs in that list so avg(3 + 4) == 3.5
+        assert response.content == b"3.5"