Repository: PrefectHQ/prefect

Issue:
Agents should submit flow runs in order of scheduled times
Right now the agents pull a list of flow runs in [arbitrary order](https://github.com/PrefectHQ/prefect/blob/master/src/prefect/agent/agent.py#L543) and submit them. In practice this doesn't matter because only a few runs are in the queue on every check, but when the work queue contains late work (say, because an agent wasn't running) it should be cleared in order - the oldest runs submitted first, then the newer ones.





Discussion:
@jlowin Would this be accomplished by an order_by on something like scheduled start time or is there some way directly in the `get_runs_in_queue` to grab based on order?
`get_runs_in_queue` will deliver in order, but when that's passed back to the query it's not guaranteed to be maintained - throwing an order_by on the query would do the trick.

Resolving patch:
diff --git a/src/prefect/agent/agent.py b/src/prefect/agent/agent.py
--- a/src/prefect/agent/agent.py
+++ b/src/prefect/agent/agent.py
@@ -554,7 +554,7 @@ def query_flow_runs(self) -> list:
                                     },
                                 },
                             ],
-                        }
+                        },
                     },
                 ): {
                     "id": True,
@@ -562,6 +562,7 @@ def query_flow_runs(self) -> list:
                     "state": True,
                     "serialized_state": True,
                     "parameters": True,
+                    "scheduled_start_time": True,
                     "flow": {
                         "id",
                         "name",
@@ -588,7 +589,11 @@ def query_flow_runs(self) -> list:
         if target_flow_run_ids:
             self.logger.debug("Querying flow run metadata")
             result = self.client.graphql(query)
-            return result.data.flow_run  # type: ignore
+
+            # Return flow runs sorted by scheduled start time
+            return sorted(
+                result.data.flow_run, key=lambda flow_run: flow_run.scheduled_start_time
+            )
         else:
             return []
 


Test patch:
diff --git a/tests/agent/test_agent.py b/tests/agent/test_agent.py
--- a/tests/agent/test_agent.py
+++ b/tests/agent/test_agent.py
@@ -143,7 +143,7 @@ def test_query_flow_runs(monkeypatch, cloud_api):
         return_value=MagicMock(
             data=MagicMock(
                 get_runs_in_queue=MagicMock(flow_run_ids=["id"]),
-                flow_run=[{"id": "id"}],
+                flow_run=[GraphQLResult({"id": "id", "scheduled_start_time": 1})],
             )
         )
     )
@@ -153,7 +153,7 @@ def test_query_flow_runs(monkeypatch, cloud_api):
 
     agent = Agent()
     flow_runs = agent.query_flow_runs()
-    assert flow_runs == [{"id": "id"}]
+    assert flow_runs == [GraphQLResult({"id": "id", "scheduled_start_time": 1})]
 
 
 def test_query_flow_runs_ignores_currently_submitting_runs(monkeypatch, cloud_api):
@@ -161,7 +161,7 @@ def test_query_flow_runs_ignores_currently_submitting_runs(monkeypatch, cloud_ap
         return_value=MagicMock(
             data=MagicMock(
                 get_runs_in_queue=MagicMock(flow_run_ids=["id1", "id2"]),
-                flow_run=[{"id1": "id1"}],
+                flow_run=[GraphQLResult({"id": "id", "scheduled_start_time": 1})],
             )
         )
     )
@@ -180,6 +180,30 @@ def test_query_flow_runs_ignores_currently_submitting_runs(monkeypatch, cloud_ap
     )
 
 
+def test_query_flow_runs_ordered_by_start_time(monkeypatch, cloud_api):
+    gql_return = MagicMock(
+        return_value=MagicMock(
+            data=MagicMock(
+                get_runs_in_queue=MagicMock(flow_run_ids=["id"]),
+                flow_run=[
+                    GraphQLResult({"id": "id2", "scheduled_start_time": 200}),
+                    GraphQLResult({"id": "id", "scheduled_start_time": 1}),
+                ],
+            )
+        )
+    )
+    client = MagicMock()
+    client.return_value.graphql = gql_return
+    monkeypatch.setattr("prefect.agent.agent.Client", client)
+
+    agent = Agent()
+    flow_runs = agent.query_flow_runs()
+    assert flow_runs == [
+        GraphQLResult({"id": "id", "scheduled_start_time": 1}),
+        GraphQLResult({"id": "id2", "scheduled_start_time": 200}),
+    ]
+
+
 def test_query_flow_runs_does_not_use_submitting_flow_runs_directly(
     monkeypatch, caplog, cloud_api
 ):
@@ -345,6 +369,7 @@ def test_agent_process(monkeypatch, cloud_api):
                                     }
                                 )
                             ],
+                            "scheduled_start_time": 1,
                         }
                     )
                 ],