Repository: PrefectHQ/prefect

Issue:
Add backoff for agent when pulling runs from Cloud
In order to interface better with the `Submitted` state bottleneck, we'll implement the backoff logic for agents. 

As long as the agent is receiving work, it'll continue to request more work rapidly. Once work stops being delivered (either because there is no work, or there's a bottleneck), the agent will back-off requests until it reaches a 5-second steady-state interval.

Note: I would recommend that we go to 10 or even 15 seconds as the steady state. That would be the maximum delay for new work to begin, but once work arrives, subsequent work would happen rapidly.

```python
intervals = {
  0: 0
  1: 0.25,
  2: 0.5,
  3: 1.0,
  4: 2.0,
  5: 4.0
  6: 8.0,
  7: 15.0,
}

def backoff_example():
    t = 0
    while True:
        runs = get_runs_from_queue()
        if runs:
            t = 0
        elif t < max(intervals):
            t += 1
        sleep(intervals[t])

```



Discussion:

Resolving patch:
diff --git a/src/prefect/agent/agent.py b/src/prefect/agent/agent.py
--- a/src/prefect/agent/agent.py
+++ b/src/prefect/agent/agent.py
@@ -36,8 +36,6 @@ class Agent:
     """
 
     def __init__(self) -> None:
-        self.loop_interval = config.cloud.agent.get("loop_interval")
-
         self.client = Client(token=config.cloud.agent.get("auth_token"))
 
         logger = logging.getLogger("agent")
@@ -58,9 +56,18 @@ def start(self) -> None:
         new flow runs to deploy
         """
         tenant_id = self.agent_connect()
+
+        # Loop intervals for query sleep backoff
+        loop_intervals = {0: 0.25, 1: 0.5, 2: 1.0, 3: 2.0, 4: 4.0, 5: 8.0, 6: 10.0}
+
+        index = 0
         while True:
-            self.agent_process(tenant_id)
-            time.sleep(self.loop_interval)
+            runs = self.agent_process(tenant_id)
+            if runs:
+                index = 0
+            elif index < max(loop_intervals.keys()):
+                index += 1
+            time.sleep(loop_intervals[index])
 
     def agent_connect(self) -> str:
         """
@@ -86,13 +93,17 @@ def agent_connect(self) -> str:
 
         return tenant_id
 
-    def agent_process(self, tenant_id: str) -> None:
+    def agent_process(self, tenant_id: str) -> bool:
         """
         Full process for finding flow runs, updating states, and deploying.
 
         Args:
             - tenant_id (str): The tenant id to use in the query
+
+        Returns:
+            - bool: whether or not flow runs were found
         """
+        flow_runs = None
         try:
             flow_runs = self.query_flow_runs(tenant_id=tenant_id)
 
@@ -111,6 +122,8 @@ def agent_process(self, tenant_id: str) -> None:
         except Exception as exc:
             self.logger.error(exc)
 
+        return bool(flow_runs)
+
     def query_tenant_id(self) -> Union[str, None]:
         """
         Query Prefect Cloud for the tenant id that corresponds to the agent's auth token
diff --git a/src/prefect/agent/kubernetes/agent.py b/src/prefect/agent/kubernetes/agent.py
--- a/src/prefect/agent/kubernetes/agent.py
+++ b/src/prefect/agent/kubernetes/agent.py
@@ -100,13 +100,12 @@ def replace_job_spec_yaml(self, flow_run: GraphQLResult) -> dict:
 
     @staticmethod
     def generate_deployment_yaml(
-        token: str = None, api: str = None, loop: str = None, namespace: str = None
+        token: str = None, api: str = None, namespace: str = None
     ) -> str:
 
         # Use defaults if not provided
         token = token or ""
         api = api or "https://api.prefect.io"
-        loop = loop or "5"
         namespace = namespace or "default"
 
         with open(
@@ -121,8 +120,7 @@ def generate_deployment_yaml(
 
         agent_env[0]["value"] = token
         agent_env[1]["value"] = api
-        agent_env[2]["value"] = loop
-        agent_env[3]["value"] = namespace
+        agent_env[2]["value"] = namespace
 
         resource_manager_env[0]["value"] = token
         resource_manager_env[1]["value"] = api
diff --git a/src/prefect/cli/agent.py b/src/prefect/cli/agent.py
--- a/src/prefect/cli/agent.py
+++ b/src/prefect/cli/agent.py
@@ -83,7 +83,6 @@ def start(name, token, no_pull):
 @click.option(
     "--api", "-a", required=False, help="A Prefect Cloud API URL.", hidden=True
 )
-@click.option("--loop", "-l", required=False, help="Agent loop interval.", hidden=True)
 @click.option(
     "--namespace",
     "-n",
@@ -91,7 +90,7 @@ def start(name, token, no_pull):
     help="Agent namespace to launch workloads.",
     hidden=True,
 )
-def install(name, token, api, loop, namespace):
+def install(name, token, api, namespace):
     """
     Install an agent. Outputs configuration text which can be used to install on various
     platforms.
@@ -105,7 +104,6 @@ def install(name, token, api, loop, namespace):
     Options:
         --token, -t         TEXT    A Prefect Cloud API token
         --api, -a           TEXT    A Prefect Cloud API URL
-        --loop, -l          TEXT    Agent loop interval
         --namespace, -n     TEXT    Agent namespace to launch workloads
     """
 
@@ -118,6 +116,6 @@ def install(name, token, api, loop, namespace):
         return
 
     deployment = from_qualified_name(retrieved_agent).generate_deployment_yaml(
-        token=token, api=api, loop=loop, namespace=namespace
+        token=token, api=api, namespace=namespace
     )
     click.echo(deployment)


Test patch:
diff --git a/tests/agent/test_agent.py b/tests/agent/test_agent.py
--- a/tests/agent/test_agent.py
+++ b/tests/agent/test_agent.py
@@ -15,11 +15,8 @@ def test_agent_init():
 
 
 def test_agent_config_options():
-    with set_temporary_config(
-        {"cloud.agent.auth_token": "TEST_TOKEN", "cloud.agent.loop_interval": 10}
-    ):
+    with set_temporary_config({"cloud.agent.auth_token": "TEST_TOKEN"}):
         agent = Agent()
-        assert agent.loop_interval == 10
         assert agent.client.token == "TEST_TOKEN"
         assert agent.logger
 
@@ -227,6 +224,27 @@ def test_agent_process(monkeypatch):
         client.return_value.graphql = gql_return
         monkeypatch.setattr("prefect.agent.agent.Client", client)
 
+        # Assert it doesn't return everything but all functions are called properly
+        agent = Agent()
+        assert agent.agent_process("id")
+
+
+def test_agent_process_no_runs_found(monkeypatch):
+    with set_temporary_config({"cloud.agent.auth_token": "token"}):
+        gql_return = MagicMock(
+            return_value=MagicMock(
+                data=MagicMock(
+                    set_flow_run_state=None,
+                    set_task_run_state=None,
+                    getRunsInQueue=MagicMock(flow_run_ids=["id"]),
+                    flow_run=[],
+                )
+            )
+        )
+        client = MagicMock()
+        client.return_value.graphql = gql_return
+        monkeypatch.setattr("prefect.agent.agent.Client", client)
+
         # Assert it doesn't return everything but all functions are called properly
         agent = Agent()
         assert not agent.agent_process("id")
diff --git a/tests/agent/test_k8s_agent.py b/tests/agent/test_k8s_agent.py
--- a/tests/agent/test_k8s_agent.py
+++ b/tests/agent/test_k8s_agent.py
@@ -25,12 +25,9 @@ def test_k8s_agent_config_options(monkeypatch):
     k8s_config = MagicMock()
     monkeypatch.setattr("kubernetes.config", k8s_config)
 
-    with set_temporary_config(
-        {"cloud.agent.auth_token": "TEST_TOKEN", "cloud.agent.loop_interval": 10}
-    ):
+    with set_temporary_config({"cloud.agent.auth_token": "TEST_TOKEN"}):
         agent = KubernetesAgent()
         assert agent
-        assert agent.loop_interval == 10
         assert agent.client.token == "TEST_TOKEN"
         assert agent.logger
         assert agent.batch_client
@@ -150,10 +147,7 @@ def test_k8s_agent_generate_deployment_yaml(monkeypatch):
     with set_temporary_config({"cloud.agent.auth_token": "token"}):
         agent = KubernetesAgent()
         deployment = agent.generate_deployment_yaml(
-            token="test_token",
-            api="test_api",
-            loop="test_loop",
-            namespace="test_namespace",
+            token="test_token", api="test_api", namespace="test_namespace"
         )
 
         deployment = yaml.safe_load(deployment)
@@ -165,8 +159,7 @@ def test_k8s_agent_generate_deployment_yaml(monkeypatch):
 
         assert agent_env[0]["value"] == "test_token"
         assert agent_env[1]["value"] == "test_api"
-        assert agent_env[2]["value"] == "test_loop"
-        assert agent_env[3]["value"] == "test_namespace"
+        assert agent_env[2]["value"] == "test_namespace"
 
         assert resource_manager_env[0]["value"] == "test_token"
         assert resource_manager_env[1]["value"] == "test_api"
diff --git a/tests/agent/test_local_agent.py b/tests/agent/test_local_agent.py
--- a/tests/agent/test_local_agent.py
+++ b/tests/agent/test_local_agent.py
@@ -21,11 +21,8 @@ def test_local_agent_config_options(monkeypatch):
     api = MagicMock()
     monkeypatch.setattr("prefect.agent.local.agent.docker.APIClient", api)
 
-    with set_temporary_config(
-        {"cloud.agent.auth_token": "TEST_TOKEN", "cloud.agent.loop_interval": 10}
-    ):
+    with set_temporary_config({"cloud.agent.auth_token": "TEST_TOKEN"}):
         agent = LocalAgent()
-        assert agent.loop_interval == 10
         assert agent.client.token == "TEST_TOKEN"
         assert agent.logger
         assert not agent.no_pull
@@ -36,11 +33,8 @@ def test_local_agent_config_options_populated(monkeypatch):
     api = MagicMock()
     monkeypatch.setattr("prefect.agent.local.agent.docker.APIClient", api)
 
-    with set_temporary_config(
-        {"cloud.agent.auth_token": "TEST_TOKEN", "cloud.agent.loop_interval": 10}
-    ):
+    with set_temporary_config({"cloud.agent.auth_token": "TEST_TOKEN"}):
         agent = LocalAgent(base_url="url", no_pull=True)
-        assert agent.loop_interval == 10
         assert agent.client.token == "TEST_TOKEN"
         assert agent.logger
         assert agent.no_pull
diff --git a/tests/agent/test_nomad_agent.py b/tests/agent/test_nomad_agent.py
--- a/tests/agent/test_nomad_agent.py
+++ b/tests/agent/test_nomad_agent.py
@@ -14,12 +14,9 @@ def test_nomad_agent_init():
 
 
 def test_nomad_agent_config_options():
-    with set_temporary_config(
-        {"cloud.agent.auth_token": "TEST_TOKEN", "cloud.agent.loop_interval": 10}
-    ):
+    with set_temporary_config({"cloud.agent.auth_token": "TEST_TOKEN"}):
         agent = NomadAgent()
         assert agent
-        assert agent.loop_interval == 10
         assert agent.client.token == "TEST_TOKEN"
         assert agent.logger
 
diff --git a/tests/cli/test_agent.py b/tests/cli/test_agent.py
--- a/tests/cli/test_agent.py
+++ b/tests/cli/test_agent.py
@@ -87,8 +87,6 @@ def test_agent_install_passes_args():
             "TEST_TOKEN",
             "--api",
             "TEST_API",
-            "--loop",
-            "TEST_LOOP",
             "--namespace",
             "TEST_NAMESPACE",
         ],
@@ -96,5 +94,4 @@ def test_agent_install_passes_args():
     assert result.exit_code == 0
     assert "TEST_TOKEN" in result.output
     assert "TEST_API" in result.output
-    assert "TEST_LOOP" in result.output
     assert "TEST_NAMESPACE" in result.output