Repository: PrefectHQ/prefect

Issue:
Enable retries for Fargate Agent
## Current behavior

We invoked `client.create_flow_run` in a for loop with 365 different parameter values. This caused 258 of them to fail with one of two different errors. In each case there was only run log message for the whole flow run.

```
March 17th 2020 at 2:32:51am | agent
ERROR 
An error occurred (ThrottlingException) when calling the RunTask operation (reached max retries: 4): Rate exceeded.
```

```
March 17th 2020 at 2:32:59am | agent
ERROR 
list index out of rangeMarch 17th 2020 at 2:32:59am | agent
ERROR 
list index out of range
```

Apparently, for the flow runs that the FargateAgent received an error from the AWS API, the FargateAgent quit and sent the error back to Prefect Cloud.

## Proposed behavior

When the FargateAgent receives a throttling error response from AWS, it could retry the request (perhaps using backoff). It appears that botocore already has retry functionality:

https://github.com/boto/botocore/pull/1260


## Example

If this were implemented, it would cover the case when a team is running a backfill workflow for every day in a given year, all at once.


Discussion:
👍 Thanks for opening. Yeah I think we should take advantage of boto's [retry configuration](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html#botocore-config)

Resolving patch:
diff --git a/src/prefect/agent/fargate/agent.py b/src/prefect/agent/fargate/agent.py
--- a/src/prefect/agent/fargate/agent.py
+++ b/src/prefect/agent/fargate/agent.py
@@ -39,6 +39,11 @@ class FargateAgent(Agent):
     prefect agent start fargate networkConfiguration="{'awsvpcConfiguration': {'assignPublicIp': 'ENABLED', 'subnets': ['my_subnet_id'], 'securityGroups': []}}"
     ```
 
+    botocore configuration options can be provided to the Fargate Agent:
+    ```
+    FargateAgent(botocore_config={"retries": {"max_attempts": 10}})
+    ```
+
     Args:
         - name (str, optional): An optional name to give this agent. Can also be set through
             the environment variable `PREFECT__CLOUD__AGENT__NAME`. Defaults to "agent"
@@ -59,6 +64,8 @@ class FargateAgent(Agent):
             `AWS_SESSION_TOKEN` or `None`
         - region_name (str, optional): AWS region name for connecting the boto3 client.
             Defaults to the value set in the environment variable `REGION_NAME` or `None`
+        - botocore_config (dict, optional): botocore configuration options to be passed to the
+            boto3 client. https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
         - enable_task_revisions (bool, optional): Enable registration of task definitions using revisions.
             When enabled, task definitions will use flow name as opposed to flow id and each new version will be a
             task definition revision. Each revision will be registered with a tag called 'PrefectFlowId'
@@ -84,6 +91,7 @@ def __init__(  # type: ignore
         aws_secret_access_key: str = None,
         aws_session_token: str = None,
         region_name: str = None,
+        botocore_config: dict = None,
         enable_task_revisions: bool = False,
         use_external_kwargs: bool = False,
         external_kwargs_s3_bucket: str = None,
@@ -96,6 +104,7 @@ def __init__(  # type: ignore
 
         from boto3 import client as boto3_client
         from boto3 import resource as boto3_resource
+        from botocore.config import Config
 
         # Config used for boto3 client initialization
         aws_access_key_id = aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID")
@@ -104,6 +113,7 @@ def __init__(  # type: ignore
         )
         aws_session_token = aws_session_token or os.getenv("AWS_SESSION_TOKEN")
         region_name = region_name or os.getenv("REGION_NAME")
+        botocore_config = botocore_config or {}
 
         # revisions and kwargs configurations
         self.enable_task_revisions = enable_task_revisions
@@ -125,6 +135,7 @@ def __init__(  # type: ignore
             aws_secret_access_key=aws_secret_access_key,
             aws_session_token=aws_session_token,
             region_name=region_name,
+            config=Config(**botocore_config),
         )
         # fetch external kwargs from s3 if needed
         if self.use_external_kwargs:
@@ -146,6 +157,7 @@ def __init__(  # type: ignore
                 aws_secret_access_key=aws_secret_access_key,
                 aws_session_token=aws_session_token,
                 region_name=region_name,
+                config=Config(**botocore_config),
             )
 
     def _override_kwargs(


Test patch:
diff --git a/tests/agent/test_fargate_agent.py b/tests/agent/test_fargate_agent.py
--- a/tests/agent/test_fargate_agent.py
+++ b/tests/agent/test_fargate_agent.py
@@ -39,6 +39,9 @@ def test_fargate_agent_config_options(monkeypatch, runner_token):
     boto3_client = MagicMock()
     monkeypatch.setattr("boto3.client", boto3_client)
 
+    botocore_config = MagicMock()
+    monkeypatch.setattr("botocore.config.Config", botocore_config)
+
     # Client args
     monkeypatch.setenv("AWS_ACCESS_KEY_ID", "")
     monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "")
@@ -59,13 +62,14 @@ def test_fargate_agent_config_options(monkeypatch, runner_token):
         assert agent.logger
         assert agent.boto3_client
 
-        boto3_client.assert_called_with(
-            "ecs",
-            aws_access_key_id=None,
-            aws_secret_access_key=None,
-            aws_session_token=None,
-            region_name=None,
-        )
+        assert boto3_client.call_args[0][0] == "ecs"
+        assert boto3_client.call_args[1]["aws_access_key_id"] == None
+        assert boto3_client.call_args[1]["aws_secret_access_key"] == None
+        assert boto3_client.call_args[1]["aws_session_token"] == None
+        assert boto3_client.call_args[1]["region_name"] == None
+
+        assert botocore_config.called
+        assert botocore_config.call_args == {}
 
 
 def test_parse_task_definition_kwargs(monkeypatch, runner_token):
@@ -300,6 +304,9 @@ def test_fargate_agent_config_options_init(monkeypatch, runner_token):
     boto3_client = MagicMock()
     monkeypatch.setattr("boto3.client", boto3_client)
 
+    botocore_config = MagicMock()
+    monkeypatch.setattr("botocore.config.Config", botocore_config)
+
     def_kwarg_dict = {
         "taskRoleArn": "test",
         "executionRoleArn": "test",
@@ -366,6 +373,7 @@ def test_fargate_agent_config_options_init(monkeypatch, runner_token):
         aws_secret_access_key="secret",
         aws_session_token="token",
         region_name="region",
+        botocore_config={"test": "config"},
         **kwarg_dict
     )
     assert agent
@@ -374,19 +382,23 @@ def test_fargate_agent_config_options_init(monkeypatch, runner_token):
     assert agent.task_run_kwargs == run_kwarg_dict
     assert agent.container_definitions_kwargs == container_def_kwargs_dict
 
-    boto3_client.assert_called_with(
-        "ecs",
-        aws_access_key_id="id",
-        aws_secret_access_key="secret",
-        aws_session_token="token",
-        region_name="region",
-    )
+    assert boto3_client.call_args[0][0] == "ecs"
+    assert boto3_client.call_args[1]["aws_access_key_id"] == "id"
+    assert boto3_client.call_args[1]["aws_secret_access_key"] == "secret"
+    assert boto3_client.call_args[1]["aws_session_token"] == "token"
+    assert boto3_client.call_args[1]["region_name"] == "region"
+
+    assert botocore_config.called
+    assert botocore_config.call_args[1] == {"test": "config"}
 
 
 def test_fargate_agent_config_env_vars(monkeypatch, runner_token):
     boto3_client = MagicMock()
     monkeypatch.setattr("boto3.client", boto3_client)
 
+    botocore_config = MagicMock()
+    monkeypatch.setattr("botocore.config.Config", botocore_config)
+
     def_kwarg_dict = {
         "taskRoleArn": "test",
         "executionRoleArn": "test",
@@ -458,19 +470,23 @@ def test_fargate_agent_config_env_vars(monkeypatch, runner_token):
     assert agent.task_run_kwargs == run_kwarg_dict
     assert agent.container_definitions_kwargs == container_def_kwargs_dict
 
-    boto3_client.assert_called_with(
-        "ecs",
-        aws_access_key_id="id",
-        aws_secret_access_key="secret",
-        aws_session_token="token",
-        region_name="region",
-    )
+    assert boto3_client.call_args[0][0] == "ecs"
+    assert boto3_client.call_args[1]["aws_access_key_id"] == "id"
+    assert boto3_client.call_args[1]["aws_secret_access_key"] == "secret"
+    assert boto3_client.call_args[1]["aws_session_token"] == "token"
+    assert boto3_client.call_args[1]["region_name"] == "region"
+
+    assert botocore_config.called
+    assert botocore_config.call_args == {}
 
 
 def test_fargate_agent_config_env_vars_lists_dicts(monkeypatch, runner_token):
     boto3_client = MagicMock()
     monkeypatch.setattr("boto3.client", boto3_client)
 
+    botocore_config = MagicMock()
+    monkeypatch.setattr("botocore.config.Config", botocore_config)
+
     def_kwarg_dict = {
         "placementConstraints": ["test"],
         "proxyConfiguration": {"test": "test"},
@@ -516,13 +532,14 @@ def test_fargate_agent_config_env_vars_lists_dicts(monkeypatch, runner_token):
     assert agent.task_run_kwargs == run_kwarg_dict
     assert agent.container_definitions_kwargs == container_def_kwargs_dict
 
-    boto3_client.assert_called_with(
-        "ecs",
-        aws_access_key_id="id",
-        aws_secret_access_key="secret",
-        aws_session_token="token",
-        region_name="region",
-    )
+    assert boto3_client.call_args[0][0] == "ecs"
+    assert boto3_client.call_args[1]["aws_access_key_id"] == "id"
+    assert boto3_client.call_args[1]["aws_secret_access_key"] == "secret"
+    assert boto3_client.call_args[1]["aws_session_token"] == "token"
+    assert boto3_client.call_args[1]["region_name"] == "region"
+
+    assert botocore_config.called
+    assert botocore_config.call_args == {}
 
 
 def test_deploy_flow_raises(monkeypatch, runner_token):