Repository: AnalogJ/lexicon

Issue:
DNS Made Easy needs retry logic, to recover from rate limiting errors
Originally certbot/certbot#7411.

DNS Made Easy has [restrictive rate limits](https://api-docs.dnsmadeeasy.com/?version=latest#f6f3c489-422d-4cf0-bccb-1933e6d655ac):
>To prevent unwanted flooding of the API system, there is a maximum number of requests that can be sent in a given time period. This limit is 150 requests per 5 minute scrolling window

Lexicon should be able to recover from hitting the rate limit without failing the operation.

The response received is an HTTP 400 with a response body of:

>{"error": ["Rate limit exceeded"]}

@adferrand suggested retry logic in [this comment](https://github.com/certbot/certbot/issues/7411#issuecomment-536438100):

>How about defining a short sleep (like 5 seconds), and a retry strategy with a max attempts, triggered in case of 400 with body {"error": ["Rate limit exceeded"]}?


Discussion:
> @adferrand I have some DME sandbox credentials you can use which has certbot-dnsmadeeasy-test.ga's DNS hosted with it - how can I send them to you?

I think the best is to send them to me as PM on EFF Mattermost.

Resolving patch:
diff --git a/lexicon/providers/dnsmadeeasy.py b/lexicon/providers/dnsmadeeasy.py
--- a/lexicon/providers/dnsmadeeasy.py
+++ b/lexicon/providers/dnsmadeeasy.py
@@ -6,8 +6,10 @@
 from builtins import bytes
 from email.utils import formatdate
 from hashlib import sha1
+from urllib3.util.retry import Retry
 
 import requests
+from requests.adapters import HTTPAdapter
 from lexicon.providers.base import Provider as BaseProvider
 
 
@@ -16,6 +18,21 @@
 NAMESERVER_DOMAINS = ['dnsmadeeasy']
 
 
+class _RetryRateLimit(Retry):
+    # Standard urllib3 Retry objects trigger retries only based on HTTP status code or HTTP method.
+    # However we need to differentiate 400 errors with body `{"error": ["Rate limit exceeded"]}`
+    # from the other 400 errors. The internal _RetryRateLimit class does that.
+    def increment(self, method=None, url=None, response=None,
+                  error=None, _pool=None, _stacktrace=None):
+        if response:
+            body = json.loads(response.data)
+            if 'Rate limit exceeded' in body.get('error', []):
+                return super(_RetryRateLimit, self).increment(
+                    method, url, response, error, _pool, _stacktrace)
+
+        raise RuntimeError('URL {0} returned a HTTP 400 status code.'.format(url))
+
+
 def provider_parser(subparser):
     """Configure provider parser for DNSMadeEasy"""
     subparser.add_argument(
@@ -165,14 +182,30 @@ def _request(self, action='GET', url='/', data=None, query_params=None):
         default_headers['x-dnsme-requestDate'] = request_date
         default_headers['x-dnsme-hmac'] = hashed.hexdigest()
 
-        response = requests.request(action, self.api_endpoint + url, params=query_params,
-                                    data=json.dumps(data),
-                                    headers=default_headers,
-                                    auth=default_auth)
-        # if the request fails for any reason, throw an error.
-        response.raise_for_status()
-
-        # PUT and DELETE actions dont return valid json.
-        if action in ['DELETE', 'PUT']:
-            return response.text
-        return response.json()
+        session = requests.Session()
+        try:
+            # DNSMadeEasy allows only 150 requests in a floating 5 min time window.
+            # So we implement a retry strategy on requests returned as 400 with body
+            # `{"error": ["Rate limit exceeded"]}`.
+            # 10 retries with backoff = 0.6 gives following retry delays after first attempt:
+            # 1.2s, 2.4s, 4.8s, 9.6s, 19.2s, 38.4s, 76.8s, 153.6s, 307.2s
+            # So last attempt is done 5 min 7 seconds after first try, so the
+            # size of the floating window.
+            # Beyond it we can assume something else is wrong and so give up.
+            session_retries = _RetryRateLimit(total=10, backoff_factor=0.6, status_forcelist=[400])
+            session_adapter = HTTPAdapter(max_retries=session_retries)
+            session.mount('http://', session_adapter)
+            session.mount('https://', session_adapter)
+            response = session.request(action, self.api_endpoint + url, params=query_params,
+                                       data=json.dumps(data),
+                                       headers=default_headers,
+                                       auth=default_auth)
+            # if the request fails for any reason, throw an error.
+            response.raise_for_status()
+
+            # PUT and DELETE actions dont return valid json.
+            if action in ['DELETE', 'PUT']:
+                return response.text
+            return response.json()
+        finally:
+            session.close()


Test patch: