Repository: Azure/WALinuxAgent
Pull request #913 (2017-10-18 23:40:00)

Address issues
[#897] -- 'Target handler state' wall of errors
[#896] -- End of Line Comments are Not Supported nor Handled
[#891] -- Create a Telemetry Event to Track Custom Data Execution
[#884] -- Cleanup Old Goal State and Extension Cache
[#876] -- The agent should use a scaling back-off when retrying HTTP requests
[#869] -- The agent should report OS information in the correct JSON format.

Updated version to 2.2.18.1
Replaced errno.EREMOTEIO with fixed value (since the constant is not present in all Python 2.7 versions)

Signed-off-by: Brendan Dixon <brendandixon@me.com>


Patch:
diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py
index df1ca5b4..045352dd 100644
--- a/azurelinuxagent/common/conf.py
+++ b/azurelinuxagent/common/conf.py
@@ -44,7 +44,7 @@ class ConfigurationProvider(object):
                 if len(parts) < 2:
                     continue
                 key = parts[0].strip()
-                value = parts[1].strip("\" ")
+                value = parts[1].split('#')[0].strip("\" ")
                 self.values[key] = value if value != "None" else None
 
     def get(self, key, default_val):
diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py
index 23b9aa5f..228e55ef 100644
--- a/azurelinuxagent/common/event.py
+++ b/azurelinuxagent/common/event.py
@@ -47,6 +47,7 @@ class WALAEventOperation:
     AgentBlacklisted = "AgentBlacklisted"
     AgentEnabled = "AgentEnabled"
     AutoUpdate = "AutoUpdate"
+    CustomData = "CustomData"
     Deploy = "Deploy"
     Disable = "Disable"
     Download = "Download"
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index 9b395792..35f58889 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -17,6 +17,7 @@
 # Requires Python 2.4+ and Openssl 1.0+
 
 import json
+import operator
 import os
 import re
 import time
@@ -55,11 +56,18 @@ AGENTS_MANIFEST_FILE_NAME = "{0}.{1}.agentsManifest"
 TRANSPORT_CERT_FILE_NAME = "TransportCert.pem"
 TRANSPORT_PRV_FILE_NAME = "TransportPrivate.pem"
 
+CACHE_PATTERNS = [
+    re.compile("^(.*)\.(\d+)\.(agentsManifest)$", re.IGNORECASE),
+    re.compile("^(.*)\.(\d+)\.(manifest\.xml)$", re.IGNORECASE),
+    re.compile("^(.*)\.(\d+)\.(xml)$", re.IGNORECASE)
+]
+
+MAXIMUM_CACHED_FILES = 50
+
 PROTOCOL_VERSION = "2012-11-30"
 ENDPOINT_FINE_NAME = "WireServer"
 
 SHORT_WAITING_INTERVAL = 1  # 1 second
-LONG_WAITING_INTERVAL = 15  # 15 seconds
 
 
 class UploadError(HttpError):
@@ -257,22 +265,28 @@ Convert VMStatus object to status blob format
 """
 
 
+def ga_status_to_guest_info(ga_status):
+    v1_ga_guest_info = {
+        "computerName" : ga_status.hostname,
+        "osName" : ga_status.osname,
+        "osVersion" : ga_status.osversion,
+        "version" : ga_status.version,
+    }
+    return v1_ga_guest_info
+
+
 def ga_status_to_v1(ga_status):
     formatted_msg = {
         'lang': 'en-US',
         'message': ga_status.message
     }
     v1_ga_status = {
-        'version': ga_status.version,
-        'status': ga_status.status,
-        'osversion': ga_status.osversion,
-        'osname': ga_status.osname,
-        'hostname': ga_status.hostname,
-        'formattedMessage': formatted_msg
+        "version" : ga_status.version,
+        "status" : ga_status.status,
+        "formattedMessage" : formatted_msg
     }
     return v1_ga_status
 
-
 def ext_substatus_to_v1(sub_status_list):
     status_list = []
     for substatus in sub_status_list:
@@ -346,6 +360,7 @@ def ext_handler_status_to_v1(handler_status, ext_statuses, timestamp):
 def vm_status_to_v1(vm_status, ext_statuses):
     timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
 
+    v1_ga_guest_info = ga_status_to_guest_info(vm_status.vmAgent)
     v1_ga_status = ga_status_to_v1(vm_status.vmAgent)
     v1_handler_status_list = []
     for handler_status in vm_status.vmAgent.extensionHandlers:
@@ -361,7 +376,8 @@ def vm_status_to_v1(vm_status, ext_statuses):
     v1_vm_status = {
         'version': '1.1',
         'timestampUTC': timestamp,
-        'aggregateStatus': v1_agg_status
+        'aggregateStatus': v1_agg_status,
+        'guestOSInfo' : v1_ga_guest_info
     }
     return v1_vm_status
 
@@ -524,33 +540,10 @@ class WireClient(object):
         self.shared_conf = None
         self.certs = None
         self.ext_conf = None
-        self.last_request = 0
-        self.req_count = 0
         self.host_plugin = None
         self.status_blob = StatusBlob(self)
 
-    def prevent_throttling(self):
-        """
-        Try to avoid throttling of wire server
-        """
-        now = time.time()
-        if now - self.last_request < 1:
-            logger.verbose("Last request issued less than 1 second ago")
-            logger.verbose("Sleep {0} second to avoid throttling.",
-                           SHORT_WAITING_INTERVAL)
-            time.sleep(SHORT_WAITING_INTERVAL)
-        self.last_request = now
-
-        self.req_count += 1
-        if self.req_count % 3 == 0:
-            logger.verbose("Sleep {0} second to avoid throttling.",
-                           SHORT_WAITING_INTERVAL)
-            time.sleep(SHORT_WAITING_INTERVAL)
-            self.req_count = 0
-
     def call_wireserver(self, http_req, *args, **kwargs):
-        self.prevent_throttling()
-
         try:
             # Never use the HTTP proxy for wireserver
             kwargs['use_proxy'] = False
@@ -742,6 +735,7 @@ class WireClient(object):
                 if self.host_plugin is not None:
                     self.host_plugin.container_id = goal_state.container_id
                     self.host_plugin.role_config_name = goal_state.role_config_name
+                self.purge_cache()
                 return
 
             except ProtocolError:
@@ -756,6 +750,40 @@ class WireClient(object):
 
         raise ProtocolError("Exceeded max retry updating goal state")
 
+    def purge_cache(self):
+        '''
+        Ensure the number of cached files does not exceed a maximum
+        '''
+        # Create a list of file tuples of (prefix, suffix, incarnation, name)
+        # -- Convert the incarnation to an integer for proper sorting
+        files = []
+        for f in os.listdir(conf.get_lib_dir()):
+            for pattern in CACHE_PATTERNS:
+                m = pattern.match(f)
+                if m is not None:
+                    t = (m.group(1), m.group(3), int(m.group(2)), f)
+                    files.append(t)
+                    break
+
+        if len(files) <= 0:
+            return
+
+        # Sort by (prefix, suffix, incarnation) in reverse order
+        # (This puts more recent incarnations first in the list)
+        files = sorted(files, key=operator.itemgetter(0,1,2), reverse=True)
+
+        # Remove any files in excess of the maximum allowed
+        # -- Restart then whenever the (prefix, suffix) change
+        last_match = [None, None]
+        for f in files:
+            if last_match != f[0:2]:
+                last_match = f[0:2]
+                count = 0
+            count += 1
+
+            if count > MAXIMUM_CACHED_FILES:
+                os.remove(os.path.join(conf.get_lib_dir(), f[3]))
+
     def get_goal_state(self):
         if self.goal_state is None:
             incarnation_file = os.path.join(conf.get_lib_dir(),
diff --git a/azurelinuxagent/common/utils/fileutil.py b/azurelinuxagent/common/utils/fileutil.py
index 7541b6fc..7846506a 100644
--- a/azurelinuxagent/common/utils/fileutil.py
+++ b/azurelinuxagent/common/utils/fileutil.py
@@ -42,7 +42,7 @@ KNOWN_IOERRORS = [
     errno.ENOSPC,       # Out of space
     errno.ENAMETOOLONG, # Name too long
     errno.ELOOP,        # Too many symbolic links encountered
-    errno.EREMOTEIO     # Remote I/O error
+    121                 # Remote I/O error (errno.EREMOTEIO -- not present in all Python 2.7+)
 ]
 
 
diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py
index ddd930bb..e9923661 100644
--- a/azurelinuxagent/common/utils/restutil.py
+++ b/azurelinuxagent/common/utils/restutil.py
@@ -32,10 +32,11 @@ from azurelinuxagent.common.version import PY_VERSION_MAJOR
 
 SECURE_WARNING_EMITTED = False
 
-DEFAULT_RETRIES = 3
+DEFAULT_RETRIES = 4
 
-SHORT_DELAY_IN_SECONDS = 5
-LONG_DELAY_IN_SECONDS = 15
+DELAY_IN_SECONDS = 1
+THROTTLE_DELAY_IN_SECONDS = 1
+MINIMUM_THROTTLE_RETRY = 15
 
 RETRY_CODES = [
     httpclient.RESET_CONTENT,
@@ -63,7 +64,8 @@ OK_CODES = [
 
 THROTTLE_CODES = [
     httpclient.FORBIDDEN,
-    httpclient.SERVICE_UNAVAILABLE
+    httpclient.SERVICE_UNAVAILABLE,
+    429, # Request Rate Limit Exceeded
 ]
 
 RETRY_EXCEPTIONS = [
@@ -77,6 +79,12 @@ HTTP_PROXY_ENV = "http_proxy"
 HTTPS_PROXY_ENV = "https_proxy"
 
 
+def _compute_delay(retry_attempt=1, delay=DELAY_IN_SECONDS):
+    fib = (1, 1)
+    for n in range(retry_attempt):
+        fib = (fib[1], fib[0]+fib[1])
+    return delay*fib[1]
+
 def _is_retry_status(status, retry_codes=RETRY_CODES):
     return status in retry_codes
 
@@ -166,7 +174,7 @@ def http_request(method,
                 use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
 
     global SECURE_WARNING_EMITTED
 
@@ -219,7 +227,7 @@ def http_request(method,
             time.sleep(delay)
 
         attempt += 1
-        delay = retry_delay
+        delay = _compute_delay(retry_attempt=attempt, delay=retry_delay)
 
         try:
             resp = _http_request(method,
@@ -237,8 +245,17 @@ def http_request(method,
                 if _is_retry_status(resp.status, retry_codes=retry_codes):
                     msg = '[HTTP Retry] HTTP {0} Status Code {1}'.format(
                         method, resp.status)
+
+                    # Always delay a short, constant time for throttling
+                    # -- Most (all?) throttling comes from wireserver or
+                    #    the HostPlugin; both throttle when receiving more than
+                    #    1 message per second; longer delays impact deployment
+                    #    peformance
+                    # -- Since the delay for throttled requests is constant,
+                    #    ensure a safe, minimum number of retries is performed
                     if _is_throttle_status(resp.status):
-                        delay = LONG_DELAY_IN_SECONDS
+                        delay = THROTTLE_DELAY_IN_SECONDS
+                        max_retry = max(max_retry, MINIMUM_THROTTLE_RETRY)
                         logger.info("[HTTP Delay] Delay {0} seconds for " \
                                     "Status Code {1}".format(
                                         delay, resp.status))
@@ -265,7 +282,7 @@ def http_request(method,
 def http_get(url, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("GET",
                         url, None, headers=headers,
                         use_proxy=use_proxy,
@@ -277,7 +294,7 @@ def http_get(url, headers=None, use_proxy=False,
 def http_head(url, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("HEAD",
                         url, None, headers=headers,
                         use_proxy=use_proxy,
@@ -289,7 +306,7 @@ def http_head(url, headers=None, use_proxy=False,
 def http_post(url, data, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("POST",
                         url, data, headers=headers,
                         use_proxy=use_proxy,
@@ -301,7 +318,7 @@ def http_post(url, data, headers=None, use_proxy=False,
 def http_put(url, data, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("PUT",
                         url, data, headers=headers,
                         use_proxy=use_proxy,
@@ -313,7 +330,7 @@ def http_put(url, data, headers=None, use_proxy=False,
 def http_delete(url, headers=None, use_proxy=False,
                 max_retry=DEFAULT_RETRIES,
                 retry_codes=RETRY_CODES,
-                retry_delay=SHORT_DELAY_IN_SECONDS):
+                retry_delay=DELAY_IN_SECONDS):
     return http_request("DELETE",
                         url, None, headers=headers,
                         use_proxy=use_proxy,
diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py
index cc41cc45..c109d584 100644
--- a/azurelinuxagent/common/version.py
+++ b/azurelinuxagent/common/version.py
@@ -113,7 +113,7 @@ def get_distro():
 
 AGENT_NAME = "WALinuxAgent"
 AGENT_LONG_NAME = "Azure Linux Agent"
-AGENT_VERSION = '2.2.18'
+AGENT_VERSION = '2.2.18.1'
 AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION)
 AGENT_DESCRIPTION = """
 The Azure Linux Agent supports the provisioning and running of Linux
diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
index 3a78900d..f8302e07 100644
--- a/azurelinuxagent/ga/exthandlers.py
+++ b/azurelinuxagent/ga/exthandlers.py
@@ -306,7 +306,8 @@ class ExtHandlersHandler(object):
                     not self.is_new_guid(ext_handler):
                 logger.info("New GUID is the same as the old GUID. Exiting without upgrading.")
                 return
-            ext_handler_i.decide_version()
+
+            ext_handler_i.decide_version(target_state=state)
             if not ext_handler_i.is_upgrade and self.last_etag == etag:
                 if self.log_etag:
                     ext_handler_i.logger.verbose("Version {0} is current for etag {1}",
@@ -453,7 +454,7 @@ class ExtHandlerInstance(object):
         self.logger.add_appender(logger.AppenderType.FILE,
                                  logger.LogLevel.INFO, log_file)
 
-    def decide_version(self):
+    def decide_version(self, target_state=None):
         self.logger.verbose("Decide which version to use")
         try:
             pkg_list = self.protocol.get_ext_handler_pkgs(self.ext_handler)
@@ -538,7 +539,14 @@ class ExtHandlerInstance(object):
         # Note:
         #  - A downgrade, which will be bound to the same major version,
         #    is allowed if the installed version is no longer available
-        if selected_pkg is None \
+        if target_state == u"uninstall":
+            if installed_pkg is None:
+                msg = "Failed to find installed version of {0} " \
+                    "to uninstall".format(self.ext_handler.name)
+                self.logger.warn(msg)
+            self.pkg = installed_pkg
+            self.ext_handler.properties.version = installed_version
+        elif selected_pkg is None \
             or (installed_pkg is not None and selected_version < installed_version):
             self.pkg = installed_pkg
             self.ext_handler.properties.version = installed_version
@@ -1031,7 +1039,7 @@ class ExtHandlerInstance(object):
 
     def get_log_dir(self):
         return os.path.join(conf.get_ext_log_dir(), self.ext_handler.name,
-                            self.ext_handler.properties.version)
+                            str(self.ext_handler.properties.version))
 
 class HandlerEnvironment(object):
     def __init__(self, data):
diff --git a/azurelinuxagent/pa/provision/default.py b/azurelinuxagent/pa/provision/default.py
index 2f7ec18c..d2789750 100644
--- a/azurelinuxagent/pa/provision/default.py
+++ b/azurelinuxagent/pa/provision/default.py
@@ -22,6 +22,7 @@ Provision handler
 import os
 import os.path
 import re
+import time
 
 from datetime import datetime
 
@@ -244,9 +245,14 @@ class ProvisionHandler(object):
         fileutil.write_file(customdata_file, customdata)
 
         if conf.get_execute_customdata():
+            start = time.time()
             logger.info("Execute custom data")
             os.chmod(customdata_file, 0o700)
             shellutil.run(customdata_file)
+            add_event(name=AGENT_NAME,
+                        duration=int(time.time() - start),
+                        is_success=True,
+                        op=WALAEventOperation.CustomData)
 
     def deploy_ssh_pubkeys(self, ovfenv):
         for pubkey in ovfenv.ssh_pubkeys:


Test patch:
diff --git a/tests/data/test_waagent.conf b/tests/data/test_waagent.conf
index e35f8c41..24d05f8e 100644
--- a/tests/data/test_waagent.conf
+++ b/tests/data/test_waagent.conf
@@ -20,13 +20,13 @@ Provisioning.DeleteRootPassword=y
 Provisioning.RegenerateSshHostKeyPair=y
 
 # Supported values are "rsa", "dsa" and "ecdsa".
-Provisioning.SshHostKeyPairType=rsa
+Provisioning.SshHostKeyPairType=rsa # An EOL comment that should be ignored
 
 # Monitor host name changes and publish changes via DHCP requests.
 Provisioning.MonitorHostName=y
 
 # Decode CustomData from Base64.
-Provisioning.DecodeCustomData=n
+Provisioning.DecodeCustomData=n#Another EOL comment that should be ignored
 
 # Execute CustomData after provisioning.
 Provisioning.ExecuteCustomData=n
@@ -63,7 +63,7 @@ ResourceDisk.MountOptions=None
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.EnableFIPS=y
+OS.EnableFIPS=y#Another EOL comment that should be ignored
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
@@ -72,7 +72,7 @@ OS.RootDeviceScsiTimeout=300
 OS.OpensslPath=None
 
 # Set the SSH ClientAliveInterval
-OS.SshClientAliveInterval=42
+OS.SshClientAliveInterval=42#Yet another EOL comment with a '#' that should be ignored
 
 # Set the path to SSH keys and configuration files
 OS.SshDir=/notareal/path
diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py
index 93b2c955..da15c263 100644
--- a/tests/protocol/test_wire.py
+++ b/tests/protocol/test_wire.py
@@ -14,6 +14,9 @@
 #
 # Requires Python 2.4+ and Openssl 1.0+
 #
+
+import glob
+
 from azurelinuxagent.common import event
 from azurelinuxagent.common.protocol.wire import *
 from tests.protocol.mockwiredata import *
@@ -25,10 +28,10 @@ wireserver_url = '168.63.129.16'
 
 @patch("time.sleep")
 @patch("azurelinuxagent.common.protocol.wire.CryptUtil")
-class TestWireProtocolGetters(AgentTestCase):
+class TestWireProtocol(AgentTestCase):
 
     def setUp(self):
-        super(TestWireProtocolGetters, self).setUp()
+        super(TestWireProtocol, self).setUp()
         HostPluginProtocol.set_default_channel(False)
     
     def _test_getters(self, test_data, MockCryptUtil, _):
@@ -367,11 +370,14 @@ class TestWireProtocolGetters(AgentTestCase):
         v1_ga_status = {
             'version': str(CURRENT_VERSION),
             'status': status,
-            'osversion': DISTRO_VERSION,
-            'osname': DISTRO_NAME,
-            'hostname': socket.gethostname(),
             'formattedMessage': formatted_msg
         }
+        v1_ga_guest_info = {
+            'computerName' : socket.gethostname(),
+            'osName' : DISTRO_NAME,
+            'osVersion' : DISTRO_VERSION,
+            'version' : str(CURRENT_VERSION),
+        }
         v1_agg_status = {
             'guestAgentStatus': v1_ga_status,
             'handlerAggregateStatus': []
@@ -379,10 +385,44 @@ class TestWireProtocolGetters(AgentTestCase):
         v1_vm_status = {
             'version': '1.1',
             'timestampUTC': timestamp,
-            'aggregateStatus': v1_agg_status
+            'aggregateStatus': v1_agg_status,
+            'guestOSInfo' : v1_ga_guest_info
         }
         self.assertEqual(json.dumps(v1_vm_status), actual.to_json())
 
+    def _create_files(self, tmp_dir, prefix, suffix, count):
+        for i in range(count):
+            f = os.path.join(tmp_dir, '.'.join((prefix, str(i), suffix)))
+            fileutil.write_file(f, "faux content")
+
+    @patch("azurelinuxagent.common.conf.get_lib_dir")
+    def test_purge_cache_purges(self, mock_conf, *args):
+        names = [
+            ("Prod", "agentsManifest"),
+            ("Test", "agentsManifest"),
+            ("FauxExtension1", "manifest.xml"),
+            ("FauxExtension2", "manifest.xml"),
+            ("GoalState", "xml"),
+            ("ExtensionsConfig", "xml")
+        ]
+
+        tmp_dir = tempfile.mkdtemp()
+        mock_conf.return_value = tmp_dir
+
+        for t in names:
+            self._create_files(tmp_dir, t[0], t[1], 2 * MAXIMUM_CACHED_FILES)
+
+        for t in names:
+            p = os.path.join(tmp_dir, '{0}.*.{1}'.format(*t))
+            self.assertEqual(2 * MAXIMUM_CACHED_FILES, len(glob.glob(p)))
+
+        client = WireProtocol(wireserver_url).client
+        client.purge_cache()
+
+        for t in names:
+            p = os.path.join(tmp_dir, '{0}.*.{1}'.format(*t))
+            self.assertEqual(MAXIMUM_CACHED_FILES, len(glob.glob(p)))
+
 
 class MockResponse:
     def __init__(self, body, status_code):
diff --git a/tests/utils/test_rest_util.py b/tests/utils/test_rest_util.py
index 52674da8..3f736d87 100644
--- a/tests/utils/test_rest_util.py
+++ b/tests/utils/test_rest_util.py
@@ -266,6 +266,68 @@ class TestHttpOperations(AgentTestCase):
         self.assertEqual(2, _http_request.call_count)
         self.assertEqual(1, _sleep.call_count)
 
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.utils.restutil._http_request")
+    def test_http_request_retries_with_fibonacci_delay(self, _http_request, _sleep):
+        # Ensure the code is not a throttle code
+        self.assertFalse(httpclient.BAD_GATEWAY in restutil.THROTTLE_CODES)
+
+        _http_request.side_effect = [
+                Mock(status=httpclient.BAD_GATEWAY)
+                    for i in range(restutil.DEFAULT_RETRIES)
+            ] + [Mock(status=httpclient.OK)]
+
+        restutil.http_get("https://foo.bar",
+                            max_retry=restutil.DEFAULT_RETRIES+1)
+
+        self.assertEqual(restutil.DEFAULT_RETRIES+1, _http_request.call_count)
+        self.assertEqual(restutil.DEFAULT_RETRIES, _sleep.call_count)
+        self.assertEqual(
+            [
+                call(restutil._compute_delay(i+1, restutil.DELAY_IN_SECONDS))
+                    for i in range(restutil.DEFAULT_RETRIES)],
+            _sleep.call_args_list)
+
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.utils.restutil._http_request")
+    def test_http_request_retries_with_constant_delay_when_throttled(self, _http_request, _sleep):
+        # Ensure the code is a throttle code
+        self.assertTrue(httpclient.SERVICE_UNAVAILABLE in restutil.THROTTLE_CODES)
+
+        _http_request.side_effect = [
+                Mock(status=httpclient.SERVICE_UNAVAILABLE)
+                    for i in range(restutil.DEFAULT_RETRIES)
+            ] + [Mock(status=httpclient.OK)]
+
+        restutil.http_get("https://foo.bar",
+                            max_retry=restutil.DEFAULT_RETRIES+1)
+
+        self.assertEqual(restutil.DEFAULT_RETRIES+1, _http_request.call_count)
+        self.assertEqual(restutil.DEFAULT_RETRIES, _sleep.call_count)
+        self.assertEqual(
+            [call(1) for i in range(restutil.DEFAULT_RETRIES)],
+            _sleep.call_args_list)
+
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.utils.restutil._http_request")
+    def test_http_request_retries_for_safe_minimum_number_when_throttled(self, _http_request, _sleep):
+        # Ensure the code is a throttle code
+        self.assertTrue(httpclient.SERVICE_UNAVAILABLE in restutil.THROTTLE_CODES)
+
+        _http_request.side_effect = [
+                Mock(status=httpclient.SERVICE_UNAVAILABLE)
+                    for i in range(restutil.MINIMUM_THROTTLE_RETRY-1)
+            ] + [Mock(status=httpclient.OK)]
+
+        restutil.http_get("https://foo.bar",
+                            max_retry=1)
+
+        self.assertEqual(restutil.MINIMUM_THROTTLE_RETRY, _http_request.call_count)
+        self.assertEqual(restutil.MINIMUM_THROTTLE_RETRY-1, _sleep.call_count)
+        self.assertEqual(
+            [call(1) for i in range(restutil.MINIMUM_THROTTLE_RETRY-1)],
+            _sleep.call_args_list)
+
     @patch("time.sleep")
     @patch("azurelinuxagent.common.utils.restutil._http_request")
     def test_http_request_raises_for_bad_request(self, _http_request, _sleep):