Repository: Azure/WALinuxAgent
Pull request #837 (2017-08-09 22:31:03)

Fix issues
[#823] -- Show config options in use
[#784] -- Adjust http retry and logging

Signed-off-by: Brendan Dixon <brendandixon@me.com>

boumenot: :shipit:

Patch:
diff --git a/azurelinuxagent/agent.py b/azurelinuxagent/agent.py
index f6d54067..55c28633 100644
--- a/azurelinuxagent/agent.py
+++ b/azurelinuxagent/agent.py
@@ -128,6 +128,11 @@ class Agent(object):
         update_handler = get_update_handler()
         update_handler.run()
 
+    def show_configuration(self):
+        configuration = conf.get_configuration()
+        for k in sorted(configuration.keys()):
+            print("{0} = {1}".format(k, configuration[k]))
+
 def main(args=[]):
     """
     Parse command line arguments, exit with usage() on error.
@@ -157,6 +162,8 @@ def main(args=[]):
                 agent.daemon()
             elif command == "run-exthandlers":
                 agent.run_exthandlers()
+            elif command == "show-configuration":
+                agent.show_configuration()
         except Exception:
             logger.error(u"Failed to run '{0}': {1}",
                          command,
@@ -198,6 +205,8 @@ def parse_args(sys_args):
             verbose = True
         elif re.match("^([-/]*)force", a):
             force = True
+        elif re.match("^([-/]*show-configuration", a):
+            cmd = "show-configuration"
         elif re.match("^([-/]*)(help|usage|\\?)", a):
             cmd = "help"
         else:
diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py
index 87f65807..5dd9f702 100644
--- a/azurelinuxagent/common/conf.py
+++ b/azurelinuxagent/common/conf.py
@@ -85,6 +85,69 @@ def load_conf_from_file(conf_file_path, conf=__conf__):
         raise AgentConfigError(("Failed to load conf file:{0}, {1}"
                                 "").format(conf_file_path, err))
 
+__SWITCH_OPTIONS__ = {
+    "OS.EnableRDMA" : False,
+    "OS.UpdateRdmaDriver" : False,
+    "OS.CheckRdmaDriver" : False,
+    "Logs.Verbose" : False,
+    "OS.EnableFIPS" : False,
+    "Provisioning.Enabled" : True,
+    "Provisioning.UseCloudInit" : False,
+    "Provisioning.AllowResetSysUser" : False,
+    "Provisioning.RegenerateSshHostKeyPair" : False,
+    "Provisioning.DeleteRootPassword" : False,
+    "Provisioning.DecodeCustomData" : False,
+    "Provisioning.ExecuteCustomData" : False,
+    "Provisioning.MonitorHostName" : False,
+    "DetectScvmmEnv" : False,
+    "ResourceDisk.Format" : False,
+    "DetectScvmmEnv" : False,
+    "ResourceDisk.Format" : False,
+    "ResourceDisk.EnableSwap" : False,
+    "AutoUpdate.Enabled" : True,
+    "EnableOverProvisioning" : False,
+    "OS.AllowHTTP" : False
+}
+
+__STRING_OPTIONS__ = {
+    "Lib.Dir" : "/var/lib/waagent",
+    "DVD.MountPoint" : "/mnt/cdrom/secure",
+    "Pid.File" : "/var/run/waagent.pid",
+    "Extension.LogDir" : "/var/log/azure",
+    "OS.OpensslPath" : "/usr/bin/openssl",
+    "OS.SshDir" : "/etc/ssh",
+    "OS.HomeDir" : "/home",
+    "OS.PasswordPath" : "/etc/shadow",
+    "OS.SudoersDir" : "/etc/sudoers.d",
+    "OS.RootDeviceScsiTimeout" : None,
+    "Provisioning.SshHostKeyPairType" : "rsa",
+    "Provisioning.PasswordCryptId" : "6",
+    "HttpProxy.Host" : None,
+    "ResourceDisk.MountPoint" : "/mnt/resource",
+    "ResourceDisk.MountOptions" : None,
+    "ResourceDisk.Filesystem" : "ext3",
+    "AutoUpdate.GAFamily" : "Prod"
+}
+
+__INTEGER_OPTIONS__ = {
+    "Provisioning.PasswordCryptSaltLength" : 10,
+    "HttpProxy.Port" : None,
+    "ResourceDisk.SwapSizeMB" : 0,
+    "Autoupdate.Frequency" : 3600
+}
+
+def get_configuration(conf=__conf__):
+    options = {}
+    for option in __SWITCH_OPTIONS__:
+        options[option] = conf.get_switch(option, __SWITCH_OPTIONS__[option])
+
+    for option in __STRING_OPTIONS__:
+        options[option] = conf.get(option, __STRING_OPTIONS__[option])
+
+    for option in __INTEGER_OPTIONS__:
+        options[option] = conf.get_int(option, __INTEGER_OPTIONS__[option])
+
+    return options
 
 def enable_rdma(conf=__conf__):
     return conf.get_switch("OS.EnableRDMA", False) or \
@@ -251,4 +314,7 @@ def get_autoupdate_frequency(conf=__conf__):
     return conf.get_int("Autoupdate.Frequency", 3600)
 
 def get_enable_overprovisioning(conf=__conf__):
-    return conf.get_switch("EnableOverProvisioning", False)
\ No newline at end of file
+    return conf.get_switch("EnableOverProvisioning", False)
+
+def get_allow_http(conf=__conf__):
+    return conf.get_switch("OS.AllowHTTP", False)
diff --git a/azurelinuxagent/common/exception.py b/azurelinuxagent/common/exception.py
index 7a0c75e5..91972ebb 100644
--- a/azurelinuxagent/common/exception.py
+++ b/azurelinuxagent/common/exception.py
@@ -86,7 +86,6 @@ class DhcpError(AgentError):
     def __init__(self, msg=None, inner=None):
         super(DhcpError, self).__init__('000006', msg, inner)
 
-
 class OSUtilError(AgentError):
     """
     Failed to perform operation to OS configuration
@@ -148,3 +147,12 @@ class UpdateError(AgentError):
 
     def __init__(self, msg=None, inner=None):
         super(UpdateError, self).__init__('000012', msg, inner)
+
+
+class BadRequestError(HttpError):
+    """
+    The server rejected the request (i.e., status code 400)
+    """
+
+    def __init__(self, msg=None, inner=None):
+        super(BadRequestError, self).__init__(msg, inner)
diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py
index 9af8a973..56bb984f 100644
--- a/azurelinuxagent/common/protocol/hostplugin.py
+++ b/azurelinuxagent/common/protocol/hostplugin.py
@@ -22,7 +22,8 @@ import json
 import traceback
 
 from azurelinuxagent.common import logger
-from azurelinuxagent.common.exception import ProtocolError, HttpError
+from azurelinuxagent.common.exception import BadRequestError, \
+                                            HttpError, ProtocolError
 from azurelinuxagent.common.future import ustr, httpclient
 from azurelinuxagent.common.utils import restutil
 from azurelinuxagent.common.utils import textutil
@@ -85,10 +86,10 @@ class HostPluginProtocol(object):
         try:
             headers = {HEADER_CONTAINER_ID: self.container_id}
             response = restutil.http_get(url, headers)
-            if response.status != httpclient.OK:
+            if restutil.request_failed(response):
                 logger.error(
                     "HostGAPlugin: Failed Get API versions: {0}".format(
-                        self.read_response_error(response)))
+                        restutil.read_response_error(response)))
             else:
                 return_val = ustr(remove_bom(response.read()), encoding='utf-8')
 
@@ -117,42 +118,7 @@ class HostPluginProtocol(object):
         return url, headers
 
     def put_vm_log(self, content):
-        """
-        Try to upload the given content to the host plugin
-        :param deployment_id: the deployment id, which is obtained from the
-        goal state (tenant name)
-        :param container_id: the container id, which is obtained from the
-        goal state
-        :param content: the binary content of the zip file to upload
-        :return:
-        """
-        if not self.ensure_initialized():
-            raise ProtocolError("HostGAPlugin: Host plugin channel is not available")
-
-        if content is None \
-                or self.container_id is None \
-                or self.deployment_id is None:
-            logger.error(
-                "HostGAPlugin: Invalid arguments passed: "
-                "[{0}], [{1}], [{2}]".format(
-                    content,
-                    self.container_id,
-                    self.deployment_id))
-            return
-        url = URI_FORMAT_PUT_LOG.format(self.endpoint, HOST_PLUGIN_PORT)
-
-        headers = {"x-ms-vmagentlog-deploymentid": self.deployment_id,
-                   "x-ms-vmagentlog-containerid": self.container_id}
-        logger.periodic(
-            logger.EVERY_FIFTEEN_MINUTES,
-            "HostGAPlugin: Put VM log to [{0}]".format(url))
-        try:
-            response = restutil.http_put(url, content, headers)
-            if response.status != httpclient.OK:
-                logger.error("HostGAPlugin: Put log failed: Code {0}".format(
-                    response.status))
-        except HttpError as e:
-            logger.error("HostGAPlugin: Put log exception: {0}".format(e))
+        raise NotImplementedError("Unimplemented")
 
     def put_vm_status(self, status_blob, sas_url, config_blob_type=None):
         """
@@ -169,6 +135,7 @@ class HostPluginProtocol(object):
 
         logger.verbose("HostGAPlugin: Posting VM status")
         try:
+
             blob_type = status_blob.type if status_blob.type else config_blob_type
 
             if blob_type == "BlockBlob":
@@ -176,17 +143,14 @@ class HostPluginProtocol(object):
             else:
                 self._put_page_blob_status(sas_url, status_blob)
 
-            if not HostPluginProtocol.is_default_channel():
+        except Exception as e:
+            # If the HostPlugin rejects the request,
+            # let the error continue, but set to use the HostPlugin
+            if isinstance(e, BadRequestError):
                 logger.verbose("HostGAPlugin: Setting host plugin as default channel")
                 HostPluginProtocol.set_default_channel(True)
-        except Exception as e:
-            message = "HostGAPlugin: Exception Put VM status: {0}, {1}".format(e, traceback.format_exc())
-            from azurelinuxagent.common.event import WALAEventOperation, report_event
-            report_event(op=WALAEventOperation.ReportStatus,
-                         is_success=False,
-                         message=message)
-            logger.warn("HostGAPlugin: resetting default channel")
-            HostPluginProtocol.set_default_channel(False)
+
+            raise
 
     def _put_block_blob_status(self, sas_url, status_blob):
         url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
@@ -198,9 +162,9 @@ class HostPluginProtocol(object):
                                     bytearray(status_blob.data, encoding='utf-8')),
                         headers=self._build_status_headers())
 
-        if response.status != httpclient.OK:
+        if restutil.request_failed(response):
             raise HttpError("HostGAPlugin: Put BlockBlob failed: {0}".format(
-                self.read_response_error(response)))
+                restutil.read_response_error(response)))
         else:
             logger.verbose("HostGAPlugin: Put BlockBlob status succeeded")
 
@@ -219,10 +183,10 @@ class HostPluginProtocol(object):
                                     status_blob.get_page_blob_create_headers(status_size)),
                         headers=self._build_status_headers())
 
-        if response.status != httpclient.OK:
+        if restutil.request_failed(response):
             raise HttpError(
                 "HostGAPlugin: Failed PageBlob clean-up: {0}".format(
-                    self.read_response_error(response)))
+                    restutil.read_response_error(response)))
         else:
             logger.verbose("HostGAPlugin: PageBlob clean-up succeeded")
         
@@ -249,11 +213,11 @@ class HostPluginProtocol(object):
                                         buf),
                             headers=self._build_status_headers())
 
-            if response.status != httpclient.OK:
+            if restutil.request_failed(response):
                 raise HttpError(
                     "HostGAPlugin Error: Put PageBlob bytes [{0},{1}]: " \
                     "{2}".format(
-                        start, end, self.read_response_error(response)))
+                        start, end, restutil.read_response_error(response)))
 
             # Advance to the next page (if any)
             start = end
@@ -287,26 +251,3 @@ class HostPluginProtocol(object):
         if PY_VERSION_MAJOR > 2:
             return s.decode('utf-8')
         return s
-    
-    @staticmethod
-    def read_response_error(response):
-        result = ''
-        if response is not None:
-            try:
-                body = remove_bom(response.read())
-                result = "[{0}: {1}] {2}".format(response.status,
-                                                 response.reason,
-                                                 body)
-
-                # this result string is passed upstream to several methods
-                # which do a raise HttpError() or a format() of some kind;
-                # as a result it cannot have any unicode characters
-                if PY_VERSION_MAJOR < 3:
-                    result = ustr(result, encoding='ascii', errors='ignore')
-                else:
-                    result = result\
-                        .encode(encoding='ascii', errors='ignore')\
-                        .decode(encoding='ascii', errors='ignore')
-            except Exception:
-                logger.warn(traceback.format_exc())
-        return result
diff --git a/azurelinuxagent/common/protocol/metadata.py b/azurelinuxagent/common/protocol/metadata.py
index b0b6f672..4de7ecfd 100644
--- a/azurelinuxagent/common/protocol/metadata.py
+++ b/azurelinuxagent/common/protocol/metadata.py
@@ -88,7 +88,7 @@ class MetadataProtocol(Protocol):
         except HttpError as e:
             raise ProtocolError(ustr(e))
 
-        if resp.status != httpclient.OK:
+        if restutil.request_failed(resp):
             raise ProtocolError("{0} - GET: {1}".format(resp.status, url))
 
         data = resp.read()
@@ -103,7 +103,7 @@ class MetadataProtocol(Protocol):
             resp = restutil.http_put(url, json.dumps(data), headers=headers)
         except HttpError as e:
             raise ProtocolError(ustr(e))
-        if resp.status != httpclient.OK:
+        if restutil.request_failed(resp):
             raise ProtocolError("{0} - PUT: {1}".format(resp.status, url))
 
     def _post_data(self, url, data, headers=None):
diff --git a/azurelinuxagent/common/protocol/restapi.py b/azurelinuxagent/common/protocol/restapi.py
index a42db377..dbacfd99 100644
--- a/azurelinuxagent/common/protocol/restapi.py
+++ b/azurelinuxagent/common/protocol/restapi.py
@@ -318,7 +318,7 @@ class Protocol(DataContract):
     def download_ext_handler_pkg(self, uri, headers=None):
         try:
             resp = restutil.http_get(uri, chk_proxy=True, headers=headers)
-            if resp.status == restutil.httpclient.OK:
+            if restutil.request_succeeded(resp):
                 return resp.read()
         except Exception as e:
             logger.warn("Failed to download from: {0}".format(uri), e)
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index 69716831..4b9ef569 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -26,7 +26,8 @@ import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.textutil as textutil
 
-from azurelinuxagent.common.exception import ProtocolNotFoundError
+from azurelinuxagent.common.exception import BadRequestError, \
+                                            ProtocolNotFoundError
 from azurelinuxagent.common.future import httpclient, bytebuffer
 from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
 from azurelinuxagent.common.protocol.restapi import *
@@ -96,7 +97,10 @@ class WireProtocol(Protocol):
         cryptutil = CryptUtil(conf.get_openssl_cmd())
         cryptutil.gen_transport_cert(trans_prv_file, trans_cert_file)
 
-        self.client.update_goal_state(forced=True)
+        self.update_goal_state(forced=True)
+
+    def update_goal_state(self, forced=False, max_retry=3):
+        self.client.update_goal_state(forced=forced, max_retry=max_retry)
 
     def get_vminfo(self):
         goal_state = self.client.get_goal_state()
@@ -117,7 +121,7 @@ class WireProtocol(Protocol):
 
     def get_vmagent_manifests(self):
         # Update goal state to get latest extensions config
-        self.client.update_goal_state()
+        self.update_goal_state()
         goal_state = self.client.get_goal_state()
         ext_conf = self.client.get_ext_conf()
         return ext_conf.vmagent_manifests, goal_state.incarnation
@@ -130,7 +134,7 @@ class WireProtocol(Protocol):
     def get_ext_handlers(self):
         logger.verbose("Get extension handler config")
         # Update goal state to get latest extensions config
-        self.client.update_goal_state()
+        self.update_goal_state()
         goal_state = self.client.get_goal_state()
         ext_conf = self.client.get_ext_conf()
         # In wire protocol, incarnation is equivalent to ETag
@@ -533,29 +537,27 @@ class WireClient(object):
             self.req_count = 0
 
     def call_wireserver(self, http_req, *args, **kwargs):
-        """
-        Call wire server; handle throttling (403), resource gone (410) and
-        service unavailable (503).
-        """
         self.prevent_throttling()
-        for retry in range(0, 3):
+
+        try:
+            # Never use the HTTP proxy for wireserver
+            kwargs['chk_proxy'] = False
             resp = http_req(*args, **kwargs)
-            if resp.status == httpclient.FORBIDDEN:
-                logger.warn("Sending too many requests to wire server. ")
-                logger.info("Sleeping {0}s to avoid throttling.",
-                            LONG_WAITING_INTERVAL)
-                time.sleep(LONG_WAITING_INTERVAL)
-            elif resp.status == httpclient.SERVICE_UNAVAILABLE:
-                logger.warn("Service temporarily unavailable, sleeping {0}s "
-                            "before retrying.", LONG_WAITING_INTERVAL)
-                time.sleep(LONG_WAITING_INTERVAL)
-            elif resp.status == httpclient.GONE:
-                msg = args[0] if len(args) > 0 else ""
-                raise WireProtocolResourceGone(msg)
-            else:
-                return resp
-        raise ProtocolError(("Calling wire server failed: "
-                             "{0}").format(resp.status))
+        except Exception as e:
+            raise ProtocolError("[Wireserver Exception] {0}".format(
+                ustr(e)))
+
+        if resp is not None and resp.status == httpclient.GONE:
+            msg = args[0] if len(args) > 0 else ""
+            raise WireProtocolResourceGone(msg)
+
+        elif restutil.request_failed(resp):
+            msg = "[Wireserver Failed] URI {0} ".format(args[0])
+            if resp is not None:
+                msg += " [HTTP Failed] Status Code {0}".format(resp.status)
+            raise ProtocolError(msg)
+
+        return resp
 
     def decode_config(self, data):
         if data is None:
@@ -565,16 +567,9 @@ class WireClient(object):
         return xml_text
 
     def fetch_config(self, uri, headers):
-        try:
-            resp = self.call_wireserver(restutil.http_get,
-                                        uri,
-                                        headers=headers)
-        except HttpError as e:
-            raise ProtocolError(ustr(e))
-
-        if resp.status != httpclient.OK:
-            raise ProtocolError("{0} - {1}".format(resp.status, uri))
-
+        resp = self.call_wireserver(restutil.http_get,
+                                    uri,
+                                    headers=headers)
         return self.decode_config(resp.read())
 
     def fetch_cache(self, local_file):
@@ -595,25 +590,11 @@ class WireClient(object):
 
     @staticmethod
     def call_storage_service(http_req, *args, **kwargs):
-        """ 
-        Call storage service, handle SERVICE_UNAVAILABLE(503)
-        """
-
         # Default to use the configured HTTP proxy
         if not 'chk_proxy' in kwargs or kwargs['chk_proxy'] is None:
             kwargs['chk_proxy'] = True
 
-        for retry in range(0, 3):
-            resp = http_req(*args, **kwargs)
-            if resp.status == httpclient.SERVICE_UNAVAILABLE:
-                logger.warn("Storage service is temporarily unavailable. ")
-                logger.info("Will retry in {0} seconds. ",
-                            LONG_WAITING_INTERVAL)
-                time.sleep(LONG_WAITING_INTERVAL)
-            else:
-                return resp
-        raise ProtocolError(("Calling storage endpoint failed: "
-                             "{0}").format(resp.status))
+        return http_req(*args, **kwargs)
 
     def fetch_manifest(self, version_uris):
         logger.verbose("Fetch manifest")
@@ -621,47 +602,61 @@ class WireClient(object):
             response = None
             if not HostPluginProtocol.is_default_channel():
                 response = self.fetch(version.uri)
+
             if not response:
                 if HostPluginProtocol.is_default_channel():
                     logger.verbose("Using host plugin as default channel")
                 else:
-                    logger.verbose("Manifest could not be downloaded, falling back to host plugin")
-                host = self.get_host_plugin()
-                uri, headers = host.get_artifact_request(version.uri)
-                response = self.fetch(uri, headers, chk_proxy=False)
-                if not response:
-                    host = self.get_host_plugin(force_update=True)
-                    logger.info("Retry fetch in {0} seconds",
-                                SHORT_WAITING_INTERVAL)
-                    time.sleep(SHORT_WAITING_INTERVAL)
-                else:
-                    host.manifest_uri = version.uri
-                    logger.verbose("Manifest downloaded successfully from host plugin")
-                    if not HostPluginProtocol.is_default_channel():
-                        logger.info("Setting host plugin as default channel")
-                        HostPluginProtocol.set_default_channel(True)
+                    logger.verbose("Failed to download manifest, "
+                                        "switching to host plugin")
+
+                try:
+                    host = self.get_host_plugin()
+                    uri, headers = host.get_artifact_request(version.uri)
+                    response = self.fetch(uri, headers, chk_proxy=False)
+
+                # If the HostPlugin rejects the request,
+                # let the error continue, but set to use the HostPlugin
+                except BadRequestError:
+                    HostPluginProtocol.set_default_channel(True)
+                    raise
+
+                host.manifest_uri = version.uri
+                logger.verbose("Manifest downloaded successfully from host plugin")
+                if not HostPluginProtocol.is_default_channel():
+                    logger.info("Setting host plugin as default channel")
+                    HostPluginProtocol.set_default_channel(True)
+
             if response:
                 return response
+
         raise ProtocolError("Failed to fetch manifest from all sources")
 
     def fetch(self, uri, headers=None, chk_proxy=None):
         logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers)
-        return_value = None
         try:
             resp = self.call_storage_service(
-                restutil.http_get,
-                uri,
-                headers,
-                chk_proxy=chk_proxy)
-            if resp.status == httpclient.OK:
-                return_value = self.decode_config(resp.read())
-            else:
-                logger.warn("Could not fetch {0} [{1}]",
-                            uri,
-                            HostPluginProtocol.read_response_error(resp))
+                        restutil.http_get,
+                        uri,
+                        headers=headers,
+                        chk_proxy=chk_proxy)
+
+            if restutil.request_failed(resp):
+                msg = "[Storage Failed] URI {0} ".format(uri)
+                if resp is not None:
+                    msg += restutil.read_response_error(resp)
+                logger.warn(msg)
+                raise ProtocolError(msg)
+
+            return self.decode_config(resp.read())
+
         except (HttpError, ProtocolError) as e:
             logger.verbose("Fetch failed from [{0}]: {1}", uri, e)
-        return return_value
+
+            if isinstance(e, BadRequestError):
+                raise
+
+        return None
 
     def update_hosting_env(self, goal_state):
         if goal_state.hosting_env_uri is None:
@@ -793,20 +788,45 @@ class WireClient(object):
         return self.ext_conf
 
     def get_ext_manifest(self, ext_handler, goal_state):
-        local_file = MANIFEST_FILE_NAME.format(ext_handler.name,
-                                               goal_state.incarnation)
-        local_file = os.path.join(conf.get_lib_dir(), local_file)
-        xml_text = self.fetch_manifest(ext_handler.versionUris)
-        self.save_cache(local_file, xml_text)
-        return ExtensionManifest(xml_text)
+        for update_goal_state in [False, True]:
+            try:
+                if update_goal_state:
+                    self.update_goal_state(forced=True)
+                    goal_state = self.get_goal_state()
+
+                local_file = MANIFEST_FILE_NAME.format(
+                                ext_handler.name,
+                                goal_state.incarnation)
+                local_file = os.path.join(conf.get_lib_dir(), local_file)
+                xml_text = self.fetch_manifest(ext_handler.versionUris)
+                self.save_cache(local_file, xml_text)
+                return ExtensionManifest(xml_text)
+
+            except BadRequestError:
+                continue
+
+        raise ProtocolError("Failed to retrieve extension manifest")
 
     def get_gafamily_manifest(self, vmagent_manifest, goal_state):
-        local_file = MANIFEST_FILE_NAME.format(vmagent_manifest.family,
-                                               goal_state.incarnation)
-        local_file = os.path.join(conf.get_lib_dir(), local_file)
-        xml_text = self.fetch_manifest(vmagent_manifest.versionsManifestUris)
-        fileutil.write_file(local_file, xml_text)
-        return ExtensionManifest(xml_text)
+        for update_goal_state in [False, True]:
+            try:
+                if update_goal_state:
+                    self.update_goal_state(forced=True)
+                    goal_state = self.get_goal_state()
+
+                local_file = MANIFEST_FILE_NAME.format(
+                                vmagent_manifest.family,
+                                goal_state.incarnation)
+                local_file = os.path.join(conf.get_lib_dir(), local_file)
+                xml_text = self.fetch_manifest(
+                            vmagent_manifest.versionsManifestUris)
+                fileutil.write_file(local_file, xml_text)
+                return ExtensionManifest(xml_text)
+
+            except BadRequestError:
+                continue
+
+        raise ProtocolError("Failed to retrieve GAFamily manifest")
 
     def check_wire_protocol_version(self):
         uri = VERSION_INFO_URI.format(self.endpoint)
@@ -825,39 +845,55 @@ class WireClient(object):
             raise ProtocolNotFoundError(error)
 
     def upload_status_blob(self):
-        ext_conf = self.get_ext_conf()
+        for update_goal_state in [False, True]:
+            try:
+                if update_goal_state:
+                    self.update_goal_state(forced=True)
 
-        blob_uri = ext_conf.status_upload_blob
-        blob_type = ext_conf.status_upload_blob_type
+                ext_conf = self.get_ext_conf()
 
-        if blob_uri is not None:
+                blob_uri = ext_conf.status_upload_blob
+                blob_type = ext_conf.status_upload_blob_type
 
-            if not blob_type in ["BlockBlob", "PageBlob"]:
-                blob_type = "BlockBlob"
-                logger.verbose("Status Blob type is unspecified "
-                    "-- assuming it is a BlockBlob")
+                if blob_uri is not None:
+
+                    if not blob_type in ["BlockBlob", "PageBlob"]:
+                        blob_type = "BlockBlob"
+                        logger.verbose("Status Blob type is unspecified "
+                            "-- assuming it is a BlockBlob")
+
+                    try:
+                        self.status_blob.prepare(blob_type)
+                    except Exception as e:
+                        self.report_status_event(
+                            "Exception creating status blob: {0}", ustr(e))
+                        return
+
+                    if not HostPluginProtocol.is_default_channel():
+                        try:
+                            if self.status_blob.upload(blob_uri):
+                                return
+                        except HttpError as e:
+                            pass
+
+                    host = self.get_host_plugin()
+                    host.put_vm_status(self.status_blob,
+                                    ext_conf.status_upload_blob,
+                                    ext_conf.status_upload_blob_type)
+                    HostPluginProtocol.set_default_channel(True)
+                    return
 
-            try:
-                self.status_blob.prepare(blob_type)
             except Exception as e:
+                # If the HostPlugin rejects the request,
+                # let the error continue, but set to use the HostPlugin
+                if isinstance(e, BadRequestError):
+                    HostPluginProtocol.set_default_channel(True)
+                    continue
+
                 self.report_status_event(
-                    "Exception creating status blob: {0}",
-                    e)
+                    "Exception uploading status blob: {0}", ustr(e))
                 return
 
-            uploaded = False
-            if not HostPluginProtocol.is_default_channel():
-                try:
-                    uploaded = self.status_blob.upload(blob_uri)
-                except HttpError as e:
-                    pass
-
-            if not uploaded:
-                host = self.get_host_plugin()
-                host.put_vm_status(self.status_blob,
-                                   ext_conf.status_upload_blob,
-                                   ext_conf.status_upload_blob_type)
-
     def report_role_prop(self, thumbprint):
         goal_state = self.get_goal_state()
         role_prop = _build_role_properties(goal_state.container_id,
@@ -898,11 +934,12 @@ class WireClient(object):
                                         health_report_uri,
                                         health_report,
                                         headers=headers,
-                                        max_retry=30)
+                                        max_retry=30,
+                                        retry_delay=15)
         except HttpError as e:
             raise ProtocolError((u"Failed to send provision status: "
                                  u"{0}").format(e))
-        if resp.status != httpclient.OK:
+        if restutil.request_failed(resp):
             raise ProtocolError((u"Failed to send provision status: "
                                  u",{0}: {1}").format(resp.status,
                                                       resp.read()))
@@ -921,7 +958,7 @@ class WireClient(object):
         except HttpError as e:
             raise ProtocolError("Failed to send events:{0}".format(e))
 
-        if resp.status != httpclient.OK:
+        if restutil.request_failed(resp):
             logger.verbose(resp.read())
             raise ProtocolError(
                 "Failed to send events:{0}".format(resp.status))
@@ -981,12 +1018,8 @@ class WireClient(object):
             "x-ms-guest-agent-public-x509-cert": cert
         }
 
-    def get_host_plugin(self, force_update=False):
-        if self.host_plugin is None or force_update:
-            if force_update:
-                logger.warn("Forcing update of goal state")
-                self.goal_state = None
-                self.update_goal_state(forced=True)
+    def get_host_plugin(self):
+        if self.host_plugin is None:
             goal_state = self.get_goal_state()
             self.host_plugin = HostPluginProtocol(self.endpoint,
                                                   goal_state.container_id,
@@ -999,23 +1032,47 @@ class WireClient(object):
 
     def get_artifacts_profile(self):
         artifacts_profile = None
-        if self.has_artifacts_profile_blob():
-            blob = self.ext_conf.artifacts_profile_blob
-            logger.verbose("Getting the artifacts profile")
-            profile = self.fetch(blob)
+        for update_goal_state in [False, True]:
+            try:
+                if update_goal_state:
+                    self.update_goal_state(forced=True)
 
-            if profile is None:
-                logger.warn("Download failed, falling back to host plugin")
-                host = self.get_host_plugin()
-                uri, headers = host.get_artifact_request(blob)
-                profile = self.decode_config(self.fetch(uri, headers, chk_proxy=False))
+                if self.has_artifacts_profile_blob():
+                    blob = self.ext_conf.artifacts_profile_blob
 
-            if not textutil.is_str_none_or_whitespace(profile):
-                    logger.verbose("Artifacts profile downloaded successfully")
-                    artifacts_profile = InVMArtifactsProfile(profile)
+                    profile = None
+                    if not HostPluginProtocol.is_default_channel():
+                        logger.verbose("Retrieving the artifacts profile")
+                        profile = self.fetch(blob)
 
-        return artifacts_profile
+                    if profile is None:
+                        if HostPluginProtocol.is_default_channel():
+                            logger.verbose("Using host plugin as default channel")
+                        else:
+                            logger.verbose("Failed to download artifacts profile, "
+                                                "switching to host plugin")
 
+                        host = self.get_host_plugin()
+                        uri, headers = host.get_artifact_request(blob)
+                        config = self.fetch(uri, headers, chk_proxy=False)
+                        profile = self.decode_config(config)
+
+                    if not textutil.is_str_none_or_whitespace(profile):
+                        logger.verbose("Artifacts profile downloaded")
+                        artifacts_profile = InVMArtifactsProfile(profile)
+
+                return artifacts_profile
+
+            except BadRequestError:
+                HostPluginProtocol.set_default_channel(True)
+                continue
+
+            except Exception as e:
+                logger.warn(
+                    "Exception retrieving artifacts profile: {0}".format(
+                        ustr(e)))
+
+        return None
 
 class VersionInfo(object):
     def __init__(self, xml_text):
diff --git a/azurelinuxagent/common/utils/restutil.py b/azurelinuxagent/common/utils/restutil.py
index 49d2d68f..7214e1aa 100644
--- a/azurelinuxagent/common/utils/restutil.py
+++ b/azurelinuxagent/common/utils/restutil.py
@@ -18,19 +18,83 @@
 #
 
 import time
+import traceback
 
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
-from azurelinuxagent.common.exception import HttpError
-from azurelinuxagent.common.future import httpclient, urlparse
-
-"""
-REST api util functions
-"""
-
-RETRY_WAITING_INTERVAL = 3
-secure_warning = True
-
+import azurelinuxagent.common.utils.textutil as textutil
+
+from azurelinuxagent.common.exception import BadRequestError, HttpError
+from azurelinuxagent.common.future import httpclient, urlparse, ustr
+from azurelinuxagent.common.version import PY_VERSION_MAJOR
+
+
+SECURE_WARNING_EMITTED = False
+
+DEFAULT_RETRIES = 3
+
+SHORT_DELAY_IN_SECONDS = 5
+LONG_DELAY_IN_SECONDS = 15
+
+RETRY_CODES = [
+    httpclient.RESET_CONTENT,
+    httpclient.PARTIAL_CONTENT,
+    httpclient.FORBIDDEN,
+    httpclient.INTERNAL_SERVER_ERROR,
+    httpclient.NOT_IMPLEMENTED,
+    httpclient.SERVICE_UNAVAILABLE,
+    httpclient.GATEWAY_TIMEOUT,
+    httpclient.INSUFFICIENT_STORAGE
+]
+
+OK_CODES = [
+    httpclient.OK,
+    httpclient.CREATED,
+    httpclient.ACCEPTED
+]
+
+THROTTLE_CODES = [
+    httpclient.FORBIDDEN,
+    httpclient.SERVICE_UNAVAILABLE
+]
+
+RETRY_EXCEPTIONS = [
+    httpclient.NotConnected,
+    httpclient.IncompleteRead,
+    httpclient.ImproperConnectionState,
+    httpclient.BadStatusLine
+]
+
+# Note:
+# - The Python library does not define constants for all possible
+#   errno values; these come from the standard C/C++ header
+RETRY_IOERRORS = [
+    64,     # ENONET -- Machine is not on the network 
+    67,     # ENOLINK -- Link has been severed 
+    70,     # ECOMM -- Communication error on send 
+    78,     # EREMCHG -- Remote address changed 
+    85,     # ERESTART -- Interrupted system call should be restarted 
+    100,    # ENETDOWN -- Network is down 
+    101,    # ENETUNREACH -- Network is unreachable 
+    102,    # ENETRESET -- Network dropped connection because of reset 
+    103,    # ECONNABORTED -- Software caused connection abort 
+    104,    # ECONNRESET -- Connection reset by peer 
+    111,    # ECONNREFUSED -- Connection refused 
+    112     # EHOSTDOWN -- Host is down 
+]
+
+
+def _is_retry_status(status, retry_codes=RETRY_CODES):
+    return status in retry_codes
+
+def _is_retry_errno(errno):
+    return errno in RETRY_IOERRORS
+
+def _is_retry_exception(e):
+    return len([x for x in RETRY_EXCEPTIONS if isinstance(e, x)]) > 0
+
+def _is_throttle_status(status):
+    return status in THROTTLE_CODES
 
 def _parse_url(url):
     o = urlparse(url)
@@ -45,11 +109,7 @@ def _parse_url(url):
     return o.hostname, o.port, secure, rel_uri
 
 
-def get_http_proxy():
-    """
-    Get http_proxy and https_proxy from environment variables.
-    Username and password is not supported now.
-    """
+def _get_http_proxy():
     host = conf.get_httpproxy_host()
     port = conf.get_httpproxy_port()
     return host, port
@@ -98,43 +158,63 @@ def _http_request(method, host, rel_uri, port=None, data=None, secure=False,
     return resp
 
 
-def http_request(method, url, data, headers=None, max_retry=3,
-                 chk_proxy=False):
-    """
-    Sending http request to server
-    On error, sleep 10 and retry max_retry times.
-    """
+def http_request(method,
+                url, data, headers=None,
+                chk_proxy=False,
+                max_retry=DEFAULT_RETRIES,
+                retry_codes=RETRY_CODES,
+                retry_delay=SHORT_DELAY_IN_SECONDS):
+
+    global SECURE_WARNING_EMITTED
+
     host, port, secure, rel_uri = _parse_url(url)
-    global secure_warning
 
     # Check proxy
     proxy_host, proxy_port = (None, None)
     if chk_proxy:
-        proxy_host, proxy_port = get_http_proxy()
+        proxy_host, proxy_port = _get_http_proxy()
 
-    # If httplib module is not built with ssl support. Fallback to http
+    # If httplib module is not built with ssl support,
+    # fallback to HTTP if allowed
     if secure and not hasattr(httpclient, "HTTPSConnection"):
+        if not conf.get_allow_http():
+            raise HttpError("HTTPS is unavailable and required")
+
         secure = False
-        if secure_warning:
-            logger.warn("httplib is not built with ssl support")
-            secure_warning = False
+        if not SECURE_WARNING_EMITTED:
+            logger.warn("Python does not include SSL support")
+            SECURE_WARNING_EMITTED = True
 
-    # If httplib module doesn't support https tunnelling. Fallback to http
+    # If httplib module doesn't support HTTPS tunnelling,
+    # fallback to HTTP if allowed
     if secure and proxy_host is not None and proxy_port is not None \
             and not hasattr(httpclient.HTTPSConnection, "set_tunnel"):
+        if not conf.get_allow_http():
+            raise HttpError("HTTPS tunnelling is unavailable and required")
+
         secure = False
-        if secure_warning:
-            logger.warn("httplib does not support https tunnelling "
-                        "(new in python 2.7)")
-            secure_warning = False
+        if not SECURE_WARNING_EMITTED:
+            logger.warn("Python does not support HTTPS tunnelling")
+            SECURE_WARNING_EMITTED = True
 
     if proxy_host or proxy_port:
         logger.verbose("HTTP proxy: [{0}:{1}]", proxy_host, proxy_port)
 
-    retry_msg = ''
-    log_msg = "HTTP {0}".format(method)
-    for retry in range(0, max_retry):
-        retry_interval = RETRY_WAITING_INTERVAL
+    msg = ''
+    attempt = 0
+    delay = retry_delay
+
+    while attempt < max_retry:
+        if attempt > 0:
+            logger.info("[HTTP Retry] Attempt {0} of {1}: {2}",
+                        attempt+1,
+                        max_retry,
+                        msg)
+            time.sleep(delay)
+
+        attempt += 1
+        delay = retry_delay
+
         try:
             resp = _http_request(method,
                                  host,
@@ -145,55 +225,125 @@ def http_request(method, url, data, headers=None, max_retry=3,
                                  headers=headers,
                                  proxy_host=proxy_host,
                                  proxy_port=proxy_port)
-            logger.verbose("HTTP response status: [{0}]", resp.status)
+            logger.verbose("[HTTP Response] Status Code {0}", resp.status)
+
+            if request_failed(resp):
+                if _is_retry_status(resp.status, retry_codes=retry_codes):
+                    msg = '[HTTP Retry] HTTP {0} Status Code {1}'.format(
+                        method, resp.status)
+                    if _is_throttle_status(resp.statue):
+                        delay = LONG_DELAY_IN_SECONDS
+                        logger.info("[HTTP Delay] Delay {0} seconds for " \
+                                    "Status Code {1}".format(
+                                        delay, resp.status))
+                    continue
+
+            if resp.status == httpclient.BAD_REQUEST:
+                raise BadRequestError()
+
             return resp
+
         except httpclient.HTTPException as e:
-            retry_msg = 'HTTP exception: {0} {1}'.format(log_msg, e)
-            retry_interval = 5
+            msg = '[HTTP Failed] HTTP {0} HttpException {1}'.format(method, e)
+            if _is_retry_exception(e):
+                continue
+            break
+
         except IOError as e:
-            retry_msg = 'IO error: {0} {1}'.format(log_msg, e)
-            # error 101: network unreachable; when the adapter resets we may
-            # see this transient error for a short time, retry once.
-            if e.errno == 101:
-                retry_interval = RETRY_WAITING_INTERVAL
-                max_retry = 1
+            msg = '[HTTP Failed] HTTP {0} IOError {1}'.format(method, e)
+            if _is_retry_errno(e.errno):
+                continue
+            break
+
+    raise HttpError(msg)
+
+
+def http_get(url, headers=None, chk_proxy=False,
+                max_retry=DEFAULT_RETRIES,
+                retry_codes=RETRY_CODES,
+                retry_delay=SHORT_DELAY_IN_SECONDS):
+    return http_request("GET",
+                        url, None, headers=headers,
+                        chk_proxy=chk_proxy,
+                        max_retry=max_retry,
+                        retry_codes=retry_codes,
+                        retry_delay=retry_delay)
+
+
+def http_head(url, headers=None, chk_proxy=False,
+                max_retry=DEFAULT_RETRIES,
+                retry_codes=RETRY_CODES,
+                retry_delay=SHORT_DELAY_IN_SECONDS):
+    return http_request("HEAD",
+                        url, None, headers=headers,
+                        chk_proxy=chk_proxy,
+                        max_retry=max_retry,
+                        retry_codes=retry_codes,
+                        retry_delay=retry_delay)
+
+
+def http_post(url, data, headers=None, chk_proxy=False,
+                max_retry=DEFAULT_RETRIES,
+                retry_codes=RETRY_CODES,
+                retry_delay=SHORT_DELAY_IN_SECONDS):
+    return http_request("POST",
+                        url, data, headers=headers,
+                        chk_proxy=chk_proxy,
+                        max_retry=max_retry,
+                        retry_codes=retry_codes,
+                        retry_delay=retry_delay)
+
+
+def http_put(url, data, headers=None, chk_proxy=False,
+                max_retry=DEFAULT_RETRIES,
+                retry_codes=RETRY_CODES,
+                retry_delay=SHORT_DELAY_IN_SECONDS):
+    return http_request("PUT",
+                        url, data, headers=headers,
+                        chk_proxy=chk_proxy,
+                        max_retry=max_retry,
+                        retry_codes=retry_codes,
+                        retry_delay=retry_delay)
+
+
+def http_delete(url, headers=None, chk_proxy=False,
+                max_retry=DEFAULT_RETRIES,
+                retry_codes=RETRY_CODES,
+                retry_delay=SHORT_DELAY_IN_SECONDS):
+    return http_request("DELETE",
+                        url, None, headers=headers,
+                        chk_proxy=chk_proxy,
+                        max_retry=max_retry,
+                        retry_codes=retry_codes,
+                        retry_delay=retry_delay)
+
+def request_failed(resp, ok_codes=OK_CODES):
+    return not request_succeeded(resp, ok_codes=ok_codes)
+
+def request_succeeded(resp, ok_codes=OK_CODES):
+    return resp is not None and resp.status in ok_codes
+
+def read_response_error(resp):
+    result = ''
+    if resp is not None:
+        try:
+            result = "[HTTP Failed] [{0}: {1}] {2}".format(
+                        resp.status,
+                        resp.reason,
+                        resp.read())
+
+            # this result string is passed upstream to several methods
+            # which do a raise HttpError() or a format() of some kind;
+            # as a result it cannot have any unicode characters
+            if PY_VERSION_MAJOR < 3:
+                result = ustr(result, encoding='ascii', errors='ignore')
             else:
-                retry_interval = 0
-                max_retry = 0
-
-        if retry < max_retry:
-            logger.info("Retry [{0}/{1} - {3}]",
-                        retry+1,
-                        max_retry,
-                        retry_interval,
-                        retry_msg)
-            time.sleep(retry_interval)
-
-    raise HttpError("{0} failed".format(log_msg))
-
-
-def http_get(url, headers=None, max_retry=3, chk_proxy=False):
-    return http_request("GET", url, data=None, headers=headers,
-                        max_retry=max_retry, chk_proxy=chk_proxy)
-
-
-def http_head(url, headers=None, max_retry=3, chk_proxy=False):
-    return http_request("HEAD", url, None, headers=headers,
-                        max_retry=max_retry, chk_proxy=chk_proxy)
-
-
-def http_post(url, data, headers=None, max_retry=3, chk_proxy=False):
-    return http_request("POST", url, data, headers=headers,
-                        max_retry=max_retry, chk_proxy=chk_proxy)
-
-
-def http_put(url, data, headers=None, max_retry=3, chk_proxy=False):
-    return http_request("PUT", url, data, headers=headers,
-                        max_retry=max_retry, chk_proxy=chk_proxy)
-
+                result = result\
+                    .encode(encoding='ascii', errors='ignore')\
+                    .decode(encoding='ascii', errors='ignore')
 
-def http_delete(url, headers=None, max_retry=3, chk_proxy=False):
-    return http_request("DELETE", url, None, headers=headers,
-                        max_retry=max_retry, chk_proxy=chk_proxy)
+            result = textutil.replace_non_ascii(result)
 
-# End REST api util functions
+        except Exception:
+            logger.warn(traceback.format_exc())
+    return result
diff --git a/azurelinuxagent/common/utils/textutil.py b/azurelinuxagent/common/utils/textutil.py
index 2d99f6f6..d552ce06 100644
--- a/azurelinuxagent/common/utils/textutil.py
+++ b/azurelinuxagent/common/utils/textutil.py
@@ -259,6 +259,17 @@ def set_ini_config(config, name, val):
         config.insert(length - 1, text)
 
 
+def replace_non_ascii(incoming, replace_char=''):
+    outgoing = ''
+    if incoming is not None:
+        for c in incoming:
+            if str_to_ord(c) > 128:
+                outgoing += replace_char
+            else:
+                outgoing += c
+    return outgoing
+
+
 def remove_bom(c):
     '''
     bom is comprised of a sequence of three chars,0xef, 0xbb, 0xbf, in case of utf-8.
diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py
index cf460d5e..9447d40a 100644
--- a/azurelinuxagent/common/version.py
+++ b/azurelinuxagent/common/version.py
@@ -113,7 +113,7 @@ def get_distro():
 
 AGENT_NAME = "WALinuxAgent"
 AGENT_LONG_NAME = "Azure Linux Agent"
-AGENT_VERSION = '2.2.14.1'
+AGENT_VERSION = '2.2.14.2'
 AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION)
 AGENT_DESCRIPTION = """
 The Azure Linux Agent supports the provisioning and running of Linux
diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py
index e5a7b8fe..9ad016f6 100644
--- a/azurelinuxagent/ga/update.py
+++ b/azurelinuxagent/ga/update.py
@@ -41,7 +41,8 @@ import azurelinuxagent.common.utils.textutil as textutil
 from azurelinuxagent.common.event import add_event, add_periodic, \
                                     elapsed_milliseconds, \
                                     WALAEventOperation
-from azurelinuxagent.common.exception import UpdateError, ProtocolError
+from azurelinuxagent.common.exception import BadRequestError, \
+                                    ProtocolError, UpdateError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
 from azurelinuxagent.common.protocol import get_protocol_util
@@ -349,84 +350,6 @@ class UpdateHandler(object):
         self._set_sentinal() 
         return
 
-    def _upgrade_available(self, base_version=CURRENT_VERSION):
-        # Emit an event expressing the state of AutoUpdate
-        # Note:
-        # - Duplicate events get suppressed; state transitions always emit
-        add_event(
-            AGENT_NAME,
-            version=CURRENT_VERSION,
-            op=WALAEventOperation.AutoUpdate,
-            is_success=conf.get_autoupdate_enabled())
-
-        # Ignore new agents if updating is disabled
-        if not conf.get_autoupdate_enabled():
-            return False
-
-        now = time.time()
-        if self.last_attempt_time is not None:
-            next_attempt_time = self.last_attempt_time + conf.get_autoupdate_frequency()
-        else:
-            next_attempt_time = now
-        if next_attempt_time > now:
-            return False
-
-        family = conf.get_autoupdate_gafamily()
-        logger.verbose("Checking for agent family {0} updates", family)
-
-        self.last_attempt_time = now
-        try:
-            protocol = self.protocol_util.get_protocol()
-            manifest_list, etag = protocol.get_vmagent_manifests()
-        except Exception as e:
-            msg = u"Exception retrieving agent manifests: {0}".format(ustr(e))
-            logger.warn(msg)
-            add_event(
-                AGENT_NAME,
-                op=WALAEventOperation.Download,
-                version=CURRENT_VERSION,
-                is_success=False,
-                message=msg)
-            return False
-
-        manifests = [m for m in manifest_list.vmAgentManifests \
-                        if m.family == family and len(m.versionsManifestUris) > 0]
-        if len(manifests) == 0:
-            logger.verbose(u"Incarnation {0} has no agent family {1} updates", etag, family)
-            return False
-
-        try:
-            pkg_list = protocol.get_vmagent_pkgs(manifests[0])
-        except ProtocolError as e:
-            msg = u"Incarnation {0} failed to get {1} package list: " \
-                  u"{2}".format(
-                etag,
-                family,
-                ustr(e))
-            logger.warn(msg)
-            add_event(
-                AGENT_NAME,
-                op=WALAEventOperation.Download,
-                version=CURRENT_VERSION,
-                is_success=False,
-                message=msg)
-            return False
-
-        # Set the agents to those available for download at least as current
-        # as the existing agent and remove from disk any agent no longer
-        # reported to the VM.
-        # Note:
-        #  The code leaves on disk available, but blacklisted, agents so as to
-        #  preserve the state. Otherwise, those agents could be again
-        #  downloaded and inappropriately retried.
-        host = self._get_host_plugin(protocol=protocol)
-        self._set_agents([GuestAgent(pkg=pkg, host=host) for pkg in pkg_list.versions])
-        self._purge_agents()
-        self._filter_blacklisted_agents()
-
-        # Return True if agents more recent than the current are available
-        return len(self.agents) > 0 and self.agents[0].version > base_version
-
     def _ensure_no_orphans(self, orphan_wait_interval=ORPHAN_WAIT_INTERVAL):
         pid_files, ignored = self._write_pid_file()
         for pid_file in pid_files:
@@ -615,6 +538,85 @@ class UpdateHandler(object):
                 str(e))
         return
 
+    def _upgrade_available(self, base_version=CURRENT_VERSION):
+        # Emit an event expressing the state of AutoUpdate
+        # Note:
+        # - Duplicate events get suppressed; state transitions always emit
+        add_event(
+            AGENT_NAME,
+            version=CURRENT_VERSION,
+            op=WALAEventOperation.AutoUpdate,
+            is_success=conf.get_autoupdate_enabled())
+
+        # Ignore new agents if updating is disabled
+        if not conf.get_autoupdate_enabled():
+            return False
+
+        now = time.time()
+        if self.last_attempt_time is not None:
+            next_attempt_time = self.last_attempt_time + \
+                                    conf.get_autoupdate_frequency()
+        else:
+            next_attempt_time = now
+        if next_attempt_time > now:
+            return False
+
+        family = conf.get_autoupdate_gafamily()
+        logger.verbose("Checking for agent family {0} updates", family)
+
+        self.last_attempt_time = now
+        protocol = self.protocol_util.get_protocol()
+
+        for update_goal_state in [False, True]:
+            try:
+                if update_goal_state:
+                    protocol.update_goal_state(forced=True)
+
+                manifest_list, etag = protocol.get_vmagent_manifests()
+
+                manifests = [m for m in manifest_list.vmAgentManifests \
+                                if m.family == family and \
+                                    len(m.versionsManifestUris) > 0]
+                if len(manifests) == 0:
+                    logger.verbose(u"Incarnation {0} has no {1} agent updates",
+                                    etag, family)
+                    return False
+
+                pkg_list = protocol.get_vmagent_pkgs(manifests[0])
+
+                # Set the agents to those available for download at least as
+                # current as the existing agent and remove from disk any agent
+                # no longer reported to the VM.
+                # Note:
+                #  The code leaves on disk available, but blacklisted, agents
+                #  so as to preserve the state. Otherwise, those agents could be
+                #  again downloaded and inappropriately retried.
+                host = self._get_host_plugin(protocol=protocol)
+                self._set_agents([GuestAgent(pkg=pkg, host=host) \
+                                     for pkg in pkg_list.versions])
+
+                self._purge_agents()
+                self._filter_blacklisted_agents()
+
+                # Return True if more recent agents are available
+                return len(self.agents) > 0 and \
+                            self.agents[0].version > base_version
+
+            except Exception as e:
+                if isinstance(e, BadRequestError):
+                    continue
+
+                msg = u"Exception retrieving agent manifests: {0}".format(
+                            ustr(e))
+                logger.warn(msg)
+                add_event(
+                    AGENT_NAME,
+                    op=WALAEventOperation.Download,
+                    version=CURRENT_VERSION,
+                    is_success=False,
+                    message=msg)
+                return False
+
     def _write_pid_file(self):
         pid_files = self._get_pid_files()
 
@@ -671,12 +673,16 @@ class GuestAgent(object):
             self._ensure_downloaded()
             self._ensure_loaded()
         except Exception as e:
+            if isinstance(e, BadRequestError):
+                raise
+
             # Note the failure, blacklist the agent if the package downloaded
             # - An exception with a downloaded package indicates the package
             #   is corrupt (e.g., missing the HandlerManifest.json file)
             self.mark_failure(is_fatal=os.path.isfile(self.get_agent_pkg_path()))
 
-            msg = u"Agent {0} download / load failed with exception: {1}".format(self.name, ustr(e))
+            msg = u"Agent {0} install failed with exception: {1}".format(
+                        self.name, ustr(e))
             logger.warn(msg)
             add_event(
                 AGENT_NAME,
@@ -795,20 +801,29 @@ class GuestAgent(object):
         for uri in self.pkg.uris:
             if not HostPluginProtocol.is_default_channel() and self._fetch(uri.uri):
                 break
+
             elif self.host is not None and self.host.ensure_initialized():
                 if not HostPluginProtocol.is_default_channel():
-                    logger.warn("Download unsuccessful, falling back to host plugin")
+                    logger.warn("Download failed, switching to host plugin")
                 else:
                     logger.verbose("Using host plugin as default channel")
 
                 uri, headers = self.host.get_artifact_request(uri.uri, self.host.manifest_uri)
-                if self._fetch(uri, headers=headers, chk_proxy=False):
-                    if not HostPluginProtocol.is_default_channel():
-                        logger.verbose("Setting host plugin as default channel")
-                        HostPluginProtocol.set_default_channel(True)
-                    break
-                else:
-                    logger.warn("Host plugin download unsuccessful")
+                try:
+                    if self._fetch(uri, headers=headers, chk_proxy=False):
+                        if not HostPluginProtocol.is_default_channel():
+                            logger.verbose("Setting host plugin as default channel")
+                            HostPluginProtocol.set_default_channel(True)
+                        break
+                    else:
+                        logger.warn("Host plugin download failed")
+
+                # If the HostPlugin rejects the request,
+                # let the error continue, but set to use the HostPlugin
+                except BadRequestError:
+                    HostPluginProtocol.set_default_channel(True)
+                    raise
+
             else:
                 logger.error("No download channels available")
 
@@ -821,13 +836,14 @@ class GuestAgent(object):
                 is_success=False,
                 message=msg)
             raise UpdateError(msg)
+
         return
 
     def _fetch(self, uri, headers=None, chk_proxy=True):
         package = None
         try:
             resp = restutil.http_get(uri, chk_proxy=chk_proxy, headers=headers)
-            if resp.status == restutil.httpclient.OK:
+            if restutil.request_succeeded(resp):
                 package = resp.read()
                 fileutil.write_file(self.get_agent_pkg_path(),
                                     bytearray(package),
@@ -835,12 +851,16 @@ class GuestAgent(object):
                 logger.verbose(u"Agent {0} downloaded from {1}", self.name, uri)
             else:
                 logger.verbose("Fetch was unsuccessful [{0}]",
-                               HostPluginProtocol.read_response_error(resp))
+                               restutil.read_response_error(resp))
         except restutil.HttpError as http_error:
+            if isinstance(http_error, BadRequestError):
+                raise
+
             logger.verbose(u"Agent {0} download from {1} failed [{2}]",
                            self.name,
                            uri,
                            http_error)
+
         return package is not None
 
     def _load_error(self):
diff --git a/config/coreos/waagent.conf b/config/coreos/waagent.conf
index 664d037f..ac19b5f2 100644
--- a/config/coreos/waagent.conf
+++ b/config/coreos/waagent.conf
@@ -107,3 +107,7 @@ OS.OpensslPath=None
 # handling until inVMArtifactsProfile.OnHold is false.
 # Default is disabled
 # EnableOverProvisioning=n
+
+# Allow fallback to HTTP if HTTPS is unavailable
+# Note: Allowing HTTP (vs. HTTPS) may cause security risks
+OS.AllowHTTP=y
diff --git a/config/waagent.conf b/config/waagent.conf
index 59a47787..bd11d461 100644
--- a/config/waagent.conf
+++ b/config/waagent.conf
@@ -104,3 +104,7 @@ OS.SshDir=/etc/ssh
 # handling until inVMArtifactsProfile.OnHold is false.
 # Default is disabled
 # EnableOverProvisioning=n
+
+# Allow fallback to HTTP if HTTPS is unavailable
+# Note: Allowing HTTP (vs. HTTPS) may cause security risks
+# OS.AllowHTTP=n


Test patch:
diff --git a/tests/common/test_conf.py b/tests/common/test_conf.py
index 1287b0dd..6a0beb0d 100644
--- a/tests/common/test_conf.py
+++ b/tests/common/test_conf.py
@@ -24,6 +24,48 @@ from tests.tools import *
 
 
 class TestConf(AgentTestCase):
+    # Note:
+    # -- These values *MUST* match those from data/test_waagent.conf
+    EXPECTED_CONFIGURATION = {
+        "Provisioning.Enabled" : True,
+        "Provisioning.UseCloudInit" : True,
+        "Provisioning.DeleteRootPassword" : True,
+        "Provisioning.RegenerateSshHostKeyPair" : True,
+        "Provisioning.SshHostKeyPairType" : "rsa",
+        "Provisioning.MonitorHostName" : True,
+        "Provisioning.DecodeCustomData" : False,
+        "Provisioning.ExecuteCustomData" : False,
+        "Provisioning.PasswordCryptId" : '6',
+        "Provisioning.PasswordCryptSaltLength" : 10,
+        "Provisioning.AllowResetSysUser" : False,
+        "ResourceDisk.Format" : True,
+        "ResourceDisk.Filesystem" : "ext4",
+        "ResourceDisk.MountPoint" : "/mnt/resource",
+        "ResourceDisk.EnableSwap" : False,
+        "ResourceDisk.SwapSizeMB" : 0,
+        "ResourceDisk.MountOptions" : None,
+        "Logs.Verbose" : False,
+        "OS.EnableFIPS" : True,
+        "OS.RootDeviceScsiTimeout" : '300',
+        "OS.OpensslPath" : '/usr/bin/openssl',
+        "OS.SshDir" : "/notareal/path",
+        "HttpProxy.Host" : None,
+        "HttpProxy.Port" : None,
+        "DetectScvmmEnv" : False,
+        "Lib.Dir" : "/var/lib/waagent",
+        "DVD.MountPoint" : "/mnt/cdrom/secure",
+        "Pid.File" : "/var/run/waagent.pid",
+        "Extension.LogDir" : "/var/log/azure",
+        "OS.HomeDir" : "/home",
+        "OS.EnableRDMA" : False,
+        "OS.UpdateRdmaDriver" : False,
+        "OS.CheckRdmaDriver" : False,
+        "AutoUpdate.Enabled" : True,
+        "AutoUpdate.GAFamily" : "Prod",
+        "EnableOverProvisioning" : False,
+        "OS.AllowHTTP" : False
+    }
+
     def setUp(self):
         AgentTestCase.setUp(self)
         self.conf = ConfigurationProvider()
@@ -59,3 +101,11 @@ class TestConf(AgentTestCase):
 
     def test_get_provision_cloudinit(self):
         self.assertTrue(get_provision_cloudinit(self.conf))
+
+    def test_get_configuration(self):
+        configuration = conf.get_configuration(self.conf)
+        self.assertTrue(len(configuration.keys()) > 0)
+        for k in TestConf.EXPECTED_CONFIGURATION.keys():
+            self.assertEqual(
+                TestConf.EXPECTED_CONFIGURATION[k],
+                configuration[k])
diff --git a/tests/data/test_waagent.conf b/tests/data/test_waagent.conf
index 6368c39a..c7bcc9e3 100644
--- a/tests/data/test_waagent.conf
+++ b/tests/data/test_waagent.conf
@@ -94,10 +94,13 @@ OS.SshDir=/notareal/path
 # Extension.LogDir=/var/log/azure
 
 #
-# Home.Dir=/home
+# OS.HomeDir=/home
 
 # Enable RDMA management and set up, should only be used in HPC images
-# OS.EnableRDMA=y
+# OS.EnableRDMA=n
+# OS.UpdateRdmaDriver=n
+# OS.CheckRdmaDriver=n
+
 
 # Enable or disable goal state processing auto-update, default is enabled
 # AutoUpdate.Enabled=y
diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py
index fb4b513d..5473a7ee 100644
--- a/tests/ga/test_update.py
+++ b/tests/ga/test_update.py
@@ -828,6 +828,12 @@ class TestUpdate(UpdateTestCase):
         self.event_patch.stop()
         return
 
+    def _create_protocol(self, count=5, versions=None):
+        latest_version = self.prepare_agents(count=count)
+        if versions is None or len(versions) <= 0:
+            versions = [latest_version]
+        return ProtocolMock(versions=versions)
+
     def _test_upgrade_available(
             self,
             base_version=FlexibleVersion(AGENT_VERSION),
@@ -835,12 +841,9 @@ class TestUpdate(UpdateTestCase):
             versions=None,
             count=5):
 
-        latest_version = self.prepare_agents(count=count)
-        if versions is None or len(versions) <= 0:
-            versions = [latest_version]
-
         if protocol is None:
-            protocol = ProtocolMock(versions=versions)
+            protocol = self._create_protocol(count=count, versions=versions)
+
         self.update_handler.protocol_util = protocol
         conf.get_autoupdate_gafamily = Mock(return_value=protocol.family)
 
@@ -850,6 +853,16 @@ class TestUpdate(UpdateTestCase):
         self.assertTrue(self._test_upgrade_available())
         return
 
+    def test_upgrade_available_will_refresh_goal_state(self):
+        protocol = self._create_protocol()
+        protocol.emulate_stale_goal_state()
+        self.assertTrue(self._test_upgrade_available(protocol=protocol))
+        self.assertEqual(2, protocol.call_counts["get_vmagent_manifests"])
+        self.assertEqual(1, protocol.call_counts["get_vmagent_pkgs"])
+        self.assertEqual(1, protocol.call_counts["update_goal_state"])
+        self.assertTrue(protocol.goal_state_forced)
+        return
+
     def test_get_latest_agent_excluded(self):
         self.prepare_agent(AGENT_VERSION)
         self.assertFalse(self._test_upgrade_available(
@@ -1549,12 +1562,22 @@ class ProtocolMock(object):
     def __init__(self, family="TestAgent", etag=42, versions=None, client=None):
         self.family = family
         self.client = client
+        self.call_counts = {
+            "get_vmagent_manifests" : 0,
+            "get_vmagent_pkgs" : 0,
+            "update_goal_state" : 0
+        }
+        self.goal_state_is_stale = False
+        self.goal_state_forced = False
         self.etag = etag
         self.versions = versions if versions is not None else []
         self.create_manifests()
         self.create_packages()
         return
 
+    def emulate_stale_goal_state(self):
+        self.goal_state_is_stale = True
+
     def create_manifests(self):
         self.agent_manifests = VMAgentManifestList()
         if len(self.versions) <= 0:
@@ -1585,11 +1608,23 @@ class ProtocolMock(object):
         return self
 
     def get_vmagent_manifests(self):
+        self.call_counts["get_vmagent_manifests"] += 1
+        if self.goal_state_is_stale:
+            self.goal_state_is_stale = False
+            raise BadRequestError()
         return self.agent_manifests, self.etag
 
     def get_vmagent_pkgs(self, manifest):
+        self.call_counts["get_vmagent_pkgs"] += 1
+        if self.goal_state_is_stale:
+            self.goal_state_is_stale = False
+            raise BadRequestError()
         return self.agent_packages
 
+    def update_goal_state(self, forced=False, max_retry=3):
+        self.call_counts["update_goal_state"] += 1
+        self.goal_state_forced = self.goal_state_forced or forced
+        return
 
 class ResponseMock(Mock):
     def __init__(self, status=restutil.httpclient.OK, response=None, reason=None):
diff --git a/tests/protocol/mockwiredata.py b/tests/protocol/mockwiredata.py
index 4e45623c..6a70a90f 100644
--- a/tests/protocol/mockwiredata.py
+++ b/tests/protocol/mockwiredata.py
@@ -16,6 +16,7 @@
 #
 
 from tests.tools import *
+from azurelinuxagent.common.exception import BadRequestError, HttpError
 from azurelinuxagent.common.future import httpclient
 from azurelinuxagent.common.utils.cryptutil import CryptUtil
 
@@ -53,6 +54,20 @@ DATA_FILE_EXT_AUTOUPGRADE_INTERNALVERSION["ext_conf"] = "wire/ext_conf_autoupgra
 
 class WireProtocolData(object):
     def __init__(self, data_files=DATA_FILE):
+        self.emulate_stale_goal_state = False
+        self.call_counts = {
+            "comp=versions" : 0,
+            "/versions" : 0,
+            "goalstate" : 0,
+            "hostingenvuri" : 0,
+            "sharedconfiguri" : 0,
+            "certificatesuri" : 0,
+            "extensionsconfiguri" : 0,
+            "extensionArtifact" : 0,
+            "manifest.xml" : 0,
+            "manifest_of_ga.xml" : 0,
+            "ExampleHandlerLinux" : 0
+        }
         self.version_info = load_data(data_files.get("version_info"))
         self.goal_state = load_data(data_files.get("goal_state"))
         self.hosting_env = load_data(data_files.get("hosting_env"))
@@ -67,32 +82,70 @@ class WireProtocolData(object):
 
     def mock_http_get(self, url, *args, **kwargs):
         content = None
-        if "versions" in url:
+
+        resp = MagicMock()
+        resp.status = httpclient.OK
+
+        # wire server versions
+        if "comp=versions" in url:
             content = self.version_info
+            self.call_counts["comp=versions"] += 1
+
+        # HostPlugin versions
+        elif "/versions" in url:
+            content = '["2015-09-01"]'
+            self.call_counts["/versions"] += 1
         elif "goalstate" in url:
             content = self.goal_state
+            self.call_counts["goalstate"] += 1
         elif "hostingenvuri" in url:
             content = self.hosting_env
+            self.call_counts["hostingenvuri"] += 1
         elif "sharedconfiguri" in url:
             content = self.shared_config
+            self.call_counts["sharedconfiguri"] += 1
         elif "certificatesuri" in url:
             content = self.certs
+            self.call_counts["certificatesuri"] += 1
         elif "extensionsconfiguri" in url:
             content = self.ext_conf
-        elif "manifest.xml" in url:
-            content = self.manifest
-        elif "manifest_of_ga.xml" in url:
-            content = self.ga_manifest
-        elif "ExampleHandlerLinux" in url:
-            content = self.ext
-            resp = MagicMock()
-            resp.status = httpclient.OK
-            resp.read = Mock(return_value=content)
-            return resp
+            self.call_counts["extensionsconfiguri"] += 1
+
         else:
-            raise Exception("Bad url {0}".format(url))
-        resp = MagicMock()
-        resp.status = httpclient.OK
+            # A stale GoalState results in a 400 from the HostPlugin
+            # for which the HTTP handler in restutil raises BadRequestError
+            if self.emulate_stale_goal_state:
+                if "extensionArtifact" in url:
+                    self.emulate_stale_goal_state = False
+                    self.call_counts["extensionArtifact"] += 1
+                    raise BadRequestError()
+                else:
+                    raise HttpError()
+
+            # For HostPlugin requests, replace the URL with that passed
+            # via the x-ms-artifact-location header
+            if "extensionArtifact" in url:
+                self.call_counts["extensionArtifact"] += 1
+                if "headers" not in kwargs or \
+                    "x-ms-artifact-location" not in kwargs["headers"]:
+                    raise Exception("Bad HEADERS passed to HostPlugin: {0}",
+                            kwargs)
+                url = kwargs["headers"]["x-ms-artifact-location"]
+
+            if "manifest.xml" in url:
+                content = self.manifest
+                self.call_counts["manifest.xml"] += 1
+            elif "manifest_of_ga.xml" in url:
+                content = self.ga_manifest
+                self.call_counts["manifest_of_ga.xml"] += 1
+            elif "ExampleHandlerLinux" in url:
+                content = self.ext
+                self.call_counts["ExampleHandlerLinux"] += 1
+                resp.read = Mock(return_value=content)
+                return resp
+            else:
+                raise Exception("Bad url {0}".format(url))
+
         resp.read = Mock(return_value=content.encode("utf-8"))
         return resp
 
diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py
index b18b6914..74f7f24e 100644
--- a/tests/protocol/test_hostplugin.py
+++ b/tests/protocol/test_hostplugin.py
@@ -146,6 +146,7 @@ class TestHostPlugin(AgentTestCase):
         test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
         status = restapi.VMStatus(status="Ready",
                                   message="Guest Agent is running")
+        wire.HostPluginProtocol.set_default_channel(False)
         with patch.object(wire.HostPluginProtocol,
                           "ensure_initialized",
                           return_value=True):
@@ -173,6 +174,7 @@ class TestHostPlugin(AgentTestCase):
         test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
         status = restapi.VMStatus(status="Ready",
                                   message="Guest Agent is running")
+        wire.HostPluginProtocol.set_default_channel(False)
         with patch.object(wire.StatusBlob,
                           "upload",
                           return_value=False):
@@ -211,6 +213,8 @@ class TestHostPlugin(AgentTestCase):
                         bytearray(faux_status, encoding='utf-8'))
 
         with patch.object(restutil, "http_request") as patch_http:
+            patch_http.return_value = Mock(status=httpclient.OK)
+
             wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state)
             plugin = wire_protocol_client.get_host_plugin()
 
@@ -224,61 +228,6 @@ class TestHostPlugin(AgentTestCase):
                     test_goal_state,
                     exp_method, exp_url, exp_data)
 
-    def test_read_response_error(self):
-        """
-        Validate the read_response_error method handles encoding correctly
-        """
-        responses = ['message', b'message', '\x80message\x80']
-        response = MagicMock()
-        response.status = 'status'
-        response.reason = 'reason'
-        with patch.object(response, 'read') as patch_response:
-            for s in responses:
-                patch_response.return_value = s
-                result = hostplugin.HostPluginProtocol.read_response_error(response)
-                self.assertTrue('[status: reason]' in result)
-                self.assertTrue('message' in result)
-
-    def test_read_response_bytes(self):
-        response_bytes = '7b:0a:20:20:20:20:22:65:72:72:6f:72:43:6f:64:65:22:' \
-                         '3a:20:22:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:' \
-                         '69:73:20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:' \
-                         '69:73:20:6f:70:65:72:61:74:69:6f:6e:2e:22:2c:0a:20:' \
-                         '20:20:20:22:6d:65:73:73:61:67:65:22:3a:20:22:c3:af:' \
-                         'c2:bb:c2:bf:3c:3f:78:6d:6c:20:76:65:72:73:69:6f:6e:' \
-                         '3d:22:31:2e:30:22:20:65:6e:63:6f:64:69:6e:67:3d:22:' \
-                         '75:74:66:2d:38:22:3f:3e:3c:45:72:72:6f:72:3e:3c:43:' \
-                         '6f:64:65:3e:49:6e:76:61:6c:69:64:42:6c:6f:62:54:79:' \
-                         '70:65:3c:2f:43:6f:64:65:3e:3c:4d:65:73:73:61:67:65:' \
-                         '3e:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:69:73:' \
-                         '20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:69:73:' \
-                         '20:6f:70:65:72:61:74:69:6f:6e:2e:0a:52:65:71:75:65:' \
-                         '73:74:49:64:3a:63:37:34:32:39:30:63:62:2d:30:30:30:' \
-                         '31:2d:30:30:62:35:2d:30:36:64:61:2d:64:64:36:36:36:' \
-                         '61:30:30:30:22:2c:0a:20:20:20:20:22:64:65:74:61:69:' \
-                         '6c:73:22:3a:20:22:22:0a:7d'.split(':')
-        expected_response = '[status: reason] {\n    "errorCode": "The blob ' \
-                            'type is invalid for this operation.",\n    ' \
-                            '"message": "<?xml version="1.0" ' \
-                            'encoding="utf-8"?>' \
-                            '<Error><Code>InvalidBlobType</Code><Message>The ' \
-                            'blob type is invalid for this operation.\n' \
-                            'RequestId:c74290cb-0001-00b5-06da-dd666a000",' \
-                            '\n    "details": ""\n}'
-
-        response_string = ''.join(chr(int(b, 16)) for b in response_bytes)
-        response = MagicMock()
-        response.status = 'status'
-        response.reason = 'reason'
-        with patch.object(response, 'read') as patch_response:
-            patch_response.return_value = response_string
-            result = hostplugin.HostPluginProtocol.read_response_error(response)
-            self.assertEqual(result, expected_response)
-            try:
-                raise HttpError("{0}".format(result))
-            except HttpError as e:
-                self.assertTrue(result in ustr(e))
-
     def test_no_fallback(self):
         """
         Validate fallback to upload status using HostGAPlugin is not happening
@@ -318,6 +267,8 @@ class TestHostPlugin(AgentTestCase):
                         bytearray(faux_status, encoding='utf-8'))
 
         with patch.object(restutil, "http_request") as patch_http:
+            patch_http.return_value = Mock(status=httpclient.OK)
+
             with patch.object(wire.HostPluginProtocol,
                           "get_api_versions") as patch_get:
                 patch_get.return_value = api_versions
diff --git a/tests/protocol/test_metadata.py b/tests/protocol/test_metadata.py
index ee4ba3e6..5047b86e 100644
--- a/tests/protocol/test_metadata.py
+++ b/tests/protocol/test_metadata.py
@@ -31,17 +31,15 @@ class TestMetadataProtocolGetters(AgentTestCase):
         return json.loads(ustr(load_data(path)), encoding="utf-8")
 
     @patch("time.sleep")
-    @patch("azurelinuxagent.common.protocol.metadata.restutil")
-    def _test_getters(self, test_data, mock_restutil ,_):
-        mock_restutil.http_get.side_effect = test_data.mock_http_get
-
-        protocol = MetadataProtocol()
-        protocol.detect()
-        protocol.get_vminfo()
-        protocol.get_certs()
-        ext_handlers, etag = protocol.get_ext_handlers()
-        for ext_handler in ext_handlers.extHandlers:
-            protocol.get_ext_handler_pkgs(ext_handler)
+    def _test_getters(self, test_data ,_):
+        with patch.object(restutil, 'http_get', test_data.mock_http_get):
+            protocol = MetadataProtocol()
+            protocol.detect()
+            protocol.get_vminfo()
+            protocol.get_certs()
+            ext_handlers, etag = protocol.get_ext_handlers()
+            for ext_handler in ext_handlers.extHandlers:
+                protocol.get_ext_handler_pkgs(ext_handler)
 
     def test_getters(self, *args):
         test_data = MetadataProtocolData(DATA_FILE)
diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py
index 02976cac..4efdc5ed 100644
--- a/tests/protocol/test_wire.py
+++ b/tests/protocol/test_wire.py
@@ -25,30 +25,34 @@ wireserver_url = '168.63.129.16'
 
 @patch("time.sleep")
 @patch("azurelinuxagent.common.protocol.wire.CryptUtil")
-@patch("azurelinuxagent.common.protocol.wire.restutil")
 class TestWireProtocolGetters(AgentTestCase):
-    def _test_getters(self, test_data, mock_restutil, MockCryptUtil, _):
-        mock_restutil.http_get.side_effect = test_data.mock_http_get
+
+    def setUp(self):
+        super(TestWireProtocolGetters, self).setUp()
+        HostPluginProtocol.set_default_channel(False)
+    
+    def _test_getters(self, test_data, MockCryptUtil, _):
         MockCryptUtil.side_effect = test_data.mock_crypt_util
 
-        protocol = WireProtocol(wireserver_url)
-        protocol.detect()
-        protocol.get_vminfo()
-        protocol.get_certs()
-        ext_handlers, etag = protocol.get_ext_handlers()
-        for ext_handler in ext_handlers.extHandlers:
-            protocol.get_ext_handler_pkgs(ext_handler)
-
-        crt1 = os.path.join(self.tmp_dir,
-                            '33B0ABCE4673538650971C10F7D7397E71561F35.crt')
-        crt2 = os.path.join(self.tmp_dir,
-                            '4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.crt')
-        prv2 = os.path.join(self.tmp_dir,
-                            '4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.prv')
-
-        self.assertTrue(os.path.isfile(crt1))
-        self.assertTrue(os.path.isfile(crt2))
-        self.assertTrue(os.path.isfile(prv2))
+        with patch.object(restutil, 'http_get', test_data.mock_http_get):
+            protocol = WireProtocol(wireserver_url)
+            protocol.detect()
+            protocol.get_vminfo()
+            protocol.get_certs()
+            ext_handlers, etag = protocol.get_ext_handlers()
+            for ext_handler in ext_handlers.extHandlers:
+                protocol.get_ext_handler_pkgs(ext_handler)
+
+            crt1 = os.path.join(self.tmp_dir,
+                                '33B0ABCE4673538650971C10F7D7397E71561F35.crt')
+            crt2 = os.path.join(self.tmp_dir,
+                                '4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.crt')
+            prv2 = os.path.join(self.tmp_dir,
+                                '4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3.prv')
+
+            self.assertTrue(os.path.isfile(crt1))
+            self.assertTrue(os.path.isfile(crt2))
+            self.assertTrue(os.path.isfile(prv2))
 
     def test_getters(self, *args):
         """Normal case"""
@@ -70,8 +74,21 @@ class TestWireProtocolGetters(AgentTestCase):
         test_data = WireProtocolData(DATA_FILE_EXT_NO_PUBLIC)
         self._test_getters(test_data, *args)
 
+    def test_getters_with_stale_goal_state(self, *args):
+        test_data = WireProtocolData(DATA_FILE)
+        test_data.emulate_stale_goal_state = True
+
+        self._test_getters(test_data, *args)
+        # Ensure HostPlugin was invoked
+        self.assertEqual(1, test_data.call_counts["/versions"])
+        self.assertEqual(2, test_data.call_counts["extensionArtifact"])
+        # Ensure the expected number of HTTP calls were made
+        # -- Tracking calls to retrieve GoalState is problematic since it is
+        #    fetched often; however, the dependent documents, such as the
+        #    HostingEnvironmentConfig, will be retrieved the expected number
+        self.assertEqual(2, test_data.call_counts["hostingenvuri"])
+
     def test_call_storage_kwargs(self,
-                                 mock_restutil,
                                  mock_cryptutil,
                                  mock_sleep):
         from azurelinuxagent.common.utils import restutil
diff --git a/tests/test_agent.py b/tests/test_agent.py
index 9b0d5f11..e6622647 100644
--- a/tests/test_agent.py
+++ b/tests/test_agent.py
@@ -17,12 +17,55 @@
 
 import mock
 import os.path
+import sys
 
 from azurelinuxagent.agent import *
 from azurelinuxagent.common.conf import *
 
 from tests.tools import *
 
+EXPECTED_CONFIGURATION = \
+"""AutoUpdate.Enabled = True
+AutoUpdate.GAFamily = Prod
+Autoupdate.Frequency = 3600
+DVD.MountPoint = /mnt/cdrom/secure
+DetectScvmmEnv = False
+EnableOverProvisioning = False
+Extension.LogDir = /var/log/azure
+HttpProxy.Host = None
+HttpProxy.Port = None
+Lib.Dir = /var/lib/waagent
+Logs.Verbose = False
+OS.AllowHTTP = False
+OS.CheckRdmaDriver = False
+OS.EnableFIPS = True
+OS.EnableRDMA = False
+OS.HomeDir = /home
+OS.OpensslPath = /usr/bin/openssl
+OS.PasswordPath = /etc/shadow
+OS.RootDeviceScsiTimeout = 300
+OS.SshDir = /notareal/path
+OS.SudoersDir = /etc/sudoers.d
+OS.UpdateRdmaDriver = False
+Pid.File = /var/run/waagent.pid
+Provisioning.AllowResetSysUser = False
+Provisioning.DecodeCustomData = False
+Provisioning.DeleteRootPassword = True
+Provisioning.Enabled = True
+Provisioning.ExecuteCustomData = False
+Provisioning.MonitorHostName = True
+Provisioning.PasswordCryptId = 6
+Provisioning.PasswordCryptSaltLength = 10
+Provisioning.RegenerateSshHostKeyPair = True
+Provisioning.SshHostKeyPairType = rsa
+Provisioning.UseCloudInit = True
+ResourceDisk.EnableSwap = False
+ResourceDisk.Filesystem = ext4
+ResourceDisk.Format = True
+ResourceDisk.MountOptions = None
+ResourceDisk.MountPoint = /mnt/resource
+ResourceDisk.SwapSizeMB = 0
+""".split('\n')
 
 class TestAgent(AgentTestCase):
 
@@ -115,3 +158,11 @@ class TestAgent(AgentTestCase):
         self.assertTrue(os.path.isfile(ext_log_dir))
         self.assertFalse(os.path.isdir(ext_log_dir))
         mock_log.assert_called_once()
+
+    def test_agent_show_configuration(self):
+        if not hasattr(sys.stdout, 'getvalue'):
+            self.fail('Test requires at least Python 2.7 with buffered output')
+        agent = Agent(False,
+                    conf_file_path=os.path.join(data_dir, "test_waagent.conf"))
+        agent.show_configuration()
+        self.assertEqual(EXPECTED_CONFIGURATION, sys.stdout.getvalue().split('\n'))
diff --git a/tests/utils/test_rest_util.py b/tests/utils/test_rest_util.py
index 5f084a61..ee56bd5b 100644
--- a/tests/utils/test_rest_util.py
+++ b/tests/utils/test_rest_util.py
@@ -16,8 +16,12 @@
 #
 
 import unittest
+
+from azurelinuxagent.common.exception import BadRequestError, \
+                                        HttpError, ProtocolError
 import azurelinuxagent.common.utils.restutil as restutil
-from azurelinuxagent.common.future import httpclient
+
+from azurelinuxagent.common.future import httpclient, ustr
 from tests.tools import AgentTestCase, patch, Mock, MagicMock
 
 
@@ -115,6 +119,175 @@ class TestHttpOperations(AgentTestCase):
         self.assertRaises(restutil.HttpError, restutil.http_get,
                           "http://foo.bar")
 
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.utils.restutil._http_request")
+    def test_http_request_retries_status_codes(self, _http_request, _sleep):
+        _http_request.side_effect = [
+            Mock(status=httpclient.SERVICE_UNAVAILABLE),
+            Mock(status=httpclient.OK)
+        ]
+
+        restutil.http_get("https://foo.bar")
+        self.assertEqual(2, _http_request.call_count)
+        self.assertEqual(1, _sleep.call_count)
+
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.utils.restutil._http_request")
+    def test_http_request_retries_passed_status_codes(self, _http_request, _sleep):
+        # Ensure the code is not part of the standard set
+        self.assertFalse(httpclient.UNAUTHORIZED in restutil.RETRY_CODES)
+
+        _http_request.side_effect = [
+            Mock(status=httpclient.UNAUTHORIZED),
+            Mock(status=httpclient.OK)
+        ]
+
+        restutil.http_get("https://foo.bar", retry_codes=[httpclient.UNAUTHORIZED])
+        self.assertEqual(2, _http_request.call_count)
+        self.assertEqual(1, _sleep.call_count)
+
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.utils.restutil._http_request")
+    def test_http_request_raises_for_bad_request(self, _http_request, _sleep):
+        _http_request.side_effect = [
+            Mock(status=httpclient.BAD_REQUEST)
+        ]
+
+        self.assertRaises(BadRequestError, restutil.http_get, "https://foo.bar")
+        self.assertEqual(1, _http_request.call_count)
+
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.utils.restutil._http_request")
+    def test_http_request_retries_exceptions(self, _http_request, _sleep):
+        # Testing each exception is difficult because they have varying
+        # signatures; for now, test one and ensure the set is unchanged
+        recognized_exceptions = [
+            httpclient.NotConnected,
+            httpclient.IncompleteRead,
+            httpclient.ImproperConnectionState,
+            httpclient.BadStatusLine
+        ]
+        self.assertEqual(recognized_exceptions, restutil.RETRY_EXCEPTIONS)
+
+        _http_request.side_effect = [
+            httpclient.IncompleteRead(''),
+            Mock(status=httpclient.OK)
+        ]
+
+        restutil.http_get("https://foo.bar")
+        self.assertEqual(2, _http_request.call_count)
+        self.assertEqual(1, _sleep.call_count)
+
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.utils.restutil._http_request")
+    def test_http_request_retries_ioerrors(self, _http_request, _sleep):
+        ioerror = IOError()
+        
+        for errno in restutil.RETRY_IOERRORS:
+            _http_request.reset_mock()
+            _sleep.reset_mock()
+
+            ioerror.errno = errno
+
+            _http_request.side_effect = [
+                ioerror,
+                Mock(status=httpclient.OK)
+            ]
+
+            restutil.http_get("https://foo.bar")
+            self.assertEqual(2, _http_request.call_count)
+            self.assertEqual(1, _sleep.call_count)
+
+    def test_request_failed(self):
+        self.assertTrue(restutil.request_failed(None))
+
+        resp = Mock()
+        for status in restutil.OK_CODES:
+            resp.status = status
+            self.assertFalse(restutil.request_failed(resp))
+
+        self.assertFalse(httpclient.BAD_REQUEST in restutil.OK_CODES)
+        resp.status = httpclient.BAD_REQUEST
+        self.assertTrue(restutil.request_failed(resp))
+
+        self.assertFalse(
+            restutil.request_failed(
+                resp, ok_codes=[httpclient.BAD_REQUEST]))
+
+    def test_request_succeeded(self):
+        self.assertFalse(restutil.request_succeeded(None))
+
+        resp = Mock()
+        for status in restutil.OK_CODES:
+            resp.status = status
+            self.assertTrue(restutil.request_succeeded(resp))
+
+        self.assertFalse(httpclient.BAD_REQUEST in restutil.OK_CODES)
+        resp.status = httpclient.BAD_REQUEST
+        self.assertFalse(restutil.request_succeeded(resp))
+
+        self.assertTrue(
+            restutil.request_succeeded(
+                resp, ok_codes=[httpclient.BAD_REQUEST]))
+
+    def test_read_response_error(self):
+        """
+        Validate the read_response_error method handles encoding correctly
+        """
+        responses = ['message', b'message', '\x80message\x80']
+        response = MagicMock()
+        response.status = 'status'
+        response.reason = 'reason'
+        with patch.object(response, 'read') as patch_response:
+            for s in responses:
+                patch_response.return_value = s
+                result = restutil.read_response_error(response)
+                print("RESPONSE: {0}".format(s))
+                print("RESULT: {0}".format(result))
+                print("PRESENT: {0}".format('[status: reason]' in result))
+                self.assertTrue('[status: reason]' in result)
+                self.assertTrue('message' in result)
+
+    def test_read_response_bytes(self):
+        response_bytes = '7b:0a:20:20:20:20:22:65:72:72:6f:72:43:6f:64:65:22:' \
+                         '3a:20:22:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:' \
+                         '69:73:20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:' \
+                         '69:73:20:6f:70:65:72:61:74:69:6f:6e:2e:22:2c:0a:20:' \
+                         '20:20:20:22:6d:65:73:73:61:67:65:22:3a:20:22:c3:af:' \
+                         'c2:bb:c2:bf:3c:3f:78:6d:6c:20:76:65:72:73:69:6f:6e:' \
+                         '3d:22:31:2e:30:22:20:65:6e:63:6f:64:69:6e:67:3d:22:' \
+                         '75:74:66:2d:38:22:3f:3e:3c:45:72:72:6f:72:3e:3c:43:' \
+                         '6f:64:65:3e:49:6e:76:61:6c:69:64:42:6c:6f:62:54:79:' \
+                         '70:65:3c:2f:43:6f:64:65:3e:3c:4d:65:73:73:61:67:65:' \
+                         '3e:54:68:65:20:62:6c:6f:62:20:74:79:70:65:20:69:73:' \
+                         '20:69:6e:76:61:6c:69:64:20:66:6f:72:20:74:68:69:73:' \
+                         '20:6f:70:65:72:61:74:69:6f:6e:2e:0a:52:65:71:75:65:' \
+                         '73:74:49:64:3a:63:37:34:32:39:30:63:62:2d:30:30:30:' \
+                         '31:2d:30:30:62:35:2d:30:36:64:61:2d:64:64:36:36:36:' \
+                         '61:30:30:30:22:2c:0a:20:20:20:20:22:64:65:74:61:69:' \
+                         '6c:73:22:3a:20:22:22:0a:7d'.split(':')
+        expected_response = '[HTTP Failed] [status: reason] {\n    "errorCode": "The blob ' \
+                            'type is invalid for this operation.",\n    ' \
+                            '"message": "<?xml version="1.0" ' \
+                            'encoding="utf-8"?>' \
+                            '<Error><Code>InvalidBlobType</Code><Message>The ' \
+                            'blob type is invalid for this operation.\n' \
+                            'RequestId:c74290cb-0001-00b5-06da-dd666a000",' \
+                            '\n    "details": ""\n}'
+
+        response_string = ''.join(chr(int(b, 16)) for b in response_bytes)
+        response = MagicMock()
+        response.status = 'status'
+        response.reason = 'reason'
+        with patch.object(response, 'read') as patch_response:
+            patch_response.return_value = response_string
+            result = restutil.read_response_error(response)
+            self.assertEqual(result, expected_response)
+            try:
+                raise HttpError("{0}".format(result))
+            except HttpError as e:
+                self.assertTrue(result in ustr(e))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/utils/test_text_util.py b/tests/utils/test_text_util.py
index 6f204c78..8ae4d143 100644
--- a/tests/utils/test_text_util.py
+++ b/tests/utils/test_text_util.py
@@ -34,6 +34,19 @@ class TestTextUtil(AgentTestCase):
                 password_hash = textutil.gen_password_hash(data, 6, 10)
                 self.assertNotEquals(None, password_hash)
 
+    def test_replace_non_ascii(self):
+        data = ustr(b'\xef\xbb\xbfhehe', encoding='utf-8')
+        self.assertEqual('hehe', textutil.replace_non_ascii(data))
+
+        data = "abcd\xa0e\xf0fghijk\xbblm"
+        self.assertEqual("abcdefghijklm", textutil.replace_non_ascii(data))
+
+        data = "abcd\xa0e\xf0fghijk\xbblm"
+        self.assertEqual("abcdXeXfghijkXlm",
+            textutil.replace_non_ascii(data, replace_char='X'))
+
+        self.assertEqual('', textutil.replace_non_ascii(None))
+
     def test_remove_bom(self):
         #Test bom could be removed
         data = ustr(b'\xef\xbb\xbfhehe', encoding='utf-8')