Repository: Azure/WALinuxAgent
Pull request #2046 (2020-10-20 17:58:52)

Unstick HGAP channel as default
<!-- DO NOT DELETE THIS TEMPLATE -->

## Description

Today, our logic is to use the direct channel as default unless it fails. If it fails, we try HGAP and if that succeeds, we stick to it indefinitely.

This change introduces alternating between default channels for each scenario using `send_request_using_appropriate_channel`, which is downloading extension manifests and packages, downloading artifacts profile, and downloading agent manifests. Agent upgrade scenario is unchanged.

Initially, direct channel is the default channel.
Scenario 1: Direct succeeds. HGAP is not called, direct remains default channel.
Scenario 2: Direct fails, HGAP succeeds -> HGAP is set to default and subsequent requests use HGAP first. Only if HGAP fails, we try to use direct. If direct fails, HGAP remains default as the last known good.
Scenario 3: Direct fails, HGAP fails -> direct remains default as the last known good. Each subsequent request goes through direct first.

This PR fixes the following things:
1) Correct alternating between channels.
2) Fixes handling failed responses (callers of `_fetch_response` never accounted for a failed response and propagated the value thinking it's successful).

<!--
Please add an informative description that covers that changes made by the pull request. 
This checklist is used to make sure that common issues in a pull request are addressed.
This will expedite the process of getting your pull request merged and avoid extra work on your part to fix issues discovered during the review process.
-->

---

### PR information
- [ ] The title of the PR is clear and informative.
- [ ] There are a small number of commits, each of which has an informative message. This means that previously merged commits do not appear in the history of the PR. For information on cleaning up the commits in your pull request, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).
- [ ] Except for special cases involving multiple contributors, the PR is started from a fork of the main repository, not a branch.
- [ ] If applicable, the PR references the bug/issue that it fixes in the description.
- [ ] New Unit tests were added for the changes made and Travis.CI is passing.

### Quality of Code and Contribution Guidelines
- [ ] I have read the [contribution guidelines](https://github.com/Azure/WALinuxAgent/blob/master/.github/CONTRIBUTING.md).


Patch:
diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py
index 892df3c2..f9544ec3 100644
--- a/azurelinuxagent/common/event.py
+++ b/azurelinuxagent/common/event.py
@@ -79,6 +79,7 @@ class WALAEventOperation:  # pylint: disable=R0903,no-init
     CollectEventUnicodeErrors = "CollectEventUnicodeErrors"
     ConfigurationChange = "ConfigurationChange"
     CustomData = "CustomData"
+    DefaultChannelChange = "DefaultChannelChange"
     Deploy = "Deploy"
     Disable = "Disable"
     Downgrade = "Downgrade"
diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py
index 41062557..856af2b7 100644
--- a/azurelinuxagent/common/protocol/hostplugin.py
+++ b/azurelinuxagent/common/protocol/hostplugin.py
@@ -56,8 +56,9 @@ _HEADER_ARTIFACT_MANIFEST_LOCATION = "x-ms-artifact-manifest-location"
 MAXIMUM_PAGEBLOB_PAGE_SIZE = 4 * 1024 * 1024  # Max page size: 4MB
 
 
+# too-many-instance-attributes<R0902> Disabled: all attributes are necessary
 class HostPluginProtocol(object):  # pylint: disable=R0902
-    _is_default_channel = False
+    is_default_channel = False
 
     FETCH_REPORTING_PERIOD = datetime.timedelta(minutes=1)
     STATUS_REPORTING_PERIOD = datetime.timedelta(minutes=1)
@@ -84,14 +85,6 @@ class HostPluginProtocol(object):  # pylint: disable=R0902
         # Role config name consists of: <deployment id>.<incarnation>(...)
         return role_config_name.split(".")[0] if role_config_name is not None else None
 
-    @staticmethod
-    def is_default_channel():
-        return HostPluginProtocol._is_default_channel
-
-    @staticmethod
-    def set_default_channel(is_default):
-        HostPluginProtocol._is_default_channel = is_default
-
     def update_container_id(self, new_container_id):
         self.container_id = new_container_id
 
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index fa36e79d..71ff84d5 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -28,8 +28,8 @@ import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.textutil as textutil
 from azurelinuxagent.common.datacontract import validate_param
-from azurelinuxagent.common.event import add_event, add_periodic, WALAEventOperation, report_event, \
-    CollectOrReportEventDebugInfo
+from azurelinuxagent.common.event import add_event, WALAEventOperation, report_event, \
+    CollectOrReportEventDebugInfo, add_periodic
 from azurelinuxagent.common.exception import ProtocolNotFoundError, \
     ResourceGoneError, ExtensionDownloadError, InvalidContainerError, ProtocolError, HttpError
 from azurelinuxagent.common.future import httpclient, bytebuffer, ustr
@@ -129,7 +129,7 @@ class WireProtocol(DataContract):
 
     def get_vmagent_pkgs(self, vmagent_manifest):
         goal_state = self.client.get_goal_state()
-        ga_manifest = self.client.get_gafamily_manifest(vmagent_manifest, goal_state)
+        ga_manifest = self.client.fetch_gafamily_manifest(vmagent_manifest, goal_state)
         valid_pkg_list = ga_manifest.pkg_list
         return valid_pkg_list
 
@@ -162,7 +162,7 @@ class WireProtocol(DataContract):
         host_func = lambda: self._download_ext_handler_pkg_through_host(uri, destination)
 
         try:
-            success = self.client.send_request_using_appropriate_channel(direct_func, host_func)
+            success = self.client.send_request_using_appropriate_channel(direct_func, host_func) is not None
         except Exception:
             success = False
 
@@ -629,14 +629,13 @@ class WireClient(object):  # pylint: disable=R0904
             host_func = lambda: self.fetch_manifest_through_host(version.uri)  # pylint: disable=W0640
 
             try:
-                response = self.send_request_using_appropriate_channel(direct_func, host_func)
-
-                if response:
+                manifest = self.send_request_using_appropriate_channel(direct_func, host_func)
+                if manifest is not None:
                     host = self.get_host_plugin()
                     host.update_manifest_uri(version.uri)
-                    return response
-            except Exception as e:  # pylint: disable=C0103
-                logger.warn("Exception when fetching manifest. Error: {0}".format(ustr(e)))
+                    return manifest
+            except Exception as error:
+                logger.warn("Failed to fetch manifest from {0}. Error: {1}", version.uri, ustr(error))
 
         raise ExtensionDownloadError("Failed to fetch manifest from all sources")
 
@@ -645,7 +644,7 @@ class WireClient(object):  # pylint: disable=R0904
         logger.verbose("Fetch [{0}] with headers [{1}] to file [{2}]", uri, headers, destination)
 
         response = self._fetch_response(uri, headers, use_proxy)
-        if response is not None:
+        if response is not None and not restutil.request_failed(response):
             chunk_size = 1024 * 1024  # 1MB buffer
             try:
                 with open(destination, 'wb', chunk_size) as destination_fh:
@@ -655,8 +654,8 @@ class WireClient(object):  # pylint: disable=R0904
                         destination_fh.write(chunk)
                         complete = len(chunk) < chunk_size
                 success = True
-            except Exception as e:  # pylint: disable=C0103
-                logger.error('Error streaming {0} to {1}: {2}'.format(uri, destination, ustr(e)))
+            except Exception as error:
+                logger.error('Error streaming {0} to {1}: {2}'.format(uri, destination, ustr(error)))
 
         return success
 
@@ -664,7 +663,7 @@ class WireClient(object):  # pylint: disable=R0904
         logger.verbose("Fetch [{0}] with headers [{1}]", uri, headers)
         content = None
         response = self._fetch_response(uri, headers, use_proxy)
-        if response is not None:
+        if response is not None and not restutil.request_failed(response):
             response_content = response.read()
             content = self.decode_config(response_content) if decode else response_content
         return content
@@ -695,10 +694,12 @@ class WireClient(object):  # pylint: disable=R0904
                 if host_plugin is not None:
                     host_plugin.report_fetch_health(uri, source='WireClient')
 
-        except (HttpError, ProtocolError, IOError) as e:  # pylint: disable=C0103
-            logger.verbose("Fetch failed from [{0}]: {1}", uri, e)
-            if isinstance(e, ResourceGoneError) or isinstance(e, InvalidContainerError):  # pylint: disable=R1701
+        except (HttpError, ProtocolError, IOError) as error:
+            logger.verbose("Fetch failed from [{0}]: {1}", uri, error)
+            if isinstance(error, (InvalidContainerError, ResourceGoneError)):
+                # These are retryable errors that should force a goal state refresh in the host plugin
                 raise
+
         return resp
 
     def update_host_plugin_from_goal_state(self):
@@ -830,7 +831,7 @@ class WireClient(object):  # pylint: disable=R0904
             raise ProtocolError("Trying to fetch Remote Access before initialization!")
         return self._goal_state.remote_access
 
-    def get_gafamily_manifest(self, vmagent_manifest, goal_state):
+    def fetch_gafamily_manifest(self, vmagent_manifest, goal_state):
         local_file = MANIFEST_FILE_NAME.format(vmagent_manifest.family, goal_state.incarnation)
         local_file = os.path.join(conf.get_lib_dir(), local_file)
 
@@ -858,47 +859,56 @@ class WireClient(object):  # pylint: disable=R0904
             raise ProtocolNotFoundError(error)
 
     def _call_hostplugin_with_container_check(self, host_func):
-        ret = None
+        """
+        Calls host_func on host channel and accounts for stale resource (ResourceGoneError or InvalidContainerError).
+        If stale, it refreshes the goal state and retries host_func.
+        This method can throw, so the callers need to handle that.
+        """
         try:
             ret = host_func()
-        except (ResourceGoneError, InvalidContainerError) as e:  # pylint: disable=C0103
+            if ret in (None, False):
+                raise Exception("Request failed using the host channel.")
+
+            return ret
+        except (ResourceGoneError, InvalidContainerError) as error:
             host_plugin = self.get_host_plugin()
-            old_container_id = host_plugin.container_id
-            old_role_config_name = host_plugin.role_config_name
 
+            old_container_id, old_role_config_name = host_plugin.container_id, host_plugin.role_config_name
             msg = "[PERIODIC] Request failed with the current host plugin configuration. " \
                   "ContainerId: {0}, role config file: {1}. Fetching new goal state and retrying the call." \
-                  "Error: {2}".format(old_container_id, old_role_config_name, ustr(e))
+                  "Error: {2}".format(old_container_id, old_role_config_name, ustr(error))
             logger.periodic_info(logger.EVERY_SIX_HOURS, msg)
 
             self.update_host_plugin_from_goal_state()
 
-            new_container_id = host_plugin.container_id
-            new_role_config_name = host_plugin.role_config_name
+            new_container_id, new_role_config_name = host_plugin.container_id, host_plugin.role_config_name
             msg = "[PERIODIC] Host plugin reconfigured with new parameters. " \
                   "ContainerId: {0}, role config file: {1}.".format(new_container_id, new_role_config_name)
             logger.periodic_info(logger.EVERY_SIX_HOURS, msg)
 
             try:
                 ret = host_func()
-                if ret:
-                    msg = "[PERIODIC] Request succeeded using the host plugin channel after goal state refresh. " \
-                          "ContainerId changed from {0} to {1}, " \
-                          "role config file changed from {2} to {3}.".format(old_container_id, new_container_id,
-                                                                             old_role_config_name, new_role_config_name)
-                    add_periodic(delta=logger.EVERY_SIX_HOURS,
-                                 name=AGENT_NAME,
-                                 version=CURRENT_VERSION,
-                                 op=WALAEventOperation.HostPlugin,
-                                 is_success=True,
-                                 message=msg,
-                                 log_event=True)
 
-            except (ResourceGoneError, InvalidContainerError) as e:  # pylint: disable=C0103
+                if ret in (None, False):
+                    raise Exception("Request failed using the host channel after goal state refresh.")
+
+                msg = "[PERIODIC] Request succeeded using the host plugin channel after goal state refresh. " \
+                      "ContainerId changed from {0} to {1}, " \
+                      "role config file changed from {2} to {3}.".format(old_container_id, new_container_id,
+                                                                         old_role_config_name, new_role_config_name)
+                add_periodic(delta=logger.EVERY_SIX_HOURS,
+                             name=AGENT_NAME,
+                             version=CURRENT_VERSION,
+                             op=WALAEventOperation.HostPlugin,
+                             is_success=True,
+                             message=msg,
+                             log_event=True)
+                return ret
+            except (ResourceGoneError, InvalidContainerError) as error:
                 msg = "[PERIODIC] Request failed using the host plugin channel after goal state refresh. " \
                       "ContainerId changed from {0} to {1}, role config file changed from {2} to {3}. " \
                       "Exception type: {4}.".format(old_container_id, new_container_id, old_role_config_name,
-                                                    new_role_config_name, type(e).__name__)
+                                                    new_role_config_name, type(error).__name__)
                 add_periodic(delta=logger.EVERY_SIX_HOURS,
                              name=AGENT_NAME,
                              version=CURRENT_VERSION,
@@ -908,50 +918,69 @@ class WireClient(object):  # pylint: disable=R0904
                              log_event=True)
                 raise
 
+    def __send_request_using_host_channel(self, host_func):
+        """
+        Calls the host_func on host channel with retries for stale goal state and handles any exceptions, consistent with the caller for direct channel.
+        At the time of writing, host_func internally calls either:
+        1) WireClient.stream which returns a boolean, or
+        2) WireClient.fetch which returns None or a HTTP response.
+        This method returns either None (failure case where host_func returned None or False), True or an HTTP response.
+        """
+        ret = None
+        try:
+            ret = self._call_hostplugin_with_container_check(host_func)
+        except Exception as error:
+            logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the host channel. Error: {0}".format(ustr(error)))
+
         return ret
 
-    def send_request_using_appropriate_channel(self, direct_func, host_func):
-        # A wrapper method for all function calls that send HTTP requests. The purpose of the method is to
-        # define which channel to use, direct or through the host plugin. For the host plugin channel,
-        # also implement a retry mechanism.
+    @staticmethod
+    def __send_request_using_direct_channel(direct_func):
+        """
+        Calls the direct_func on direct channel and handles any exceptions, consistent with the caller for host channel.
+        At the time of writing, direct_func internally calls either:
+        1) WireClient.stream which returns a boolean, or
+        2) WireClient.fetch which returns None or a HTTP response.
+        This method returns either None (failure case where direct_func returned None or False), True or an HTTP response.
+        """
+        ret = None
+        try:
+            ret = direct_func()
 
-        # By default, the direct channel is the default channel. If that is the case, try getting a response
-        # through that channel. On failure, fall back to the host plugin channel.
+            if ret in (None, False):
+                logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the direct channel.")
+                return None
+        except Exception as error:
+            logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the direct channel. Error: {0}".format(ustr(error)))
 
-        # When using the host plugin channel, regardless if it's set as default or not, try sending the request first.
-        # On specific failures that indicate a stale goal state (such as resource gone or invalid container parameter),
-        # refresh the goal state and try again. If successful, set the host plugin channel as default. If failed,
-        # raise the exception.
+        return ret
 
-        # NOTE: direct_func and host_func are passed as lambdas. Be careful about capturing goal state data in them as
-        # they will not be refreshed even if a goal state refresh is called before retrying the host_func.
+    def send_request_using_appropriate_channel(self, direct_func, host_func):
+        """
+        Determines which communication channel to use. By default, the primary channel is direct, host channel is secondary.
+        We call the primary channel first and return on success. If primary fails, we try secondary. If secondary fails,
+        we return and *don't* switch the default channel. If secondary succeeds, we change the default channel.
+        This method doesn't raise since the calls to direct_func and host_func are already wrapped and handle any exceptions.
+        Possible return values are manifest, artifacts profile, True or None.
+        """
+        direct_channel = lambda: self.__send_request_using_direct_channel(direct_func)
+        host_channel = lambda: self.__send_request_using_host_channel(host_func)
 
-        if not HostPluginProtocol.is_default_channel():
-            ret = None
-            try:
-                ret = direct_func()
-
-                # Different direct channel functions report failure in different ways: by returning None, False,
-                # or raising ResourceGone or InvalidContainer exceptions.
-                if not ret:
-                    logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the direct channel, "
-                                                            "switching to host plugin.")
-            except (ResourceGoneError, InvalidContainerError) as e:  # pylint: disable=C0103
-                logger.periodic_info(logger.EVERY_HOUR, "[PERIODIC] Request failed using the direct channel, "
-                                                        "switching to host plugin. Error: {0}".format(ustr(e)))
-
-            if ret:
-                return ret
+        if HostPluginProtocol.is_default_channel:
+            primary_channel, secondary_channel = host_channel, direct_channel
         else:
-            logger.periodic_info(logger.EVERY_HALF_DAY, "[PERIODIC] Using host plugin as default channel.")
-
-        ret = self._call_hostplugin_with_container_check(host_func)
-
-        if not HostPluginProtocol.is_default_channel():
-            logger.info("Setting host plugin as default channel from now on. "
-                        "Restart the agent to reset the default channel.")
-            HostPluginProtocol.set_default_channel(True)
-
+            primary_channel, secondary_channel = direct_channel, host_channel
+
+        ret = primary_channel()
+        if ret is not None:
+            return ret
+
+        ret = secondary_channel()
+        if ret is not None:
+            HostPluginProtocol.is_default_channel = not HostPluginProtocol.is_default_channel
+            message = "Default channel changed to {0} channel.".format("HostGA" if HostPluginProtocol.is_default_channel else "direct")
+            logger.info(message)
+            add_event(AGENT_NAME, op=WALAEventOperation.DefaultChannelChange, version=CURRENT_VERSION, is_success=True, message=message, log_event=False)
         return ret
 
     def upload_status_blob(self):
@@ -1194,8 +1223,11 @@ class WireClient(object):  # pylint: disable=R0904
 
             try:
                 profile = self.send_request_using_appropriate_channel(direct_func, host_func)
-            except Exception as e:  # pylint: disable=C0103
-                logger.warn("Exception retrieving artifacts profile: {0}".format(ustr(e)))
+                if profile is None:
+                    logger.warn("Failed to fetch artifacts profile from blob {0}", blob)
+                    return None
+            except Exception as error:
+                logger.warn("Exception retrieving artifacts profile from blob {0}. Error: {1}".format(blob, ustr(error)))
                 return None
 
             if not textutil.is_str_empty(profile):
diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
index bfac3bfa..57e1a878 100644
--- a/azurelinuxagent/ga/exthandlers.py
+++ b/azurelinuxagent/ga/exthandlers.py
@@ -883,7 +883,7 @@ class ExtHandlerInstance(object):  # pylint: disable=R0904
         self.logger.info("Downloading extension package: {0}", source_uri)
         try:
             if not self.protocol.download_ext_handler_pkg(source_uri, target_file):
-                raise Exception("Failed to download extension package - no error information is available")
+                raise Exception("Failed to download extension package from {0}".format(source_uri))
         except Exception as exception:
             self.logger.info("Error downloading extension package: {0}", ustr(exception))
             if os.path.exists(target_file):
diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py
index abae8e77..34993cb2 100644
--- a/azurelinuxagent/ga/update.py
+++ b/azurelinuxagent/ga/update.py
@@ -959,11 +959,11 @@ class GuestAgent(object):
         uris_shuffled = self.pkg.uris
         random.shuffle(uris_shuffled)
         for uri in uris_shuffled:
-            if not HostPluginProtocol.is_default_channel() and self._fetch(uri.uri):  # pylint: disable=R1723
+            if not HostPluginProtocol.is_default_channel and self._fetch(uri.uri):  # pylint: disable=R1723
                 break
 
             elif self.host is not None and self.host.ensure_initialized():
-                if not HostPluginProtocol.is_default_channel():
+                if not HostPluginProtocol.is_default_channel:
                     logger.warn("Download failed, switching to host plugin")
                 else:
                     logger.verbose("Using host plugin as default channel")
@@ -971,9 +971,9 @@ class GuestAgent(object):
                 uri, headers = self.host.get_artifact_request(uri.uri, self.host.manifest_uri)
                 try:
                     if self._fetch(uri, headers=headers, use_proxy=False):  # pylint: disable=R1723
-                        if not HostPluginProtocol.is_default_channel():
+                        if not HostPluginProtocol.is_default_channel:
                             logger.verbose("Setting host plugin as default channel")
-                            HostPluginProtocol.set_default_channel(True)
+                            HostPluginProtocol.is_default_channel = True
                         break
                     else:
                         logger.warn("Host plugin download failed")
@@ -981,7 +981,7 @@ class GuestAgent(object):
                 # If the HostPlugin rejects the request,
                 # let the error continue, but set to use the HostPlugin
                 except ResourceGoneError:
-                    HostPluginProtocol.set_default_channel(True)
+                    HostPluginProtocol.is_default_channel = True
                     raise
 
             else:


Test patch:
diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py
index d3c892cf..5088393b 100644
--- a/tests/protocol/test_hostplugin.py
+++ b/tests/protocol/test_hostplugin.py
@@ -264,7 +264,7 @@ class TestHostPlugin(HttpRequestPredicates, AgentTestCase):  # pylint: disable=t
             self.assertEqual(sas_url, patch_put.call_args[0][0])
 
             # ensure host plugin is not set as default
-            self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+            self.assertFalse(wire.HostPluginProtocol.is_default_channel)
 
     @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized",
            return_value=True)
@@ -296,7 +296,7 @@ class TestHostPlugin(HttpRequestPredicates, AgentTestCase):  # pylint: disable=t
             self.assertEqual(sas_url, patch_put.call_args[0][0])
 
             # ensure host plugin is not set as default
-            self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+            self.assertFalse(wire.HostPluginProtocol.is_default_channel)
 
     @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized",
            return_value=True)
@@ -330,7 +330,7 @@ class TestHostPlugin(HttpRequestPredicates, AgentTestCase):  # pylint: disable=t
             self.assertEqual(sas_url, patch_put.call_args[0][0])
 
             # ensure host plugin is not set as default
-            self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+            self.assertFalse(wire.HostPluginProtocol.is_default_channel)
 
     @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized",
            return_value=True)
@@ -362,14 +362,14 @@ class TestHostPlugin(HttpRequestPredicates, AgentTestCase):  # pylint: disable=t
             self.assertEqual(sas_url, patch_put.call_args[0][0])
 
             # ensure host plugin is not set as default
-            self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+            self.assertFalse(wire.HostPluginProtocol.is_default_channel)
 
     @patch("azurelinuxagent.common.event.add_event")
     def test_put_status_error_reporting(self, patch_add_event):
         """
         Validate the telemetry when uploading status fails
         """
-        wire.HostPluginProtocol.set_default_channel(False)
+        wire.HostPluginProtocol.is_default_channel = False
         with patch.object(wire.StatusBlob, "upload", return_value=False):
             with self.create_mock_protocol() as wire_protocol:
                 wire_protocol_client = wire_protocol.client
@@ -386,7 +386,7 @@ class TestHostPlugin(HttpRequestPredicates, AgentTestCase):  # pylint: disable=t
                         # The agent tries to upload using a direct connection, and that succeeds.
                         self.assertEqual(1, wire_protocol_client.status_blob.upload.call_count)  # pylint: disable=no-member
                         # The agent never touches the default protocol is this code path, so no change.
-                        self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+                        self.assertFalse(wire.HostPluginProtocol.is_default_channel)
                         # The agent never logs telemetry event for direct fallback
                         self.assertEqual(1, patch_add_event.call_count)
                         self.assertEqual('ReportStatus', patch_add_event.call_args[1]['op'])
diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py
index 375c8c73..df46289f 100644
--- a/tests/protocol/test_wire.py
+++ b/tests/protocol/test_wire.py
@@ -27,7 +27,7 @@ import uuid
 from datetime import datetime, timedelta
 
 from azurelinuxagent.common import conf
-from azurelinuxagent.common.exception import InvalidContainerError, ResourceGoneError, ProtocolError, \
+from azurelinuxagent.common.exception import ResourceGoneError, ProtocolError, \
     ExtensionDownloadError, HttpError
 from azurelinuxagent.common.protocol.goal_state import ExtensionsConfig
 from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
@@ -87,7 +87,7 @@ class TestWireProtocol(AgentTestCase):
 
     def setUp(self):
         super(TestWireProtocol, self).setUp()
-        HostPluginProtocol.set_default_channel(False)
+        HostPluginProtocol.is_default_channel = False
 
     def _test_getters(self, test_data, certsMustBePresent, __, MockCryptUtil, _):  # pylint: disable=invalid-name
         MockCryptUtil.side_effect = test_data.mock_crypt_util
@@ -229,7 +229,7 @@ class TestWireProtocol(AgentTestCase):
                 return MockResponse(body=b'', status_code=200)
 
         with mock_wire_protocol(mockwiredata.DATA_FILE, http_put_handler=http_put_handler) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
             protocol.client.status_blob.vm_status = VMStatus(message="Ready", status="Ready")
 
             protocol.client.upload_status_blob()
@@ -244,11 +244,11 @@ class TestWireProtocol(AgentTestCase):
             with patch.object(HostPluginProtocol, "ensure_initialized", return_value=True):
                 with patch.object(StatusBlob, "upload", return_value=False) as patch_default_upload:
                     with patch.object(HostPluginProtocol, "_put_block_blob_status") as patch_http:
-                        HostPluginProtocol.set_default_channel(False)
+                        HostPluginProtocol.is_default_channel = False
                         protocol.client.upload_status_blob()
                         patch_default_upload.assert_not_called()
                         patch_http.assert_called_once_with(testurl, protocol.client.status_blob)
-                        self.assertFalse(HostPluginProtocol.is_default_channel())
+                        self.assertFalse(HostPluginProtocol.is_default_channel)
 
     @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized")
     def test_upload_status_blob_unknown_type_assumes_block(self, *_):
@@ -257,7 +257,7 @@ class TestWireProtocol(AgentTestCase):
 
             with patch.object(StatusBlob, "prepare") as patch_prepare:
                 with patch.object(StatusBlob, "upload") as patch_default_upload:
-                    HostPluginProtocol.set_default_channel(False)
+                    HostPluginProtocol.is_default_channel = False
                     protocol.client.upload_status_blob()
 
                     patch_prepare.assert_called_once_with("BlockBlob")
@@ -484,16 +484,16 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             return None
 
         with mock_wire_protocol(mockwiredata.DATA_FILE, http_get_handler=http_get_handler) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             success = protocol.download_ext_handler_pkg(extension_url, target_file)
 
             urls = protocol.get_tracked_urls()
-            self.assertEqual(success, True, 'The download should have succeeded')
+            self.assertTrue(success, "The download should have succeeded")
             self.assertEqual(len(urls), 1, "Unexpected number of HTTP requests: [{0}]".format(urls))
             self.assertEqual(urls[0], extension_url, "The extension should have been downloaded over the direct channel")
-            self.assertTrue(os.path.exists(target_file), 'The extension package was not downloaded')
-            self.assertEqual(HostPluginProtocol.is_default_channel(), False, "The host channel should not have been set as the default")
+            self.assertTrue(os.path.exists(target_file), "The extension package was not downloaded")
+            self.assertFalse(HostPluginProtocol.is_default_channel, "The host channel should not have been set as the default")
 
     def test_download_ext_handler_pkg_should_use_host_channel_when_direct_channel_fails_and_set_host_as_default(self):
         extension_url = 'https://fake_host/fake_extension.zip'
@@ -507,23 +507,23 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             return None
 
         with mock_wire_protocol(mockwiredata.DATA_FILE, http_get_handler=http_get_handler) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             success = protocol.download_ext_handler_pkg(extension_url, target_file)
 
             urls = protocol.get_tracked_urls()
-            self.assertEqual(success, True, 'The download should have succeeded')
+            self.assertTrue(success, "The download should have succeeded")
             self.assertEqual(len(urls), 2, "Unexpected number of HTTP requests: [{0}]".format(urls))
             self.assertEqual(urls[0], extension_url, "The first attempt should have been over the direct channel")
             self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The retry attempt should have been over the host channel")
             self.assertTrue(os.path.exists(target_file), 'The extension package was not downloaded')
-            self.assertEqual(HostPluginProtocol.is_default_channel(), True, "The host channel should have been set as the default")
+            self.assertTrue(HostPluginProtocol.is_default_channel, "The host channel should have been set as the default")
 
     def test_download_ext_handler_pkg_should_retry_the_host_channel_after_refreshing_host_plugin(self):
         extension_url = 'https://fake_host/fake_extension.zip'
         target_file = os.path.join(self.tmp_dir, 'fake_extension.zip')
 
-        def http_get_handler(url, *args, **kwargs):  # pylint: disable=unused-argument
+        def http_get_handler(url, *_, **kwargs):
             if url == extension_url:
                 return HttpError("Exception to fake an error on the direct channel")
             if self.is_host_plugin_extension_request(url, kwargs, extension_url):
@@ -538,7 +538,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
         http_get_handler.goal_state_requests = 0
 
         with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             try:
                 # initialization of the host plugin triggers a request for the goal state; do it here before we start tracking those requests.
@@ -549,47 +549,45 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
                 success = protocol.download_ext_handler_pkg(extension_url, target_file)
 
                 urls = protocol.get_tracked_urls()
-                self.assertEqual(success, True, 'The download should have succeeded')
+                self.assertTrue(success, "The download should have succeeded")
                 self.assertEqual(len(urls), 4, "Unexpected number of HTTP requests: [{0}]".format(urls))
                 self.assertEqual(urls[0], extension_url, "The first attempt should have been over the direct channel")
                 self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The second attempt should have been over the host channel")
                 self.assertTrue(self.is_goal_state_request(urls[2]), "The host channel should have been refreshed the goal state")
                 self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[3]), "The third attempt should have been over the host channel")
                 self.assertTrue(os.path.exists(target_file), 'The extension package was not downloaded')
-                self.assertEqual(HostPluginProtocol.is_default_channel(), True, "The host channel should have been set as the default")
+                self.assertTrue(HostPluginProtocol.is_default_channel, "The host channel should have been set as the default")
             finally:
-                HostPluginProtocol.set_default_channel(False)
+                HostPluginProtocol.is_default_channel = False
 
     def test_download_ext_handler_pkg_should_not_change_default_channel_when_all_channels_fail(self):
         extension_url = 'https://fake_host/fake_extension.zip'
+        target_file = os.path.join(self.tmp_dir, "fake_extension.zip")
 
         def http_get_handler(url, *_, **kwargs):
-            if url == extension_url:
-                return HttpError("Exception to fake error on direct channel")
-            if self.is_host_plugin_extension_request(url, kwargs, extension_url):
-                return ResourceGoneError("Exception to fake error on host channel")
+            if url == extension_url or self.is_host_plugin_extension_request(url, kwargs, extension_url):
+                return MockResponse(body=b"content not found", status_code=404, reason="Not Found")
             if self.is_goal_state_request(url):
                 protocol.track_url(url)  # keep track of goal state requests
             return None
 
         with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             # initialization of the host plugin triggers a request for the goal state; do it here before we start tracking those requests.
             protocol.client.get_host_plugin()
 
             protocol.set_http_handlers(http_get_handler=http_get_handler)
 
-            success = protocol.download_ext_handler_pkg(extension_url, "/an-invalid-directory/an-invalid-file.zip")
+            success = protocol.download_ext_handler_pkg(extension_url, target_file)
 
             urls = protocol.get_tracked_urls()
-            self.assertEqual(success, False, "The download should have failed")
-            self.assertEqual(len(urls), 4, "Unexpected number of HTTP requests: [{0}]".format(urls))
+            self.assertFalse(success, "The download should have failed")
+            self.assertEqual(len(urls), 2, "Unexpected number of HTTP requests: [{0}]".format(urls))
             self.assertEqual(urls[0], extension_url, "The first attempt should have been over the direct channel")
             self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The second attempt should have been over the host channel")
-            self.assertTrue(self.is_goal_state_request(urls[2]), "The host channel should have been refreshed the goal state")
-            self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[3]), "The third attempt should have been over the host channel")
-            self.assertEqual(HostPluginProtocol.is_default_channel(), False, "The host channel should not have been set as the default")
+            self.assertFalse(os.path.exists(target_file), "The extension package was downloaded and it shouldn't have")
+            self.assertFalse(HostPluginProtocol.is_default_channel, "The host channel should not have been set as the default")
 
     def test_fetch_manifest_should_not_invoke_host_channel_when_direct_channel_succeeds(self):
         manifest_url = 'https://fake_host/fake_manifest.xml'
@@ -603,7 +601,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             return None
 
         with mock_wire_protocol(mockwiredata.DATA_FILE, http_get_handler=http_get_handler) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             manifest = protocol.client.fetch_manifest([VMAgentManifestUri(uri=manifest_url)])
 
@@ -611,7 +609,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             self.assertEqual(manifest, manifest_xml, 'The expected manifest was not downloaded')
             self.assertEqual(len(urls), 1, "Unexpected number of HTTP requests: [{0}]".format(urls))
             self.assertEqual(urls[0], manifest_url, "The manifest should have been downloaded over the direct channel")
-            self.assertEqual(HostPluginProtocol.is_default_channel(), False, "The default channel should not have changed")
+            self.assertFalse(HostPluginProtocol.is_default_channel, "The default channel should not have changed")
 
     def test_fetch_manifest_should_use_host_channel_when_direct_channel_fails_and_set_it_to_default(self):
         manifest_url = 'https://fake_host/fake_manifest.xml'
@@ -625,7 +623,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             return None
 
         with mock_wire_protocol(mockwiredata.DATA_FILE, http_get_handler=http_get_handler) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             try:
                 manifest = protocol.client.fetch_manifest([VMAgentManifestUri(uri=manifest_url)])
@@ -635,9 +633,9 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
                 self.assertEqual(len(urls), 2, "Unexpected number of HTTP requests: [{0}]".format(urls))
                 self.assertEqual(urls[0], manifest_url, "The first attempt should have been over the direct channel")
                 self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The retry should have been over the host channel")
-                self.assertEqual(HostPluginProtocol.is_default_channel(), True, "The host should have been set as the default channel")
+                self.assertTrue(HostPluginProtocol.is_default_channel, "The host should have been set as the default channel")
             finally:
-                HostPluginProtocol.set_default_channel(False)  # Reset default channel
+                HostPluginProtocol.is_default_channel = False  # Reset default channel
 
     def test_fetch_manifest_should_retry_the_host_channel_after_refreshing_the_host_plugin_and_set_the_host_as_default(self):
         manifest_url = 'https://fake_host/fake_manifest.xml'
@@ -658,7 +656,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
         http_get_handler.goal_state_requests = 0
 
         with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             try:
                 # initialization of the host plugin triggers a request for the goal state; do it here before we start tracking those requests.
@@ -674,9 +672,9 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
                 self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The second attempt should have been over the host channel")
                 self.assertTrue(self.is_goal_state_request(urls[2]), "The host channel should have been refreshed the goal state")
                 self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[3]), "The third attempt should have been over the host channel")
-                self.assertEqual(HostPluginProtocol.is_default_channel(), True, "The host should have been set as the default channel")
+                self.assertTrue(HostPluginProtocol.is_default_channel, "The host should have been set as the default channel")
             finally:
-                HostPluginProtocol.set_default_channel(False)  # Reset default channel
+                HostPluginProtocol.is_default_channel = False  # Reset default channel
 
     def test_fetch_manifest_should_update_goal_state_and_not_change_default_channel_if_host_fails(self):
         manifest_url = 'https://fake_host/fake_manifest.xml'
@@ -690,7 +688,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
 
         # Everything fails. Goal state should have been updated and host channel should not have been set as default.
         with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             # initialization of the host plugin triggers a request for the goal state; do it here before we start
             # tracking those requests.
@@ -704,16 +702,12 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             urls = protocol.get_tracked_urls()
             self.assertEqual(len(urls), 4, "Unexpected number of HTTP requests: [{0}]".format(urls))
             self.assertEqual(urls[0], manifest_url, "The first attempt should have been over the direct channel")
-            self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]),
-                            "The second attempt should have been over the host channel")
-            self.assertTrue(self.is_goal_state_request(urls[2]),
-                            "The host channel should have been refreshed the goal state")
-            self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[3]),
-                            "The third attempt should have been over the host channel")
-            self.assertEqual(HostPluginProtocol.is_default_channel(), False,
-                              "The host should not have been set as the default channel")
+            self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The second attempt should have been over the host channel")
+            self.assertTrue(self.is_goal_state_request(urls[2]), "The host channel should have been refreshed the goal state")
+            self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[3]), "The third attempt should have been over the host channel")
+            self.assertFalse(HostPluginProtocol.is_default_channel, "The host should not have been set as the default channel")
 
-            self.assertEqual(HostPluginProtocol.is_default_channel(), False)
+            self.assertEqual(HostPluginProtocol.is_default_channel, False)
 
     def test_get_artifacts_profile_should_not_invoke_host_channel_when_direct_channel_succeeds(self):
         def http_get_handler(url, *_, **__):  # pylint: disable=useless-return
@@ -722,14 +716,14 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             return None
 
         with mock_wire_protocol(mockwiredata.DATA_FILE_IN_VM_ARTIFACTS_PROFILE, http_get_handler=http_get_handler) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             return_value = protocol.client.get_artifacts_profile()
 
             self.assertIsInstance(return_value, InVMArtifactsProfile, 'The request did not return a valid artifacts profile: {0}'.format(return_value))
             urls = protocol.get_tracked_urls()
             self.assertEqual(len(urls), 1, "Unexpected HTTP requests: [{0}]".format(urls))
-            self.assertEqual(HostPluginProtocol.is_default_channel(), False)
+            self.assertFalse(HostPluginProtocol.is_default_channel, "The host should not have been set as the default channel")
 
     def test_get_artifacts_profile_should_use_host_channel_when_direct_channel_fails(self):
         def http_get_handler(url, *_, **kwargs):
@@ -740,7 +734,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             return None
 
         with mock_wire_protocol(mockwiredata.DATA_FILE_IN_VM_ARTIFACTS_PROFILE) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             try:
                 protocol.set_http_handlers(http_get_handler=http_get_handler)
@@ -754,9 +748,9 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
                 self.assertEqual(len(urls), 2, "Invalid number of requests: [{0}]".format(urls))
                 self.assertTrue(self.is_in_vm_artifacts_profile_request(urls[0]), "The first request should have been over the direct channel")
                 self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The second request should have been over the host channel")
-                self.assertEqual(HostPluginProtocol.is_default_channel(), True, "The default channel should have changed to the host")
+                self.assertTrue(HostPluginProtocol.is_default_channel, "The default channel should have changed to the host")
             finally:
-                HostPluginProtocol.set_default_channel(False)
+                HostPluginProtocol.is_default_channel = False
 
     def test_get_artifacts_profile_should_retry_the_host_channel_after_refreshing_the_host_plugin(self):
         def http_get_handler(url, *_, **kwargs):
@@ -773,7 +767,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
         http_get_handler.host_plugin_calls = 0
 
         with mock_wire_protocol(mockwiredata.DATA_FILE_IN_VM_ARTIFACTS_PROFILE) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             try:
                 # initialization of the host plugin triggers a request for the goal state; do it here before we start tracking those requests.
@@ -792,9 +786,9 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
                 self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The second request should have been over the host channel")
                 self.assertTrue(self.is_goal_state_request(urls[2]), "The goal state should have been refreshed before retrying the host channel")
                 self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[3]), "The retry request should have been over the host channel")
-                self.assertEqual(HostPluginProtocol.is_default_channel(), True, "The default channel should have changed to the host")
+                self.assertTrue(HostPluginProtocol.is_default_channel, "The default channel should have changed to the host")
             finally:
-                HostPluginProtocol.set_default_channel(False)
+                HostPluginProtocol.is_default_channel = False
 
     def test_get_artifacts_profile_should_refresh_the_host_plugin_and_not_change_default_channel_if_host_plugin_fails(self):
         def http_get_handler(url, *_, **kwargs):
@@ -807,7 +801,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             return None
 
         with mock_wire_protocol(mockwiredata.DATA_FILE_IN_VM_ARTIFACTS_PROFILE) as protocol:
-            HostPluginProtocol.set_default_channel(False)
+            HostPluginProtocol.is_default_channel = False
 
             # initialization of the host plugin triggers a request for the goal state; do it here before we start tracking those requests.
             protocol.client.get_host_plugin()
@@ -823,7 +817,7 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
             self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[1]), "The second request should have been over the host channel")
             self.assertTrue(self.is_goal_state_request(urls[2]), "The goal state should have been refreshed before retrying the host channel")
             self.assertTrue(self.is_host_plugin_extension_artifact_request(urls[3]), "The retry request should have been over the host channel")
-            self.assertEqual(HostPluginProtocol.is_default_channel(), False, "The default channel should not have changed")
+            self.assertFalse(HostPluginProtocol.is_default_channel, "The default channel should not have changed")
 
     def test_upload_logs_should_not_refresh_plugin_when_first_attempt_succeeds(self):
         def http_put_handler(url, *_, **__):  # pylint: disable=inconsistent-return-statements
@@ -854,104 +848,122 @@ class TestWireClient(HttpRequestPredicates, AgentTestCase):
 
             urls = protocol.get_tracked_urls()
             self.assertEqual(len(urls), 2, "Invalid number of requests: [{0}]".format(urls))
-            self.assertTrue(self.is_host_plugin_put_logs_request(urls[0]),
-                            "The first request should have been over the host channel")
-            self.assertTrue(self.is_host_plugin_put_logs_request(urls[1]),
-                            "The second request should have been over the host channel")
-
-    def test_send_request_using_appropriate_channel_should_not_invoke_host_channel_when_direct_channel_succeeds(self):
-        with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
-            protocol.client.get_host_plugin().set_default_channel(False)
+            self.assertTrue(self.is_host_plugin_put_logs_request(urls[0]), "The first request should have been over the host channel")
+            self.assertTrue(self.is_host_plugin_put_logs_request(urls[1]), "The second request should have been over the host channel")
 
-            def direct_func(*args):  # pylint: disable=unused-argument
-                direct_func.counter += 1
-                return 42
+    @staticmethod
+    def _set_and_fail_helper_channel_functions(fail_direct=False, fail_host=False):
+        def direct_func(*_):
+            direct_func.counter += 1
+            if direct_func.fail:
+                return None
+            return "direct"
 
-            def host_func(*args):  # pylint: disable=useless-return,unused-argument
-                host_func.counter += 1
+        def host_func(*_):
+            host_func.counter += 1
+            if host_func.fail:
                 return None
+            return "host"
 
-            direct_func.counter = 0
-            host_func.counter = 0
+        direct_func.counter = 0
+        direct_func.fail = fail_direct
 
-            # Assert we've only called the direct channel functions and that it succeeded.
-            ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
-            self.assertEqual(42, ret)
-            self.assertEqual(1, direct_func.counter)
-            self.assertEqual(0, host_func.counter)
+        host_func.counter = 0
+        host_func.fail = fail_host
 
-    def test_send_request_using_appropriate_channel_should_not_use_direct_channel_when_host_channel_is_default(self):
-        with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
-            protocol.client.get_host_plugin().set_default_channel(True)
+        return direct_func, host_func
 
-            def direct_func(*args):  # pylint: disable=unused-argument
-                direct_func.counter += 1
-                return 42
+    def test_send_request_using_appropriate_channel_should_not_invoke_secondary_when_primary_channel_succeeds(self):
+        with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
+            # Scenario #1: Direct channel default
+            HostPluginProtocol.is_default_channel = False
 
-            def host_func(*args):  # pylint: disable=unused-argument
-                host_func.counter += 1
-                return 43
+            direct_func, host_func = self._set_and_fail_helper_channel_functions()
+            # Assert we're only calling the primary channel (direct) and that it succeeds.
+            for iteration in range(5):
+                ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
+                self.assertEqual("direct", ret)
+                self.assertEqual(iteration + 1, direct_func.counter)
+                self.assertEqual(0, host_func.counter)
+                self.assertFalse(HostPluginProtocol.is_default_channel)
 
-            direct_func.counter = 0
-            host_func.counter = 0
+            # Scenario #2: Host channel default
+            HostPluginProtocol.is_default_channel = True
+            direct_func, host_func = self._set_and_fail_helper_channel_functions()
 
-            # Assert we've only called the host channel function since it's the default channel
-            ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
-            self.assertEqual(43, ret)
-            self.assertEqual(0, direct_func.counter)
-            self.assertEqual(1, host_func.counter)
+            # Assert we're only calling the primary channel (host) and that it succeeds.
+            for iteration in range(5):
+                ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
+                self.assertEqual("host", ret)
+                self.assertEqual(0, direct_func.counter)
+                self.assertEqual(iteration + 1, host_func.counter)
+                self.assertTrue(HostPluginProtocol.is_default_channel)
 
-    def test_send_request_using_appropriate_channel_should_use_host_channel_when_direct_channel_fails(self):
+    def test_send_request_using_appropriate_channel_should_not_change_default_channel_if_none_succeeds(self):
         with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
-            host = protocol.client.get_host_plugin()
-            host.set_default_channel(False)
+            # Scenario #1: Direct channel is default
+            HostPluginProtocol.is_default_channel = False
+            direct_func, host_func = self._set_and_fail_helper_channel_functions(fail_direct=True, fail_host=True)
+
+            # Assert we keep trying both channels, but the default channel doesn't change
+            for iteration in range(5):
+                ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
+                self.assertEqual(None, ret)
+                self.assertEqual(iteration + 1, direct_func.counter)
+                self.assertEqual(iteration + 1, host_func.counter)
+                self.assertFalse(HostPluginProtocol.is_default_channel)
 
-            def direct_func(*args):  # pylint: disable=unused-argument
-                direct_func.counter += 1
-                raise InvalidContainerError()
+            # Scenario #2: Host channel is default
+            HostPluginProtocol.is_default_channel = True
+            direct_func, host_func = self._set_and_fail_helper_channel_functions(fail_direct=True, fail_host=True)
 
-            def host_func(*args):  # pylint: disable=unused-argument
-                host_func.counter += 1
-                return 42
+            # Assert we keep trying both channels, but the default channel doesn't change
+            for iteration in range(5):
+                ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
+                self.assertEqual(None, ret)
+                self.assertEqual(iteration + 1, direct_func.counter)
+                self.assertEqual(iteration + 1, host_func.counter)
+                self.assertTrue(HostPluginProtocol.is_default_channel)
 
-            direct_func.counter = 0
-            host_func.counter = 0
+    def test_send_request_using_appropriate_channel_should_change_default_channel_when_secondary_succeeds(self):
+        with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
+            # Scenario #1: Direct channel is default
+            HostPluginProtocol.is_default_channel = False
+            direct_func, host_func = self._set_and_fail_helper_channel_functions(fail_direct=True, fail_host=False)
 
-            # Assert we've called both the direct channel function and the host channel function, which succeeded.
-            # After the host channel succeeds, the host plugin should have been set as the default channel.
+            # Assert we've called both channels and the default channel changed
             ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
-            self.assertEqual(42, ret)
+            self.assertEqual("host", ret)
             self.assertEqual(1, direct_func.counter)
             self.assertEqual(1, host_func.counter)
-            self.assertEqual(True, host.is_default_channel())
-
-    def test_send_request_using_appropriate_channel_should_retry_the_host_channel_after_reloading_goal_state(self):
-        with mock_wire_protocol(mockwiredata.DATA_FILE) as protocol:
-            protocol.client.get_host_plugin().set_default_channel(False)
+            self.assertTrue(HostPluginProtocol.is_default_channel)
 
-            def direct_func(*args):  # pylint: disable=unused-argument
-                direct_func.counter += 1
-                raise InvalidContainerError()
+            # If host keeps succeeding, assert we keep calling only that channel and not changing the default.
+            for iteration in range(5):
+                ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
+                self.assertEqual("host", ret)
+                self.assertEqual(1, direct_func.counter)
+                self.assertEqual(1 + iteration + 1, host_func.counter)
+                self.assertTrue(HostPluginProtocol.is_default_channel)
 
-            def host_func(*args):  # pylint: disable=unused-argument
-                host_func.counter += 1
-                if host_func.counter == 1:
-                    raise ResourceGoneError("Resource is gone")
-                return 42
+            # Scenario #2: Host channel is default
+            HostPluginProtocol.is_default_channel = True
+            direct_func, host_func = self._set_and_fail_helper_channel_functions(fail_direct=False, fail_host=True)
 
-            direct_func.counter = 0
-            host_func.counter = 0
+            # Assert we've called both channels and the default channel changed
+            ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
+            self.assertEqual("direct", ret)
+            self.assertEqual(1, direct_func.counter)
+            self.assertEqual(1, host_func.counter)
+            self.assertFalse(HostPluginProtocol.is_default_channel)
 
-            # Assert we've called both the direct channel function (once) and the host channel function (twice).
-            # After the host channel succeeds, the host plugin should have been set as the default channel.
-            with patch(
-                    'azurelinuxagent.common.protocol.wire.WireClient.update_host_plugin_from_goal_state') as mock_update_host_plugin_from_goal_state:
+            # If direct keeps succeeding, assert we keep calling only that channel and not changing the default.
+            for iteration in range(5):
                 ret = protocol.client.send_request_using_appropriate_channel(direct_func, host_func)
-                self.assertEqual(42, ret)
-                self.assertEqual(1, direct_func.counter)
-                self.assertEqual(2, host_func.counter)
-                self.assertEqual(1, mock_update_host_plugin_from_goal_state.call_count)
-                self.assertEqual(True, protocol.client.get_host_plugin().is_default_channel())
+                self.assertEqual("direct", ret)
+                self.assertEqual(1 + iteration + 1, direct_func.counter)
+                self.assertEqual(1, host_func.counter)
+                self.assertFalse(HostPluginProtocol.is_default_channel)
 
 
 class UpdateGoalStateTestCase(AgentTestCase):
@@ -1246,9 +1258,10 @@ class UpdateHostPluginFromGoalStateTestCase(AgentTestCase):
 
 
 class MockResponse:  # pylint: disable=too-few-public-methods
-    def __init__(self, body, status_code):
+    def __init__(self, body, status_code, reason=None):
         self.body = body
         self.status = status_code
+        self.reason = reason
 
     def read(self, *_):
         return self.body