Repository: Azure/WALinuxAgent
Pull request #919 (2017-10-26 17:32:42)

Use cached extension version manifest if available

brendandixon: @uiri I don't believe we can accept this without a very thorough review. The agent learns of new extension versions *only* by re-fetching the manifest. The manifest, sadly, is not versioned nor is the agent given any indicator that a new manifest is available. As a result, the agent must fetch it fresh every time just in case new versions of the extension are available.
uiri: You are right. I should have looked at this method more closely. The manifest cache is versioned on the GoalState incarnation number. That is probably not enough to prevent this method from accidentally picking up cached versions though.

Is it ever necessary to update the version manifest in a situation where the GoalState (and thus its incarnation number) has _not_ changed?
brendandixon: @uiri The manifest is *stored* under the incarnation number, but is not versioned by it. For example, two successive incarnations *may* retrieve the *same* manifest, but both will be stored.

The general idea is this: When a GoalState arrives, the agent should retrieve all related files (the entire document tree), including extension manifests. It should cache those to disk and use only those until the next incarnation (GoalState change) arrives.

This code predates my involvement and I'd need to review it closely to see it aligns with this goal. We'll accept changes to make it align if it does not.
uiri: Thank you for the clarification surrounding the intended behavior of this code.

It appears to me that:

- the very first thing the ext handlers handler does is update the goal state (via calling `get_ext_handlers` on the wire protocol)
- Unless extension handling is on hold, it will go through each extension handler. This then gets called indirectly by `decide_version`

It looks like this might update the goal state in the middle of processing one of the extensions. That seems broken to me. I think a better behavior would be passing in the GoalState incarnation from the extension handler handling code. If it does that, then the cache becomes reliable.

Patch:
diff --git a/azurelinuxagent/common/errorstate.py b/azurelinuxagent/common/errorstate.py
new file mode 100644
index 00000000..750aa77b
--- /dev/null
+++ b/azurelinuxagent/common/errorstate.py
@@ -0,0 +1,31 @@
+from datetime import datetime, timedelta
+
+ERROR_STATE_DELTA = timedelta(minutes=15)
+
+
+class ErrorState(object):
+    def __init__(self, min_timedelta = ERROR_STATE_DELTA):
+        self.min_timedelta = min_timedelta
+
+        self.count = 0
+        self.timestamp = None
+
+    def incr(self):
+        if self.count == 0:
+            self.timestamp = datetime.utcnow()
+
+        self.count += 1
+
+    def reset(self):
+        self.count = 0
+        self.timestamp = None
+
+    def is_triggered(self):
+        if self.timestamp is None:
+            return False
+
+        delta = datetime.utcnow() - self.timestamp
+        if delta >= self.min_timedelta:
+            return True
+
+        return False
diff --git a/azurelinuxagent/common/exception.py b/azurelinuxagent/common/exception.py
index 2e5a78a4..51411935 100644
--- a/azurelinuxagent/common/exception.py
+++ b/azurelinuxagent/common/exception.py
@@ -113,6 +113,15 @@ class ProtocolNotFoundError(ProtocolError):
         super(ProtocolNotFoundError, self).__init__(msg, inner)
 
 
+class RestartError(ProtocolError):
+    """
+    Variant of ProtocolError used to restart processing if the GoalState
+    becomes stale.
+    """
+
+    pass
+
+
 class HttpError(AgentError):
     """
     Http request failure
diff --git a/azurelinuxagent/common/protocol/metadata.py b/azurelinuxagent/common/protocol/metadata.py
index ed4fff98..0cfa5ede 100644
--- a/azurelinuxagent/common/protocol/metadata.py
+++ b/azurelinuxagent/common/protocol/metadata.py
@@ -240,7 +240,7 @@ class MetadataProtocol(Protocol):
             set_properties("extensionHandlers", ext_list.extHandlers, data)
         return ext_list, etag
 
-    def get_ext_handler_pkgs(self, ext_handler):
+    def get_ext_handler_pkgs(self, ext_handler, etag):
         logger.verbose("Get extension handler packages")
         pkg_list = ExtHandlerPackageList()
 
diff --git a/azurelinuxagent/common/protocol/restapi.py b/azurelinuxagent/common/protocol/restapi.py
index 275cedb0..5a678d75 100644
--- a/azurelinuxagent/common/protocol/restapi.py
+++ b/azurelinuxagent/common/protocol/restapi.py
@@ -315,15 +315,15 @@ class Protocol(DataContract):
     def get_ext_handlers(self):
         raise NotImplementedError()
 
-    def get_ext_handler_pkgs(self, extension):
+    def get_ext_handler_pkgs(self, extension, etag):
         raise NotImplementedError()
 
     def get_artifacts_profile(self):
         raise NotImplementedError()
 
-    def download_ext_handler_pkg(self, uri, headers=None):
+    def download_ext_handler_pkg(self, uri, headers=None, use_proxy=True):
         try:
-            resp = restutil.http_get(uri, use_proxy=True, headers=headers)
+            resp = restutil.http_get(uri, headers=headers, use_proxy=use_proxy)
             if restutil.request_succeeded(resp):
                 return resp.read()
         except Exception as e:
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index a92e0b89..133576cd 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -28,7 +28,7 @@ import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.textutil as textutil
 
 from azurelinuxagent.common.exception import ProtocolNotFoundError, \
-                                            ResourceGoneError
+                                            ResourceGoneError, RestartError
 from azurelinuxagent.common.future import httpclient, bytebuffer
 from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
 from azurelinuxagent.common.protocol.restapi import *
@@ -153,10 +153,9 @@ class WireProtocol(Protocol):
         # In wire protocol, incarnation is equivalent to ETag
         return ext_conf.ext_handlers, goal_state.incarnation
 
-    def get_ext_handler_pkgs(self, ext_handler):
+    def get_ext_handler_pkgs(self, ext_handler, etag):
         logger.verbose("Get extension handler package")
-        goal_state = self.client.get_goal_state()
-        man = self.client.get_ext_manifest(ext_handler, goal_state)
+        man = self.client.get_ext_manifest(ext_handler, etag)
         return man.pkg_list
 
     def get_artifacts_profile(self):
@@ -172,7 +171,7 @@ class WireProtocol(Protocol):
             logger.warn("Download did not succeed, falling back to host plugin")
             host = self.client.get_host_plugin()
             uri, headers = host.get_artifact_request(uri, host.manifest_uri)
-            package = super(WireProtocol, self).download_ext_handler_pkg(uri, headers=headers)
+            package = super(WireProtocol, self).download_ext_handler_pkg(uri, headers=headers, use_proxy=False)
         return package
 
     def report_provision_status(self, provision_status):
@@ -840,25 +839,35 @@ class WireClient(object):
                 self.ext_conf = ExtensionsConfig(xml_text)
         return self.ext_conf
 
-    def get_ext_manifest(self, ext_handler, goal_state):
+    def get_ext_manifest(self, ext_handler, incarnation):
+
         for update_goal_state in [False, True]:
             try:
                 if update_goal_state:
                     self.update_goal_state(forced=True)
-                    goal_state = self.get_goal_state()
+                    incarnation = self.get_goal_state().incarnation
 
                 local_file = MANIFEST_FILE_NAME.format(
                                 ext_handler.name,
-                                goal_state.incarnation)
+                                incarnation)
                 local_file = os.path.join(conf.get_lib_dir(), local_file)
-                xml_text = self.fetch_manifest(ext_handler.versionUris)
-                self.save_cache(local_file, xml_text)
+
+                xml_text = None
+                if not update_goal_state:
+                    try:
+                        xml_text = self.fetch_cache(local_file)
+                    except ProtocolError:
+                        pass
+
+                if xml_text is None:
+                    xml_text = self.fetch_manifest(ext_handler.versionUris)
+                    self.save_cache(local_file, xml_text)
                 return ExtensionManifest(xml_text)
 
             except ResourceGoneError:
                 continue
 
-        raise ProtocolError("Failed to retrieve extension manifest")
+        raise RestartError("Failed to retrieve extension manifest")
 
     def filter_package_list(self, family, ga_manifest, goal_state):
         complete_list = ga_manifest.pkg_list
diff --git a/azurelinuxagent/ga/env.py b/azurelinuxagent/ga/env.py
index 0456cb06..26487818 100644
--- a/azurelinuxagent/ga/env.py
+++ b/azurelinuxagent/ga/env.py
@@ -86,7 +86,7 @@ class EnvHandler(object):
                     version=CURRENT_VERSION,
                     op=WALAEventOperation.Firewall,
                     is_success=success,
-                    log_event=True)
+                    log_event=False)
 
             timeout = conf.get_root_device_scsi_timeout()
             if timeout is not None:
diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
index df5db7e5..ea539007 100644
--- a/azurelinuxagent/ga/exthandlers.py
+++ b/azurelinuxagent/ga/exthandlers.py
@@ -35,9 +35,10 @@ import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.restutil as restutil
 import azurelinuxagent.common.utils.shellutil as shellutil
 import azurelinuxagent.common.version as version
+from azurelinuxagent.common.errorstate import ErrorState, ERROR_STATE_DELTA
 
 from azurelinuxagent.common.event import add_event, WALAEventOperation, elapsed_milliseconds
-from azurelinuxagent.common.exception import ExtensionError, ProtocolError, HttpError
+from azurelinuxagent.common.exception import ExtensionError, ProtocolError, HttpError, RestartError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.version import AGENT_VERSION
 from azurelinuxagent.common.protocol.restapi import ExtHandlerStatus, \
@@ -180,6 +181,9 @@ class ExtHandlersHandler(object):
         self.last_upgrade_guids = {}
         self.log_report = False
         self.log_etag = True
+        self.log_process = False
+
+        self.report_status_error_state = ErrorState()
 
     def run(self):
         self.ext_handlers, etag = None, None
@@ -207,6 +211,8 @@ class ExtHandlersHandler(object):
 
             self.report_ext_handlers_status()
             self.cleanup_outdated_handlers()
+        except RestartError:
+            raise
         except Exception as e:
             msg = u"Exception processing extension handlers: {0}".format(
                 ustr(e))
@@ -344,7 +350,7 @@ class ExtHandlersHandler(object):
                 return
 
             self.set_log_upgrade_guid(ext_handler, True)
-            ext_handler_i.decide_version(target_state=state)
+            ext_handler_i.decide_version(etag=etag, target_state=state)
             if not ext_handler_i.is_upgrade and self.last_etag == etag:
                 if self.log_etag:
                     ext_handler_i.logger.verbose("Version {0} is current for etag {1}",
@@ -372,11 +378,17 @@ class ExtHandlersHandler(object):
             else:
                 message = u"Unknown ext handler state:{0}".format(state)
                 raise ExtensionError(message)
+        except RestartError:
+            ext_handler_i.logger.info("GoalState became stale during "
+                                      "processing. Restarting with new "
+                                      "GoalState")
+            raise
         except Exception as e:
             ext_handler_i.set_handler_status(message=ustr(e), code=-1)
             ext_handler_i.report_event(message=ustr(e), is_success=False)
     
     def handle_enable(self, ext_handler_i):
+        self.log_process = True
         old_ext_handler_i = ext_handler_i.get_installed_ext_handler()
         if old_ext_handler_i is not None and \
            old_ext_handler_i.version_gt(ext_handler_i):
@@ -403,6 +415,7 @@ class ExtHandlersHandler(object):
         ext_handler_i.enable() 
 
     def handle_disable(self, ext_handler_i):
+        self.log_process = True
         handler_state = ext_handler_i.get_handler_state()
         ext_handler_i.logger.info("[Disable] current handler state is: {0}",
                                   handler_state.lower())
@@ -410,6 +423,7 @@ class ExtHandlersHandler(object):
             ext_handler_i.disable()
 
     def handle_uninstall(self, ext_handler_i):
+        self.log_process = True
         handler_state = ext_handler_i.get_handler_state()
         ext_handler_i.logger.info("[Uninstall] current handler state is: {0}",
                                   handler_state.lower())
@@ -418,7 +432,7 @@ class ExtHandlersHandler(object):
                 ext_handler_i.disable()
             ext_handler_i.uninstall()
         ext_handler_i.rm_ext_handler_dir()
-    
+
     def report_ext_handlers_status(self):
         """Go through handler_state dir, collect and report status"""
         vm_status = VMStatus(status="Ready", message="Guest Agent is running")
@@ -439,7 +453,9 @@ class ExtHandlersHandler(object):
             self.protocol.report_vm_status(vm_status)
             if self.log_report:
                 logger.verbose("Completed vm agent status report")
+            self.report_status_error_state.reset()
         except ProtocolError as e:
+            self.report_status_error_state.incr()
             message = "Failed to report vm agent status: {0}".format(e)
             add_event(AGENT_NAME,
                 version=CURRENT_VERSION,
@@ -447,6 +463,18 @@ class ExtHandlersHandler(object):
                 is_success=False,
                 message=message)
 
+        if self.report_status_error_state.is_triggered():
+            message = "Failed to report vm agent status for more than {0}"\
+                .format(ERROR_STATE_DELTA)
+
+            add_event(AGENT_NAME,
+                version=CURRENT_VERSION,
+                op=WALAEventOperation.ExtensionProcessing,
+                is_success=False,
+                message=message)
+
+            self.report_status_error_state.reset()
+
     def report_ext_handler_status(self, vm_status, ext_handler):
         ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
         
@@ -494,12 +522,9 @@ class ExtHandlerInstance(object):
         self.logger.add_appender(logger.AppenderType.FILE,
                                  logger.LogLevel.INFO, log_file)
 
-    def decide_version(self, target_state=None):
+    def decide_version(self, etag, target_state=None):
         self.logger.verbose("Decide which version to use")
-        try:
-            pkg_list = self.protocol.get_ext_handler_pkgs(self.ext_handler)
-        except ProtocolError as e:
-            raise ExtensionError("Failed to get ext handler pkgs", e)
+        pkg_list = self.protocol.get_ext_handler_pkgs(self.ext_handler, etag)
 
         # Determine the desired and installed versions
         requested_version = FlexibleVersion(
diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py
index e3ef292f..71ac9b0b 100644
--- a/azurelinuxagent/ga/monitor.py
+++ b/azurelinuxagent/ga/monitor.py
@@ -203,26 +203,31 @@ class MonitorHandler(object):
                     version=CURRENT_VERSION,
                     op=WALAEventOperation.HeartBeat,
                     is_success=True,
-                    message=msg)
+                    message=msg,
+                    log_event=False)
 
                 counter += 1
 
-                ioerrors = IOErrorCounter.get_and_reset()
-                hostplugin_errors = ioerrors.get("hostplugin")
-                protocol_errors = ioerrors.get("protocol")
-                other_errors = ioerrors.get("other")
+                io_errors = IOErrorCounter.get_and_reset()
+                hostplugin_errors = io_errors.get("hostplugin")
+                protocol_errors = io_errors.get("protocol")
+                other_errors = io_errors.get("other")
 
-                if hostplugin_errors > 0 or \
-                    protocol_errors > 0 or \
-                    other_errors > 0:
-                    msg = "hostplugin:{0};protocol:{1};other:{2}".format(
-                        hostplugin_errors, protocol_errors, other_errors)
+                if hostplugin_errors > 0 \
+                        or protocol_errors > 0 \
+                        or other_errors > 0:
+
+                    msg = "hostplugin:{0};protocol:{1};other:{2}"\
+                        .format(hostplugin_errors,
+                                protocol_errors,
+                                other_errors)
                     add_event(
                         name=AGENT_NAME,
                         version=CURRENT_VERSION,
                         op=WALAEventOperation.HttpErrors,
-                        is_success=False,
-                        msg=msg)
+                        is_success=True,
+                        message=msg,
+                        log_event=False)
 
             try:
                 self.collect_and_send_events()
diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py
index 03d94c39..b6414913 100644
--- a/azurelinuxagent/ga/update.py
+++ b/azurelinuxagent/ga/update.py
@@ -44,12 +44,13 @@ from azurelinuxagent.common.event import add_event, add_periodic, \
                                     WALAEventOperation
 from azurelinuxagent.common.exception import ProtocolError, \
                                             ResourceGoneError, \
+                                            RestartError, \
                                             UpdateError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
 from azurelinuxagent.common.protocol import get_protocol_util
 from azurelinuxagent.common.protocol.hostplugin import HostPluginProtocol
-from azurelinuxagent.common.protocol.wire import WireProtocol
+from azurelinuxagent.common.protocol.wire import WireProtocol, RestartError
 from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
 from azurelinuxagent.common.version import AGENT_NAME, AGENT_VERSION, AGENT_LONG_VERSION, \
                                             AGENT_DIR_GLOB, AGENT_PKG_GLOB, \
@@ -283,7 +284,10 @@ class UpdateHandler(object):
                 utc_start = datetime.utcnow()
 
                 last_etag = exthandlers_handler.last_etag
-                exthandlers_handler.run()
+                try:
+                    exthandlers_handler.run()
+                except RestartError:
+                    continue
 
                 if last_etag != exthandlers_handler.last_etag:
                     self._ensure_readonly_files()
@@ -295,7 +299,7 @@ class UpdateHandler(object):
                         duration=elapsed_milliseconds(utc_start),
                         message="Incarnation {0}".format(
                                             exthandlers_handler.last_etag),
-                        log_event=True)
+                        log_event=exthandlers_handler.log_process)
 
                 time.sleep(GOAL_STATE_INTERVAL)
 
@@ -660,7 +664,7 @@ class UpdateHandler(object):
                     continue
 
                 msg = u"Exception retrieving agent manifests: {0}".format(
-                            ustr(e))
+                            ustr(traceback.format_exc()))
                 logger.warn(msg)
                 add_event(
                     AGENT_NAME,
@@ -704,13 +708,13 @@ class GuestAgent(object):
         version = None
         if path is not None:
             m = AGENT_DIR_PATTERN.match(path)
-            if m == None:
+            if m is None:
                 raise UpdateError(u"Illegal agent directory: {0}".format(path))
             version = m.group(1)
         elif self.pkg is not None:
             version = pkg.version
 
-        if version == None:
+        if version is None:
             raise UpdateError(u"Illegal agent version: {0}".format(version))
         self.version = FlexibleVersion(version)
 
@@ -727,6 +731,13 @@ class GuestAgent(object):
             if isinstance(e, ResourceGoneError):
                 raise
 
+            # The agent was improperly blacklisting versions due to a timeout
+            # encountered while downloading a later version. Errors of type
+            # socket.error are IOError, so this should provide sufficient
+            # protection against a large class of I/O operation failures.
+            if isinstance(e, IOError):
+                raise
+
             # Note the failure, blacklist the agent if the package downloaded
             # - An exception with a downloaded package indicates the package
             #   is corrupt (e.g., missing the HandlerManifest.json file)
diff --git a/azurelinuxagent/pa/provision/cloudinit.py b/azurelinuxagent/pa/provision/cloudinit.py
index 22c3f9ca..fa47799a 100644
--- a/azurelinuxagent/pa/provision/cloudinit.py
+++ b/azurelinuxagent/pa/provision/cloudinit.py
@@ -64,7 +64,7 @@ class CloudInitProvisionHandler(ProvisionHandler):
             logger.info("Finished provisioning")
 
             self.report_ready(thumbprint)
-            self.report_event("Provision succeed",
+            self.report_event("Provisioning with cloud-init succeeded",
                 is_success=True,
                 duration=elapsed_milliseconds(utc_start))
             self.report_event(self.create_guest_state_telemetry_messsage(),
diff --git a/azurelinuxagent/pa/provision/default.py b/azurelinuxagent/pa/provision/default.py
index 5d6f1565..44e171b4 100644
--- a/azurelinuxagent/pa/provision/default.py
+++ b/azurelinuxagent/pa/provision/default.py
@@ -89,7 +89,7 @@ class ProvisionHandler(object):
 
             self.write_provisioned()
 
-            self.report_event("Provision succeeded",
+            self.report_event("Provisioning succeeded",
                 is_success=True,
                 duration=elapsed_milliseconds(utc_start))
 
diff --git a/azurelinuxagent/pa/provision/factory.py b/azurelinuxagent/pa/provision/factory.py
index d87765f3..9e88618f 100644
--- a/azurelinuxagent/pa/provision/factory.py
+++ b/azurelinuxagent/pa/provision/factory.py
@@ -16,9 +16,7 @@
 #
 
 import azurelinuxagent.common.conf as conf
-import azurelinuxagent.common.logger as logger
 
-from azurelinuxagent.common.utils.textutil import Version
 from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \
                                      DISTRO_FULL_NAME
 
diff --git a/azurelinuxagent/pa/rdma/suse.py b/azurelinuxagent/pa/rdma/suse.py
index 729c8428..2b6ae294 100644
--- a/azurelinuxagent/pa/rdma/suse.py
+++ b/azurelinuxagent/pa/rdma/suse.py
@@ -91,7 +91,7 @@ class SUSERDMAHandler(RDMAHandler):
                 logger.info("Package '%s' is not a match." % entry)
             else:
                 logger.info("Package '%s' is a match. Installing." % entry)
-                complete_name = '%s-%s' % (package_name, version)
+                complete_name = '%s-%s' % (package_name, entry)
                 cmd = zypper_install % complete_name
                 result = shellutil.run(cmd)
                 if result:
diff --git a/config/alpine/waagent.conf b/config/alpine/waagent.conf
index 37f19c92..883c81b5 100644
--- a/config/alpine/waagent.conf
+++ b/config/alpine/waagent.conf
@@ -89,5 +89,5 @@ OS.SshDir=/etc/ssh
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/arch/waagent.conf b/config/arch/waagent.conf
index 1c0f264f..cf14e0ac 100644
--- a/config/arch/waagent.conf
+++ b/config/arch/waagent.conf
@@ -115,5 +115,5 @@ OS.SshDir=/etc/ssh
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/bigip/waagent.conf b/config/bigip/waagent.conf
index 25bb2f07..1e59a6ae 100644
--- a/config/bigip/waagent.conf
+++ b/config/bigip/waagent.conf
@@ -93,5 +93,5 @@ AutoUpdate.Enabled=y
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/clearlinux/waagent.conf b/config/clearlinux/waagent.conf
index 61c15722..c63f92ba 100644
--- a/config/clearlinux/waagent.conf
+++ b/config/clearlinux/waagent.conf
@@ -87,5 +87,5 @@ AutoUpdate.GAFamily=Prod
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/coreos/waagent.conf b/config/coreos/waagent.conf
index 1ae78243..ee4ebf2c 100644
--- a/config/coreos/waagent.conf
+++ b/config/coreos/waagent.conf
@@ -115,5 +115,5 @@ OS.AllowHTTP=y
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/freebsd/waagent.conf b/config/freebsd/waagent.conf
index faefc3ed..74ad8430 100644
--- a/config/freebsd/waagent.conf
+++ b/config/freebsd/waagent.conf
@@ -113,5 +113,5 @@ OS.SudoersDir=/usr/local/etc/sudoers.d
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/gaia/waagent.conf b/config/gaia/waagent.conf
index f1a5cb6a..0fd78a80 100644
--- a/config/gaia/waagent.conf
+++ b/config/gaia/waagent.conf
@@ -112,5 +112,5 @@ AutoUpdate.Enabled=n
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/openbsd/waagent.conf b/config/openbsd/waagent.conf
index 6dcd9e93..580b4a32 100644
--- a/config/openbsd/waagent.conf
+++ b/config/openbsd/waagent.conf
@@ -111,5 +111,5 @@ OS.PasswordPath=/etc/master.passwd
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/suse/waagent.conf b/config/suse/waagent.conf
index 4b0d2c2a..e83caf36 100644
--- a/config/suse/waagent.conf
+++ b/config/suse/waagent.conf
@@ -115,5 +115,5 @@ OS.SshDir=/etc/ssh
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/ubuntu/waagent.conf b/config/ubuntu/waagent.conf
index 5c6b7b3d..4fa2d124 100644
--- a/config/ubuntu/waagent.conf
+++ b/config/ubuntu/waagent.conf
@@ -103,5 +103,5 @@ OS.SshDir=/etc/ssh
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/config/waagent.conf b/config/waagent.conf
index ffd4d064..4317ed3d 100644
--- a/config/waagent.conf
+++ b/config/waagent.conf
@@ -115,5 +115,5 @@ OS.SshDir=/etc/ssh
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y


Test patch:
diff --git a/tests/common/test_errorstate.py b/tests/common/test_errorstate.py
new file mode 100644
index 00000000..7513fe59
--- /dev/null
+++ b/tests/common/test_errorstate.py
@@ -0,0 +1,69 @@
+from datetime import timedelta
+
+from azurelinuxagent.common.errorstate import *
+from tests.tools import *
+
+
+class TestErrorState(unittest.TestCase):
+    def test_errorstate00(self):
+        """
+        If ErrorState is never incremented, it will never trigger.
+        """
+        test_subject = ErrorState(timedelta(seconds=10000))
+        self.assertFalse(test_subject.is_triggered())
+        self.assertEqual(0, test_subject.count)
+
+    def test_errorstate01(self):
+        """
+        If ErrorState is never incremented, and the timedelta is zero it will
+        not trigger.
+        """
+        test_subject = ErrorState(timedelta(seconds=0))
+        self.assertFalse(test_subject.is_triggered())
+        self.assertEqual(0, test_subject.count)
+
+    def test_errorstate02(self):
+        """
+        If ErrorState is triggered, and the current time is within timedelta
+        of now it will trigger.
+        """
+        test_subject = ErrorState(timedelta(seconds=0))
+        test_subject.incr()
+
+
+        self.assertTrue(test_subject.is_triggered())
+        self.assertEqual(1, test_subject.count)
+
+    @patch('azurelinuxagent.common.errorstate.datetime')
+    def test_errorstate03(self, mock_time):
+        """
+        ErrorState will not trigger until
+         1. ErrorState has been incr() at least once.
+         2. The timedelta from the first incr() has elapsed.
+        """
+        test_subject = ErrorState(timedelta(minutes=15))
+
+        for x in range(1, 10):
+            mock_time.utcnow = Mock(return_value=datetime.utcnow() + timedelta(minutes=x))
+
+            test_subject.incr()
+            self.assertFalse(test_subject.is_triggered())
+
+        mock_time.utcnow = Mock(return_value=datetime.utcnow() + timedelta(minutes=30))
+        test_subject.incr()
+        self.assertTrue(test_subject.is_triggered())
+
+    def test_errorstate04(self):
+        """
+        If ErrorState is reset the timestamp of the last incr() is reset to
+        None.
+        """
+
+        test_subject = ErrorState(timedelta(minutes=15))
+        self.assertTrue(test_subject.timestamp is None)
+
+        test_subject.incr()
+        self.assertTrue(test_subject.timestamp is not None)
+
+        test_subject.reset()
+        self.assertTrue(test_subject.timestamp is None)
diff --git a/tests/data/test_waagent.conf b/tests/data/test_waagent.conf
index 05159872..6dfef694 100644
--- a/tests/data/test_waagent.conf
+++ b/tests/data/test_waagent.conf
@@ -123,5 +123,5 @@ OS.SshDir=/notareal/path
 
 # Add firewall rules to protect access to Azure host node services
 # Note:
-# - The default is false to protect the state of exising VMs
+# - The default is false to protect the state of existing VMs
 OS.EnableFirewall=y
diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py
index bc9a351d..9a729890 100644
--- a/tests/ga/test_extension.py
+++ b/tests/ga/test_extension.py
@@ -579,12 +579,12 @@ class TestExtension(AgentTestCase):
                         decision_version = '1.0.0'
 
                 _, protocol = self._create_mock(WireProtocolData(datafile), *args)
-                ext_handlers, _ = protocol.get_ext_handlers()
+                ext_handlers, etag = protocol.get_ext_handlers()
                 self.assertEqual(1, len(ext_handlers.extHandlers))
                 ext_handler = ext_handlers.extHandlers[0]
                 self.assertEqual('OSTCExtensions.ExampleHandlerLinux', ext_handler.name)
                 self.assertEqual(config_version, ext_handler.properties.version, "config version.")
-                ExtHandlerInstance(ext_handler, protocol).decide_version()
+                ExtHandlerInstance(ext_handler, protocol).decide_version(etag)
                 self.assertEqual(decision_version, ext_handler.properties.version, "decision version.")
 
     def test_ext_handler_version_decide_between_minor_versions(self, *args):
@@ -611,6 +611,7 @@ class TestExtension(AgentTestCase):
         _, protocol = self._create_mock(WireProtocolData(DATA_FILE), *args)
         version_uri = Mock()
         version_uri.uri = 'http://some/Microsoft.OSTCExtensions_ExampleHandlerLinux_asiaeast_manifest.xml'
+        incarnation = 1
 
         for (installed_version, config_version, expected_version, autoupgrade_expected_version) in cases:
             ext_handler = Mock()
@@ -622,8 +623,9 @@ class TestExtension(AgentTestCase):
             ext_handler_instance = ExtHandlerInstance(ext_handler, protocol)
             ext_handler_instance.get_installed_version = Mock(return_value=installed_version)
 
-            ext_handler_instance.decide_version()
+            ext_handler_instance.decide_version(incarnation)
             self.assertEqual(expected_version, ext_handler.properties.version)
+            incarnation += 1
 
             ext_handler.properties.version = config_version
             ext_handler.properties.upgradePolicy = 'auto'
@@ -631,8 +633,9 @@ class TestExtension(AgentTestCase):
             ext_handler_instance = ExtHandlerInstance(ext_handler, protocol)
             ext_handler_instance.get_installed_version = Mock(return_value=installed_version)
 
-            ext_handler_instance.decide_version()
+            ext_handler_instance.decide_version(incarnation)
             self.assertEqual(autoupgrade_expected_version, ext_handler.properties.version)
+            incarnation += 1
 
 
 if __name__ == '__main__':
diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py
index 88535f5b..a40278c7 100644
--- a/tests/ga/test_update.py
+++ b/tests/ga/test_update.py
@@ -387,6 +387,30 @@ class TestGuestAgent(UpdateTestCase):
         self.assertTrue(agent.is_blacklisted)
         self.assertEqual(agent.is_blacklisted, agent.error.is_blacklisted)
 
+    @patch("azurelinuxagent.ga.update.GuestAgent._ensure_downloaded")
+    @patch("azurelinuxagent.ga.update.GuestAgent._ensure_loaded")
+    def test_resource_gone_error_not_blacklisted(self, mock_loaded, mock_downloaded):
+        try:
+            mock_downloaded.side_effect = ResourceGoneError()
+            agent = GuestAgent(path=self.agent_path)
+            self.assertFalse(agent.is_blacklisted)
+        except ResourceGoneError:
+            pass
+        except:
+            self.fail("Exception was not expected!")
+
+    @patch("azurelinuxagent.ga.update.GuestAgent._ensure_downloaded")
+    @patch("azurelinuxagent.ga.update.GuestAgent._ensure_loaded")
+    def test_ioerror_not_blacklisted(self, mock_loaded, mock_downloaded):
+        try:
+            mock_downloaded.side_effect = IOError()
+            agent = GuestAgent(path=self.agent_path)
+            self.assertFalse(agent.is_blacklisted)
+        except IOError:
+            pass
+        except:
+            self.fail("Exception was not expected!")
+
     @patch("azurelinuxagent.ga.update.GuestAgent._ensure_downloaded")
     @patch("azurelinuxagent.ga.update.GuestAgent._ensure_loaded")
     def test_is_downloaded(self, mock_loaded, mock_downloaded):
@@ -1550,6 +1574,7 @@ class ProtocolMock(object):
         self.call_counts["update_goal_state"] += 1
         self.goal_state_forced = self.goal_state_forced or forced
 
+
 class ResponseMock(Mock):
     def __init__(self, status=restutil.httpclient.OK, response=None, reason=None):
         Mock.__init__(self)
@@ -1579,5 +1604,6 @@ class TimeMock(Mock):
         self.next_time += self.time_increment
         return current_time
 
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/pa/test_provision.py b/tests/pa/test_provision.py
index ab0a9102..2c2d2c9b 100644
--- a/tests/pa/test_provision.py
+++ b/tests/pa/test_provision.py
@@ -151,7 +151,7 @@ class TestProvision(AgentTestCase):
 
         ph.run()
 
-        call1 = call("Provision succeeded", duration=ANY, is_success=True)
+        call1 = call("Provisioning succeeded", duration=ANY, is_success=True)
         call2 = call(ANY, is_success=True, operation=WALAEventOperation.GuestState)
         ph.report_event.assert_has_calls([call1, call2])
 
diff --git a/tests/protocol/test_metadata.py b/tests/protocol/test_metadata.py
index 5047b86e..5f90f128 100644
--- a/tests/protocol/test_metadata.py
+++ b/tests/protocol/test_metadata.py
@@ -39,7 +39,7 @@ class TestMetadataProtocolGetters(AgentTestCase):
             protocol.get_certs()
             ext_handlers, etag = protocol.get_ext_handlers()
             for ext_handler in ext_handlers.extHandlers:
-                protocol.get_ext_handler_pkgs(ext_handler)
+                protocol.get_ext_handler_pkgs(ext_handler, etag)
 
     def test_getters(self, *args):
         test_data = MetadataProtocolData(DATA_FILE)
diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py
index da15c263..b9bf7032 100644
--- a/tests/protocol/test_wire.py
+++ b/tests/protocol/test_wire.py
@@ -43,8 +43,9 @@ class TestWireProtocol(AgentTestCase):
             protocol.get_vminfo()
             protocol.get_certs()
             ext_handlers, etag = protocol.get_ext_handlers()
+            self.assertEqual("1", etag)
             for ext_handler in ext_handlers.extHandlers:
-                protocol.get_ext_handler_pkgs(ext_handler)
+                protocol.get_ext_handler_pkgs(ext_handler, etag)
 
             crt1 = os.path.join(self.tmp_dir,
                                 '33B0ABCE4673538650971C10F7D7397E71561F35.crt')
@@ -93,6 +94,7 @@ class TestWireProtocol(AgentTestCase):
         #    HostingEnvironmentConfig, will be retrieved the expected number
         self.assertEqual(2, test_data.call_counts["hostingenvuri"])
 
+
     def test_call_storage_kwargs(self,
                                  mock_cryptutil,
                                  mock_sleep):