Repository: Azure/WALinuxAgent
Pull request #702 (2017-05-08 21:47:41)

Added Provisioning telemetry duration
Correctly handle cleaning directories that include symlinks
Reduced polling interval #701
Enable selective agent testing
Reduced cloud-init files removed on deprovision #631

Signed-off-by: Brendan Dixon <brendandixon@me.com>


Patch:
diff --git a/README.md b/README.md
index fd160737..c9211904 100644
--- a/README.md
+++ b/README.md
@@ -188,7 +188,7 @@ ResourceDisk.SwapSizeMB=0
 LBProbeResponder=y
 Logs.Verbose=n
 OS.RootDeviceScsiTimeout=300
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 OS.OpensslPath=None
 OS.SshDir=/etc/ssh
 HttpProxy.Host=None
@@ -353,7 +353,7 @@ _Default: n_
 If set, the agent will attempt to install and then load an RDMA kernel driver
 that matches the version of the firmware on the underlying hardware.
 
-* __OS.FIPSEnabled__
+* __OS.EnableFIPS__
 _Type: Boolean_
 _Default: n_
 
diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py
index a8a22172..1979a4a5 100644
--- a/azurelinuxagent/common/conf.py
+++ b/azurelinuxagent/common/conf.py
@@ -116,7 +116,7 @@ def get_ext_log_dir(conf=__conf__):
     return conf.get("Extension.LogDir", "/var/log/azure")
 
 def get_fips_enabled(conf=__conf__):
-    return conf.get_switch("OS.FIPSEnabled", False)
+    return conf.get_switch("OS.EnableFIPS", False)
 
 def get_openssl_cmd(conf=__conf__):
     return conf.get("OS.OpensslPath", "/usr/bin/openssl")
diff --git a/azurelinuxagent/common/utils/fileutil.py b/azurelinuxagent/common/utils/fileutil.py
index 8713d0c6..bae19571 100644
--- a/azurelinuxagent/common/utils/fileutil.py
+++ b/azurelinuxagent/common/utils/fileutil.py
@@ -119,16 +119,20 @@ def rm_files(*args):
 
 def rm_dirs(*args):
     """
-    Remove all the contents under the directry
+    Remove the contents of each directry
     """
-    for dir_name in args:
-        if os.path.isdir(dir_name):
-            for item in os.listdir(dir_name):
-                path = os.path.join(dir_name, item)
-                if os.path.isfile(path):
-                    os.remove(path)
-                elif os.path.isdir(path):
-                    shutil.rmtree(path)
+    for p in args:
+        if not os.path.isdir(p):
+            continue
+
+        for pp in os.listdir(p):
+            path = os.path.join(p, pp)
+            if os.path.isfile(path):
+                os.remove(path)
+            elif os.path.islink(path):
+                os.unlink(path)
+            elif os.path.isdir(path):
+                shutil.rmtree(path)
 
 def trim_ext(path, ext):
     if not ext.startswith("."):
diff --git a/azurelinuxagent/daemon/main.py b/azurelinuxagent/daemon/main.py
index 9d7130c1..66806647 100644
--- a/azurelinuxagent/daemon/main.py
+++ b/azurelinuxagent/daemon/main.py
@@ -30,7 +30,8 @@ from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
 from azurelinuxagent.common.protocol import get_protocol_util
 from azurelinuxagent.common.rdma import setup_rdma_device
-from azurelinuxagent.common.version import AGENT_LONG_NAME, AGENT_VERSION, \
+from azurelinuxagent.common.version import AGENT_NAME, AGENT_LONG_NAME, \
+    AGENT_VERSION, \
     DISTRO_NAME, DISTRO_VERSION, PY_VERSION_MAJOR, PY_VERSION_MINOR, \
     PY_VERSION_MICRO
 from azurelinuxagent.daemon.resourcedisk import get_resourcedisk_handler
@@ -74,7 +75,7 @@ class DaemonHandler(object):
                 self.daemon()
             except Exception as e:
                 err_msg = traceback.format_exc()
-                add_event("WALA", is_success=False, message=ustr(err_msg),
+                add_event(name=AGENT_NAME, is_success=False, message=ustr(err_msg),
                           op=WALAEventOperation.UnhandledError)
                 logger.info("Sleep 15 seconds and restart daemon")
                 time.sleep(15)
diff --git a/azurelinuxagent/daemon/resourcedisk/default.py b/azurelinuxagent/daemon/resourcedisk/default.py
index aebe2650..dadd49c4 100644
--- a/azurelinuxagent/daemon/resourcedisk/default.py
+++ b/azurelinuxagent/daemon/resourcedisk/default.py
@@ -29,6 +29,7 @@ import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.shellutil as shellutil
 from azurelinuxagent.common.exception import ResourceDiskError
 from azurelinuxagent.common.osutil import get_osutil
+from azurelinuxagent.common.version import AGENT_NAME
 
 DATALOSS_WARNING_FILE_NAME = "DATALOSS_WARNING_README.txt"
 DATA_LOSS_WARNING = """\
@@ -74,7 +75,7 @@ class ResourceDiskHandler(object):
             return mount_point
         except ResourceDiskError as e:
             logger.error("Failed to mount resource disk {0}", e)
-            add_event(name="WALA", is_success=False, message=ustr(e),
+            add_event(name=AGENT_NAME, is_success=False, message=ustr(e),
                       op=WALAEventOperation.ActivateResourceDisk)
 
     def enable_swap(self, mount_point):
diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py
index 8d9478fb..735279ed 100644
--- a/azurelinuxagent/ga/update.py
+++ b/azurelinuxagent/ga/update.py
@@ -57,6 +57,7 @@ from azurelinuxagent.ga.exthandlers import HandlerManifest
 
 AGENT_ERROR_FILE = "error.json" # File name for agent error record
 AGENT_MANIFEST_FILE = "HandlerManifest.json"
+AGENT_SUPPORTED_FILE = "supported.json"
 
 CHILD_HEALTH_INTERVAL = 15 * 60
 CHILD_LAUNCH_INTERVAL = 5 * 60
@@ -64,9 +65,8 @@ CHILD_LAUNCH_RESTART_MAX = 3
 CHILD_POLL_INTERVAL = 60
 
 MAX_FAILURE = 3 # Max failure allowed for agent before blacklisted
-RETAIN_INTERVAL = 24 * 60 * 60 # Retain interval for black list
 
-GOAL_STATE_INTERVAL = 25
+GOAL_STATE_INTERVAL = 3
 REPORT_STATUS_INTERVAL = 15
 
 ORPHAN_WAIT_INTERVAL = 15 * 60 * 60
@@ -129,7 +129,7 @@ class UpdateHandler(object):
             agent_dir = latest_agent.get_agent_dir()
             agent_name = latest_agent.name
             agent_version = latest_agent.version
-
+        
         try:
 
             # Launch the correct Python version for python-based agents
@@ -243,7 +243,6 @@ class UpdateHandler(object):
             self._ensure_no_orphans()
             self._emit_restart_event()
 
-            # TODO: Add means to stop running
             while self.running:
                 if self._is_orphaned:
                     logger.info("Goal state agent {0} was orphaned -- exiting", CURRENT_AGENT)
@@ -267,7 +266,13 @@ class UpdateHandler(object):
                     op=WALAEventOperation.ProcessGoalState,
                     is_success=True,
                     duration=elapsed_milliseconds(utc_start))
-                
+
+                test_agent = self.get_test_agent()
+                if test_agent is not None and test_agent.in_slice:
+                    test_agent.enable()
+                    logger.info(u"Enabled Agent {0} as test agent", test_agent.name)
+                    break
+
                 time.sleep(GOAL_STATE_INTERVAL)
 
         except Exception as e:
@@ -317,13 +322,21 @@ class UpdateHandler(object):
         if not conf.get_autoupdate_enabled():
             return None
         
-        self._load_agents()
+        self._find_agents()
         available_agents = [agent for agent in self.agents
                             if agent.is_available
                             and agent.version > FlexibleVersion(AGENT_VERSION)]
 
         return available_agents[0] if len(available_agents) >= 1 else None
 
+    def get_test_agent(self):
+        agent = None
+        agents = [agent for agent in self._load_agents() if agent.is_test]
+        if len(agents) > 0:
+            agents.sort(key=lambda agent: agent.version, reverse=True)
+            agent = agents[0]
+        return agent
+
     def _emit_restart_event(self):
         if not self._is_clean_start:
             msg = u"{0} did not terminate cleanly".format(CURRENT_AGENT)
@@ -469,6 +482,17 @@ class UpdateHandler(object):
         self.agents = [agent for agent in self.agents if not agent.is_blacklisted]
         return
 
+    def _find_agents(self):
+        """
+        Load all non-blacklisted agents currently on disk.
+        """
+        try:
+            self._set_agents(self._load_agents())
+            self._filter_blacklisted_agents()
+        except Exception as e:
+            logger.warn(u"Exception occurred loading available agents: {0}", ustr(e))
+        return
+
     def _get_pid_files(self):
         pid_file = conf.get_agent_pid_file_path()
         
@@ -514,17 +538,9 @@ class UpdateHandler(object):
         return fileutil.read_file(conf.get_agent_pid_file_path()) != ustr(parent_pid)
 
     def _load_agents(self):
-        """
-        Load all non-blacklisted agents currently on disk.
-        """
-        try:
-            path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME))
-            self._set_agents([GuestAgent(path=agent_dir)
-                                for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)])
-            self._filter_blacklisted_agents()
-        except Exception as e:
-            logger.warn(u"Exception occurred loading available agents: {0}", ustr(e))
-        return
+        path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME))
+        return [GuestAgent(path=agent_dir)
+                        for agent_dir in glob.iglob(path) if os.path.isdir(agent_dir)]
 
     def _purge_agents(self):
         """
@@ -622,7 +638,11 @@ class GuestAgent(object):
         logger.verbose(u"Instantiating Agent {0} from {1}", self.name, location)
 
         self.error = None
+        self.supported = None
+
         self._load_error()
+        self._load_supported()
+
         self._ensure_downloaded()
         return
 
@@ -645,10 +665,18 @@ class GuestAgent(object):
     def get_agent_pkg_path(self):
         return ".".join((os.path.join(conf.get_lib_dir(), self.name), "zip"))
 
+    def get_agent_supported_file(self):
+        return os.path.join(conf.get_lib_dir(), self.name, AGENT_SUPPORTED_FILE)
+
     def clear_error(self):
         self.error.clear()
         return
 
+    def enable(self):
+        if self.error.is_sentinel:
+            self.error.clear()
+        return
+
     @property
     def is_available(self):
         return self.is_downloaded and not self.is_blacklisted
@@ -661,6 +689,14 @@ class GuestAgent(object):
     def is_downloaded(self):
         return self.is_blacklisted or os.path.isfile(self.get_agent_manifest_path())
 
+    @property
+    def is_test(self):
+        return self.error.is_sentinel and self.supported.is_supported
+
+    @property
+    def in_slice(self):
+        return self.is_test and self.supported.in_slice
+
     def mark_failure(self, is_fatal=False):
         try:
             if not os.path.isdir(self.get_agent_dir()):
@@ -694,6 +730,7 @@ class GuestAgent(object):
             self._unpack()
             self._load_manifest()
             self._load_error()
+            self._load_supported()
 
             msg = u"Agent {0} downloaded successfully".format(self.name)
             logger.verbose(msg)
@@ -782,6 +819,12 @@ class GuestAgent(object):
             logger.warn(u"Agent {0} failed loading error state: {1}", self.name, ustr(e))
         return
 
+    def _load_supported(self):
+        try:
+            self.supported = Supported(self.get_agent_supported_file())
+        except Exception as e:
+            self.supported = Supported()
+
     def _load_manifest(self):
         path = self.get_agent_manifest_path()
         if not os.path.isfile(path):
@@ -871,18 +914,15 @@ class GuestAgentError(object):
         self.failure_count = 0
         self.was_fatal = False
         return
-    
-    def clear_old_failure(self):
-        if self.last_failure <= 0.0:
-            return
-        if self.last_failure < (time.time() - RETAIN_INTERVAL):
-            self.clear()
-        return
 
     @property
     def is_blacklisted(self):
         return self.was_fatal or self.failure_count >= MAX_FAILURE
 
+    @property
+    def is_sentinel(self):
+        return self.was_fatal and self.last_failure == 0.0
+
     def load(self):
         if self.path is not None and os.path.isfile(self.path):
             with open(self.path, 'r') as f:
@@ -918,3 +958,61 @@ class GuestAgentError(object):
             self.last_failure,
             self.failure_count,
             self.was_fatal)
+
+class Supported(object):
+    def __init__(self, path):
+        if path is None:
+            raise UpdateError(u"Supported requires a path")
+        self.path = path
+
+        self._load()
+        return
+
+    @property
+    def is_supported(self):
+        return self._supported_distribution is not None
+
+    @property
+    def in_slice(self):
+        d = self._supported_distribution
+        return d is not None and d.in_slice
+
+    @property
+    def _supported_distribution(self):
+        for d in self.distributions:
+            dd = self.distributions[d]
+            if dd.is_supported:
+                return dd
+        return None
+
+    def _load(self):
+        self.distributions = {}
+        try:
+            if self.path is not None and os.path.isfile(self.path):
+                j = json.loads(fileutil.read_file(self.path))
+                for d in j:
+                    self.distributions[d] = SupportedDistribution(j[d])
+        except Exception as e:
+            logger.warn("Failed JSON parse of {0}: {1}".format(self.path, e))
+        return
+
+class SupportedDistribution(object):
+    def __init__(self, s):
+        if s is None or not isinstance(s, dict):
+            raise UpdateError(u"SupportedDisribution requires a dictionary")
+
+        self.slice = s['slice']
+        self.versions = s['versions']
+
+    @property
+    def is_supported(self):
+        d = ','.join(platform.linux_distribution())
+        for v in self.versions:
+            if re.match(v, d):
+                return True
+        return False
+
+    @property
+    def in_slice(self):
+        n = int((60 * self.slice) / 100)
+        return (n - datetime.utcnow().second) > 0
diff --git a/azurelinuxagent/pa/deprovision/default.py b/azurelinuxagent/pa/deprovision/default.py
index b5f5069f..35634365 100644
--- a/azurelinuxagent/pa/deprovision/default.py
+++ b/azurelinuxagent/pa/deprovision/default.py
@@ -100,7 +100,9 @@ class DeprovisionHandler(object):
         actions.append(DeprovisionAction(fileutil.rm_dirs, dirs_to_del))
 
     def cloud_init_directories(self):
-        return ["/var/lib/cloud"]
+        return ["/var/lib/cloud/instance",
+            "/var/lib/cloud/instances/",
+            "/var/lib/cloud/data"]
 
     def cloud_init_files(self):
         return ["/etc/sudoers.d/90-cloud-init-users"]
diff --git a/azurelinuxagent/pa/provision/cloudinit.py b/azurelinuxagent/pa/provision/cloudinit.py
index 69b0bc3f..b325f64d 100644
--- a/azurelinuxagent/pa/provision/cloudinit.py
+++ b/azurelinuxagent/pa/provision/cloudinit.py
@@ -20,10 +20,13 @@
 import os
 import time
 
+from datetime import datetime
+
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.fileutil as fileutil
 
+from azurelinuxagent.common.event import elapsed_milliseconds
 from azurelinuxagent.common.exception import ProvisionError, ProtocolError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.protocol import OVF_FILE_NAME
@@ -48,6 +51,7 @@ class CloudInitProvisionHandler(ProvisionHandler):
             logger.info("Provisioning already completed, skipping.")
             return
 
+        utc_start = datetime.utcnow()
         logger.info("Running CloudInit provisioning handler")
         self.wait_for_ovfenv()
         self.protocol_util.get_protocol()
@@ -63,7 +67,9 @@ class CloudInitProvisionHandler(ProvisionHandler):
             return
 
         self.report_ready(thumbprint)
-        self.report_event("Provision succeed", is_success=True)
+        self.report_event("Provision succeed",
+            is_success=True,
+            duration=elapsed_milliseconds(utc_start))
 
     def wait_for_ovfenv(self, max_retry=360, sleep_time=5):
         """
diff --git a/azurelinuxagent/pa/provision/default.py b/azurelinuxagent/pa/provision/default.py
index af27967c..36eab136 100644
--- a/azurelinuxagent/pa/provision/default.py
+++ b/azurelinuxagent/pa/provision/default.py
@@ -23,18 +23,22 @@ import os
 import os.path
 import re
 
+from datetime import datetime
+
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.shellutil as shellutil
 import azurelinuxagent.common.utils.fileutil as fileutil
 
 from azurelinuxagent.common.future import ustr
-from azurelinuxagent.common.event import add_event, WALAEventOperation
+from azurelinuxagent.common.event import add_event, WALAEventOperation, \
+    elapsed_milliseconds
 from azurelinuxagent.common.exception import ProvisionError, ProtocolError, \
     OSUtilError
 from azurelinuxagent.common.osutil import get_osutil
 from azurelinuxagent.common.protocol.restapi import ProvisionStatus
 from azurelinuxagent.common.protocol import get_protocol_util
+from azurelinuxagent.common.version import AGENT_NAME
 
 CUSTOM_DATA_FILE = "CustomData"
 CLOUD_INIT_PATTERN = b".*/bin/cloud-init.*"
@@ -53,6 +57,7 @@ class ProvisionHandler(object):
             logger.info("Provisioning already completed, skipping.")
             return
 
+        utc_start = datetime.utcnow()
         thumbprint = None
         # If provision is not enabled, report ready and then return
         if not conf.get_provision_enabled():
@@ -71,12 +76,15 @@ class ProvisionHandler(object):
                 self.provision(ovf_env)
                 thumbprint = self.reg_ssh_host_key()
                 self.osutil.restart_ssh_service()
-                self.report_event("Provision succeed", is_success=True)
+                self.report_event("Provision succeed",
+                    is_success=True,
+                    duration=elapsed_milliseconds(utc_start))
             except (ProtocolError, ProvisionError) as e:
                 self.report_not_ready("ProvisioningFailed", ustr(e))
                 self.report_event(ustr(e))
                 logger.error("Provisioning failed: {0}", ustr(e))
                 return
+
         # write out provisioned file and report Ready
         fileutil.write_file(provisioned, "")
         self.report_ready(thumbprint)
@@ -189,9 +197,12 @@ class ProvisionHandler(object):
             logger.info("Deploy ssh key pairs.")
             self.osutil.deploy_ssh_keypair(ovfenv.username, keypair)
 
-    def report_event(self, message, is_success=False):
-        add_event(name="WALA", message=message, is_success=is_success,
-                  op=WALAEventOperation.Provision)
+    def report_event(self, message, is_success=False, duration=0):
+        add_event(name=AGENT_NAME,
+                    message=message,
+                    duration=duration,
+                    is_success=is_success,
+                    op=WALAEventOperation.Provision)
 
     def report_not_ready(self, sub_status, description):
         status = ProvisionStatus(status="NotReady", subStatus=sub_status,
diff --git a/config/alpine/waagent.conf b/config/alpine/waagent.conf
index 95004c37..2e3f6a5f 100644
--- a/config/alpine/waagent.conf
+++ b/config/alpine/waagent.conf
@@ -60,7 +60,7 @@ Logs.Verbose=n
 Network.Interface=eth0
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
diff --git a/config/arch/waagent.conf b/config/arch/waagent.conf
index 88ee0f8b..686b90c3 100644
--- a/config/arch/waagent.conf
+++ b/config/arch/waagent.conf
@@ -61,7 +61,7 @@ LBProbeResponder=y
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
diff --git a/config/bigip/waagent.conf b/config/bigip/waagent.conf
index 65271e7d..a6a380b7 100644
--- a/config/bigip/waagent.conf
+++ b/config/bigip/waagent.conf
@@ -55,7 +55,7 @@ LBProbeResponder=y
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
diff --git a/config/clearlinux/waagent.conf b/config/clearlinux/waagent.conf
index 87d3f540..6606cd73 100644
--- a/config/clearlinux/waagent.conf
+++ b/config/clearlinux/waagent.conf
@@ -60,7 +60,7 @@ ResourceDisk.SwapSizeMB=0
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
diff --git a/config/coreos/waagent.conf b/config/coreos/waagent.conf
index 2c73e395..664d037f 100644
--- a/config/coreos/waagent.conf
+++ b/config/coreos/waagent.conf
@@ -61,7 +61,7 @@ LBProbeResponder=y
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Set the path to SSH keys and configuration files
 OS.SshDir=/etc/ssh
diff --git a/config/freebsd/waagent.conf b/config/freebsd/waagent.conf
index 60772939..5149573e 100644
--- a/config/freebsd/waagent.conf
+++ b/config/freebsd/waagent.conf
@@ -55,7 +55,7 @@ ResourceDisk.MountOptions=None
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
diff --git a/config/suse/waagent.conf b/config/suse/waagent.conf
index 813990ae..b2e90a88 100644
--- a/config/suse/waagent.conf
+++ b/config/suse/waagent.conf
@@ -61,7 +61,7 @@ LBProbeResponder=y
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
diff --git a/config/ubuntu/waagent.conf b/config/ubuntu/waagent.conf
index 0c92daa2..734a4035 100644
--- a/config/ubuntu/waagent.conf
+++ b/config/ubuntu/waagent.conf
@@ -61,7 +61,7 @@ LBProbeResponder=y
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
diff --git a/config/waagent.conf b/config/waagent.conf
index bf853511..b1b1ba3f 100644
--- a/config/waagent.conf
+++ b/config/waagent.conf
@@ -58,7 +58,7 @@ ResourceDisk.MountOptions=None
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=n
+OS.EnableFIPS=n
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300


Test patch:
diff --git a/tests/data/ga/supported.json b/tests/data/ga/supported.json
new file mode 100644
index 00000000..2ae37533
--- /dev/null
+++ b/tests/data/ga/supported.json
@@ -0,0 +1,8 @@
+{
+    "ubuntu.16.10-x64": {
+        "versions": [
+            "^Ubuntu,16.10,yakkety$"
+        ],
+        "slice": 10
+    }
+}
diff --git a/tests/data/test_waagent.conf b/tests/data/test_waagent.conf
index 30f94b0c..fe5d4a7a 100644
--- a/tests/data/test_waagent.conf
+++ b/tests/data/test_waagent.conf
@@ -58,7 +58,7 @@ ResourceDisk.MountOptions=None
 Logs.Verbose=n
 
 # Is FIPS enabled
-OS.FIPSEnabled=y
+OS.EnableFIPS=y
 
 # Root device timeout in seconds.
 OS.RootDeviceScsiTimeout=300
diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py
index e7a7af4e..cc9e7a89 100644
--- a/tests/ga/test_update.py
+++ b/tests/ga/test_update.py
@@ -17,9 +17,16 @@
 
 from __future__ import print_function
 
+from datetime import datetime
+
+import json
+import shutil
+
 from azurelinuxagent.common.protocol.hostplugin import *
 from azurelinuxagent.common.protocol.wire import *
+from azurelinuxagent.common.utils.fileutil import *
 from azurelinuxagent.ga.update import *
+
 from tests.tools import *
 
 NO_ERROR = {
@@ -28,6 +35,18 @@ NO_ERROR = {
     "was_fatal" : False
 }
 
+FATAL_ERROR = {
+    "last_failure" : 42.42,
+    "failure_count" : 2,
+    "was_fatal" : True
+}
+
+SENTINEL_ERROR = {
+    "last_failure" : 0.0,
+    "failure_count" : 0,
+    "was_fatal" : True
+}
+
 WITH_ERROR = {
     "last_failure" : 42.42,
     "failure_count" : 2,
@@ -137,19 +156,25 @@ class UpdateTestCase(AgentTestCase):
             fileutil.copy_file(agent, to_dir=self.tmp_dir)
         return
 
-    def expand_agents(self):
+    def expand_agents(self, mark_test=False):
         for agent in self.agent_pkgs():
-            zipfile.ZipFile(agent).extractall(os.path.join(
-                self.tmp_dir,
-                fileutil.trim_ext(agent, "zip")))
+            path = os.path.join(self.tmp_dir, fileutil.trim_ext(agent, "zip"))
+            zipfile.ZipFile(agent).extractall(path)
+            if mark_test:
+                src = os.path.join(data_dir, 'ga', 'supported.json')
+                dst = os.path.join(path, 'supported.json')
+                shutil.copy(src, dst)
+
+                dst = os.path.join(path, 'error.json')
+                fileutil.write_file(dst, json.dumps(SENTINEL_ERROR))
         return
 
-    def prepare_agent(self, version):
+    def prepare_agent(self, version, mark_test=False):
         """
         Create a download for the current agent version, copied from test data
         """
         self.copy_agents(get_agent_pkgs()[0])
-        self.expand_agents()
+        self.expand_agents(mark_test=mark_test)
 
         versions = self.agent_versions()
         src_v = FlexibleVersion(str(versions[0]))
@@ -214,6 +239,64 @@ class UpdateTestCase(AgentTestCase):
         return dst_v
 
 
+class TestSupportedDistribution(UpdateTestCase):
+    def setUp(self):
+        UpdateTestCase.setUp(self)
+        self.sd = SupportedDistribution({
+                    'slice':10,
+                    'versions': ['^Ubuntu,16.10,yakkety$']})
+        
+
+    def test_creation(self):
+        self.assertRaises(TypeError, SupportedDistribution)
+        self.assertRaises(UpdateError, SupportedDistribution, None)
+
+        self.assertEqual(self.sd.slice, 10)
+        self.assertEqual(self.sd.versions, ['^Ubuntu,16.10,yakkety$'])
+
+    @patch('platform.linux_distribution')
+    def test_is_supported(self, mock_dist):
+        mock_dist.return_value = ['Ubuntu', '16.10', 'yakkety']
+        self.assertTrue(self.sd.is_supported)
+
+        mock_dist.return_value = ['something', 'else', 'entirely']
+        self.assertFalse(self.sd.is_supported)
+
+    @patch('azurelinuxagent.ga.update.datetime')
+    def test_in_slice(self, mock_dt):
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5))
+        self.assertTrue(self.sd.in_slice)
+
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 42))
+        self.assertFalse(self.sd.in_slice)
+
+
+class TestSupported(UpdateTestCase):
+    def setUp(self):
+        UpdateTestCase.setUp(self)
+        self.sp = Supported(os.path.join(data_dir, 'ga', 'supported.json'))
+
+    def test_creation(self):
+        self.assertRaises(TypeError, Supported)
+        self.assertRaises(UpdateError, Supported, None)
+
+    @patch('platform.linux_distribution')
+    def test_is_supported(self, mock_dist):
+        mock_dist.return_value = ['Ubuntu', '16.10', 'yakkety']
+        self.assertTrue(self.sp.is_supported)
+
+        mock_dist.return_value = ['something', 'else', 'entirely']
+        self.assertFalse(self.sp.is_supported)
+
+    @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety'])
+    @patch('azurelinuxagent.ga.update.datetime')
+    def test_in_slice(self, mock_dt, mock_dist):
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5))
+        self.assertTrue(self.sp.in_slice)
+
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 42))
+        self.assertFalse(self.sp.in_slice)
+
 class TestGuestAgentError(UpdateTestCase):
     def test_creation(self):
         self.assertRaises(TypeError, GuestAgentError)
@@ -241,6 +324,17 @@ class TestGuestAgentError(UpdateTestCase):
         self.assertEqual(NO_ERROR["was_fatal"], err.was_fatal)
         return
 
+    def test_is_sentinel(self):
+        with self.get_error_file(error_data=SENTINEL_ERROR) as path:
+            err = GuestAgentError(path.name)
+            self.assertTrue(err.is_blacklisted)
+            self.assertTrue(err.is_sentinel)
+        
+        with self.get_error_file(error_data=FATAL_ERROR) as path:
+            err = GuestAgentError(path.name)
+            self.assertTrue(err.is_blacklisted)
+            self.assertFalse(err.is_sentinel)
+
     def test_load_preserves_error_state(self):
         with self.get_error_file(error_data=WITH_ERROR) as path:
             err = GuestAgentError(path.name)
@@ -274,15 +368,6 @@ class TestGuestAgentError(UpdateTestCase):
         # Agent failed >= MAX_FAILURE, it should be blacklisted
         self.assertTrue(err.is_blacklisted)
         self.assertEqual(MAX_FAILURE, err.failure_count)
-
-        # Clear old failure does not clear recent failure
-        err.clear_old_failure()
-        self.assertTrue(err.is_blacklisted)
-
-        # Clear does remove old, outdated failures
-        err.last_failure -= RETAIN_INTERVAL * 2
-        err.clear_old_failure()
-        self.assertFalse(err.is_blacklisted)
         return
 
     def test_mark_failure_permanent(self):
@@ -334,6 +419,9 @@ class TestGuestAgent(UpdateTestCase):
         self.assertEqual(get_agent_name(), agent.name)
         self.assertEqual(get_agent_version(), agent.version)
 
+        self.assertFalse(agent.is_test)
+        self.assertFalse(agent.in_slice)
+
         self.assertEqual(self.agent_path, agent.get_agent_dir())
 
         path = os.path.join(self.agent_path, AGENT_MANIFEST_FILE)
@@ -403,6 +491,43 @@ class TestGuestAgent(UpdateTestCase):
         self.assertTrue(agent.is_downloaded)
         return
 
+    @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety'])
+    def test_is_test(self, mock_dist):
+        self.expand_agents(mark_test=True)
+        agent = GuestAgent(path=self.agent_path)
+
+        self.assertTrue(agent.is_blacklisted)
+        self.assertTrue(agent.is_test)
+
+    @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety'])
+    @patch('azurelinuxagent.ga.update.datetime')
+    def test_in_slice(self, mock_dt, mock_dist):
+        self.expand_agents(mark_test=True)
+        agent = GuestAgent(path=self.agent_path)
+
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5))
+        self.assertTrue(agent.in_slice)
+
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 42))
+        self.assertFalse(agent.in_slice)
+
+    @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety'])
+    @patch('azurelinuxagent.ga.update.datetime')
+    def test_enable(self, mock_dt, mock_dist):
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5))
+
+        self.expand_agents(mark_test=True)
+        agent = GuestAgent(path=self.agent_path)
+
+        self.assertTrue(agent.is_blacklisted)
+        self.assertTrue(agent.is_test)
+        self.assertTrue(agent.in_slice)
+
+        agent.enable()
+
+        self.assertFalse(agent.is_blacklisted)
+        self.assertFalse(agent.is_test)
+
     @patch("azurelinuxagent.ga.update.GuestAgent._ensure_downloaded")
     def test_mark_failure(self, mock_ensure):
         agent = GuestAgent(path=self.agent_path)
@@ -933,6 +1058,14 @@ class TestUpdate(UpdateTestCase):
         self.assertEqual("1250_waagent.pid", os.path.basename(pid_file))
         return
 
+    @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety'])
+    @patch('azurelinuxagent.ga.update.datetime')
+    def test_get_test_agent(self, mock_dt, mock_dist):
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5))
+        self.prepare_agent(AGENT_VERSION, mark_test=True)
+
+        self.assertNotEqual(None, self.update_handler.get_test_agent())
+
     def test_is_clean_start_returns_true_when_no_sentinal(self):
         self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path()))
         self.assertTrue(self.update_handler._is_clean_start)
@@ -972,27 +1105,27 @@ class TestUpdate(UpdateTestCase):
             self.assertTrue(self.update_handler._is_orphaned)
         return
 
-    def test_load_agents(self):
+    def test_find_agents(self):
         self.prepare_agents()
 
         self.assertTrue(0 <= len(self.update_handler.agents))
-        self.update_handler._load_agents()
+        self.update_handler._find_agents()
         self.assertEqual(len(get_agents(self.tmp_dir)), len(self.update_handler.agents))
         return
 
-    def test_load_agents_does_reload(self):
+    def test_find_agents_does_reload(self):
         self.prepare_agents()
 
-        self.update_handler._load_agents()
+        self.update_handler._find_agents()
         agents = self.update_handler.agents
 
-        self.update_handler._load_agents()
+        self.update_handler._find_agents()
         self.assertNotEqual(agents, self.update_handler.agents)
         return
 
-    def test_load_agents_sorts(self):
+    def test_find_agents_sorts(self):
         self.prepare_agents()
-        self.update_handler._load_agents()
+        self.update_handler._find_agents()
 
         v = FlexibleVersion("100000")
         for a in self.update_handler.agents:
@@ -1002,7 +1135,7 @@ class TestUpdate(UpdateTestCase):
 
     def test_purge_agents(self):
         self.prepare_agents()
-        self.update_handler._load_agents()
+        self.update_handler._find_agents()
 
         # Ensure at least three agents initially exist
         self.assertTrue(2 < len(self.update_handler.agents))
@@ -1014,7 +1147,7 @@ class TestUpdate(UpdateTestCase):
         # Reload and assert only the kept agents remain on disk
         self.update_handler.agents = kept_agents
         self.update_handler._purge_agents()
-        self.update_handler._load_agents()
+        self.update_handler._find_agents()
         self.assertEqual(
             [agent.version for agent in kept_agents],
             [agent.version for agent in self.update_handler.agents])
@@ -1198,8 +1331,6 @@ class TestUpdate(UpdateTestCase):
                 self.update_handler.running = False
             return
 
-        calls = calls * invocations
-
         fileutil.write_file(conf.get_agent_pid_file_path(), ustr(42))
 
         with patch('azurelinuxagent.ga.exthandlers.get_exthandlers_handler') as mock_handler:
@@ -1227,13 +1358,30 @@ class TestUpdate(UpdateTestCase):
         return
 
     def test_run_keeps_running(self):
-        self._test_run(invocations=15)
+        self._test_run(invocations=15, calls=[call.run()]*15)
         return
 
     def test_run_stops_if_update_available(self):
         self.update_handler._upgrade_available = Mock(return_value=True)
         self._test_run(invocations=0, calls=[], enable_updates=True)
         return
+    
+    @patch('platform.linux_distribution', return_value=['Ubuntu', '16.10', 'yakkety'])
+    @patch('azurelinuxagent.ga.update.datetime')
+    def test_run_stops_if_test_agent_available(self, mock_dt, mock_dist):
+        mock_dt.utcnow = Mock(return_value=datetime(2017, 1, 1, 0, 0, 5))
+        self.prepare_agent(AGENT_VERSION, mark_test=True)
+
+        agent = GuestAgent(path=self.agent_dir(AGENT_VERSION))
+        agent.enable = Mock()
+        self.assertTrue(agent.is_test)
+        self.assertTrue(agent.in_slice)
+
+        with patch('azurelinuxagent.ga.update.UpdateHandler.get_test_agent',
+                return_value=agent) as mock_test:
+            self._test_run(invocations=0)
+            self.assertEqual(mock_test.call_count, 1)
+            self.assertEqual(agent.enable.call_count, 1)
 
     def test_run_stops_if_orphaned(self):
         with patch('os.getppid', return_value=1):
@@ -1417,6 +1565,5 @@ class TimeMock(Mock):
         self.next_time += self.time_increment
         return current_time
 
-
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/utils/test_file_util.py b/tests/utils/test_file_util.py
index 76fb15ba..0b92513c 100644
--- a/tests/utils/test_file_util.py
+++ b/tests/utils/test_file_util.py
@@ -15,9 +15,14 @@
 # Requires Python 2.4+ and Openssl 1.0+
 #
 
+import glob
+import random
+import string
+import tempfile
 import uuid
 
 import azurelinuxagent.common.utils.fileutil as fileutil
+
 from azurelinuxagent.common.future import ustr
 from tests.tools import *
 
@@ -69,9 +74,6 @@ class TestFileOperations(AgentTestCase):
         self.assertEquals('abc', filename)
 
     def test_remove_files(self):
-        import random
-        import string
-        import glob
         random_word = lambda : ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
 
         #Create 10 test files
@@ -90,9 +92,27 @@ class TestFileOperations(AgentTestCase):
         self.assertEqual(0, len(glob.glob(os.path.join(self.tmp_dir, test_file_pattern))))
         self.assertEqual(0, len(glob.glob(os.path.join(self.tmp_dir, test_file_pattern2))))
 
+    def test_remove_dirs(self):
+        dirs = []
+        for n in range(0,5):
+            dirs.append(tempfile.mkdtemp())
+        for d in dirs:
+            for n in range(0, random.choice(range(0,10))):
+                fileutil.write_file(os.path.join(d, "test"+str(n)), "content")
+            for n in range(0, random.choice(range(0,10))):
+                dd = os.path.join(d, "testd"+str(n))
+                os.mkdir(dd)
+                for nn in range(0, random.choice(range(0,10))):
+                    os.symlink(dd, os.path.join(dd, "sym"+str(nn)))
+            for n in range(0, random.choice(range(0,10))):
+                os.symlink(d, os.path.join(d, "sym"+str(n)))
+
+        fileutil.rm_dirs(*dirs)
+
+        for d in dirs:
+            self.assertEqual(len(os.listdir(d)), 0)
+
     def test_get_all_files(self):
-        import random
-        import string
         random_word = lambda: ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
 
         # Create 10 test files at the root dir and 10 other in the sub dir