Repository: Azure/WALinuxAgent
Pull request #1365 (2018-10-16 00:24:57)

Add support for provisioning SRIOV Infiniband VMs
<!-- DO NOT DELETE THIS TEMPLATE -->

## Description

Issue # <!-- if any -->
<!--
Please add an informative description that covers that changes made by the pull request. 
This checklist is used to make sure that common issues in a pull request are addressed.
This will expedite the process of getting your pull request merged and avoid extra work on your part to fix issues discovered during the review process.
-->

---

### PR information
- [ ] The title of the PR is clear and informative.
- [ ] There are a small number of commits, each of which has an informative message. This means that previously merged commits do not appear in the history of the PR. For information on cleaning up the commits in your pull request, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).
- [ ] Except for special cases involving multiple contributors, the PR is started from a fork of the main repository, not a branch.
- [ ] If applicable, the PR references the bug/issue that it fixes in the description.
- [ ] New Unit tests were added for the changes made and Travis.CI is passing.

### Quality of Code and Contribution Guidelines
- [ ] I have read the [contribution guidelines](https://github.com/Azure/WALinuxAgent/blob/master/.github/CONTRIBUTING.md).


Patch:
diff --git a/README.md b/README.md
index 8fe747ed..ccb3074f 100644
--- a/README.md
+++ b/README.md
@@ -195,6 +195,7 @@ ResourceDisk.Filesystem=ext4
 ResourceDisk.MountPoint=/mnt/resource
 ResourceDisk.MountOptions=None
 ResourceDisk.EnableSwap=n
+ResourceDisk.EnableSwapEncryption=n
 ResourceDisk.SwapSizeMB=0
 Logs.Verbose=n
 OS.AllowHTTP=n
@@ -370,6 +371,13 @@ _Default: n_
 If set, a swap file (/swapfile) is created on the resource disk and added to the
 system swap space.
 
+#### __ResourceDisk.EnableSwapEncryption__
+
+_Type: Boolean_  
+_Default: n_
+
+If set, the swap file (/swapfile) is mounted as an encrypted filesystem.
+
 #### __ResourceDisk.SwapSizeMB__
 
 _Type: Integer_  
diff --git a/__main__.py b/__main__.py
deleted file mode 100644
index 087b4080..00000000
--- a/__main__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright 2018 Microsoft Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Requires Python 2.6+ and Openssl 1.0+
-#
-
-import azurelinuxagent.agent as agent
-
-agent.main()
diff --git a/azurelinuxagent/common/cgroups.py b/azurelinuxagent/common/cgroups.py
index 91e956ff..fdd7bc15 100644
--- a/azurelinuxagent/common/cgroups.py
+++ b/azurelinuxagent/common/cgroups.py
@@ -183,6 +183,7 @@ class CGroupsTelemetry(object):
         "memory": Memory
     }
     _hierarchies = list(_metrics.keys())
+
     tracked_names = set()
 
     @staticmethod
@@ -278,10 +279,13 @@ class CGroupsTelemetry(object):
         :rtype: dict(str: [(str, str, float)])
         """
         results = {}
+        limits = {}
+
         for cgroup_name, collector in CGroupsTelemetry._tracked.copy().items():
             cgroup_name = cgroup_name if cgroup_name else WRAPPER_CGROUP_NAME
             results[cgroup_name] = collector.collect()
-        return results
+            limits[cgroup_name] = collector.cgroup.threshold
+        return results, limits
 
     @staticmethod
     def update_tracked(ext_handlers):
@@ -400,6 +404,7 @@ class CGroups(object):
             self.is_wrapper_cgroup = False
 
         self.cgroups = {}
+        self.threshold = None
 
         if not self.enabled():
             return
@@ -476,6 +481,22 @@ class CGroups(object):
             tasks_file = self._get_cgroup_file(hierarchy, 'cgroup.procs')
             fileutil.append_file(tasks_file, "{0}\n".format(pid))
 
+    def get_cpu_limits(self):
+        # default values
+        cpu_limit = DEFAULT_CPU_LIMIT_AGENT if AGENT_NAME.lower() in self.name.lower() else DEFAULT_CPU_LIMIT_EXT
+
+        return cpu_limit
+
+    def get_memory_limits(self):
+        # default values
+        mem_limit = max(DEFAULT_MEM_LIMIT_MIN_MB, round(self._osutil.get_total_mem() * DEFAULT_MEM_LIMIT_PCT / 100, 0))
+
+        # agent values
+        if AGENT_NAME.lower() in self.name.lower():
+            mem_limit = min(DEFAULT_MEM_LIMIT_MAX_MB, mem_limit)
+
+        return mem_limit
+
     def set_limits(self):
         """
         Set per-hierarchy limits based on the cgroup name (agent or particular extension)
@@ -493,13 +514,8 @@ class CGroups(object):
                 return
 
         # default values
-        cpu_limit = DEFAULT_CPU_LIMIT_EXT
-        mem_limit = max(DEFAULT_MEM_LIMIT_MIN_MB, round(self._osutil.get_total_mem() * DEFAULT_MEM_LIMIT_PCT / 100, 0))
-
-        # agent values
-        if AGENT_NAME.lower() in self.name.lower():
-            cpu_limit = DEFAULT_CPU_LIMIT_AGENT
-            mem_limit = min(DEFAULT_MEM_LIMIT_MAX_MB, mem_limit)
+        cpu_limit = self.get_cpu_limits()
+        mem_limit = self.get_memory_limits()
 
         msg = '{0}: {1}% {2}mb'.format(self.name, cpu_limit, mem_limit)
         logger.info("Setting cgroups limits for {0}".format(msg))
@@ -522,6 +538,9 @@ class CGroups(object):
                 message=msg,
                 log_event=False)
 
+        # Returning the limits -
+        self.threshold = {"cpu": cpu_limit, "memory": mem_limit}
+
     @staticmethod
     def _apply_wrapper_limits(path, hierarchy):
         """
diff --git a/azurelinuxagent/common/conf.py b/azurelinuxagent/common/conf.py
index eb3900f3..5b5f783d 100644
--- a/azurelinuxagent/common/conf.py
+++ b/azurelinuxagent/common/conf.py
@@ -108,6 +108,7 @@ __SWITCH_OPTIONS__ = {
     "DetectScvmmEnv": False,
     "ResourceDisk.Format": False,
     "ResourceDisk.EnableSwap": False,
+    "ResourceDisk.EnableSwapEncryption": False,
     "AutoUpdate.Enabled": True,
     "EnableOverProvisioning": True,
     "CGroups.EnforceLimits": False,
@@ -324,7 +325,9 @@ def get_resourcedisk_format(conf=__conf__):
 
 def get_resourcedisk_enable_swap(conf=__conf__):
     return conf.get_switch("ResourceDisk.EnableSwap", False)
-
+    
+def get_resourcedisk_enable_swap_encryption(conf=__conf__):
+    return conf.get_switch("ResourceDisk.EnableSwapEncryption", False)
 
 def get_resourcedisk_mountpoint(conf=__conf__):
     return conf.get("ResourceDisk.MountPoint", "/mnt/resource")
diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py
index f155a08b..9e06c0ca 100644
--- a/azurelinuxagent/common/event.py
+++ b/azurelinuxagent/common/event.py
@@ -46,6 +46,7 @@ class WALAEventOperation:
     ArtifactsProfileBlob = "ArtifactsProfileBlob"
     AutoUpdate = "AutoUpdate"
     CustomData = "CustomData"
+    CGroupsLimitsCrossed = "CGroupsLimitsCrossed"
     Deploy = "Deploy"
     Disable = "Disable"
     Downgrade = "Downgrade"
diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py
index a10e9aa0..f33e5ad8 100644
--- a/azurelinuxagent/common/osutil/default.py
+++ b/azurelinuxagent/common/osutil/default.py
@@ -521,6 +521,8 @@ class DefaultOSUtil(object):
         if value is not None:
             if not value.startswith("ssh-"):
                 raise OSUtilError("Bad public key: {0}".format(value))
+            if not value.endswith("\n"):
+                value += "\n"
             fileutil.write_file(path, value)
         elif thumbprint is not None:
             lib_dir = conf.get_lib_dir()
diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py
index b63e8b11..bf3294d3 100644
--- a/azurelinuxagent/common/osutil/factory.py
+++ b/azurelinuxagent/common/osutil/factory.py
@@ -46,8 +46,7 @@ def get_osutil(distro_name=DISTRO_NAME,
     if distro_name == "arch":
         return ArchUtil()
 
-    if distro_name == "clear linux os for intel architecture" \
-            or distro_name == "clear linux software for intel architecture":
+    if "Clear Linux" in distro_full_name:
         return ClearLinuxUtil()
 
     if distro_name == "ubuntu":
diff --git a/azurelinuxagent/common/osutil/redhat.py b/azurelinuxagent/common/osutil/redhat.py
index 2f0ca576..a33810f8 100644
--- a/azurelinuxagent/common/osutil/redhat.py
+++ b/azurelinuxagent/common/osutil/redhat.py
@@ -68,7 +68,7 @@ class Redhat6xOSUtil(DefaultOSUtil):
             ssh_rsa_pubkey = cryptutil.asn1_to_ssh(pubkey)
         except CryptError as e:
             raise OSUtilError(ustr(e))
-        fileutil.write_file(output_file, ssh_rsa_pubkey)
+        fileutil.append_file(output_file, ssh_rsa_pubkey)
 
     # Override
     def get_dhcp_pid(self):
diff --git a/azurelinuxagent/common/protocol/restapi.py b/azurelinuxagent/common/protocol/restapi.py
index 5497e764..08b25c65 100644
--- a/azurelinuxagent/common/protocol/restapi.py
+++ b/azurelinuxagent/common/protocol/restapi.py
@@ -151,18 +151,19 @@ class Extension(DataContract):
                  sequenceNumber=None,
                  publicSettings=None,
                  protectedSettings=None,
-                 certificateThumbprint=None):
+                 certificateThumbprint=None,
+                 dependencyLevel=0):
         self.name = name
         self.sequenceNumber = sequenceNumber
         self.publicSettings = publicSettings
         self.protectedSettings = protectedSettings
         self.certificateThumbprint = certificateThumbprint
+        self.dependencyLevel = dependencyLevel
 
 
 class ExtHandlerProperties(DataContract):
     def __init__(self):
         self.version = None
-        self.dependencyLevel = None
         self.state = None
         self.extensions = DataContractList(Extension)
 
@@ -179,9 +180,11 @@ class ExtHandler(DataContract):
         self.versionUris = DataContractList(ExtHandlerVersionUri)
 
     def sort_key(self):
-        level = self.properties.dependencyLevel
-        if level is None:
+        levels = [e.dependencyLevel for e in self.properties.extensions]
+        if len(levels) == 0:
             level = 0
+        else:
+            level = min(levels)
         # Process uninstall or disabled before enabled, in reverse order
         # remap 0 to -1, 1 to -2, 2 to -3, etc
         if self.properties.state != u"enabled":
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index 752f62c6..de39fa37 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -1591,11 +1591,6 @@ class ExtensionsConfig(object):
         ext_handler.properties.version = getattrib(plugin, "version")
         ext_handler.properties.state = getattrib(plugin, "state")
 
-        try:
-            ext_handler.properties.dependencyLevel = int(getattrib(plugin, "dependencyLevel"))
-        except ValueError:
-            ext_handler.properties.dependencyLevel = 0
-
         location = getattrib(plugin, "location")
         failover_location = getattrib(plugin, "failoverlocation")
         for uri in [location, failover_location]:
@@ -1627,6 +1622,15 @@ class ExtensionsConfig(object):
             logger.error("Invalid extension settings")
             return
 
+        depends_on_level = 0
+        depends_on_node = find(settings[0], "DependsOn")
+        if depends_on_node != None:
+            try:
+                depends_on_level = int(getattrib(depends_on_node, "dependencyLevel"))
+            except (ValueError, TypeError):
+                logger.warn("Could not parse dependencyLevel for handler {0}. Setting it to 0".format(name))
+                depends_on_level = 0
+
         for plugin_settings_list in runtime_settings["runtimeSettings"]:
             handler_settings = plugin_settings_list["handlerSettings"]
             ext = Extension()
@@ -1636,6 +1640,7 @@ class ExtensionsConfig(object):
             ext.sequenceNumber = seqNo
             ext.publicSettings = handler_settings.get("publicSettings")
             ext.protectedSettings = handler_settings.get("protectedSettings")
+            ext.dependencyLevel = depends_on_level
             thumbprint = handler_settings.get(
                 "protectedSettingsCertThumbprint")
             ext.certificateThumbprint = thumbprint
diff --git a/azurelinuxagent/common/rdma.py b/azurelinuxagent/common/rdma.py
index 5504a068..9fc6b1a5 100644
--- a/azurelinuxagent/common/rdma.py
+++ b/azurelinuxagent/common/rdma.py
@@ -39,7 +39,7 @@ dapl_config_paths = [
     '/usr/local/etc/dat.conf'
 ]
 
-def setup_rdma_device():
+def setup_rdma_device(nd_version):
     logger.verbose("Parsing SharedConfig XML contents for RDMA details")
     xml_doc = parse_doc(
         fileutil.read_file(os.path.join(conf.get_lib_dir(), SHARED_CONF_FILE_NAME)))
@@ -71,19 +71,22 @@ def setup_rdma_device():
         rdma_ipv4_addr, rdma_mac_addr))
 
     # Set up the RDMA device with collected informatino
-    RDMADeviceHandler(rdma_ipv4_addr, rdma_mac_addr).start()
+    RDMADeviceHandler(rdma_ipv4_addr, rdma_mac_addr, nd_version).start()
     logger.info("RDMA: device is set up")
     return
 
 class RDMAHandler(object):
 
     driver_module_name = 'hv_network_direct'
+    nd_version = None
 
-    @staticmethod
-    def get_rdma_version():
+    def get_rdma_version(self):
         """Retrieve the firmware version information from the system.
            This depends on information provided by the Linux kernel."""
 
+        if self.nd_version :
+            return self.nd_version
+
         kvp_key_size = 512
         kvp_value_size = 2048
         driver_info_source = '/var/lib/hyperv/.kvp_pool_0'
@@ -104,7 +107,8 @@ class RDMAHandler(object):
                 value_0 = value.split("\x00")[0]
                 if key_0 == "NdDriverVersion" :
                     f.close()
-                    return value_0
+                    self.nd_version = value_0
+                    return self.nd_version
             else :
                 break
         f.close()
@@ -148,6 +152,12 @@ class RDMAHandler(object):
         logger.info('RDMA: Loaded the kernel driver successfully.')
         return True
 
+    def install_driver_if_needed(self):
+        if self.nd_version:
+            self.install_driver()
+        else:
+            logger.info('RDMA: skip installing driver when ndversion not present\n')
+
     def install_driver(self):
         """Install the driver. This is distribution specific and must
            be overwritten in the child implementation."""
@@ -184,15 +194,20 @@ class RDMADeviceHandler(object):
     """
 
     rdma_dev = '/dev/hvnd_rdma'
+    sriov_dir = '/sys/class/infiniband'
     device_check_timeout_sec = 120
     device_check_interval_sec = 1
+    ipoib_check_timeout_sec = 60
+    ipoib_check_interval_sec = 1
 
     ipv4_addr = None
     mac_adr = None
+    nd_version = None
 
-    def __init__(self, ipv4_addr, mac_addr):
+    def __init__(self, ipv4_addr, mac_addr, nd_version):
         self.ipv4_addr = ipv4_addr
         self.mac_addr = mac_addr
+        self.nd_version = nd_version
 
     def start(self):
         """
@@ -203,42 +218,82 @@ class RDMADeviceHandler(object):
 
     def process(self):
         try:
-            RDMADeviceHandler.update_dat_conf(dapl_config_paths, self.ipv4_addr)
+            if not self.nd_version :
+                logger.info("RDMA: provisioning SRIOV RDMA device.")
+                self.provision_sriov_rdma()
+            else :
+                logger.info("RDMA: provisioning Network Direct RDMA device.")
+                self.provision_network_direct_rdma()
+        except Exception as e:
+            logger.error("RDMA: device processing failed: {0}".format(e))
 
-            skip_rdma_device = False
-            module_name = "hv_network_direct"
-            retcode,out = shellutil.run_get_output("modprobe -R %s" % module_name, chk_err=False)
-            if retcode == 0:
-                module_name = out.strip()
-            else:
-                logger.info("RDMA: failed to resolve module name. Use original name")
-            retcode,out = shellutil.run_get_output("modprobe %s" % module_name)
-            if retcode != 0:
-                logger.error("RDMA: failed to load module %s" % module_name)
-                return
-            retcode,out = shellutil.run_get_output("modinfo %s" % module_name)
-            if retcode == 0:
-                version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out, re.IGNORECASE)
-                if version:
-                    v1 = int(version.groups(0)[0])
-                    v2 = int(version.groups(0)[1])
-                    if v1>4 or v1==4 and v2>0:
-                        logger.info("Skip setting /dev/hvnd_rdma on 4.1 or later")
-                        skip_rdma_device = True
-                else:
-                    logger.info("RDMA: hv_network_direct driver version not present, assuming 4.0.x or older.")
+    def provision_network_direct_rdma(self) :
+        RDMADeviceHandler.update_dat_conf(dapl_config_paths, self.ipv4_addr)
+
+        skip_rdma_device = False
+        module_name = "hv_network_direct"
+        retcode,out = shellutil.run_get_output("modprobe -R %s" % module_name, chk_err=False)
+        if retcode == 0:
+            module_name = out.strip()
+        else:
+            logger.info("RDMA: failed to resolve module name. Use original name")
+        retcode,out = shellutil.run_get_output("modprobe %s" % module_name)
+        if retcode != 0:
+            logger.error("RDMA: failed to load module %s" % module_name)
+            return
+        retcode,out = shellutil.run_get_output("modinfo %s" % module_name)
+        if retcode == 0:
+            version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out, re.IGNORECASE)
+            if version:
+                v1 = int(version.groups(0)[0])
+                v2 = int(version.groups(0)[1])
+                if v1>4 or v1==4 and v2>0:
+                    logger.info("Skip setting /dev/hvnd_rdma on 4.1 or later")
+                    skip_rdma_device = True
             else:
-                logger.warn("RDMA: failed to get module info on hv_network_direct.")
+                logger.info("RDMA: hv_network_direct driver version not present, assuming 4.0.x or older.")
+        else:
+            logger.warn("RDMA: failed to get module info on hv_network_direct.")
+
+        if not skip_rdma_device:
+            RDMADeviceHandler.wait_rdma_device(
+                self.rdma_dev, self.device_check_timeout_sec, self.device_check_interval_sec)
+            RDMADeviceHandler.write_rdma_config_to_device(
+                self.rdma_dev, self.ipv4_addr, self.mac_addr)
+
+        RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr)
+
+    def provision_sriov_rdma(self) :
+        RDMADeviceHandler.wait_any_rdma_device(
+            self.sriov_dir, self.device_check_timeout_sec, self.device_check_interval_sec)
+        RDMADeviceHandler.update_iboip_interface(self.ipv4_addr, self.ipoib_check_timeout_sec, self.ipoib_check_interval_sec)
+        return
 
-            if not skip_rdma_device:
-                RDMADeviceHandler.wait_rdma_device(
-                    self.rdma_dev, self.device_check_timeout_sec, self.device_check_interval_sec)
-                RDMADeviceHandler.write_rdma_config_to_device(
-                    self.rdma_dev, self.ipv4_addr, self.mac_addr)
+    @staticmethod
+    def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec) :
+        logger.info("Wait for ib0 become available")
+        total_retries = timeout_sec/check_interval_sec
+        n = 0
+        found_ib0 = None
+        while not found_ib0 and n < total_retries:
+            ret, output = shellutil.run_get_output("ifconfig -a")
+            if ret != 0:
+                raise Exception("Failed to list network interfaces")
+            found_ib0 = re.search("ib0", output, re.IGNORECASE)
+            if found_ib0:
+                break
+            time.sleep(check_interval_sec)
+            n += 1
 
-            RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr)
-        except Exception as e:
-            logger.error("RDMA: device processing failed: {0}".format(e))
+        if not found_ib0:
+            raise Exception("ib0 is not available")
+
+        netmask = 16
+        logger.info("RDMA: configuring IPv4 addr and netmask on ipoib interface")
+        addr = '{0}/{1}'.format(ipv4_addr, netmask)
+        if shellutil.run("ifconfig ib0 {0}".format(addr)) != 0:
+            raise Exception("Could set addr to {0} on ib0".format(addr))
+        logger.info("RDMA: ipoib address and netmask configured on interface")
 
     @staticmethod
     def update_dat_conf(paths, ipv4_addr):
@@ -301,6 +356,26 @@ class RDMADeviceHandler(object):
         raise Exception("The device did not show up in {0} seconds ({1} retries)".format(
             timeout_sec, total_retries))
 
+    @staticmethod
+    def wait_any_rdma_device(dir, timeout_sec, check_interval_sec):
+        logger.info(
+            "RDMA: waiting for any Infiniband device at directory={0} timeout={1}s".format(
+            dir, timeout_sec))
+        total_retries = timeout_sec/check_interval_sec
+        n = 0
+        while n < total_retries:
+            r = os.listdir(dir)
+            if r:
+                logger.info("RDMA: device found in {0}".format(dir))
+                return
+            logger.verbose(
+                "RDMA: device not ready, sleep {0}s".format(check_interval_sec))
+            time.sleep(check_interval_sec)
+            n += 1
+        logger.error("RDMA device wait timed out")
+        raise Exception("The device did not show up in {0} seconds ({1} retries)".format(
+            timeout_sec, total_retries))
+
     @staticmethod
     def update_network_interface(mac_addr, ipv4_addr):
         netmask=16
diff --git a/azurelinuxagent/daemon/main.py b/azurelinuxagent/daemon/main.py
index 12531761..c53f9917 100644
--- a/azurelinuxagent/daemon/main.py
+++ b/azurelinuxagent/daemon/main.py
@@ -140,7 +140,8 @@ class DaemonHandler(object):
 
         # Enable RDMA, continue in errors
         if conf.enable_rdma():
-            self.rdma_handler.install_driver()
+            nd_version = self.rdma_handler.get_rdma_version()
+            self.rdma_handler.install_driver_if_needed()
 
             logger.info("RDMA capabilities are enabled in configuration")
             try:
@@ -154,7 +155,7 @@ class DaemonHandler(object):
                     raise Exception("Attempt to setup RDMA without Wireserver")
                 client.update_goal_state(forced=True)
 
-                setup_rdma_device()
+                setup_rdma_device(nd_version)
             except Exception as e:
                 logger.error("Error setting up rdma device: %s" % e)
         else:
diff --git a/azurelinuxagent/daemon/resourcedisk/freebsd.py b/azurelinuxagent/daemon/resourcedisk/freebsd.py
index a65d7f8f..ece166be 100644
--- a/azurelinuxagent/daemon/resourcedisk/freebsd.py
+++ b/azurelinuxagent/daemon/resourcedisk/freebsd.py
@@ -16,9 +16,11 @@
 #
 # Requires Python 2.6+ and Openssl 1.0+
 #
+import os
 import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.shellutil as shellutil
+import azurelinuxagent.common.conf as conf
 from azurelinuxagent.common.exception import ResourceDiskError
 from azurelinuxagent.daemon.resourcedisk.default import ResourceDiskHandler
 
@@ -121,3 +123,41 @@ class FreeBSDResourceDiskHandler(ResourceDiskHandler):
 
         logger.info("Resource disk partition {0} is mounted at {1} with fstype {2}", partition, mount_point, fs)
         return mount_point
+
+    def create_swap_space(self, mount_point, size_mb):
+        size_kb = size_mb * 1024
+        size = size_kb * 1024
+        swapfile = os.path.join(mount_point, 'swapfile')
+        swaplist = shellutil.run_get_output("swapctl -l")[1]
+
+        if swapfile in swaplist \
+                and os.path.isfile(swapfile) \
+                and os.path.getsize(swapfile) == size:
+            logger.info("Swap already enabled")
+            return
+
+        if os.path.isfile(swapfile) and os.path.getsize(swapfile) != size:
+            logger.info("Remove old swap file")
+            shellutil.run("swapoff -a", chk_err=False)
+            os.remove(swapfile)
+
+        if not os.path.isfile(swapfile):
+            logger.info("Create swap file")
+            self.mkfile(swapfile, size_kb * 1024)
+        
+        mddevice = shellutil.run_get_output("mdconfig -a -t vnode -f {0}".format(swapfile))[1].rstrip()
+        shellutil.run("chmod 0600 /dev/{0}".format(mddevice))
+        
+        if conf.get_resourcedisk_enable_swap_encryption():
+            shellutil.run("kldload aesni")
+            shellutil.run("kldload cryptodev")
+            shellutil.run("kldload geom_eli")
+            shellutil.run("geli onetime -e AES-XTS -l 256 -d /dev/{0}".format(mddevice))
+            shellutil.run("chmod 0600 /dev/{0}.eli".format(mddevice))
+            if shellutil.run("swapon /dev/{0}.eli".format(mddevice)):
+                raise ResourceDiskError("/dev/{0}.eli".format(mddevice))
+            logger.info("Enabled {0}KB of swap at /dev/{1}.eli ({2})".format(size_kb, mddevice, swapfile))
+        else:
+            if shellutil.run("swapon /dev/{0}".format(mddevice)):
+                raise ResourceDiskError("/dev/{0}".format(mddevice))
+            logger.info("Enabled {0}KB of swap at /dev/{1} ({2})".format(size_kb, mddevice, swapfile))
diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
index b1454a07..91b54117 100644
--- a/azurelinuxagent/ga/exthandlers.py
+++ b/azurelinuxagent/ga/exthandlers.py
@@ -57,7 +57,9 @@ from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION
 HANDLER_ENVIRONMENT_VERSION = 1.0
 
 EXTENSION_STATUS_ERROR = 'error'
+EXTENSION_STATUS_SUCCESS = 'success'
 VALID_EXTENSION_STATUS = ['transitioning', 'error', 'success', 'warning']
+EXTENSION_TERMINAL_STATUSES = ['error', 'success']
 
 VALID_HANDLER_STATUS = ['Ready', 'NotReady', "Installing", "Unresponsive"]
 
@@ -66,6 +68,7 @@ HANDLER_NAME_PATTERN = re.compile(HANDLER_PATTERN+"$", re.IGNORECASE)
 HANDLER_PKG_EXT = ".zip"
 HANDLER_PKG_PATTERN = re.compile(HANDLER_PATTERN + r"\.zip$", re.IGNORECASE)
 
+DEFAULT_EXT_TIMEOUT_MINUTES = 90
 
 def validate_has_key(obj, key, fullname):
     if key not in obj:
@@ -120,9 +123,7 @@ def parse_ext_status(ext_status, data):
     ext_status.code = status_data.get('code', 0)
     formatted_message = status_data.get('formattedMessage')
     ext_status.message = parse_formatted_message(formatted_message)
-    substatus_list = status_data.get('substatus')
-    if substatus_list is None:
-        return
+    substatus_list = status_data.get('substatus', [])
     for substatus in substatus_list:
         if substatus is not None:
             ext_status.substatusList.append(parse_ext_substatus(substatus))
@@ -318,11 +319,62 @@ class ExtHandlersHandler(object):
                 logger.info("Extension handling is on hold")
                 return
 
+        wait_until = datetime.datetime.utcnow() + datetime.timedelta(minutes=DEFAULT_EXT_TIMEOUT_MINUTES)
+        max_dep_level = max([handler.sort_key() for handler in self.ext_handlers.extHandlers])
+
         self.ext_handlers.extHandlers.sort(key=operator.methodcaller('sort_key'))
         for ext_handler in self.ext_handlers.extHandlers:
-            # TODO: handle install in sequence, enable in parallel
             self.handle_ext_handler(ext_handler, etag)
-    
+
+            # Wait for the extension installation until it is handled.
+            # This is done for the install and enable. Not for the uninstallation.
+            # If handled successfully, proceed with the current handler.
+            # Otherwise, skip the rest of the extension installation.
+            dep_level = ext_handler.sort_key()
+            if dep_level >= 0 and dep_level < max_dep_level:
+                if not self.wait_for_handler_successful_completion(ext_handler, wait_until):
+                    logger.warn("An extension failed or timed out, will skip processing the rest of the extensions")
+                    break
+
+    def wait_for_handler_successful_completion(self, ext_handler, wait_until):
+        '''
+        Check the status of the extension being handled.
+        Wait until it has a terminal state or times out.
+        Return True if it is handled successfully. False if not.
+        '''
+        handler_i = ExtHandlerInstance(ext_handler, self.protocol)
+        for ext in ext_handler.properties.extensions:
+            ext_completed, status = handler_i.is_ext_handling_complete(ext)
+
+            # Keep polling for the extension status until it becomes success or times out
+            while not ext_completed and datetime.datetime.utcnow() <= wait_until:
+                time.sleep(5)
+                ext_completed, status = handler_i.is_ext_handling_complete(ext)
+
+            # In case of timeout or terminal error state, we log it and return false
+            # so that the extensions waiting on this one can be skipped processing
+            if datetime.datetime.utcnow() > wait_until:
+                msg = "Extension {0} did not reach a terminal state within the allowed timeout. Last status was {1}".format(ext.name, status)
+                logger.warn(msg)
+                add_event(AGENT_NAME,
+                          version=CURRENT_VERSION,
+                          op=WALAEventOperation.ExtensionProcessing,
+                          is_success=False,
+                          message=msg)
+                return False
+
+            if status != EXTENSION_STATUS_SUCCESS:
+                msg = "Extension {0} did not succeed. Status was {1}".format(ext.name, status)
+                logger.warn(msg)
+                add_event(AGENT_NAME,
+                          version=CURRENT_VERSION,
+                          op=WALAEventOperation.ExtensionProcessing,
+                          is_success=False,
+                          message=msg)
+                return False
+
+        return True
+
     def handle_ext_handler(self, ext_handler, etag):
         ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
 
@@ -879,6 +931,35 @@ class ExtHandlerInstance(object):
 
         return ext_status
     
+    def get_ext_handling_status(self, ext):
+        seq_no, ext_status_file = self.get_status_file_path(ext)
+        if seq_no < 0 or ext_status_file is None:
+            return None
+
+        # Missing status file is considered a non-terminal state here
+        # so that extension sequencing can wait until it becomes existing
+        if not os.path.exists(ext_status_file):
+            status = "warning"
+        else:
+            ext_status = self.collect_ext_status(ext)
+            status = ext_status.status if ext_status is not None else None
+
+        return status
+
+    def is_ext_handling_complete(self, ext):
+        status = self.get_ext_handling_status(ext)
+
+        # when seq < 0 (i.e. no new user settings), the handling is complete and return None status
+        if status is None:
+            return (True, None)
+
+        # If not in terminal state, it is incomplete
+        if status not in EXTENSION_TERMINAL_STATUSES:
+            return (False, status)
+
+        # Extension completed, return its status
+        return (True, status)
+
     def report_ext_status(self):
         active_exts = []
         # TODO Refactor or remove this common code pattern (for each extension subordinate to an ext_handler, do X).
diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py
index d6b66921..f3967d35 100644
--- a/azurelinuxagent/ga/monitor.py
+++ b/azurelinuxagent/ga/monitor.py
@@ -37,15 +37,15 @@ from azurelinuxagent.common.protocol import get_protocol_util
 from azurelinuxagent.common.protocol.healthservice import HealthService
 from azurelinuxagent.common.protocol.imds import get_imds_client
 from azurelinuxagent.common.protocol.restapi import TelemetryEventParam, \
-                                                    TelemetryEventList, \
-                                                    TelemetryEvent, \
-                                                    set_properties
+    TelemetryEventList, \
+    TelemetryEvent, \
+    set_properties
 import azurelinuxagent.common.utils.networkutil as networkutil
 from azurelinuxagent.common.utils.restutil import IOErrorCounter
 from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, getattrib, hash_strings
 from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \
-            DISTRO_CODE_NAME, AGENT_LONG_VERSION, \
-            AGENT_NAME, CURRENT_AGENT, CURRENT_VERSION
+    DISTRO_CODE_NAME, AGENT_LONG_VERSION, \
+    AGENT_NAME, CURRENT_AGENT, CURRENT_VERSION
 
 
 def parse_event(data_str):
@@ -95,7 +95,6 @@ def get_monitor_handler():
 
 
 class MonitorHandler(object):
-
     EVENT_COLLECTION_PERIOD = datetime.timedelta(minutes=1)
     TELEMETRY_HEARTBEAT_PERIOD = datetime.timedelta(minutes=30)
     CGROUP_TELEMETRY_PERIOD = datetime.timedelta(minutes=5)
@@ -320,7 +319,8 @@ class MonitorHandler(object):
         if self.last_host_plugin_heartbeat is None:
             self.last_host_plugin_heartbeat = datetime.datetime.utcnow() - MonitorHandler.HOST_PLUGIN_HEARTBEAT_PERIOD
 
-        if datetime.datetime.utcnow() >= (self.last_host_plugin_heartbeat + MonitorHandler.HOST_PLUGIN_HEARTBEAT_PERIOD):
+        if datetime.datetime.utcnow() >= (
+                self.last_host_plugin_heartbeat + MonitorHandler.HOST_PLUGIN_HEARTBEAT_PERIOD):
             try:
                 host_plugin = self.protocol.client.get_host_plugin()
                 host_plugin.ensure_initialized()
@@ -418,10 +418,36 @@ class MonitorHandler(object):
 
         if datetime.datetime.utcnow() >= (self.last_telemetry_heartbeat + MonitorHandler.CGROUP_TELEMETRY_PERIOD):
             try:
-                for cgroup_name, metrics in CGroupsTelemetry.collect_all_tracked().items():
+                metric_reported, metric_threshold = CGroupsTelemetry.collect_all_tracked()
+                for cgroup_name, metrics in metric_reported.items():
+                    thresholds = metric_threshold[cgroup_name]
+
                     for metric_group, metric_name, value in metrics:
                         if value > 0:
                             report_metric(metric_group, metric_name, cgroup_name, value)
+
+                        if metric_group == "Memory":
+                            if value >= thresholds["memory"]:
+                                msg = "CGroup {0}: Crossed the Memory Threshold. Current Value:{1}, Threshold:{2}.".format(
+                                    cgroup_name, value, thresholds["memory"])
+                                add_event(name=AGENT_NAME,
+                                          version=CURRENT_VERSION,
+                                          op=WALAEventOperation.CGroupsLimitsCrossed,
+                                          is_success=True,
+                                          message=msg,
+                                          log_event=True)
+
+                        if metric_group == "Process":
+                            if value >= thresholds["cpu"]:
+                                msg = "CGroup {0}: Crossed the Processor Threshold. Current Value:{1}, Threshold:{2}.".format(
+                                    cgroup_name, value, thresholds["cpu"])
+                                add_event(name=AGENT_NAME,
+                                          version=CURRENT_VERSION,
+                                          op=WALAEventOperation.CGroupsLimitsCrossed,
+                                          is_success=True,
+                                          message=msg,
+                                          log_event=True)
+
             except Exception as e:
                 logger.warn("Monitor: failed to collect cgroups performance metrics: {0}", ustr(e))
                 logger.verbose(traceback.format_exc())
diff --git a/azurelinuxagent/pa/deprovision/factory.py b/azurelinuxagent/pa/deprovision/factory.py
index 21169b34..07c26bf7 100644
--- a/azurelinuxagent/pa/deprovision/factory.py
+++ b/azurelinuxagent/pa/deprovision/factory.py
@@ -40,7 +40,7 @@ def get_deprovision_handler(distro_name=DISTRO_NAME,
             return UbuntuDeprovisionHandler()
     if distro_name == "coreos":
         return CoreOSDeprovisionHandler()
-    if distro_name == "clear linux":
+    if "Clear Linux" in distro_full_name:
         return ClearLinuxDeprovisionHandler()
 
     return DeprovisionHandler()
diff --git a/azurelinuxagent/pa/rdma/centos.py b/azurelinuxagent/pa/rdma/centos.py
index 3d34ddfc..7ed50f9c 100644
--- a/azurelinuxagent/pa/rdma/centos.py
+++ b/azurelinuxagent/pa/rdma/centos.py
@@ -59,7 +59,7 @@ class CentOSRDMAHandler(RDMAHandler):
 
         # Find out RDMA firmware version and see if the existing package needs
         # updating or if the package is missing altogether (and install it)
-        fw_version = RDMAHandler.get_rdma_version()
+        fw_version = self.get_rdma_version()
         if not fw_version:
             raise Exception('Cannot determine RDMA firmware version')
         logger.info("RDMA: found firmware version: {0}".format(fw_version))
@@ -241,4 +241,4 @@ class CentOSRDMAHandler(RDMAHandler):
             raise Exception("RDMA: failed to install kvp daemon package '%s'" % kvp_pkg_to_install)
         logger.info("RDMA: package '%s' successfully installed" % kvp_pkg_to_install)
         logger.info("RDMA: Machine will now be rebooted.")
-        self.reboot_system()
\ No newline at end of file
+        self.reboot_system()
diff --git a/azurelinuxagent/pa/rdma/suse.py b/azurelinuxagent/pa/rdma/suse.py
index d09fe74f..6fdf1285 100644
--- a/azurelinuxagent/pa/rdma/suse.py
+++ b/azurelinuxagent/pa/rdma/suse.py
@@ -28,7 +28,7 @@ class SUSERDMAHandler(RDMAHandler):
     def install_driver(self):
         """Install the appropriate driver package for the RDMA firmware"""
 
-        fw_version = RDMAHandler.get_rdma_version()
+        fw_version = self.get_rdma_version()
         if not fw_version:
             error_msg = 'RDMA: Could not determine firmware version. '
             error_msg += 'Therefore, no driver will be installed.'
diff --git a/azurelinuxagent/pa/rdma/ubuntu.py b/azurelinuxagent/pa/rdma/ubuntu.py
index 7e4bae38..cef80565 100644
--- a/azurelinuxagent/pa/rdma/ubuntu.py
+++ b/azurelinuxagent/pa/rdma/ubuntu.py
@@ -32,7 +32,7 @@ class UbuntuRDMAHandler(RDMAHandler):
     def install_driver(self):
         #Install the appropriate driver package for the RDMA firmware
 
-        nd_version = RDMAHandler.get_rdma_version()
+        nd_version = self.get_rdma_version()
         if not nd_version:
             logger.error("RDMA: Could not determine firmware version. No driver will be installed")
             return
diff --git a/setup.py b/setup.py
index e15a2d46..c24180d2 100755
--- a/setup.py
+++ b/setup.py
@@ -107,8 +107,7 @@ def get_data_files(name, version, fullname):
         set_udev_files(data_files)
         set_files(data_files, dest="/usr/share/oem",
                   src=["init/coreos/cloud-config.yml"])
-    elif name == 'clear linux os for intel architecture' \
-            or name == 'clear linux software for intel architecture':
+    elif "Clear Linux" in fullname:
         set_bin_files(data_files, dest="/usr/bin")
         set_conf_files(data_files, dest="/usr/share/defaults/waagent",
                        src=["config/clearlinux/waagent.conf"])
@@ -232,7 +231,6 @@ setuptools.setup(
     url='https://github.com/Azure/WALinuxAgent',
     license='Apache License Version 2.0',
     packages=find_packages(exclude=["tests*"]),
-    py_modules=["__main__"],
     install_requires=requires,
     cmdclass={
         'install': install


Test patch:
diff --git a/tests/common/test_cgroups.py b/tests/common/test_cgroups.py
index 18e8757b..56d5a1df 100644
--- a/tests/common/test_cgroups.py
+++ b/tests/common/test_cgroups.py
@@ -41,6 +41,7 @@ def make_self_cgroups():
     :return: CGroups containing this process
     :rtype: CGroups
     """
+
     def path_maker(hierarchy, __):
         suffix = CGroups.get_my_cgroup_path(CGroups.get_hierarchy_id('cpu'))
         return os.path.join(BASE_CGROUPS, hierarchy, suffix)
@@ -55,6 +56,7 @@ def make_root_cgroups():
     :return: CGroups for most-encompassing cgroup
     :rtype: CGroups
     """
+
     def path_maker(hierarchy, _):
         return os.path.join(BASE_CGROUPS, hierarchy)
 
@@ -116,7 +118,7 @@ class TestCGroups(AgentTestCase):
             self.assertLess(cpu.current_cpu_total, cpu.current_system_cpu)
 
             consume_cpu_time()  # Eat some CPU
-            time.sleep(1)       # Generate some idle time
+            time.sleep(1)  # Generate some idle time
             cpu.update()
             self.assertLess(cpu.current_cpu_total, cpu.current_system_cpu)
 
@@ -128,7 +130,7 @@ class TestCGroups(AgentTestCase):
         self.assertTrue(CGroupsTelemetry.is_tracked(test_extension_name))
         consume_cpu_time()
         time.sleep(1)
-        metrics = CGroupsTelemetry.collect_all_tracked()
+        metrics, limits = CGroupsTelemetry.collect_all_tracked()
         my_metrics = metrics[test_extension_name]
         self.assertEqual(len(my_metrics), 2)
         for item in my_metrics:
@@ -169,6 +171,22 @@ class TestCGroups(AgentTestCase):
         """
         self.exercise_telemetry_instantiation(make_self_cgroups())
 
+    @skip_if_predicate_true(i_am_root, "Test does not run when root")
+    @patch("azurelinuxagent.common.conf.get_cgroups_enforce_limits")
+    @patch("azurelinuxagent.common.cgroups.CGroups.set_cpu_limit")
+    @patch("azurelinuxagent.common.cgroups.CGroups.set_memory_limit")
+    def test_telemetry_instantiation_as_normal_user_with_limits(self, mock_get_cgroups_enforce_limits,
+                                                                mock_set_cpu_limit,
+                                                                mock_set_memory_limit):
+        """
+        Tracking an existing cgroup for an extension; collect all metrics.
+        """
+        mock_get_cgroups_enforce_limits.return_value = True
+
+        cg = make_self_cgroups()
+        cg.set_limits()
+        self.exercise_telemetry_instantiation(cg)
+
     def test_cpu_telemetry(self):
         """
         Test Cpu telemetry class
@@ -214,7 +232,7 @@ class TestCGroups(AgentTestCase):
         self.assertEqual(2048, CGroups._format_memory_value('kilobytes', 2))
         self.assertEqual(0, CGroups._format_memory_value('kilobytes', 0))
         self.assertEqual(2048000, CGroups._format_memory_value('kilobytes', 2000))
-        self.assertEqual(2048*1024, CGroups._format_memory_value('megabytes', 2))
+        self.assertEqual(2048 * 1024, CGroups._format_memory_value('megabytes', 2))
         self.assertEqual((1024 + 512) * 1024 * 1024, CGroups._format_memory_value('gigabytes', 1.5))
         self.assertRaises(CGroupsException, CGroups._format_memory_value, 'KiloBytes', 1)
 
diff --git a/tests/common/test_conf.py b/tests/common/test_conf.py
index 0f85010b..55fd5aec 100644
--- a/tests/common/test_conf.py
+++ b/tests/common/test_conf.py
@@ -43,6 +43,7 @@ class TestConf(AgentTestCase):
         "ResourceDisk.Filesystem": "ext4",
         "ResourceDisk.MountPoint": "/mnt/resource",
         "ResourceDisk.EnableSwap": False,
+        "ResourceDisk.EnableSwapEncryption": False,
         "ResourceDisk.SwapSizeMB": 0,
         "ResourceDisk.MountOptions": None,
         "Logs.Verbose": False,
diff --git a/tests/data/test_waagent.conf b/tests/data/test_waagent.conf
index ec89e724..049c8e2b 100644
--- a/tests/data/test_waagent.conf
+++ b/tests/data/test_waagent.conf
@@ -58,6 +58,9 @@ ResourceDisk.MountPoint=/mnt/resource
 # Create and use swapfile on resource disk.
 ResourceDisk.EnableSwap=n
 
+# Use encrypted swap
+ResourceDisk.EnableSwapEncryption=n
+
 # Size of the swapfile.
 ResourceDisk.SwapSizeMB=0
 
diff --git a/tests/data/wire/ext_conf_sequencing.xml b/tests/data/wire/ext_conf_sequencing.xml
index 7f4c9bc8..3120a979 100644
--- a/tests/data/wire/ext_conf_sequencing.xml
+++ b/tests/data/wire/ext_conf_sequencing.xml
@@ -15,14 +15,19 @@
   </GAFamilies>
 </GuestAgentExtension>
 <Plugins>
-  <Plugin name="OSTCExtensions.ExampleHandlerLinux" version="1.0.0" location="http://rdfepirv2hknprdstr03.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_ExampleHandlerLinux_asiaeast_manifest.xml" config="" state="enabled" autoUpgrade="false" failoverlocation="http://rdfepirv2hknprdstr04.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_ExampleHandlerLinux_asiaeast_manifest.xml" runAsStartupTask="false" isJson="true" dependencyLevel="2"/>
-  <Plugin name="OSTCExtensions.OtherExampleHandlerLinux" version="1.0.0" location="http://rdfepirv2hknprdstr03.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_OtherExampleHandlerLinux_asiaeast_manifest.xml" config="" state="enabled" autoUpgrade="false" failoverlocation="http://rdfepirv2hknprdstr04.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_OtherExampleHandlerLinux_asiaeast_manifest.xml" runAsStartupTask="false" isJson="true" dependencyLevel="1" />
+  <Plugin name="OSTCExtensions.ExampleHandlerLinux" version="1.0.0" location="http://rdfepirv2hknprdstr03.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_ExampleHandlerLinux_asiaeast_manifest.xml" config="" state="enabled" autoUpgrade="false" failoverlocation="http://rdfepirv2hknprdstr04.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_ExampleHandlerLinux_asiaeast_manifest.xml" runAsStartupTask="false" isJson="true" />
+  <Plugin name="OSTCExtensions.OtherExampleHandlerLinux" version="1.0.0" location="http://rdfepirv2hknprdstr03.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_OtherExampleHandlerLinux_asiaeast_manifest.xml" config="" state="enabled" autoUpgrade="false" failoverlocation="http://rdfepirv2hknprdstr04.blob.core.windows.net/b01058962be54ceca550a390fa5ff064/Microsoft.OSTCExtensions_OtherExampleHandlerLinux_asiaeast_manifest.xml" runAsStartupTask="false" isJson="true" />
 </Plugins>
 <PluginSettings>
   <Plugin name="OSTCExtensions.ExampleHandlerLinux" version="1.0.0">
+    <DependsOn dependencyLevel="2">
+      <DependsOnExtension handler="OSTCExtensions.OtherExampleHandlerLinux" />
+    </DependsOn>
     <RuntimeSettings seqNo="0">{"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]}</RuntimeSettings>
   </Plugin>
   <Plugin name="OSTCExtensions.OtherExampleHandlerLinux" version="1.0.0">
+    <DependsOn dependencyLevel="1">
+    </DependsOn>
     <RuntimeSettings seqNo="0">{"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]}</RuntimeSettings>
   </Plugin>
 </PluginSettings>
diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py
index 8940419d..e77d68ec 100644
--- a/tests/ga/test_extension.py
+++ b/tests/ga/test_extension.py
@@ -18,6 +18,8 @@
 import os.path
 
 from tests.protocol.mockwiredata import *
+
+from azurelinuxagent.common.protocol.restapi import Extension
 from azurelinuxagent.ga.exthandlers import *
 from azurelinuxagent.common.protocol.wire import WireProtocol
 
@@ -409,8 +411,8 @@ class TestExtension(ExtensionTestCase):
         self.assertTrue(exthandlers_handler.ext_handlers is not None)
         self.assertTrue(exthandlers_handler.ext_handlers.extHandlers is not None)
         self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2)
-        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.dependencyLevel, 1)
-        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.dependencyLevel, 2)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 1)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 2)
 
         #Test goal state not changed
         exthandlers_handler.run()
@@ -422,6 +424,7 @@ class TestExtension(ExtensionTestCase):
                                                             "<Incarnation>2<")
         test_data.ext_conf = test_data.ext_conf.replace("seqNo=\"0\"",
                                                         "seqNo=\"1\"")
+        # Swap the dependency ordering
         test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"2\"",
                                                         "dependencyLevel=\"3\"")
         test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"1\"",
@@ -431,10 +434,13 @@ class TestExtension(ExtensionTestCase):
         self._assert_ext_status(protocol.report_ext_status, "success", 1)
 
         self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2)
-        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.dependencyLevel, 3)
-        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.dependencyLevel, 4)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 3)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 4)
 
         #Test disable
+        # In the case of disable, the last extension to be enabled should be
+        # the first extension disabled. The first extension enabled should be
+        # the last one disabled.
         test_data.goal_state = test_data.goal_state.replace("<Incarnation>2<",
                                                             "<Incarnation>3<")
         test_data.ext_conf = test_data.ext_conf.replace("enabled", "disabled")
@@ -443,13 +449,17 @@ class TestExtension(ExtensionTestCase):
                                     1, "1.0.0",
                                     expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux")
         self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2)
-        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.dependencyLevel, 4)
-        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.dependencyLevel, 3)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 4)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 3)
 
         #Test uninstall
+        # In the case of uninstall, the last extension to be installed should be
+        # the first extension uninstalled. The first extension installed
+        # should be the last one uninstalled.
         test_data.goal_state = test_data.goal_state.replace("<Incarnation>3<",
                                                             "<Incarnation>4<")
         test_data.ext_conf = test_data.ext_conf.replace("disabled", "uninstall")
+        # Swap the dependency ordering AGAIN
         test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"3\"",
                                                         "dependencyLevel=\"6\"")
         test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"4\"",
@@ -457,8 +467,32 @@ class TestExtension(ExtensionTestCase):
         exthandlers_handler.run()
         self._assert_no_handler_status(protocol.report_vm_status)
         self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2)
-        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.dependencyLevel, 6)
-        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.dependencyLevel, 5)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 6)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 5)
+
+    def test_ext_handler_sequencing_default_dependency_level(self, *args):
+        test_data = WireProtocolData(DATA_FILE)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+        exthandlers_handler.run()
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0)
+
+    def test_ext_handler_sequencing_invalid_dependency_level(self, *args):
+        test_data = WireProtocolData(DATA_FILE_EXT_SEQUENCING)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+        test_data.goal_state = test_data.goal_state.replace("<Incarnation>1<",
+                                                            "<Incarnation>2<")
+        test_data.ext_conf = test_data.ext_conf.replace("seqNo=\"0\"",
+                                                        "seqNo=\"1\"")
+        test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"1\"",
+                                                        "dependencyLevel=\"a6\"")
+        test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"2\"",
+                                                        "dependencyLevel=\"5b\"")
+        exthandlers_handler.run()
+
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0)
+        self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0)
 
     def test_ext_handler_rollingupgrade(self, *args):
         test_data = WireProtocolData(DATA_FILE_EXT_ROLLINGUPGRADE)
@@ -666,6 +700,164 @@ class TestExtension(ExtensionTestCase):
         self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0")
         self._assert_ext_status(protocol.report_ext_status, "error", 0)
 
+    def test_wait_for_handler_successful_completion_empty_exts(self, *args):
+        '''
+        Testing wait_for_handler_successful_completion() when there is no extension in a handler.
+        Expected to return True.
+        '''
+        test_data = WireProtocolData(DATA_FILE)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+        handler = ExtHandler(name="handler")
+
+        ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=None)
+        self.assertTrue(exthandlers_handler.wait_for_handler_successful_completion(handler, datetime.datetime.utcnow()))
+
+    def _helper_wait_for_handler_successful_completion(self, exthandlers_handler):
+        '''
+        Call wait_for_handler_successful_completion() passing a handler with an extension.
+        Override the wait time to be 5 seconds to minimize the timout duration.
+        Return the value returned by wait_for_handler_successful_completion().
+        '''
+        handler_name = "Handler"
+        exthandler = ExtHandler(name=handler_name)
+        extension = Extension(name=handler_name)
+        exthandler.properties.extensions.append(extension)
+
+        # Override the timeout value to minimize the test duration
+        wait_until = datetime.datetime.utcnow() + datetime.timedelta(seconds=5)
+        return exthandlers_handler.wait_for_handler_successful_completion(exthandler, wait_until)
+
+    def test_wait_for_handler_successful_completion_no_status(self, *args):
+        '''
+        Testing wait_for_handler_successful_completion() when there is no status file or seq_no is negative.
+        Expected to return False.
+        '''
+        test_data = WireProtocolData(DATA_FILE)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+        ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=None)
+        self.assertFalse(self._helper_wait_for_handler_successful_completion(exthandlers_handler))
+
+    def test_wait_for_handler_successful_completion_success_status(self, *args):
+        '''
+        Testing wait_for_handler_successful_completion() when there is successful status.
+        Expected to return True.
+        '''
+        test_data = WireProtocolData(DATA_FILE)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+        status = "success"
+
+        ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status)
+        self.assertTrue(self._helper_wait_for_handler_successful_completion(exthandlers_handler))
+
+    def test_wait_for_handler_successful_completion_error_status(self, *args):
+        '''
+        Testing wait_for_handler_successful_completion() when there is error status.
+        Expected to return False.
+        '''
+        test_data = WireProtocolData(DATA_FILE)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+        status = "error"
+
+        ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status)
+        self.assertFalse(self._helper_wait_for_handler_successful_completion(exthandlers_handler))
+
+    def test_wait_for_handler_successful_completion_timeout(self, *args):
+        '''
+        Testing wait_for_handler_successful_completion() when there is non terminal status.
+        Expected to return False.
+        '''
+        test_data = WireProtocolData(DATA_FILE)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+        # Choose a non-terminal status
+        status = "warning"
+
+        ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status)
+        self.assertFalse(self._helper_wait_for_handler_successful_completion(exthandlers_handler))
+
+    def test_get_ext_handling_status(self, *args):
+        '''
+        Testing get_ext_handling_status() function with various cases and
+        verifying against the expected values
+        '''
+        test_data = WireProtocolData(DATA_FILE)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+        handler_name = "Handler"
+        exthandler = ExtHandler(name=handler_name)
+        extension = Extension(name=handler_name)
+        exthandler.properties.extensions.append(extension)
+
+        # In the following list of test cases, the first element corresponds to seq_no.
+        # the second element is the status file name, the third element indicates if the status file exits or not.
+        # The fourth element is the expected value from get_ext_handling_status()
+        test_cases = [
+            [-5, None, False, None],
+            [-1, None, False, None],
+            [0, None, False, None],
+            [0, "filename", False, "warning"],
+            [0, "filename", True, ExtensionStatus(status="success")],
+            [5, "filename", False, "warning"],
+            [5, "filename", True, ExtensionStatus(status="success")]
+        ]
+
+        orig_state = os.path.exists
+        for case in test_cases:
+            ext_handler_i = ExtHandlerInstance(exthandler, protocol)
+            ext_handler_i.get_status_file_path = MagicMock(return_value=(case[0], case[1]))
+            os.path.exists = MagicMock(return_value=case[2])
+            if case[2]:
+                # when the status file exists, it is expected return the value from collect_ext_status()
+                ext_handler_i.collect_ext_status= MagicMock(return_value=case[3])
+
+            status = ext_handler_i.get_ext_handling_status(extension)
+            if case[2]:
+                self.assertEqual(status, case[3].status)
+            else:
+                self.assertEqual(status, case[3])
+
+        os.path.exists = orig_state
+
+    def test_is_ext_handling_complete(self, *args):
+        '''
+        Testing is_ext_handling_complete() with various input and
+        verifying against the expected output values.
+        '''
+        test_data = WireProtocolData(DATA_FILE)
+        exthandlers_handler, protocol = self._create_mock(test_data, *args)
+        
+        handler_name = "Handler"
+        exthandler = ExtHandler(name=handler_name)
+        extension = Extension(name=handler_name)
+        exthandler.properties.extensions.append(extension)
+
+        ext_handler_i = ExtHandlerInstance(exthandler, protocol)
+        
+        # Testing no status case
+        ext_handler_i.get_ext_handling_status = MagicMock(return_value=None)
+        completed, status = ext_handler_i.is_ext_handling_complete(extension)
+        self.assertTrue(completed)
+        self.assertEqual(status, None)
+
+        # Here the key represents the possible input value to is_ext_handling_complete()
+        # the value represents the output tuple from is_ext_handling_complete()
+        expected_results = {
+            "error": (True, "error"),
+            "success": (True, "success"),
+            "warning": (False, "warning"),
+            "transitioning": (False, "transitioning")
+        }
+        
+        for key in expected_results.keys():
+            ext_handler_i.get_ext_handling_status = MagicMock(return_value=key)
+            completed, status = ext_handler_i.is_ext_handling_complete(extension)
+            self.assertEqual(completed, expected_results[key][0])
+            self.assertEqual(status, expected_results[key][1])
+
     def test_ext_handler_version_decide_autoupgrade_internalversion(self, *args):
         for internal in [False, True]:
             for autoupgrade in [False, True]:
@@ -917,6 +1109,180 @@ class TestExtension(ExtensionTestCase):
 
         self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.1")
 
+@patch("azurelinuxagent.common.protocol.wire.CryptUtil")
+@patch("azurelinuxagent.common.utils.restutil.http_get")
+class TestExtensionSequencing(AgentTestCase):
+
+    def _create_mock(self, mock_http_get, MockCryptUtil):
+        test_data = WireProtocolData(DATA_FILE)
+
+        #Mock protocol to return test data
+        mock_http_get.side_effect = test_data.mock_http_get
+        MockCryptUtil.side_effect = test_data.mock_crypt_util
+
+        protocol = WireProtocol("foo.bar")
+        protocol.detect()
+        protocol.report_ext_status = MagicMock()
+        protocol.report_vm_status = MagicMock()
+        protocol.get_artifacts_profile = MagicMock()
+
+        handler = get_exthandlers_handler()
+        handler.protocol_util.get_protocol = Mock(return_value=protocol)
+        handler.ext_handlers, handler.last_etag = protocol.get_ext_handlers()
+        conf.get_enable_overprovisioning = Mock(return_value=False)
+
+        def wait_for_handler_successful_completion(prev_handler, wait_until):
+            return orig_wait_for_handler_successful_completion(prev_handler, datetime.datetime.utcnow() + datetime.timedelta(seconds=5))
+
+        orig_wait_for_handler_successful_completion = handler.wait_for_handler_successful_completion
+        handler.wait_for_handler_successful_completion = wait_for_handler_successful_completion
+        return handler
+
+    def _set_dependency_levels(self, dependency_levels, exthandlers_handler):
+        '''
+        Creates extensions with the given dependencyLevel
+        '''
+        handler_map = dict()
+        all_handlers = []
+        for h, level in dependency_levels:
+            if handler_map.get(h) is None:
+                handler = ExtHandler(name=h)
+                extension = Extension(name=h)
+                handler.properties.state = "enabled"
+                handler.properties.extensions.append(extension)
+                handler_map[h] = handler
+                all_handlers.append(handler)
+
+            handler = handler_map[h]
+            for ext in handler.properties.extensions:
+                ext.dependencyLevel = level
+
+        exthandlers_handler.ext_handlers.extHandlers = []
+        for handler in all_handlers:
+            exthandlers_handler.ext_handlers.extHandlers.append(handler)
+
+    def _validate_extension_sequence(self, expected_sequence, exthandlers_handler):
+        installed_extensions = [a[0].name for a, k in exthandlers_handler.handle_ext_handler.call_args_list]
+        self.assertListEqual(expected_sequence, installed_extensions, "Expected and actual list of extensions are not equal")
+
+    def _run_test(self, extensions_to_be_failed, expected_sequence, exthandlers_handler):
+        '''
+        Mocks get_ext_handling_status() to mimic error status for a given extension.
+        Calls ExtHandlersHandler.run()
+        Verifies if the ExtHandlersHandler.handle_ext_handler() was called with appropriate extensions
+        in the expected order.
+        '''
+
+        def get_ext_handling_status(ext):
+            status = "error" if ext.name in extensions_to_be_failed else "success"
+            return status
+
+        ExtHandlerInstance.get_ext_handling_status = MagicMock(side_effect = get_ext_handling_status)
+        exthandlers_handler.handle_ext_handler = MagicMock()
+        exthandlers_handler.run()
+        self._validate_extension_sequence(expected_sequence, exthandlers_handler)
+
+    def test_handle_ext_handlers(self, *args):
+        '''
+        Tests extension sequencing among multiple extensions with dependencies.
+        This test introduces failure in all possible levels and extensions.
+        Verifies that the sequencing is in the expected order and a failure in one extension
+        skips the rest of the extensions in the sequence.
+        '''
+        exthandlers_handler = self._create_mock(*args)
+
+        self._set_dependency_levels([("A", 3), ("B", 2), ("C", 2), ("D", 1), ("E", 1), ("F", 1), ("G", 1)],
+                                    exthandlers_handler)
+
+        extensions_to_be_failed = []
+        expected_sequence = ["D", "E", "F", "G", "B", "C", "A"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["D"]
+        expected_sequence = ["D"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["E"]
+        expected_sequence = ["D", "E"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["F"]
+        expected_sequence = ["D", "E", "F"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["G"]
+        expected_sequence = ["D", "E", "F", "G"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["B"]
+        expected_sequence = ["D", "E", "F", "G", "B"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["C"]
+        expected_sequence = ["D", "E", "F", "G", "B", "C"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["A"]
+        expected_sequence = ["D", "E", "F", "G", "B", "C", "A"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+    def test_handle_ext_handlers_with_uninstallation(self, *args):
+        '''
+        Tests extension sequencing among multiple extensions with dependencies when
+        some extension are to be uninstalled.
+        Verifies that the sequencing is in the expected order and the uninstallation takes place
+        prior to all the installation/enable.
+        '''
+        exthandlers_handler = self._create_mock(*args)
+
+        # "A", "D" and "F" are marked as to be uninstalled
+        self._set_dependency_levels([("A", 0), ("B", 2), ("C", 2), ("D", 0), ("E", 1), ("F", 0), ("G", 1)],
+                                    exthandlers_handler)
+
+        extensions_to_be_failed = []
+        expected_sequence = ["A", "D", "F", "E", "G", "B", "C"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+    def test_handle_ext_handlers_fallback(self, *args):
+        '''
+        This test makes sure that the extension sequencing is applied only when the user specifies
+        dependency information in the extension.
+        When there is no dependency specified, the agent is expected to assign dependencyLevel=0 to all extension.
+        Also, it is expected to install all the extension no matter if there is any failure in any of the extensions.
+        '''
+        exthandlers_handler = self._create_mock(*args)
+
+        self._set_dependency_levels([("A", 1), ("B", 1), ("C", 1), ("D", 1), ("E", 1), ("F", 1), ("G", 1)],
+                                    exthandlers_handler)
+
+        # Expected sequence must contain all the extensions in the given order.
+        # The following test cases verfy against this same expected sequence no matter if any extension failed
+        expected_sequence = ["A", "B", "C", "D", "E", "F", "G"]
+
+        # Make sure that failure in any extension does not prevent other extensions to be installed
+        extensions_to_be_failed = []
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["A"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["B"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["C"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["D"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["E"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["F"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+        extensions_to_be_failed = ["G"]
+        self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/test_agent.py b/tests/test_agent.py
index cf31071e..cbcef4f5 100644
--- a/tests/test_agent.py
+++ b/tests/test_agent.py
@@ -63,6 +63,7 @@ Provisioning.RegenerateSshHostKeyPair = True
 Provisioning.SshHostKeyPairType = rsa
 Provisioning.UseCloudInit = True
 ResourceDisk.EnableSwap = False
+ResourceDisk.EnableSwapEncryption = False
 ResourceDisk.Filesystem = ext4
 ResourceDisk.Format = True
 ResourceDisk.MountOptions = None