Repository: Azure/WALinuxAgent
Pull request #266 (2016-06-24 22:58:12)

CentOS RDMA driver handler
Pretty much ported what's in waagent-2.0 branch for CentOS. Did some manual validation as well on centos65 and centos71

Let's merge this after all other RDMA patches are in.

msftclas: Hi __@ahmetalpbalkan__, I'm your friendly neighborhood Microsoft Pull Request Bot (You can call me MSBOT). Thanks for your contribution!
    <p>
        It looks like you're a Microsoft contributor (Ahmet Alp Balkan). If you're full-time, we DON'T require a Contribution License Agreement. If you are a vendor, please DO sign the electronic Contribution License Agreement. It will take 2 minutes and there's no faxing! https://cla.microsoft.com.
    </p>

TTYL, MSBOT;


Patch:
diff --git a/azurelinuxagent/common/dhcp.py b/azurelinuxagent/common/dhcp.py
index 8c81f248..d5c90cb0 100644
--- a/azurelinuxagent/common/dhcp.py
+++ b/azurelinuxagent/common/dhcp.py
@@ -49,6 +49,7 @@ class DhcpHandler(object):
         self.gateway = None
         self.routes = None
         self._request_broadcast = False
+        self.skip_cache = False
 
     def run(self):
         """
@@ -56,9 +57,11 @@ class DhcpHandler(object):
         Configure default gateway and routes
         Save wire server endpoint if found
         """
-        if not self.wireserver_route_exists and not self.dhcp_lease_exists:
-            self.send_dhcp_req()
-            self.conf_routes()
+        if self.wireserver_route_exists or self.dhcp_cache_exists:
+            return
+
+        self.send_dhcp_req()
+        self.conf_routes()
 
     def wait_for_network(self):
         """
@@ -104,17 +107,23 @@ class DhcpHandler(object):
         return route_exists
 
     @property
-    def dhcp_lease_exists(self):
+    def dhcp_cache_exists(self):
         """
         Check whether the dhcp options cache exists and contains the
-        wireserver endpoint.
+        wireserver endpoint, unless skip_cache is True.
         :return: True if the cached endpoint was found in the dhcp lease
         """
+        if self.skip_cache:
+            return False
+
         exists = False
+
+        logger.info("checking for dhcp lease cache")
         cached_endpoint = self.osutil.get_dhcp_lease_endpoint()
         if cached_endpoint is not None:
             self.endpoint = cached_endpoint
             exists = True
+        logger.info("cache exists [{0}]".format(exists))
         return exists
 
     def conf_routes(self):
diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py
index 2cf34ce4..76d48ca9 100644
--- a/azurelinuxagent/common/osutil/default.py
+++ b/azurelinuxagent/common/osutil/default.py
@@ -73,7 +73,7 @@ class DefaultOSUtil(object):
         userentry = self.get_userentry(username)
         uidmin = None
         try:
-            uidmin_def = fileutil.get_line_startingwith("UID_MIN", 
+            uidmin_def = fileutil.get_line_startingwith("UID_MIN",
                                                         "/etc/login.defs")
             if uidmin_def is not None:
                 uidmin = int(uidmin_def.split()[1])
@@ -215,7 +215,7 @@ class DefaultOSUtil(object):
             pub_path = os.path.join(lib_dir, thumbprint + '.pub')
             pub = crytputil.get_pubkey_from_crt(crt_path)
             fileutil.write_file(pub_path, pub)
-            self.set_selinux_context(pub_path, 
+            self.set_selinux_context(pub_path,
                                      'unconfined_u:object_r:ssh_home_t:s0')
             self.openssl_to_openssh(pub_path, path)
             fileutil.chmod(pub_path, 0o600)
@@ -320,7 +320,7 @@ class DefaultOSUtil(object):
         retcode = self.umount(mount_point, chk_err=chk_err)
         if chk_err and retcode != 0:
             raise OSUtilError("Failed to umount dvd.")
-    
+
     def eject_dvd(self, chk_err=True):
         dvd = self.get_dvd_device()
         retcode = shellutil.run("eject {0}".format(dvd))
@@ -547,6 +547,7 @@ class DefaultOSUtil(object):
         FOOTER_LEASE = "}"
         FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S"
 
+        logger.info("looking for leases in path [{0}]".format(pathglob))
         for lease_file in glob.glob(pathglob):
             leases = open(lease_file).read()
             if HEADER_OPTION in leases:
@@ -557,6 +558,7 @@ class DefaultOSUtil(object):
                     if line.startswith(HEADER_LEASE):
                         cached_endpoint = None
                         has_option_245 = False
+                        expired = True
                     elif HEADER_DNS in line:
                         cached_endpoint = line.replace(HEADER_DNS, '').strip(" ;")
                     elif HEADER_OPTION in line:
@@ -573,10 +575,17 @@ class DefaultOSUtil(object):
                             except:
                                 logger.error("could not parse expiry token '{0}'".format(line))
                     elif FOOTER_LEASE in line:
+                        logger.info("dhcp entry:{0}, 245:{1}, expired:{2}".format(
+                            cached_endpoint, has_option_245, expired))
                         if not expired and cached_endpoint is not None and has_option_245:
                             endpoint = cached_endpoint
-                            # return the first valid entry
-                            break
+                            logger.info("found endpoint [{0}]".format(endpoint))
+                            # we want to return the last valid entry, so
+                            # keep searching
+        if endpoint is not None:
+            logger.info("cached endpoint found [{0}]".format(endpoint))
+        else:
+            logger.info("cached endpoint not found")
         return endpoint
 
     def is_missing_default_route(self):
@@ -761,7 +770,7 @@ class DefaultOSUtil(object):
             return int(ret[1])
         else:
             raise OSUtilError("Failed to get processor cores")
-    
+
     def set_admin_access_to_ip(self, dest_ip):
         #This allows root to access dest_ip
         rm_old= "iptables -D OUTPUT -d {0} -j ACCEPT -m owner --uid-owner 0"
diff --git a/azurelinuxagent/common/protocol/util.py b/azurelinuxagent/common/protocol/util.py
index 98adeeda..7e7a74f1 100644
--- a/azurelinuxagent/common/protocol/util.py
+++ b/azurelinuxagent/common/protocol/util.py
@@ -139,6 +139,7 @@ class ProtocolUtil(object):
         except ProtocolError as e:
             logger.info("WireServer is not responding. Reset endpoint")
             self.dhcp_handler.endpoint = None
+            self.dhcp_handler.skip_cache = True
             raise e
 
     def _detect_metadata_protocol(self):
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index a482e319..a488647e 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -867,7 +867,6 @@ class WireClient(object):
             "x-ms-guest-agent-public-x509-cert": cert
         }
 
-
 class VersionInfo(object):
     def __init__(self, xml_text):
         """
@@ -981,7 +980,6 @@ class SharedConfig(object):
         # Not used currently
         return self
 
-
 class Certificates(object):
     """
     Object containing certificates of host and provisioned user.
diff --git a/azurelinuxagent/common/rdma.py b/azurelinuxagent/common/rdma.py
index 32d46adb..3ba332d3 100644
--- a/azurelinuxagent/common/rdma.py
+++ b/azurelinuxagent/common/rdma.py
@@ -21,9 +21,15 @@ Handle packages and modules to enable RDMA for IB networking
 
 import os
 import re
+import threading
+import time
 import azurelinuxagent.common.logger as logger
+import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.shellutil as shellutil
 
+dapl_config_paths = ['/etc/dat.conf', '/etc/rdma/dat.conf',
+                     '/usr/local/etc/dat.conf']
+
 
 class RDMAHandler(object):
 
@@ -63,7 +69,7 @@ class RDMAHandler(object):
     def load_driver_module(self):
         """Load the kernel driver, this depends on the proper driver
            to be installed with the install_driver() method"""
-        logger.info('RDMA: Loading the kernel driver.')
+        logger.info("RDMA: probing module '%s'" % self.driver_module_name)
         result = shellutil.run('modprobe %s' % self.driver_module_name)
         if result != 0:
             error_msg = 'Could not load "%s" kernel module. '
@@ -82,7 +88,7 @@ class RDMAHandler(object):
 
     def is_driver_loaded(self):
         """Check if the network module is loaded in kernel space"""
-        cmd = 'lsmod | grep %s' % self.driver_module_name
+        cmd = 'lsmod | grep ^%s' % self.driver_module_name
         status, loaded_modules = shellutil.run_get_output(cmd)
         logger.info('RDMA: Checking if the module loaded.')
         if loaded_modules:
@@ -97,3 +103,134 @@ class RDMAHandler(object):
         ret = shellutil.run('shutdown -r now')
         if ret != 0:
             logger.error('RDMA: Failed to reboot the system')
+
+
+dapl_config_paths = [
+    '/etc/dat.conf', '/etc/rdma/dat.conf', '/usr/local/etc/dat.conf']
+
+class RDMADeviceHandler(object):
+
+    """
+    Responsible for writing RDMA IP and MAC address to the /dev/hvnd_rdma
+    interface.
+    """
+
+    rdma_dev = '/dev/hvnd_rdma'
+    device_check_timeout_sec = 120
+    device_check_interval_sec = 1
+
+    ipv4_addr = None
+    mac_adr = None
+
+    def __init__(self, ipv4_addr, mac_addr):
+        self.ipv4_addr = ipv4_addr
+        self.mac_addr = mac_addr
+
+    def start(self):
+        """
+        Start a thread in the background to process the RDMA tasks and returns.
+        """
+        logger.info("RDMA: starting device processing in the background.")
+        threading.Thread(target=self.process).start()
+
+    def process(self):
+        RDMADeviceHandler.wait_rdma_device(
+            self.rdma_dev, self.device_check_timeout_sec, self.device_check_interval_sec)
+        RDMADeviceHandler.update_dat_conf(dapl_config_paths, self.ipv4_addr)
+        RDMADeviceHandler.write_rdma_config_to_device(
+            self.rdma_dev, self.ipv4_addr, self.mac_addr)
+        RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr)
+
+    @staticmethod
+    def update_dat_conf(paths, ipv4_addr):
+        """
+        Looks at paths for dat.conf file and updates the ip address for the
+        infiniband interface.
+        """
+        logger.info("Updating DAPL configuration file")
+        for f in paths:
+            logger.info("RDMA: trying {0}".format(f))
+            if not os.path.isfile(f):
+                logger.info(
+                    "RDMA: DAPL config not found at {0}".format(f))
+                continue
+            logger.info("RDMA: DAPL config is at: {0}".format(f))
+            cfg = fileutil.read_file(f)
+            new_cfg = RDMADeviceHandler.replace_dat_conf_contents(
+                cfg, ipv4_addr)
+            fileutil.write_file(f, new_cfg)
+            logger.info("RDMA: DAPL configuration is updated")
+            return
+
+        raise Exception("RDMA: DAPL configuration file not found at predefined paths")
+
+    @staticmethod
+    def replace_dat_conf_contents(cfg, ipv4_addr):
+        old = "ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 \"\S+ 0\""
+        new = "ofa-v2-ib0 u2.0 nonthreadsafe default libdaplofa.so.2 dapl.2.0 \"{0} 0\"".format(
+            ipv4_addr)
+        return re.sub(old, new, cfg)
+
+    @staticmethod
+    def write_rdma_config_to_device(path, ipv4_addr, mac_addr):
+        data = RDMADeviceHandler.generate_rdma_config(ipv4_addr, mac_addr)
+        logger.info(
+            "RDMA: Updating device with configuration: {0}".format(data))
+        with open(path, "w") as f:
+            f.write(data)
+        logger.info("RDMA: Updated device with IPv4/MAC addr successfully")
+
+    @staticmethod
+    def generate_rdma_config(ipv4_addr, mac_addr):
+        return 'rdmaMacAddress="{0}" rdmaIPv4Address="{1}"'.format(mac_addr, ipv4_addr)
+
+    @staticmethod
+    def wait_rdma_device(path, timeout_sec, check_interval_sec):
+        logger.info("RDMA: waiting for device={0} timeout={1}s".format(path, timeout_sec))
+        total_retries = timeout_sec/check_interval_sec
+        n = 0
+        while n < total_retries:
+            if os.path.exists(path):
+                logger.info("RDMA: device ready")
+                return
+            logger.verbose(
+                "RDMA: device not ready, sleep {0}s".format(check_interval_sec))
+            time.sleep(check_interval_sec)
+            n += 1
+        logger.error("RDMA device wait timed out")
+        raise Exception("The device did not show up in {0} seconds ({1} retries)".format(
+            timeout_sec, total_retries))
+
+    @staticmethod
+    def update_network_interface(mac_addr, ipv4_addr):
+        netmask=16
+        
+        logger.info("RDMA: will update the network interface with IPv4/MAC")
+
+        if_name=RDMADeviceHandler.get_interface_by_mac(mac_addr)
+        logger.info("RDMA: network interface found: {0}", if_name)
+        logger.info("RDMA: bringing network interface up")
+        if shellutil.run("ifconfig {0} up".format(if_name)) != 0:
+            raise Exception("Could not bring up RMDA interface: {0}".format(if_name))
+
+        logger.info("RDMA: configuring IPv4 addr and netmask on interface")
+        addr = '{0}/{1}'.format(ipv4_addr, netmask)
+        if shellutil.run("ifconfig {0} {1}".format(if_name, addr)) != 0:
+            raise Exception("Could set addr to {1} on {0}".format(if_name, addr))
+        logger.info("RDMA: network address and netmask configured on interface")
+
+    @staticmethod
+    def get_interface_by_mac(mac):
+        ret, output = shellutil.run_get_output("ifconfig -a")
+        if ret != 0:
+            raise Exception("Failed to list network interfaces")
+        output = output.replace('\n', '')
+        match = re.search(r"(eth\d).*(HWaddr|ether) {0}".format(mac), 
+                          output, re.IGNORECASE)
+        if match is None:
+            raise Exception("Failed to get ifname with mac: {0}".format(mac))
+        output = match.group(0)
+        eths = re.findall(r"eth\d", output)
+        if eths is None or len(eths) == 0:
+            raise Exception("ifname with mac: {0} not found".format(mac))
+        return eths[-1]
diff --git a/azurelinuxagent/daemon/main.py b/azurelinuxagent/daemon/main.py
index 42beb4fa..431c68a2 100644
--- a/azurelinuxagent/daemon/main.py
+++ b/azurelinuxagent/daemon/main.py
@@ -30,8 +30,10 @@ from azurelinuxagent.common.version import AGENT_LONG_NAME, AGENT_VERSION, \
                                      DISTRO_NAME, DISTRO_VERSION, \
                                      DISTRO_FULL_NAME, PY_VERSION_MAJOR, \
                                      PY_VERSION_MINOR, PY_VERSION_MICRO
+from azurelinuxagent.common.protocol.wire import SHARED_CONF_FILE_NAME
 import azurelinuxagent.common.event as event
 import azurelinuxagent.common.utils.fileutil as fileutil
+from azurelinuxagent.common.utils.textutil import parse_doc, find, getattrib
 from azurelinuxagent.common.osutil import get_osutil
 from azurelinuxagent.common.protocol import get_protocol_util
 from azurelinuxagent.daemon.scvmm import get_scvmm_handler
@@ -41,6 +43,7 @@ from azurelinuxagent.daemon.env import get_env_handler
 from azurelinuxagent.pa.provision import get_provision_handler
 from azurelinuxagent.pa.rdma import get_rdma_handler
 from azurelinuxagent.ga.update import get_update_handler
+from azurelinuxagent.common.rdma import RDMADeviceHandler
 
 def get_daemon_handler():
     return DaemonHandler()
@@ -52,14 +55,6 @@ class DaemonHandler(object):
     def __init__(self):
         self.running = True
         self.osutil = get_osutil()
-        self.protocol_util = get_protocol_util()
-        self.scvmm_handler = get_scvmm_handler()
-        self.resourcedisk_handler = get_resourcedisk_handler()
-        self.rdma_handler = get_rdma_handler()
-        self.monitor_handler = get_monitor_handler()
-        self.env_handler = get_env_handler()
-        self.provision_handler = get_provision_handler()
-        self.update_handler = get_update_handler()
 
     def run(self):
         logger.info("{0} Version:{1}", AGENT_LONG_NAME, AGENT_VERSION)
@@ -95,6 +90,16 @@ class DaemonHandler(object):
 
     def daemon(self):
         logger.info("Run daemon") 
+
+        self.protocol_util = get_protocol_util()
+        self.scvmm_handler = get_scvmm_handler()
+        self.resourcedisk_handler = get_resourcedisk_handler()
+        self.rdma_handler = get_rdma_handler()
+        self.monitor_handler = get_monitor_handler()
+        self.env_handler = get_env_handler()
+        self.provision_handler = get_provision_handler()
+        self.update_handler = get_update_handler()
+
         #Create lib dir
         if not os.path.isdir(conf.get_lib_dir()):
             fileutil.mkdir(conf.get_lib_dir(), mode=0o700)
@@ -116,6 +121,52 @@ class DaemonHandler(object):
         self.monitor_handler.run()
 
         self.env_handler.run()
+
+        # Enable RDMA, continue in errors
+        if conf.enable_rdma():
+                logger.info("RDMA capabilities are enabled in configuration")
+                try:
+                    self.setup_rdma_device()
+                except Exception as e:
+                    logger.error("Error setting up rdma device: %s" % e)
+        else:
+            logger.info("RDMA capabilities are not enabled, skipping")
         
         while self.running:
             self.update_handler.run_latest()
+
+
+    def setup_rdma_device(self):
+        logger.verbose("Parsing SharedConfig XML contents for RDMA details")
+        xml_doc = parse_doc(
+            fileutil.read_file(os.path.join(conf.get_lib_dir(), SHARED_CONF_FILE_NAME)))
+        if xml_doc is None:
+            logger.error("Could not parse SharedConfig XML document")
+            return
+        instance_elem = find(xml_doc, "Instance")
+        if not instance_elem:
+            logger.error("Could not find <Instance> in SharedConfig document")
+            return
+
+        rdma_ipv4_addr = getattrib(instance_elem, "rdmaIPv4Address")
+        if not rdma_ipv4_addr:
+            logger.error(
+                "Could not find rdmaIPv4Address attribute on Instance element of SharedConfig.xml document")
+            return
+
+        rdma_mac_addr = getattrib(instance_elem, "rdmaMacAddress")
+        if not rdma_mac_addr:
+            logger.error(
+                "Could not find rdmaMacAddress attribute on Instance element of SharedConfig.xml document")
+            return
+
+        # add colons to the MAC address (e.g. 00155D33FF1D ->
+        # 00:15:5D:33:FF:1D)
+        rdma_mac_addr = ':'.join([rdma_mac_addr[i:i+2]
+                                  for i in range(0, len(rdma_mac_addr), 2)])
+        logger.info("Found RDMA details. IPv4={0} MAC={1}".format(
+            rdma_ipv4_addr, rdma_mac_addr))
+
+        # Set up the RDMA device with collected informatino
+        RDMADeviceHandler(rdma_ipv4_addr, rdma_mac_addr).start()
+        logger.info("RDMA: device is set up")
diff --git a/azurelinuxagent/daemon/monitor.py b/azurelinuxagent/daemon/monitor.py
index 40170325..9e1e1eb7 100644
--- a/azurelinuxagent/daemon/monitor.py
+++ b/azurelinuxagent/daemon/monitor.py
@@ -92,6 +92,8 @@ class MonitorHandler(object):
         self.sysinfo = []
 
     def run(self):
+        self.init_sysinfo()
+
         event_thread = threading.Thread(target=self.daemon)
         event_thread.setDaemon(True)
         event_thread.start()
@@ -175,7 +177,6 @@ class MonitorHandler(object):
             logger.error("{0}", e)
 
     def daemon(self):
-        self.init_sysinfo()
         last_heartbeat = datetime.datetime.min
         period = datetime.timedelta(minutes=30)
         while True:
diff --git a/azurelinuxagent/pa/rdma/centos.py b/azurelinuxagent/pa/rdma/centos.py
new file mode 100644
index 00000000..c527e1be
--- /dev/null
+++ b/azurelinuxagent/pa/rdma/centos.py
@@ -0,0 +1,203 @@
+# Microsoft Azure Linux Agent
+#
+# Copyright 2014 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.4+ and Openssl 1.0+
+#
+
+import glob
+import os
+import re
+import azurelinuxagent.common.logger as logger
+import azurelinuxagent.common.utils.shellutil as shellutil
+from azurelinuxagent.common.rdma import RDMAHandler
+
+
+class CentOSRDMAHandler(RDMAHandler):
+    rdma_user_mode_package_name = 'microsoft-hyper-v-rdma'
+    rdma_kernel_mode_package_name = 'kmod-microsoft-hyper-v-rdma'
+    rdma_wrapper_package_name = 'msft-rdma-drivers'
+
+    hyper_v_package_name = "hypervkvpd"
+    hyper_v_package_name_new = "microsoft-hyper-v"
+
+    version_major = None
+    version_minor = None
+
+    def __init__(self, distro_version):
+        v = distro_version.split('.')
+        if len(v) < 2:
+            raise Exception('Unexpected centos version: %s' % distro_version)
+        self.version_major, self.version_minor = v[0], v[1]
+
+    def install_driver(self):
+        """
+        Install the KVP daemon and the appropriate RDMA driver package for the
+        RDMA firmware.
+        """
+        fw_version = RDMAHandler.get_rdma_version()
+        if not fw_version:
+            raise Exception('Cannot determine RDMA firmware version')
+        logger.info("RDMA: found firmware version: {0}".format(fw_version))
+        fw_version = self.get_int_rdma_version(fw_version)
+        installed_pkg = self.get_rdma_package_info()
+        if installed_pkg:
+            logger.info(
+                'RDMA: driver package present: {0}'.format(installed_pkg))
+            if self.is_rdma_package_up_to_date(installed_pkg, fw_version):
+                logger.info('RDMA: driver package is up-to-date')
+                return
+            else:
+                logger.info('RDMA: driver package needs updating')
+                self.update_rdma_package(fw_version)
+        else:
+            logger.info('RDMA: driver package is NOT installed')
+            self.update_rdma_package(fw_version)
+
+    def is_rdma_package_up_to_date(self, pkg, fw_version):
+        # Example match (pkg name, -, followed by 3 segments, fw_version and -):
+        # - pkg=microsoft-hyper-v-rdma-4.1.0.142-20160323.x86_64
+        # - fw_version=142
+        pattern = '{0}-\d\.\d\.\d\.({1})-'.format(
+            self.rdma_user_mode_package_name, fw_version)
+        return re.match(pattern, pkg)
+
+    @staticmethod
+    def get_int_rdma_version(version):
+        s = version.split('.')
+        if len(s) == 0:
+            raise Exception('Unexpected RDMA firmware version: "%s"' % version)
+        return s[0]
+
+    def get_rdma_package_info(self):
+        """
+        Returns the installed rdma package name or None
+        """
+        ret, output = shellutil.run_get_output(
+            'rpm -q %s' % self.rdma_user_mode_package_name, chk_err=False)
+        if ret != 0:
+            return None
+        return output
+
+    def update_rdma_package(self, fw_version):
+        logger.info("RDMA: updating RDMA packages")
+        self.refresh_repos()
+        self.force_install_package(self.rdma_wrapper_package_name)
+        self.install_rdma_drivers(fw_version)
+
+    def force_install_package(self, pkg_name):
+        """
+        Attempts to remove existing package and installs the package
+        """
+        logger.info('RDMA: Force installing package: %s' % pkg_name)
+        if self.uninstall_package(pkg_name) != 0:
+            logger.info('RDMA: Erasing package failed but will continue')
+        if self.install_package(pkg_name) != 0:
+            raise Exception('Failed to install package "{0}"'.format(pkg_name))
+        logger.info('RDMA: installation completed: %s' % pkg_name)
+
+    @staticmethod
+    def uninstall_package(pkg_name):
+        return shellutil.run('yum erase -y -q {0}'.format(pkg_name))
+
+    @staticmethod
+    def install_package(pkg_name):
+        return shellutil.run('yum install -y -q {0}'.format(pkg_name))
+
+    def refresh_repos(self):
+        logger.info("RDMA: refreshing yum repos")
+        if shellutil.run('yum clean all') != 0:
+            raise Exception('Cleaning yum repositories failed')
+        if shellutil.run('yum updateinfo') != 0:
+            raise Exception('Failed to act on yum repo update information')
+        logger.info("RDMA: repositories refreshed")
+
+    def install_rdma_drivers(self, fw_version):
+        """
+        Installs the drivers from /opt/rdma/rhel[Major][Minor] directory,
+        particularly the microsoft-hyper-v-rdma-* kmod-* and (no debuginfo or
+        src). Tries to uninstall them first.
+        """
+        pkg_dir = '/opt/microsoft/rdma/rhel{0}{1}'.format(
+            self.version_major, self.version_minor)
+        logger.info('RDMA: pkgs dir: {0}'.format(pkg_dir))
+        if not os.path.isdir(pkg_dir):
+            raise Exception('RDMA packages directory %s is missing' % pkg_dir)
+
+        pkgs = os.listdir(pkg_dir)
+        logger.info('RDMA: found %d files in package directory' % len(pkgs))
+
+        # Uninstal KVP daemon first (if exists)
+        self.uninstall_kvp_driver_package_if_exists()
+
+        # Install kernel mode driver (kmod-microsoft-hyper-v-rdma-*)
+        kmod_pkg = self.get_file_by_pattern(
+            pkgs, "%s-\d\.\d\.\d\.+(%s)-\d{8}\.x86_64.rpm" % (self.rdma_kernel_mode_package_name, fw_version))
+        if not kmod_pkg:
+            raise Exception("RDMA kernel mode package not found")
+        kmod_pkg_path = os.path.join(pkg_dir, kmod_pkg)
+        self.uninstall_pkg_and_install_from(
+            'kernel mode', self.rdma_kernel_mode_package_name, kmod_pkg_path)
+
+        # Install user mode driver (microsoft-hyper-v-rdma-*)
+        umod_pkg = self.get_file_by_pattern(
+            pkgs, "%s-\d\.\d\.\d\.+(%s)-\d{8}\.x86_64.rpm" % (self.rdma_user_mode_package_name, fw_version))
+        if not umod_pkg:
+            raise Exception("RDMA user mode package not found")
+        umod_pkg_path = os.path.join(pkg_dir, umod_pkg)
+        self.uninstall_pkg_and_install_from(
+            'user mode', self.rdma_user_mode_package_name, umod_pkg_path)
+
+        logger.info("RDMA: driver packages installed")
+        self.load_driver_module()
+        if not self.is_driver_loaded():
+            logger.info("RDMA: driver module is not loaded; reboot required")
+            self.reboot_system()
+        else:
+            logger.info("RDMA: kernel module is loaded")
+
+    @staticmethod
+    def get_file_by_pattern(list, pattern):
+        for l in list:
+            if re.match(pattern, l):
+                return l
+        return None
+
+    def uninstall_pkg_and_install_from(self, pkg_type, pkg_name, pkg_path):
+        logger.info(
+            "RDMA: Processing {0} driver: {1}".format(pkg_type, pkg_path))
+        logger.info("RDMA: Try to uninstall existing version: %s" % pkg_name)
+        if self.uninstall_package(pkg_name) == 0:
+            logger.info("RDMA: Successfully uninstaled %s" % pkg_name)
+        logger.info(
+            "RDMA: Installing {0} package from {1}".format(pkg_type, pkg_path))
+        if self.install_package(pkg_path) != 0:
+            raise Exception(
+                "Failed to install RDMA {0} package".format(pkg_type))
+
+    def uninstall_kvp_driver_package_if_exists(self):
+        kvp_pkgs = [self.hyper_v_package_name,
+                    self.hyper_v_package_name_new]
+
+        for kvp_pkg in kvp_pkgs:
+            if shellutil.run("rpm -q %s" % kvp_pkg, chk_err=False) != 0:
+                logger.info(
+                    "RDMA: kvp package %s does not exist, skipping" % kvp_pkg)
+            else:
+                logger.info('RDMA: erasing kvp package "%s"' % kvp_pkg)
+                if shellutil.run("yum erase -q -y %s" % kvp_pkg, chk_err=False) == 0:
+                    logger.info("RDMA: successfully erased package")
+                else:
+                    logger.error("RDMA: failed to erase package")
diff --git a/azurelinuxagent/pa/rdma/factory.py b/azurelinuxagent/pa/rdma/factory.py
index 9a6a238d..535b3d3e 100644
--- a/azurelinuxagent/pa/rdma/factory.py
+++ b/azurelinuxagent/pa/rdma/factory.py
@@ -20,6 +20,7 @@ import azurelinuxagent.common.logger as logger
 from azurelinuxagent.common.version import DISTRO_FULL_NAME, DISTRO_VERSION
 from azurelinuxagent.common.rdma import RDMAHandler
 from .suse import SUSERDMAHandler
+from .centos import CentOSRDMAHandler
 
 
 def get_rdma_handler(
@@ -33,4 +34,8 @@ def get_rdma_handler(
     ):
         return SUSERDMAHandler()
 
+    if distro_full_name == 'CentOS Linux' or distro_full_name == 'CentOS':
+        return CentOSRDMAHandler(distro_version)
+
+    logger.info("No RDMA handler exists for distro='{0}' version='{1}'", distro_full_name, distro_version)
     return RDMAHandler()


Test patch:
diff --git a/tests/common/dhcp/test_dhcp.py b/tests/common/dhcp/test_dhcp.py
index 5e4d6085..c27ffe88 100644
--- a/tests/common/dhcp/test_dhcp.py
+++ b/tests/common/dhcp/test_dhcp.py
@@ -65,12 +65,43 @@ class TestDHCP(AgentTestCase):
         self.assertTrue(dhcp_handler.routes is None)
         self.assertTrue(dhcp_handler.gateway is None)
 
-    def test_dhcp_lease_exists(self):
+    def test_dhcp_cache_exists(self):
         dhcp_handler = dhcp.get_dhcp_handler()
         dhcp_handler.osutil = osutil.DefaultOSUtil()
-        with patch.object(osutil.DefaultOSUtil, 'get_dhcp_lease_endpoint', return_value=None):
-            self.assertFalse(dhcp_handler.dhcp_lease_exists)
+        with patch.object(osutil.DefaultOSUtil, 'get_dhcp_lease_endpoint',
+                          return_value=None):
+            self.assertFalse(dhcp_handler.dhcp_cache_exists)
             self.assertEqual(dhcp_handler.endpoint, None)
-        with patch.object(osutil.DefaultOSUtil, 'get_dhcp_lease_endpoint', return_value="foo"):
-            self.assertTrue(dhcp_handler.dhcp_lease_exists)
+        with patch.object(osutil.DefaultOSUtil, 'get_dhcp_lease_endpoint',
+                          return_value="foo"):
+            self.assertTrue(dhcp_handler.dhcp_cache_exists)
             self.assertEqual(dhcp_handler.endpoint, "foo")
+
+    def test_dhcp_skip_cache(self):
+        handler = dhcp.get_dhcp_handler()
+        handler.osutil = osutil.DefaultOSUtil()
+        with patch('os.path.exists', return_value=False):
+            with patch.object(osutil.DefaultOSUtil, 'get_dhcp_lease_endpoint')\
+                    as patch_dhcp_cache:
+                with patch.object(dhcp.DhcpHandler, 'send_dhcp_req') \
+                        as patch_dhcp_send:
+
+                    endpoint = 'foo'
+                    patch_dhcp_cache.return_value = endpoint
+
+                    # endpoint comes from cache
+                    self.assertFalse(handler.skip_cache)
+                    handler.run()
+                    self.assertTrue(patch_dhcp_cache.call_count == 1)
+                    self.assertTrue(patch_dhcp_send.call_count == 0)
+                    self.assertTrue(handler.endpoint == endpoint)
+
+                    # reset
+                    handler.skip_cache = True
+                    handler.endpoint = None
+
+                    # endpoint comes from dhcp request
+                    self.assertTrue(handler.skip_cache)
+                    handler.run()
+                    self.assertTrue(patch_dhcp_cache.call_count == 1)
+                    self.assertTrue(patch_dhcp_send.call_count == 1)
diff --git a/tests/common/osutil/test_default.py b/tests/common/osutil/test_default.py
index 23c5f57e..22813484 100644
--- a/tests/common/osutil/test_default.py
+++ b/tests/common/osutil/test_default.py
@@ -124,5 +124,12 @@ class TestOSUtil(AgentTestCase):
                 self.assertTrue(endpoint is not None)
                 self.assertEqual(endpoint, "168.63.129.16")
 
+    def test_dhcp_lease_multi(self):
+        with patch.object(glob, "glob", return_value=['/var/lib/dhcp/dhclient.eth0.leases']):
+            with patch(open_patch(), mock.mock_open(read_data=load_data("dhcp.leases.multi"))):
+                endpoint = get_osutil(distro_name='ubuntu', distro_version='12.04').get_dhcp_lease_endpoint()
+                self.assertTrue(endpoint is not None)
+                self.assertEqual(endpoint, "second")
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/data/dhcp.leases b/tests/data/dhcp.leases
index 875b13be..cb4f3961 100644
--- a/tests/data/dhcp.leases
+++ b/tests/data/dhcp.leases
@@ -7,14 +7,15 @@ lease {
   option routers 10.0.1.1;
   option dhcp-message-type 5;
   option dhcp-server-identifier 168.63.129.16;
-  option domain-name-servers expired;
+  option domain-name-servers invalid;
   option dhcp-renewal-time 4294967295;
+  option rfc3442-classless-static-routes 0,10,0,1,1,32,168,63,129,16,10,0,1,1;
   option unknown-245 a8:3f:81:10;
   option dhcp-rebinding-time 4294967295;
   option domain-name "qylsde3bnlhu5dstzf3bav5inc.fx.internal.cloudapp.net";
-  renew 4 2015/06/16 16:58:54;
-  rebind 4 2015/06/16 16:58:54;
-  expire 4 2015/06/16 16:58:54;
+  renew 0 2152/07/23 23:27:10;
+  rebind 0 2152/07/23 23:27:10;
+  expire 0 never;
 }
 lease {
   interface "eth0";
@@ -25,15 +26,14 @@ lease {
   option routers 10.0.1.1;
   option dhcp-message-type 5;
   option dhcp-server-identifier 168.63.129.16;
-  option domain-name-servers 168.63.129.16;
+  option domain-name-servers expired;
   option dhcp-renewal-time 4294967295;
-  option rfc3442-classless-static-routes 0,10,0,1,1,32,168,63,129,16,10,0,1,1;
   option unknown-245 a8:3f:81:10;
   option dhcp-rebinding-time 4294967295;
   option domain-name "qylsde3bnlhu5dstzf3bav5inc.fx.internal.cloudapp.net";
-  renew 0 2152/07/23 23:27:10;
-  rebind 0 2152/07/23 23:27:10;
-  expire 0 2152/07/23 23:27:10;
+  renew 4 2015/06/16 16:58:54;
+  rebind 4 2015/06/16 16:58:54;
+  expire 4 2015/06/16 16:58:54;
 }
 lease {
   interface "eth0";
@@ -44,7 +44,7 @@ lease {
   option routers 10.0.1.1;
   option dhcp-message-type 5;
   option dhcp-server-identifier 168.63.129.16;
-  option domain-name-servers invalid;
+  option domain-name-servers 168.63.129.16;
   option dhcp-renewal-time 4294967295;
   option rfc3442-classless-static-routes 0,10,0,1,1,32,168,63,129,16,10,0,1,1;
   option unknown-245 a8:3f:81:10;
@@ -52,5 +52,5 @@ lease {
   option domain-name "qylsde3bnlhu5dstzf3bav5inc.fx.internal.cloudapp.net";
   renew 0 2152/07/23 23:27:10;
   rebind 0 2152/07/23 23:27:10;
-  expire 0 never;
+  expire 0 2152/07/23 23:27:10;
 }
diff --git a/tests/data/dhcp.leases.multi b/tests/data/dhcp.leases.multi
new file mode 100644
index 00000000..cfe9c67d
--- /dev/null
+++ b/tests/data/dhcp.leases.multi
@@ -0,0 +1,57 @@
+lease {
+  interface "eth0";
+  fixed-address 10.0.1.4;
+  server-name "RDE41D2D9BB18C";
+  option subnet-mask 255.255.255.0;
+  option dhcp-lease-time 4294967295;
+  option routers 10.0.1.1;
+  option dhcp-message-type 5;
+  option dhcp-server-identifier 168.63.129.16;
+  option domain-name-servers first;
+  option dhcp-renewal-time 4294967295;
+  option rfc3442-classless-static-routes 0,10,0,1,1,32,168,63,129,16,10,0,1,1;
+  option unknown-245 a8:3f:81:10;
+  option dhcp-rebinding-time 4294967295;
+  option domain-name "qylsde3bnlhu5dstzf3bav5inc.fx.internal.cloudapp.net";
+  renew 0 2152/07/23 23:27:10;
+  rebind 0 2152/07/23 23:27:10;
+  expire 0 2152/07/23 23:27:10;
+}
+lease {
+  interface "eth0";
+  fixed-address 10.0.1.4;
+  server-name "RDE41D2D9BB18C";
+  option subnet-mask 255.255.255.0;
+  option dhcp-lease-time 4294967295;
+  option routers 10.0.1.1;
+  option dhcp-message-type 5;
+  option dhcp-server-identifier 168.63.129.16;
+  option domain-name-servers second;
+  option dhcp-renewal-time 4294967295;
+  option rfc3442-classless-static-routes 0,10,0,1,1,32,168,63,129,16,10,0,1,1;
+  option unknown-245 a8:3f:81:10;
+  option dhcp-rebinding-time 4294967295;
+  option domain-name "qylsde3bnlhu5dstzf3bav5inc.fx.internal.cloudapp.net";
+  renew 0 2152/07/23 23:27:10;
+  rebind 0 2152/07/23 23:27:10;
+  expire 0 2152/07/23 23:27:10;
+}
+lease {
+  interface "eth0";
+  fixed-address 10.0.1.4;
+  server-name "RDE41D2D9BB18C";
+  option subnet-mask 255.255.255.0;
+  option dhcp-lease-time 4294967295;
+  option routers 10.0.1.1;
+  option dhcp-message-type 5;
+  option dhcp-server-identifier 168.63.129.16;
+  option domain-name-servers expired;
+  option dhcp-renewal-time 4294967295;
+  option rfc3442-classless-static-routes 0,10,0,1,1,32,168,63,129,16,10,0,1,1;
+  option unknown-245 a8:3f:81:10;
+  option dhcp-rebinding-time 4294967295;
+  option domain-name "qylsde3bnlhu5dstzf3bav5inc.fx.internal.cloudapp.net";
+  renew 0 2152/07/23 23:27:10;
+  rebind 0 2152/07/23 23:27:10;
+  expire 0 2012/07/23 23:27:10;
+}