Repository: Azure/WALinuxAgent
Pull request #344 (2016-07-29 00:52:37)

rdma: log exception thrown in device processing
Exceptions thrown from rdma processing thread running in
the background were not logged properly.




Patch:
diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py
index 374b0e7e..957b686f 100644
--- a/azurelinuxagent/common/event.py
+++ b/azurelinuxagent/common/event.py
@@ -32,21 +32,23 @@ from azurelinuxagent.common.protocol.restapi import TelemetryEventParam, \
                                              TelemetryEvent, \
                                              set_properties, get_properties
 from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \
-                                     DISTRO_CODE_NAME, AGENT_VERSION
+                                     DISTRO_CODE_NAME, AGENT_VERSION, \
+                                     CURRENT_AGENT, CURRENT_VERSION
 
 
 class WALAEventOperation:
+    ActivateResourceDisk="ActivateResourceDisk"
+    Disable = "Disable"
+    Download = "Download"
+    Enable = "Enable"
     HeartBeat="HeartBeat"
-    Provision = "Provision"
     Install = "Install"
+    Provision = "Provision"
+    Restart="Restart"
+    UnhandledError="UnhandledError"
     UnInstall = "UnInstall"
-    Disable = "Disable"
-    Enable = "Enable"
-    Download = "Download"
     Upgrade = "Upgrade"
     Update = "Update"
-    ActivateResourceDisk="ActivateResourceDisk"
-    UnhandledError="UnhandledError"
 
 class EventLogger(object):
     def __init__(self):
@@ -71,7 +73,7 @@ class EventLogger(object):
         except IOError as e:
             raise EventError("Failed to write events to file:{0}", e)
 
-    def add_event(self, name, op="", is_success=True, duration=0, version=AGENT_VERSION,
+    def add_event(self, name, op="", is_success=True, duration=0, version=CURRENT_VERSION,
                   message="", evt_type="", is_internal=False):
         event = TelemetryEvent(1, "69B669B9-4AF8-4C50-BDC4-6006FA76E975")
         event.parameters.append(TelemetryEventParam('Name', name))
@@ -92,7 +94,7 @@ class EventLogger(object):
 
 __event_logger__ = EventLogger()
 
-def add_event(name, op="", is_success=True, duration=0, version=AGENT_VERSION,
+def add_event(name, op="", is_success=True, duration=0, version=CURRENT_VERSION,
               message="", evt_type="", is_internal=False,
               reporter=__event_logger__):
     log = logger.info if is_success else logger.error
diff --git a/azurelinuxagent/common/rdma.py b/azurelinuxagent/common/rdma.py
index 2b51dd5e..5dc1f669 100644
--- a/azurelinuxagent/common/rdma.py
+++ b/azurelinuxagent/common/rdma.py
@@ -197,12 +197,15 @@ class RDMADeviceHandler(object):
         threading.Thread(target=self.process).start()
 
     def process(self):
-        RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr)
-        RDMADeviceHandler.wait_rdma_device(
-            self.rdma_dev, self.device_check_timeout_sec, self.device_check_interval_sec)
-        RDMADeviceHandler.write_rdma_config_to_device(
-            self.rdma_dev, self.ipv4_addr, self.mac_addr)
-        RDMADeviceHandler.update_dat_conf(dapl_config_paths, self.ipv4_addr)
+        try:
+            RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr)
+            RDMADeviceHandler.wait_rdma_device(
+                self.rdma_dev, self.device_check_timeout_sec, self.device_check_interval_sec)
+            RDMADeviceHandler.write_rdma_config_to_device(
+                self.rdma_dev, self.ipv4_addr, self.mac_addr)
+            RDMADeviceHandler.update_dat_conf(dapl_config_paths, self.ipv4_addr)
+        except Exception as e:
+            logger.error("RDMA: device processing failed: {0}".format(e))
 
     @staticmethod
     def update_dat_conf(paths, ipv4_addr):
diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py
index 123d2e95..a4177593 100644
--- a/azurelinuxagent/common/version.py
+++ b/azurelinuxagent/common/version.py
@@ -50,7 +50,7 @@ def get_distro():
 
 AGENT_NAME = "WALinuxAgent"
 AGENT_LONG_NAME = "Azure Linux Agent"
-AGENT_VERSION = '2.1.5.0'
+AGENT_VERSION = '2.1.5.1'
 AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION)
 AGENT_DESCRIPTION = """\
 The Azure Linux Agent supports the provisioning and running of Linux
diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
index d3c8f329..642fdf24 100644
--- a/azurelinuxagent/ga/exthandlers.py
+++ b/azurelinuxagent/ga/exthandlers.py
@@ -108,6 +108,45 @@ def parse_ext_status(ext_status, data):
     for substatus in substatus_list:
         ext_status.substatusList.append(parse_ext_substatus(substatus))
 
+# This code migrates, if it exists, handler state and status from an
+# agent-owned directory into the handler-owned config directory
+#
+# Notes:
+#  - The v2.0.x branch wrote all handler-related state into the handler-owned
+#    config directory (e.g., /var/lib/waagent/Microsoft.Azure.Extensions.LinuxAsm-2.0.1/config).
+#  - The v2.1.x branch original moved that state into an agent-owned handler
+#    state directory (e.g., /var/lib/waagent/handler_state).
+#  - This move can cause v2.1.x agents to multiply invoke a handler's install
+#    command. It also makes clean-up more difficult since the agent must
+#    remove the state as well as the handler directory.
+def migrate_handler_state():
+    handler_state_path = os.path.join(conf.get_lib_dir(), "handler_state")
+    if not os.path.isdir(handler_state_path):
+        return
+
+    for handler_path in glob.iglob(os.path.join(handler_state_path, "*")):
+        handler = os.path.basename(handler_path)
+        handler_config_path = os.path.join(conf.get_lib_dir(), handler, "config")
+        if os.path.isdir(handler_config_path):
+            for file in ("State", "Status"):
+                from_path = os.path.join(handler_state_path, handler, file.lower())
+                to_path = os.path.join(handler_config_path, "Handler" + file)
+                if os.path.isfile(from_path) and not os.path.isfile(to_path):
+                    try:
+                        shutil.move(from_path, to_path)
+                    except Exception as e:
+                        logger.warn(
+                            "Exception occurred migrating {0} {1} file: {2}",
+                            handler,
+                            file,
+                            str(e))
+
+    try:
+        shutil.rmtree(handler_state_path)
+    except Exception as e:
+        logger.warn("Exception occurred removing {0}: {1}", handler_state_path, str(e))
+    return
+
 class ExtHandlerState(object):
     NotInstalled = "NotInstalled"
     Installed = "Installed"
@@ -547,10 +586,6 @@ class ExtHandlerInstance(object):
     
     def rm_ext_handler_dir(self):
         try:
-            handler_state_dir = self.get_handler_state_dir()
-            if os.path.isdir(handler_state_dir):
-                self.logger.info("Remove ext handler dir: {0}", handler_state_dir)
-                shutil.rmtree(handler_state_dir)
             base_dir = self.get_base_dir()
             if os.path.isdir(base_dir):
                 self.logger.info("Remove ext handler dir: {0}", base_dir)
@@ -746,28 +781,18 @@ class ExtHandlerInstance(object):
             fileutil.write_file(self.get_env_file(), json.dumps(env))
         except IOError as e:
             raise ExtensionError(u"Failed to save handler environment", e)
-    
-    def get_handler_state_dir(self):
-        return os.path.join(conf.get_lib_dir(), "handler_state", 
-                            self.get_full_name())
 
     def set_handler_state(self, handler_state):
-        state_dir = self.get_handler_state_dir()
-        if not os.path.exists(state_dir):
-            try:
-                fileutil.mkdir(state_dir, 0o700)
-            except IOError as e:
-                self.logger.error("Failed to create state dir: {0}", e)
-        
+        state_dir = self.get_conf_dir()
         try:
-            state_file = os.path.join(state_dir, "state")
+            state_file = os.path.join(state_dir, "HandlerState")
             fileutil.write_file(state_file, handler_state)
         except IOError as e:
             self.logger.error("Failed to set state: {0}", e)
     
     def get_handler_state(self):
-        state_dir = self.get_handler_state_dir()
-        state_file = os.path.join(state_dir, "state")
+        state_dir = self.get_conf_dir()
+        state_file = os.path.join(state_dir, "HandlerState")
         if not os.path.isfile(state_file):
             return ExtHandlerState.NotInstalled
 
@@ -777,32 +802,25 @@ class ExtHandlerInstance(object):
             self.logger.error("Failed to get state: {0}", e)
             return ExtHandlerState.NotInstalled
     
-    def set_handler_status(self, status="NotReady", message="", 
-                           code=0):
-        state_dir = self.get_handler_state_dir()
-        if not os.path.exists(state_dir):
-            try:
-                fileutil.mkdir(state_dir, 0o700)
-            except IOError as e:
-                self.logger.error("Failed to create state dir: {0}", e)
-        
+    def set_handler_status(self, status="NotReady", message="", code=0):
+        state_dir = self.get_conf_dir()
+
         handler_status = ExtHandlerStatus()
         handler_status.name = self.ext_handler.name
         handler_status.version = self.ext_handler.properties.version
         handler_status.message = message
         handler_status.code = code
         handler_status.status = status
-        status_file = os.path.join(state_dir, "status")
+        status_file = os.path.join(state_dir, "HandlerStatus")
 
         try:
-            fileutil.write_file(status_file, 
-                                json.dumps(get_properties(handler_status)))
+            fileutil.write_file(status_file, json.dumps(get_properties(handler_status)))
         except (IOError, ValueError, ProtocolError) as e:
             self.logger.error("Failed to save handler status: {0}", e)
         
     def get_handler_status(self):
-        state_dir = self.get_handler_state_dir()
-        status_file = os.path.join(state_dir, "status")
+        state_dir = self.get_conf_dir()
+        status_file = os.path.join(state_dir, "HandlerStatus")
         if not os.path.isfile(status_file):
             return None
         
diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py
index f49cef89..478a7a3f 100644
--- a/azurelinuxagent/ga/monitor.py
+++ b/azurelinuxagent/ga/monitor.py
@@ -183,8 +183,11 @@ class MonitorHandler(object):
         while True:
             if (datetime.datetime.now() - last_heartbeat) > period:
                 last_heartbeat = datetime.datetime.now()
-                add_event(op=WALAEventOperation.HeartBeat, name="WALA",
-                          is_success=True)
+                add_event(
+                    op=WALAEventOperation.HeartBeat,
+                    name=CURRENT_AGENT,
+                    version=CURRENT_VERSION,
+                    is_success=True)
             try:
                 self.collect_and_send_events()
             except Exception as e:
diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py
index e89608a0..37774d29 100644
--- a/azurelinuxagent/ga/update.py
+++ b/azurelinuxagent/ga/update.py
@@ -59,10 +59,14 @@ CHILD_LAUNCH_RESTART_MAX = 3
 CHILD_POLL_INTERVAL = 60
 
 MAX_FAILURE = 3 # Max failure allowed for agent before blacklisted
+RETAIN_INTERVAL = 24 * 60 * 60 # Retain interval for black list
 
 GOAL_STATE_INTERVAL = 25
 REPORT_STATUS_INTERVAL = 15
-RETAIN_INTERVAL = 24 * 60 * 60 # Retain interval for black list
+
+ORPHAN_WAIT_INTERVAL = 15 * 60 * 60
+
+AGENT_SENTINAL_FILE = "current_version"
 
 
 def get_update_handler():
@@ -218,12 +222,20 @@ class UpdateHandler(object):
         from azurelinuxagent.ga.env import get_env_handler
         get_env_handler().run()
 
-        from azurelinuxagent.ga.exthandlers import get_exthandlers_handler
+        from azurelinuxagent.ga.exthandlers import get_exthandlers_handler, migrate_handler_state
         exthandlers_handler = get_exthandlers_handler()
+        migrate_handler_state()
 
-        # TODO: Add means to stop running
         try:
+            self._ensure_no_orphans()
+            self._emit_restart_event()
+
+            # TODO: Add means to stop running
             while self.running:
+                if self._is_orphaned:
+                    logger.info("Goal state agent {0} was orphaned -- exiting", CURRENT_AGENT)
+                    break
+
                 if self._ensure_latest_agent():
                     if len(self.agents) > 0:
                         logger.info(
@@ -234,16 +246,26 @@ class UpdateHandler(object):
 
                 exthandlers_handler.run()
                 
-                time.sleep(25)
+                time.sleep(GOAL_STATE_INTERVAL)
 
         except Exception as e:
             logger.warn(u"Agent {0} failed with exception: {1}", CURRENT_AGENT, ustr(e))
             sys.exit(1)
+            return
 
+        self._shutdown()
         sys.exit(0)
         return
 
     def forward_signal(self, signum, frame):
+        # Note:
+        #  - At present, the handler is registered only for SIGTERM.
+        #    However, clean shutdown is both SIGTERM and SIGKILL.
+        #    A SIGKILL handler is not being registered at this time to
+        #    minimize perturbing the code.
+        if signum in (signal.SIGTERM, signal.SIGKILL):
+            self._shutdown()
+
         if self.child_process is None:
             return
         
@@ -258,6 +280,7 @@ class UpdateHandler(object):
             self.signal_handler(signum, frame)
         elif self.signal_handler is signal.SIG_DFL:
             if signum == signal.SIGTERM:
+                # TODO: This should set self.running to False vs. just exiting
                 sys.exit(0)
         return
 
@@ -275,6 +298,20 @@ class UpdateHandler(object):
         available_agents = [agent for agent in self.agents if agent.is_available]
         return available_agents[0] if len(available_agents) >= 1 else None
 
+    def _emit_restart_event(self):
+        if not self._is_clean_start:
+            msg = u"{0} unexpectedly restarted".format(CURRENT_AGENT)
+            logger.info(msg)
+            add_event(
+                AGENT_NAME,
+                version=CURRENT_VERSION,
+                op=WALAEventOperation.Restart,
+                is_success=False,
+                message=msg)
+
+        self._set_sentinal() 
+        return
+
     def _ensure_latest_agent(self, base_version=CURRENT_VERSION):
         # Ignore new agents if updating is disabled
         if not conf.get_autoupdate_enabled():
@@ -343,6 +380,34 @@ class UpdateHandler(object):
         # Return True if agents more recent than the current are available
         return len(self.agents) > 0 and self.agents[0].version > base_version
 
+    def _ensure_no_orphans(self, orphan_wait_interval=ORPHAN_WAIT_INTERVAL):
+        previous_pid_file, pid_file = self._write_pid_file()
+        if previous_pid_file is not None:
+            try:
+                pid = fileutil.read_file(previous_pid_file)
+                wait_interval = orphan_wait_interval
+                while self.osutil.check_pid_alive(pid):
+                    wait_interval -= GOAL_STATE_INTERVAL
+                    if wait_interval <= 0:
+                        logger.warn(
+                            u"{0} forcibly terminated orphan process {1}",
+                            CURRENT_AGENT,
+                            pid)
+                        os.kill(pid, signal.SIGKILL)
+                        break
+                    
+                    logger.info(
+                        u"{0} waiting for orphan process {1} to terminate",
+                        CURRENT_AGENT,
+                        pid)
+                    time.sleep(GOAL_STATE_INTERVAL)
+
+            except Exception as e:
+                logger.warn(
+                    u"Exception occurred waiting for orphan agent to terminate: {0}",
+                    ustr(e))
+        return
+
     def _evaluate_agent_health(self, latest_agent):
         """
         Evaluate the health of the selected agent: If it is restarting
@@ -375,6 +440,50 @@ class UpdateHandler(object):
         self.agents = [agent for agent in self.agents if not agent.is_blacklisted]
         return
 
+    def _get_pid_files(self):
+        pid_file = conf.get_agent_pid_file_path()
+        
+        pid_dir = os.path.dirname(pid_file)
+        pid_name = os.path.basename(pid_file)
+        
+        pid_re = re.compile("(\d+)_{0}".format(re.escape(pid_name)))
+        pid_files = [int(pid_re.match(f).group(1)) for f in os.listdir(pid_dir) if pid_re.match(f)]
+        pid_files.sort()
+
+        pid_index = -1 if len(pid_files) <= 0 else pid_files[-1]
+        previous_pid_file = None \
+                        if pid_index < 0 \
+                        else os.path.join(pid_dir, "{0}_{1}".format(pid_index, pid_name))
+        pid_file = os.path.join(pid_dir, "{0}_{1}".format(pid_index+1, pid_name))
+        return previous_pid_file, pid_file
+
+    @property
+    def _is_clean_start(self):
+        if not os.path.isfile(self._sentinal_file_path()):
+            return True
+
+        try:
+            if fileutil.read_file(self._sentinal_file_path()) != CURRENT_AGENT:
+                return True
+        except Exception as e:
+            logger.warn(
+                u"Exception reading sentinal file {0}: {1}",
+                self._sentinal_file_path(),
+                str(e))
+
+        return False
+
+    @property
+    def _is_orphaned(self):
+        parent_pid = os.getppid()
+        if parent_pid in (1, None):
+            return True
+
+        if not os.path.isfile(conf.get_agent_pid_file_path()):
+            return True
+
+        return fileutil.read_file(conf.get_agent_pid_file_path()) != ustr(parent_pid)
+
     def _load_agents(self):
         """
         Load all non-blacklisted agents currently on disk.
@@ -423,6 +532,46 @@ class UpdateHandler(object):
         self.agents.sort(key=lambda agent: agent.version, reverse=True)
         return
 
+    def _set_sentinal(self, agent=CURRENT_AGENT):
+        try:
+            fileutil.write_file(self._sentinal_file_path(), agent)
+        except Exception as e:
+            logger.warn(
+                u"Exception writing sentinal file {0}: {1}",
+                self._sentinal_file_path(),
+                str(e))
+        return
+
+    def _sentinal_file_path(self):
+        return os.path.join(conf.get_lib_dir(), AGENT_SENTINAL_FILE)
+
+    def _shutdown(self):
+        if not os.path.isfile(self._sentinal_file_path()):
+            return
+
+        try:
+            os.remove(self._sentinal_file_path())
+        except Exception as e:
+            logger.warn(
+                u"Exception removing sentinal file {0}: {1}",
+                self._sentinal_file_path(),
+                str(e))
+        return
+
+    def _write_pid_file(self):
+        previous_pid_file, pid_file = self._get_pid_files()
+        try:
+            fileutil.write_file(pid_file, ustr(os.getpid()))
+            logger.info(u"{0} running as process {1}", CURRENT_AGENT, ustr(os.getpid()))
+        except Exception as e:
+            pid_file = None
+            logger.warn(
+                u"Expection writing goal state agent {0} pid to {1}: {2}",
+                CURRENT_AGENT,
+                pid_file,
+                ustr(e))
+        return previous_pid_file, pid_file
+
 
 class GuestAgent(object):
     def __init__(self, path=None, pkg=None):


Test patch:
diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py
index 71c219c5..9e246b71 100644
--- a/tests/ga/test_extension.py
+++ b/tests/ga/test_extension.py
@@ -15,12 +15,141 @@
 # Requires Python 2.4+ and Openssl 1.0+
 #
 
+import azurelinuxagent.common.utils.fileutil as fileutil
+
 from tests.protocol.mockwiredata import *
+
 from azurelinuxagent.common.exception import *
 from azurelinuxagent.common.protocol import get_protocol_util
+from azurelinuxagent.common.protocol.restapi import ExtHandlerStatus, \
+                                                    ExtensionStatus, \
+                                                    ExtensionSubStatus, \
+                                                    Extension, \
+                                                    VMStatus, ExtHandler, \
+                                                    get_properties
 from azurelinuxagent.ga.exthandlers import *
 from azurelinuxagent.common.protocol.wire import WireProtocol
 
+class TestHandlerStateMigration(AgentTestCase):
+    def setUp(self):
+        AgentTestCase.setUp(self)
+
+        handler_name = "Not.A.Real.Extension"
+        handler_version = "1.2.3"
+
+        self.ext_handler = ExtHandler(handler_name)
+        self.ext_handler.properties.version = handler_version
+        self.ext_handler_i = ExtHandlerInstance(self.ext_handler, "dummy protocol")
+
+        self.handler_state = "Enabled"
+        self.handler_status = ExtHandlerStatus(
+            name=handler_name,
+            version=handler_version,
+            status="Ready",
+            message="Uninteresting message")
+        return
+
+    def _prepare_handler_state(self):
+        handler_state_path = os.path.join(
+                                self.tmp_dir,
+                                "handler_state",
+                                self.ext_handler_i.get_full_name())
+        os.makedirs(handler_state_path)
+        fileutil.write_file(
+            os.path.join(handler_state_path, "state"),
+            self.handler_state)
+        fileutil.write_file(
+            os.path.join(handler_state_path, "status"),
+            json.dumps(get_properties(self.handler_status)))
+        return
+
+    def _prepare_handler_config(self):
+        handler_config_path = os.path.join(
+                                self.tmp_dir,
+                                self.ext_handler_i.get_full_name(),
+                                "config")
+        os.makedirs(handler_config_path)
+        return
+
+    def test_migration_migrates(self):
+        self._prepare_handler_state()
+        self._prepare_handler_config()
+
+        migrate_handler_state()
+
+        self.assertEquals(self.ext_handler_i.get_handler_state(), self.handler_state)
+        self.assertEquals(
+            self.ext_handler_i.get_handler_status().status,
+            self.handler_status.status)
+        return
+
+    def test_migration_skips_if_empty(self):
+        self._prepare_handler_config()
+
+        migrate_handler_state()
+
+        self.assertFalse(
+            os.path.isfile(os.path.join(self.ext_handler_i.get_conf_dir(), "HandlerState")))
+        self.assertFalse(
+            os.path.isfile(os.path.join(self.ext_handler_i.get_conf_dir(), "HandlerStatus")))
+        return
+
+    def test_migration_cleans_up(self):
+        self._prepare_handler_state()
+        self._prepare_handler_config()
+
+        migrate_handler_state()
+
+        self.assertFalse(os.path.isdir(os.path.join(conf.get_lib_dir(), "handler_state")))
+        return
+
+    def test_migration_does_not_overwrite(self):
+        self._prepare_handler_state()
+        self._prepare_handler_config()
+
+        state = "Installed"
+        status = "NotReady"
+        code = 1
+        message = "A message"
+        self.assertNotEquals(state, self.handler_state)
+        self.assertNotEquals(status, self.handler_status.status)
+        self.assertNotEquals(code, self.handler_status.code)
+        self.assertNotEquals(message, self.handler_status.message)
+
+        self.ext_handler_i.set_handler_state(state)
+        self.ext_handler_i.set_handler_status(status=status, code=code, message=message)
+
+        migrate_handler_state()
+
+        self.assertEquals(self.ext_handler_i.get_handler_state(), state)
+        handler_status = self.ext_handler_i.get_handler_status()
+        self.assertEquals(handler_status.status, status)
+        self.assertEquals(handler_status.code, code)
+        self.assertEquals(handler_status.message, message)
+        return
+
+    @patch("shutil.move", side_effect=Exception)
+    def test_migration_ignores_move_errors(self, shutil_mock):
+        self._prepare_handler_state()
+        self._prepare_handler_config()
+
+        try:
+            migrate_handler_state()
+        except Exception as e:
+            self.assertTrue(False, "Unexpected exception: {0}".format(str(e)))
+        return
+
+    @patch("shutil.rmtree", side_effect=Exception)
+    def test_migration_ignores_tree_remove_errors(self, shutil_mock):
+        self._prepare_handler_state()
+        self._prepare_handler_config()
+
+        try:
+            migrate_handler_state()
+        except Exception as e:
+            self.assertTrue(False, "Unexpected exception: {0}".format(str(e)))
+        return
+
 @patch("azurelinuxagent.common.protocol.wire.CryptUtil")
 @patch("azurelinuxagent.common.utils.restutil.http_get")
 class TestExtension(AgentTestCase):
diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py
index 74804fb9..05569291 100644
--- a/tests/ga/test_update.py
+++ b/tests/ga/test_update.py
@@ -23,6 +23,7 @@ import json
 import os
 import platform
 import random
+import re
 import subprocess
 import sys
 import tempfile
@@ -598,6 +599,7 @@ class TestGuestAgent(UpdateTestCase):
 class TestUpdate(UpdateTestCase):
     def setUp(self):
         UpdateTestCase.setUp(self)
+        self.event_patch = patch('azurelinuxagent.common.event.add_event')
         self.update_handler = get_update_handler()
         return
 
@@ -617,6 +619,23 @@ class TestUpdate(UpdateTestCase):
         self.assertEqual(None, self.update_handler.signal_handler)
         return
 
+    def test_emit_restart_event_writes_sentinal_file(self):
+        self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path()))
+        self.update_handler._emit_restart_event()
+        self.assertTrue(os.path.isfile(self.update_handler._sentinal_file_path()))
+        return
+
+    def test_emit_restart_event_emits_event_if_not_clean_start(self):
+        try:
+            mock_event = self.event_patch.start()
+            self.update_handler._set_sentinal()
+            self.update_handler._emit_restart_event()
+            self.assertEqual(1, mock_event.call_count)
+        except Exception as e:
+            pass
+        self.event_patch.stop()
+        return
+
     def _test_ensure_latest_agent(
             self,
             base_version=FlexibleVersion(AGENT_VERSION),
@@ -700,6 +719,58 @@ class TestUpdate(UpdateTestCase):
             v = a.version
         return
 
+    def _test_ensure_no_orphans(self, invocations=3, interval=ORPHAN_WAIT_INTERVAL):
+        with patch.object(self.update_handler, 'osutil') as mock_util:
+            # Note:
+            # - Python only allows mutations of objects to which a function has
+            #   a reference. Incrementing an integer directly changes the
+            #   reference. Incrementing an item of a list changes an item to
+            #   which the code has a reference.
+            #   See http://stackoverflow.com/questions/26408941/python-nested-functions-and-variable-scope
+            iterations = [0]
+            def iterator(*args, **kwargs):
+                iterations[0] += 1
+                return iterations[0] < invocations
+
+            mock_util.check_pid_alive = Mock(side_effect=iterator)
+            
+            with patch('os.getpid', return_value=42):
+                with patch('time.sleep', return_value=None) as mock_sleep:
+                    self.update_handler._ensure_no_orphans(orphan_wait_interval=interval)
+                    return mock_util.check_pid_alive.call_count, mock_sleep.call_count
+        return
+
+    def test_ensure_no_orphans(self):
+        fileutil.write_file(os.path.join(self.tmp_dir, "0_waagent.pid"), ustr(41))
+        calls, sleeps = self._test_ensure_no_orphans(invocations=3)
+        self.assertEqual(3, calls)
+        self.assertEqual(2, sleeps)
+        return
+
+    def test_ensure_no_orphans_skips_if_no_orphans(self):
+        calls, sleeps = self._test_ensure_no_orphans(invocations=3)
+        self.assertEqual(0, calls)
+        self.assertEqual(0, sleeps)
+        return
+
+    def test_ensure_no_orphans_ignores_exceptions(self):
+        with patch('azurelinuxagent.common.utils.fileutil.read_file', side_effect=Exception):
+            calls, sleeps = self._test_ensure_no_orphans(invocations=3)
+            self.assertEqual(0, calls)
+            self.assertEqual(0, sleeps)
+        return
+
+    def test_ensure_no_orphans_kills_after_interval(self):
+        fileutil.write_file(os.path.join(self.tmp_dir, "0_waagent.pid"), ustr(41))
+        with patch('os.kill') as mock_kill:
+            calls, sleeps = self._test_ensure_no_orphans(
+                                        invocations=4,
+                                        interval=3*GOAL_STATE_INTERVAL)
+            self.assertEqual(3, calls)
+            self.assertEqual(2, sleeps)
+            self.assertEqual(1, mock_kill.call_count)
+        return
+
     def _test_evaluate_agent_health(self, child_agent_index=0):
         self.prepare_agents()
 
@@ -789,6 +860,59 @@ class TestUpdate(UpdateTestCase):
         self.assertEqual(latest_agent.version, prior_agent.version)
         return
 
+    def test_get_pid_files(self):
+        previous_pid_file, pid_file, = self.update_handler._get_pid_files()
+        self.assertEqual(None, previous_pid_file)
+        self.assertEqual("0_waagent.pid", os.path.basename(pid_file))
+        return
+
+    def test_get_pid_files_returns_previous(self):
+        for n in range(1250):
+            fileutil.write_file(os.path.join(self.tmp_dir, str(n)+"_waagent.pid"), ustr(n+1))
+        previous_pid_file, pid_file, = self.update_handler._get_pid_files()
+        self.assertEqual("1249_waagent.pid", os.path.basename(previous_pid_file))
+        self.assertEqual("1250_waagent.pid", os.path.basename(pid_file))
+        return
+
+    def test_is_clean_start_returns_true_when_no_sentinal(self):
+        self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path()))
+        self.assertTrue(self.update_handler._is_clean_start)
+        return
+
+    def test_is_clean_start_returns_true_sentinal_agent_is_not_current(self):
+        self.update_handler._set_sentinal(agent="Not the Current Agent")
+        self.assertTrue(os.path.isfile(self.update_handler._sentinal_file_path()))
+        self.assertTrue(self.update_handler._is_clean_start)
+        return
+
+    def test_is_clean_start_returns_false_for_current_agent(self):
+        self.update_handler._set_sentinal(agent=CURRENT_AGENT)
+        self.assertFalse(self.update_handler._is_clean_start)
+        return
+
+    def test_is_clean_start_returns_false_for_exceptions(self):
+        self.update_handler._set_sentinal()
+        with patch("azurelinuxagent.common.utils.fileutil.read_file", side_effect=Exception):
+            self.assertFalse(self.update_handler._is_clean_start)
+        return
+
+    def test_is_orphaned_returns_false_if_parent_exists(self):
+        fileutil.write_file(conf.get_agent_pid_file_path(), ustr(42))
+        with patch('os.getppid', return_value=42):
+            self.assertFalse(self.update_handler._is_orphaned)
+        return
+
+    def test_is_orphaned_returns_true_if_parent_is_init(self):
+        with patch('os.getppid', return_value=1):
+            self.assertTrue(self.update_handler._is_orphaned)
+        return
+
+    def test_is_orphaned_returns_true_if_parent_does_not_exist(self):
+        fileutil.write_file(conf.get_agent_pid_file_path(), ustr(24))
+        with patch('os.getppid', return_value=42):
+            self.assertTrue(self.update_handler._is_orphaned)
+        return
+
     def test_load_agents(self):
         self.prepare_agents()
 
@@ -991,7 +1115,7 @@ class TestUpdate(UpdateTestCase):
         #   reference. Incrementing an item of a list changes an item to
         #   which the code has a reference.
         #   See http://stackoverflow.com/questions/26408941/python-nested-functions-and-variable-scope
-        iterations = [0]
+        iterations = [0] 
         def iterator(*args, **kwargs):
             iterations[0] += 1
             if iterations[0] >= invocations:
@@ -999,14 +1123,19 @@ class TestUpdate(UpdateTestCase):
             return
 
         calls = calls * invocations
-        
+
+        fileutil.write_file(conf.get_agent_pid_file_path(), ustr(42))
+
         with patch('azurelinuxagent.ga.exthandlers.get_exthandlers_handler') as mock_handler:
             with patch('azurelinuxagent.ga.monitor.get_monitor_handler') as mock_monitor:
                 with patch('azurelinuxagent.ga.env.get_env_handler') as mock_env:
                     with patch('time.sleep', side_effect=iterator) as mock_sleep:
                         with patch('sys.exit') as mock_exit:
-
-                            self.update_handler.run()
+                            if isinstance(os.getppid, MagicMock):
+                                self.update_handler.run()
+                            else:
+                                with patch('os.getppid', return_value=42):
+                                    self.update_handler.run()
 
                             self.assertEqual(1, mock_handler.call_count)
                             self.assertEqual(mock_handler.return_value.method_calls, calls)
@@ -1014,6 +1143,7 @@ class TestUpdate(UpdateTestCase):
                             self.assertEqual(1, mock_monitor.call_count)
                             self.assertEqual(1, mock_env.call_count)
                             self.assertEqual(1, mock_exit.call_count)
+
         return
 
     def test_run(self):
@@ -1029,6 +1159,28 @@ class TestUpdate(UpdateTestCase):
         self._test_run(invocations=0, calls=[], enable_updates=True)
         return
 
+    def test_run_stops_if_orphaned(self):
+        with patch('os.getppid', return_value=1):
+            self._test_run(invocations=0, calls=[], enable_updates=True)
+        return
+
+    def test_run_clears_sentinal_on_successful_exit(self):
+        self._test_run()
+        self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path()))
+        return
+
+    def test_run_leaves_sentinal_on_unsuccessful_exit(self):
+        self.update_handler._ensure_latest_agent = Mock(side_effect=Exception)
+        self._test_run(invocations=0, calls=[], enable_updates=True)
+        self.assertTrue(os.path.isfile(self.update_handler._sentinal_file_path()))
+        return
+
+    def test_run_emits_restart_event(self):
+        self.update_handler._emit_restart_event = Mock()
+        self._test_run()
+        self.assertEqual(1, self.update_handler._emit_restart_event.call_count)
+        return
+
     def test_set_agents_sets_agents(self):
         self.prepare_agents()
 
@@ -1048,6 +1200,59 @@ class TestUpdate(UpdateTestCase):
             v = a.version
         return
 
+    def test_set_sentinal(self):
+        self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path()))
+        self.update_handler._set_sentinal()
+        self.assertTrue(os.path.isfile(self.update_handler._sentinal_file_path()))
+        return
+
+    def test_set_sentinal_writes_current_agent(self):
+        self.update_handler._set_sentinal()
+        self.assertTrue(
+            fileutil.read_file(self.update_handler._sentinal_file_path()),
+            CURRENT_AGENT)
+        return
+
+    def test_shutdown(self):
+        self.update_handler._set_sentinal()
+        self.update_handler._shutdown()
+        self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path()))
+        return
+
+    def test_shutdown_ignores_missing_sentinal_file(self):
+        self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path()))
+        self.update_handler._shutdown()
+        self.assertFalse(os.path.isfile(self.update_handler._sentinal_file_path()))
+        return
+
+    def test_shutdown_ignores_exceptions(self):
+        self.update_handler._set_sentinal()
+
+        try:
+            with patch("os.remove", side_effect=Exception):
+                self.update_handler._shutdown()
+        except Exception as e:
+            self.assertTrue(False, "Unexpected exception")
+        return
+
+    def test_write_pid_file(self):
+        for n in range(1112):
+            fileutil.write_file(os.path.join(self.tmp_dir, str(n)+"_waagent.pid"), ustr(n+1))
+        with patch('os.getpid', return_value=1112):
+            previous_pid_file, pid_file = self.update_handler._write_pid_file()
+            self.assertEqual("1111_waagent.pid", os.path.basename(previous_pid_file))
+            self.assertEqual("1112_waagent.pid", os.path.basename(pid_file))
+            self.assertEqual(fileutil.read_file(pid_file), ustr(1112))
+        return
+
+    def test_write_pid_file_ignores_exceptions(self):
+        with patch('azurelinuxagent.common.utils.fileutil.write_file', side_effect=Exception):
+            with patch('os.getpid', return_value=42):
+                previous_pid_file, pid_file = self.update_handler._write_pid_file()
+                self.assertEqual(None, previous_pid_file)
+                self.assertEqual(None, pid_file)
+        return
+
 
 class ChildMock(Mock):
     def __init__(self, return_value=0, side_effect=None):
diff --git a/tests/tools.py b/tests/tools.py
index 8bf23ed5..2732f761 100644
--- a/tests/tools.py
+++ b/tests/tools.py
@@ -60,6 +60,7 @@ class AgentTestCase(unittest.TestCase):
         conf.get_lib_dir = Mock(return_value=self.tmp_dir)
         ext_log_dir = os.path.join(self.tmp_dir, "azure")
         conf.get_ext_log_dir = Mock(return_value=ext_log_dir)
+        conf.get_agent_pid_file_path = Mock(return_value=os.path.join(self.tmp_dir, "waagent.pid"))
 
     def tearDown(self):
         if not debug and self.tmp_dir is not None: