Repository: Azure/WALinuxAgent
Pull request #1208 (2018-06-12 20:16:01)

Fix host plugin status reporting
Couple issues here:

- when we receive 410, the goal state is never force updated, due to a simple logic bug
- in this and other cases, we never raise a protocol error high enough to trigger the error state, effectively hiding extended report status failures

This addresses both, and updates unit tests.


Patch:
diff --git a/azurelinuxagent/common/protocol/hostplugin.py b/azurelinuxagent/common/protocol/hostplugin.py
index c958c020..295d6aa7 100644
--- a/azurelinuxagent/common/protocol/hostplugin.py
+++ b/azurelinuxagent/common/protocol/hostplugin.py
@@ -213,23 +213,12 @@ class HostPluginProtocol(object):
             raise ProtocolError("HostGAPlugin: Status blob was not provided")
 
         logger.verbose("HostGAPlugin: Posting VM status")
-        try:
-
-            blob_type = status_blob.type if status_blob.type else config_blob_type
-
-            if blob_type == "BlockBlob":
-                self._put_block_blob_status(sas_url, status_blob)
-            else:
-                self._put_page_blob_status(sas_url, status_blob)
-
-        except Exception as e:
-            # If the HostPlugin rejects the request,
-            # let the error continue, but set to use the HostPlugin
-            if isinstance(e, ResourceGoneError):
-                logger.verbose("HostGAPlugin: Setting host plugin as default channel")
-                HostPluginProtocol.set_default_channel(True)
+        blob_type = status_blob.type if status_blob.type else config_blob_type
 
-            raise
+        if blob_type == "BlockBlob":
+            self._put_block_blob_status(sas_url, status_blob)
+        else:
+            self._put_page_blob_status(sas_url, status_blob)
 
     def _put_block_blob_status(self, sas_url, status_blob):
         url = URI_FORMAT_PUT_VM_STATUS.format(self.endpoint, HOST_PLUGIN_PORT)
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index 267be74e..fb8216ff 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -935,74 +935,57 @@ class WireClient(object):
             raise ProtocolNotFoundError(error)
 
     def upload_status_blob(self):
-        for update_goal_state in [False, True]:
-            try:
-                if update_goal_state:
-                    self.update_goal_state(forced=True)
+        try:
+            self.update_goal_state()
+            ext_conf = self.get_ext_conf()
 
-                ext_conf = self.get_ext_conf()
-
-                blob_uri = ext_conf.status_upload_blob
-                blob_type = ext_conf.status_upload_blob_type
-
-                if blob_uri is not None:
-
-                    if not blob_type in ["BlockBlob", "PageBlob"]:
-                        blob_type = "BlockBlob"
-                        logger.verbose("Status Blob type is unspecified "
-                            "-- assuming it is a BlockBlob")
-
-                    try:
-                        self.status_blob.prepare(blob_type)
-                    except Exception as e:
-                        self.report_status_event(
-                            "Exception creating status blob: {0}", ustr(e))
-                        return
-
-                    # Swap the order of use for the HostPlugin vs. the "direct" route.
-                    # Prefer the use of HostPlugin. If HostPlugin fails fall back to the
-                    # direct route.
-                    #
-                    # The code previously preferred the "direct" route always, and only fell back
-                    # to the HostPlugin *if* there was an error.  We would like to move to
-                    # the HostPlugin for all traffic, but this is a big change.  We would like
-                    # to see how this behaves at scale, and have a fallback should things go
-                    # wrong.  This is why we try HostPlugin then direct.
-                    try:
-                        host = self.get_host_plugin()
-                        host.put_vm_status(self.status_blob,
-                                           ext_conf.status_upload_blob,
-                                           ext_conf.status_upload_blob_type)
-                        return
-                    except HttpError:
-                        pass
-                    except Exception as e:
-                        if not isinstance(e, ResourceGoneError):
-                            self.report_status_event("Exception uploading status blob (HostPlugin): {0}", ustr(e))
-
-                    try:
-                        if self.status_blob.upload(blob_uri):
-                            from azurelinuxagent.common.event import add_event, WALAEventOperation
-                            add_event(
-                                name=AGENT_NAME,
-                                version=CURRENT_VERSION,
-                                op=WALAEventOperation.ReportStatus,
-                                is_success=True,
-                                message="direct",
-                                log_event=False)
-                            return
-                    except HttpError:
-                        pass
+            blob_uri = ext_conf.status_upload_blob
+            blob_type = ext_conf.status_upload_blob_type
+
+            if blob_uri is None:
+                raise ProtocolError("No blob uri found")
 
-                    return
+            if blob_type not in ["BlockBlob", "PageBlob"]:
+                blob_type = "BlockBlob"
+                logger.verbose("Status Blob type is unspecified, assuming BlockBlob")
+
+            try:
+                self.status_blob.prepare(blob_type)
             except Exception as e:
-                if isinstance(e, ResourceGoneError):
-                    continue
+                raise ProtocolError("Exception creating status blob: {0}", ustr(e))
+
+            # Swap the order of use for the HostPlugin vs. the "direct" route.
+            # Prefer the use of HostPlugin. If HostPlugin fails fall back to the
+            # direct route.
+            #
+            # The code previously preferred the "direct" route always, and only fell back
+            # to the HostPlugin *if* there was an error.  We would like to move to
+            # the HostPlugin for all traffic, but this is a big change.  We would like
+            # to see how this behaves at scale, and have a fallback should things go
+            # wrong.  This is why we try HostPlugin then direct.
+            try:
+                host = self.get_host_plugin()
+                host.put_vm_status(self.status_blob,
+                                   ext_conf.status_upload_blob,
+                                   ext_conf.status_upload_blob_type)
+                return
+            except ResourceGoneError:
+                # do not attempt direct, force goal state update and wait to try again
+                self.update_goal_state(forced=True)
+                return
+            except Exception as e:
+                # for all other errors, fall back to direct
+                pass
 
-                self.report_status_event(
-                    "Exception uploading status blob: {0}", ustr(e))
+            self.report_status_event("direct")
+            if self.status_blob.upload(blob_uri):
                 return
 
+        except Exception as e:
+            self.report_status_event("Exception uploading status blob: {0}", ustr(e))
+
+        raise ProtocolError("Failed to upload status blob via either channel")
+
     def report_role_prop(self, thumbprint):
         goal_state = self.get_goal_state()
         role_prop = _build_role_properties(goal_state.container_id,


Test patch:
diff --git a/tests/common/test_cgroups.py b/tests/common/test_cgroups.py
index 01812ce1..8ad8a37d 100644
--- a/tests/common/test_cgroups.py
+++ b/tests/common/test_cgroups.py
@@ -179,7 +179,9 @@ class TestCGroups(AgentTestCase):
         self.assertGreater(ticks_after, ticks_before)
         p2 = cpu.get_cpu_percent()
         self.assertGreater(p2, 0)
-        self.assertLess(p2, 100)
+        # when running under PyCharm, this is often > 100
+        # on a multi-core machine
+        self.assertLess(p2, 200)
 
     def test_format_memory_value(self):
         """
diff --git a/tests/protocol/test_hostplugin.py b/tests/protocol/test_hostplugin.py
index ad4a308b..dae2523c 100644
--- a/tests/protocol/test_hostplugin.py
+++ b/tests/protocol/test_hostplugin.py
@@ -25,7 +25,7 @@ import azurelinuxagent.common.protocol.wire as wire
 import azurelinuxagent.common.protocol.hostplugin as hostplugin
 from azurelinuxagent.common.errorstate import ErrorState
 
-from azurelinuxagent.common.exception import HttpError
+from azurelinuxagent.common.exception import HttpError, ResourceGoneError
 from azurelinuxagent.common.protocol.hostplugin import API_VERSION
 from azurelinuxagent.common.utils import restutil
 from tests.protocol.mockwiredata import WireProtocolData, DATA_FILE
@@ -129,7 +129,11 @@ class TestHostPlugin(AgentTestCase):
         self.assertEqual(headers['x-ms-containerid'], goal_state.container_id)
         self.assertEqual(headers['x-ms-host-config-name'], goal_state.role_config_name)
 
-    def test_fallback(self):
+    @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized", return_value=True)
+    @patch("azurelinuxagent.common.protocol.wire.StatusBlob.upload", return_value=False)
+    @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol._put_page_blob_status")
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
+    def test_default_channel(self, patch_update, patch_put, patch_upload, _):
         """
         Status now defaults to HostPlugin. Validate that any errors on the public
         channel are ignored.  Validate that the default channel is never changed
@@ -137,57 +141,157 @@ class TestHostPlugin(AgentTestCase):
         """
         test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
         status = restapi.VMStatus(status="Ready", message="Guest Agent is running")
-        with patch.object(wire.HostPluginProtocol,
-                          "ensure_initialized",
-                          return_value=True):
-            with patch.object(wire.StatusBlob, "upload", return_value=False) as patch_upload:
-                with patch.object(wire.HostPluginProtocol,
-                                  "_put_page_blob_status") as patch_put:
-                    wire_protocol_client = wire.WireProtocol(wireserver_url).client
-                    wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state)
-                    wire_protocol_client.ext_conf = wire.ExtensionsConfig(None)
-                    wire_protocol_client.ext_conf.status_upload_blob = sas_url
-                    wire_protocol_client.ext_conf.status_upload_blob_type = page_blob_type
-                    wire_protocol_client.status_blob.set_vm_status(status)
-                    wire_protocol_client.upload_status_blob()
-                    self.assertEqual(patch_upload.call_count, 0)
-                    self.assertTrue(patch_put.call_count == 1,
-                                    "Fallback was not engaged")
-                    self.assertTrue(patch_put.call_args[0][0] == sas_url)
-                    self.assertFalse(wire.HostPluginProtocol.is_default_channel())
 
-    def test_fallback_failure(self):
+        wire_protocol_client = wire.WireProtocol(wireserver_url).client
+        wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state)
+        wire_protocol_client.ext_conf = wire.ExtensionsConfig(None)
+        wire_protocol_client.ext_conf.status_upload_blob = sas_url
+        wire_protocol_client.ext_conf.status_upload_blob_type = page_blob_type
+        wire_protocol_client.status_blob.set_vm_status(status)
+
+        # act
+        wire_protocol_client.upload_status_blob()
+
+        # assert direct route is not called
+        self.assertEqual(0, patch_upload.call_count, "Direct channel was used")
+
+        # assert host plugin route is called
+        self.assertEqual(1, patch_put.call_count, "Host plugin was not used")
+
+        # assert update goal state is only called once, non-forced
+        self.assertEqual(1, patch_update.call_count, "Unexpected call count")
+        self.assertEqual(0, len(patch_update.call_args[1]), "Unexpected parameters")
+
+        # ensure the correct url is used
+        self.assertEqual(sas_url, patch_put.call_args[0][0])
+
+        # ensure host plugin is not set as default
+        self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+
+    @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized",
+           return_value=True)
+    @patch("azurelinuxagent.common.protocol.wire.StatusBlob.upload",
+           return_value=True)
+    @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol._put_page_blob_status",
+           side_effect=HttpError("503"))
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
+    def test_fallback_channel_503(self, patch_update, patch_put, patch_upload, _):
         """
-        Validate that if host plugin fails and the direct connection fails the
-        default channel remains unchanged.
+        When host plugin returns a 503, we should fall back to the direct channel
         """
         test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
-        status = restapi.VMStatus(status="Ready",
-                                  message="Guest Agent is running")
-        wire.HostPluginProtocol.set_default_channel(False)
-        with patch.object(wire.HostPluginProtocol,
-                          "ensure_initialized",
-                          return_value=True):
-            with patch.object(wire.StatusBlob,
-                              "upload",
-                              return_value=False) as patch_upload:
-                with patch.object(wire.HostPluginProtocol,
-                                  "_put_page_blob_status",
-                                  side_effect=wire.HttpError("put failure")) as patch_put:
-                    self.assertFalse(wire.HostPluginProtocol.is_default_channel())
-                    client = wire.WireProtocol(wireserver_url).client
-                    client.get_goal_state = Mock(return_value=test_goal_state)
-                    client.ext_conf = wire.ExtensionsConfig(None)
-                    client.ext_conf.status_upload_blob = sas_url
-                    client.ext_conf.status_upload_blob_type = page_blob_type
-                    client.status_blob.set_vm_status(status)
-                    client.upload_status_blob()
-                    self.assertEqual(1, patch_put.call_count)
-                    self.assertEqual(1, patch_upload.call_count)
-                    self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+        status = restapi.VMStatus(status="Ready", message="Guest Agent is running")
+
+        wire_protocol_client = wire.WireProtocol(wireserver_url).client
+        wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state)
+        wire_protocol_client.ext_conf = wire.ExtensionsConfig(None)
+        wire_protocol_client.ext_conf.status_upload_blob = sas_url
+        wire_protocol_client.ext_conf.status_upload_blob_type = page_blob_type
+        wire_protocol_client.status_blob.set_vm_status(status)
+
+        # act
+        wire_protocol_client.upload_status_blob()
+
+        # assert direct route is called
+        self.assertEqual(1, patch_upload.call_count, "Direct channel was not used")
+
+        # assert host plugin route is called
+        self.assertEqual(1, patch_put.call_count, "Host plugin was not used")
+
+        # assert update goal state is only called once, non-forced
+        self.assertEqual(1, patch_update.call_count, "Update goal state unexpected call count")
+        self.assertEqual(0, len(patch_update.call_args[1]), "Update goal state unexpected call count")
+
+        # ensure the correct url is used
+        self.assertEqual(sas_url, patch_put.call_args[0][0])
+
+        # ensure host plugin is not set as default
+        self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+
+    @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized",
+           return_value=True)
+    @patch("azurelinuxagent.common.protocol.wire.StatusBlob.upload",
+           return_value=True)
+    @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol._put_page_blob_status",
+           side_effect=ResourceGoneError("410"))
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
+    def test_fallback_channel_410(self, patch_update, patch_put, patch_upload, _):
+        """
+        When host plugin returns a 410, we should force the goal state update and return
+        """
+        test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
+        status = restapi.VMStatus(status="Ready", message="Guest Agent is running")
+
+        wire_protocol_client = wire.WireProtocol(wireserver_url).client
+        wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state)
+        wire_protocol_client.ext_conf = wire.ExtensionsConfig(None)
+        wire_protocol_client.ext_conf.status_upload_blob = sas_url
+        wire_protocol_client.ext_conf.status_upload_blob_type = page_blob_type
+        wire_protocol_client.status_blob.set_vm_status(status)
+
+        # act
+        wire_protocol_client.upload_status_blob()
+
+        # assert direct route is not called
+        self.assertEqual(0, patch_upload.call_count, "Direct channel was used")
+
+        # assert host plugin route is called
+        self.assertEqual(1, patch_put.call_count, "Host plugin was not used")
+
+        # assert update goal state is called twice, forced=True on the second
+        self.assertEqual(2, patch_update.call_count, "Update goal state unexpected call count")
+        self.assertEqual(1, len(patch_update.call_args[1]), "Update goal state unexpected call count")
+        self.assertTrue(patch_update.call_args[1]['forced'], "Update goal state unexpected call count")
+
+        # ensure the correct url is used
+        self.assertEqual(sas_url, patch_put.call_args[0][0])
+
+        # ensure host plugin is not set as default
+        self.assertFalse(wire.HostPluginProtocol.is_default_channel())
+
+    @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized",
+           return_value=True)
+    @patch("azurelinuxagent.common.protocol.wire.StatusBlob.upload",
+           return_value=False)
+    @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol._put_page_blob_status",
+           side_effect=HttpError("500"))
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
+    def test_fallback_channel_failure(self, patch_update, patch_put, patch_upload, _):
+        """
+        When host plugin returns a 500, and direct fails, we should raise a ProtocolError
+        """
+        test_goal_state = wire.GoalState(WireProtocolData(DATA_FILE).goal_state)
+        status = restapi.VMStatus(status="Ready", message="Guest Agent is running")
+
+        wire_protocol_client = wire.WireProtocol(wireserver_url).client
+        wire_protocol_client.get_goal_state = Mock(return_value=test_goal_state)
+        wire_protocol_client.ext_conf = wire.ExtensionsConfig(None)
+        wire_protocol_client.ext_conf.status_upload_blob = sas_url
+        wire_protocol_client.ext_conf.status_upload_blob_type = page_blob_type
+        wire_protocol_client.status_blob.set_vm_status(status)
+
+        # act
+        self.assertRaises(wire.ProtocolError, wire_protocol_client.upload_status_blob)
+
+        # assert direct route is not called
+        self.assertEqual(1, patch_upload.call_count, "Direct channel was not used")
+
+        # assert host plugin route is called
+        self.assertEqual(1, patch_put.call_count, "Host plugin was not used")
+
+        # assert update goal state is called twice, forced=True on the second
+        self.assertEqual(1, patch_update.call_count, "Update goal state unexpected call count")
+        self.assertEqual(0, len(patch_update.call_args[1]), "Update goal state unexpected call count")
+
+        # ensure the correct url is used
+        self.assertEqual(sas_url, patch_put.call_args[0][0])
+
+        # ensure host plugin is not set as default
+        self.assertFalse(wire.HostPluginProtocol.is_default_channel())
 
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
     @patch("azurelinuxagent.common.event.add_event")
-    def test_put_status_error_reporting(self, patch_add_event):
+    def test_put_status_error_reporting(self, patch_add_event, _):
         """
         Validate the telemetry when uploading status fails
         """
@@ -209,7 +313,7 @@ class TestHostPlugin(AgentTestCase):
                               side_effect=put_error) as patch_http_put:
                 with patch.object(wire.HostPluginProtocol,
                                   "ensure_initialized", return_value=True):
-                    wire_protocol_client.upload_status_blob()
+                    self.assertRaises(wire.ProtocolError, wire_protocol_client.upload_status_blob)
 
                     # The agent tries to upload via HostPlugin and that fails due to
                     # http_put having a side effect of "put_error"
@@ -218,8 +322,11 @@ class TestHostPlugin(AgentTestCase):
                     self.assertEqual(1, wire_protocol_client.status_blob.upload.call_count)
                     # The agent never touches the default protocol is this code path, so no change.
                     self.assertFalse(wire.HostPluginProtocol.is_default_channel())
-                    # The agent never logs a telemetry event for a bad HTTP call
-                    self.assertEqual(patch_add_event.call_count, 0)
+                    # The agent never logs telemetry event for direct fallback
+                    self.assertEqual(1, patch_add_event.call_count)
+                    self.assertEqual('ReportStatus', patch_add_event.call_args[1]['op'])
+                    self.assertEqual('direct', patch_add_event.call_args[1]['message'])
+                    self.assertEqual(False, patch_add_event.call_args[1]['is_success'])
 
     def test_validate_http_request(self):
         """Validate correct set of data is sent to HostGAPlugin when reporting VM status"""
@@ -259,7 +366,8 @@ class TestHostPlugin(AgentTestCase):
                 self.assertEqual('POST', patch_http.call_args_list[1][0][0])
                 self.assertEqual(health_service_url, patch_http.call_args_list[1][0][1])
 
-    def test_no_fallback(self):
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
+    def test_no_fallback(self, _):
         """
         Validate fallback to upload status using HostGAPlugin is not happening
         when status reporting via default method is successful
diff --git a/tests/protocol/test_wire.py b/tests/protocol/test_wire.py
index 91de3620..030f5418 100644
--- a/tests/protocol/test_wire.py
+++ b/tests/protocol/test_wire.py
@@ -172,6 +172,7 @@ class TestWireProtocol(AgentTestCase):
         self.assertEqual(patch_http.call_args_list[0][0][1], ext_uri)
         self.assertEqual(patch_http.call_args_list[1][0][1], host_uri)
 
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
     def test_upload_status_blob_default(self, *args):
         """
         Default status blob method is HostPlugin.
@@ -196,6 +197,7 @@ class TestWireProtocol(AgentTestCase):
                     # host plugin uploads the status blob
                     patch_host_ga_plugin_upload.assert_called_once_with(ANY, testurl, 'BlockBlob')
 
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
     def test_upload_status_blob_host_ga_plugin(self, *args):
         vmstatus = VMStatus(message="Ready", status="Ready")
         wire_protocol_client = WireProtocol(wireserver_url).client
@@ -221,8 +223,9 @@ class TestWireProtocol(AgentTestCase):
                     patch_http.assert_called_once_with(testurl, wire_protocol_client.status_blob)
                     self.assertFalse(HostPluginProtocol.is_default_channel())
 
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
     @patch("azurelinuxagent.common.protocol.hostplugin.HostPluginProtocol.ensure_initialized")
-    def test_upload_status_blob_unknown_type_assumes_block(self, _, *args):
+    def test_upload_status_blob_unknown_type_assumes_block(self, _, __, *args):
         vmstatus = VMStatus(message="Ready", status="Ready")
         wire_protocol_client = WireProtocol(wireserver_url).client
         wire_protocol_client.ext_conf = ExtensionsConfig(None)
@@ -240,6 +243,7 @@ class TestWireProtocol(AgentTestCase):
                     patch_default_upload.assert_called_once_with(testurl)
                     patch_get_goal_state.assert_called_once_with()
 
+    @patch("azurelinuxagent.common.protocol.wire.WireClient.update_goal_state")
     def test_upload_status_blob_reports_prepare_error(self, *args):
         vmstatus = VMStatus(message="Ready", status="Ready")
         wire_protocol_client = WireProtocol(wireserver_url).client
@@ -249,10 +253,9 @@ class TestWireProtocol(AgentTestCase):
         wire_protocol_client.status_blob.vm_status = vmstatus
         goal_state = GoalState(WireProtocolData(DATA_FILE).goal_state)
 
-        with patch.object(StatusBlob, "prepare",
-                    side_effect=Exception) as mock_prepare:
+        with patch.object(StatusBlob, "prepare", side_effect=Exception) as mock_prepare:
             with patch.object(WireClient, "report_status_event") as mock_event:
-                wire_protocol_client.upload_status_blob()
+                self.assertRaises(ProtocolError, wire_protocol_client.upload_status_blob)
 
                 self.assertEqual(1, mock_prepare.call_count)
                 self.assertEqual(1, mock_event.call_count)
diff --git a/tests/utils/test_process_util.py b/tests/utils/test_process_util.py
index 2f298c29..85abdb5a 100644
--- a/tests/utils/test_process_util.py
+++ b/tests/utils/test_process_util.py
@@ -23,7 +23,7 @@ from azurelinuxagent.common.utils.processutil \
 from tests.tools import *
 import sys
 
-process_target = "tests/utils/process_target.sh"
+process_target = "{0}/process_target.sh".format(os.path.abspath(os.path.join(__file__, os.pardir)))
 process_cmd_template = "{0} -o '{1}' -e '{2}'"