Repository: Azure/WALinuxAgent (python)

Issue:
Telemetry Event to Report Detected CPU and Memory
There are cases where customers are paying for X CPUs and Y Memory, but they are not actually getting it. This is due to their kernel being improperly configured, or just being too old. The telemetry we have reports what Hyper-V gave the customer, which only tells half the story. I would like the agent to report the values from /proc/cpuinfo and /proc/meminfo (or something comparable), so we can detect these types of conditions.


Pull request:
Emit a GuestState telemetry event after provision
Closes #920

Patch:
diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py
index 108cf009..af4dee6c 100644
--- a/azurelinuxagent/common/event.py
+++ b/azurelinuxagent/common/event.py
@@ -63,6 +63,7 @@ class WALAEventOperation:
     Partition = "Partition"
     ProcessGoalState = "ProcessGoalState"
     Provision = "Provision"
+    GuestState = "GuestState"
     ReportStatus = "ReportStatus"
     Restart = "Restart"
     UnhandledError = "UnhandledError"
diff --git a/azurelinuxagent/pa/provision/cloudinit.py b/azurelinuxagent/pa/provision/cloudinit.py
index c6745260..22c3f9ca 100644
--- a/azurelinuxagent/pa/provision/cloudinit.py
+++ b/azurelinuxagent/pa/provision/cloudinit.py
@@ -26,9 +26,9 @@ from datetime import datetime
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.fileutil as fileutil
-import azurelinuxagent.common.utils.shellutil as shellutil
 
-from azurelinuxagent.common.event import elapsed_milliseconds
+from azurelinuxagent.common.event import elapsed_milliseconds, \
+    WALAEventOperation
 from azurelinuxagent.common.exception import ProvisionError, ProtocolError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.protocol import OVF_FILE_NAME
@@ -67,6 +67,9 @@ class CloudInitProvisionHandler(ProvisionHandler):
             self.report_event("Provision succeed",
                 is_success=True,
                 duration=elapsed_milliseconds(utc_start))
+            self.report_event(self.create_guest_state_telemetry_messsage(),
+                  is_success=True,
+                  operation=WALAEventOperation.GuestState)
 
         except ProvisionError as e:
             logger.error("Provisioning failed: {0}", ustr(e))
diff --git a/azurelinuxagent/pa/provision/default.py b/azurelinuxagent/pa/provision/default.py
index d2789750..5d6f1565 100644
--- a/azurelinuxagent/pa/provision/default.py
+++ b/azurelinuxagent/pa/provision/default.py
@@ -22,6 +22,7 @@ Provision handler
 import os
 import os.path
 import re
+import socket
 import time
 
 from datetime import datetime
@@ -88,10 +89,14 @@ class ProvisionHandler(object):
 
             self.write_provisioned()
 
-            self.report_event("Provision succeed",
+            self.report_event("Provision succeeded",
                 is_success=True,
                 duration=elapsed_milliseconds(utc_start))
 
+            self.report_event(self.create_guest_state_telemetry_messsage(),
+                is_success=True,
+                operation=WALAEventOperation.GuestState)
+
             self.report_ready(thumbprint)
             logger.info("Provisioning complete")
 
@@ -264,12 +269,53 @@ class ProvisionHandler(object):
             logger.info("Deploy ssh key pairs.")
             self.osutil.deploy_ssh_keypair(ovfenv.username, keypair)
 
-    def report_event(self, message, is_success=False, duration=0):
+    def report_event(self, message, is_success=False, duration=0,
+                     operation=WALAEventOperation.Provision):
         add_event(name=AGENT_NAME,
                     message=message,
                     duration=duration,
                     is_success=is_success,
-                    op=WALAEventOperation.Provision)
+                    op=operation)
+
+    def get_cpu_count(self):
+        try:
+            count = len([x for x in open('/proc/cpuinfo').readlines()
+                         if x.startswith("processor")])
+            return count
+        except Exception as e:
+            logger.verbose(u"Failed to determine the CPU count: {0}.", ustr(e))
+            pass
+        return -1
+
+    def get_mem_size_mb(self):
+        try:
+            for line in open('/proc/meminfo').readlines():
+                m = re.match('^MemTotal:\s*(\d+) kB$', line)
+                if m is not None:
+                    return int(int(m.group(1)) / 1024)
+        except Exception as e:
+            logger.verbose(u"Failed to determine the memory size: {0}..", ustr(e))
+            pass
+        return -1
+
+    def create_guest_state_telemetry_messsage(self):
+        """
+        Create a GuestState JSON message that contains the current CPU, Memory
+        (MB), and hostname of the guest.
+
+        e.g.
+
+        {
+         "cpu": 1,
+         "mem": 1024,
+         "hostname": "server1234"
+        }
+        """
+        cpu = self.get_cpu_count()
+        mem = self.get_mem_size_mb()
+
+        return """{{"cpu": {0}, "mem": {1}, "hostname": "{2}"}}"""\
+            .format(cpu, mem, socket.gethostname())
 
     def report_not_ready(self, sub_status, description):
         status = ProvisionStatus(status="NotReady", subStatus=sub_status,


Test patch:
diff --git a/tests/pa/test_provision.py b/tests/pa/test_provision.py
index 7045fccf..ab0a9102 100644
--- a/tests/pa/test_provision.py
+++ b/tests/pa/test_provision.py
@@ -15,9 +15,13 @@
 # Requires Python 2.4+ and Openssl 1.0+
 #
 
+import json
+import socket
+
 import azurelinuxagent.common.utils.fileutil as fileutil
 
-from azurelinuxagent.common.exception import ProtocolError
+from azurelinuxagent.common.event import WALAEventOperation
+from azurelinuxagent.common.exception import ProvisionError
 from azurelinuxagent.common.osutil.default import DefaultOSUtil
 from azurelinuxagent.common.protocol import OVF_FILE_NAME
 from azurelinuxagent.pa.provision import get_provision_handler
@@ -115,6 +119,85 @@ class TestProvision(AgentTestCase):
         ph.osutil.is_current_instance_id.assert_called_once()
         deprovision_handler.run_changed_unique_id.assert_called_once()
 
+    @distros()
+    @patch('azurelinuxagent.common.osutil.default.DefaultOSUtil.get_instance_id',
+        return_value='B9F3C233-9913-9F42-8EB3-BA656DF32502')
+    def test_provision_telemetry_success(self, mock_util, distro_name, distro_version,
+                       distro_full_name):
+        """
+        Assert that the agent issues two telemetry messages as part of a
+        successful provisioning.
+
+         1. Provision
+         2. GuestState
+        """
+        ph = get_provision_handler(distro_name, distro_version,
+                                   distro_full_name)
+        ph.report_event = MagicMock()
+        ph.reg_ssh_host_key = MagicMock(return_value='--thumprint--')
+
+        mock_osutil = MagicMock()
+        mock_osutil.decode_customdata = Mock(return_value="")
+
+        ph.osutil = mock_osutil
+        ph.protocol_util.osutil = mock_osutil
+        ph.protocol_util.get_protocol_by_file = MagicMock()
+        ph.protocol_util.get_protocol = MagicMock()
+
+        conf.get_dvd_mount_point = Mock(return_value=self.tmp_dir)
+        ovfenv_file = os.path.join(self.tmp_dir, OVF_FILE_NAME)
+        ovfenv_data = load_data("ovf-env.xml")
+        fileutil.write_file(ovfenv_file, ovfenv_data)
+
+        ph.run()
+
+        call1 = call("Provision succeeded", duration=ANY, is_success=True)
+        call2 = call(ANY, is_success=True, operation=WALAEventOperation.GuestState)
+        ph.report_event.assert_has_calls([call1, call2])
+
+        args, kwargs = ph.report_event.call_args_list[1]
+        guest_state_json = json.loads(args[0])
+        self.assertTrue(1 <= guest_state_json['cpu'])
+        self.assertTrue(1 <= guest_state_json['mem'])
+        self.assertEqual(socket.gethostname(), guest_state_json['hostname'])
+
+    @distros()
+    @patch(
+        'azurelinuxagent.common.osutil.default.DefaultOSUtil.get_instance_id',
+        return_value='B9F3C233-9913-9F42-8EB3-BA656DF32502')
+    def test_provision_telemetry_fail(self, mock_util, distro_name,
+                                         distro_version,
+                                         distro_full_name):
+        """
+        Assert that the agent issues one telemetry message as part of a
+        failed provisioning.
+
+         1. Provision
+        """
+        ph = get_provision_handler(distro_name, distro_version,
+                                   distro_full_name)
+        ph.report_event = MagicMock()
+        ph.reg_ssh_host_key = MagicMock(side_effect=ProvisionError(
+            "--unit-test--"))
+
+        mock_osutil = MagicMock()
+        mock_osutil.decode_customdata = Mock(return_value="")
+
+        ph.osutil = mock_osutil
+        ph.protocol_util.osutil = mock_osutil
+        ph.protocol_util.get_protocol_by_file = MagicMock()
+        ph.protocol_util.get_protocol = MagicMock()
+
+        conf.get_dvd_mount_point = Mock(return_value=self.tmp_dir)
+        ovfenv_file = os.path.join(self.tmp_dir, OVF_FILE_NAME)
+        ovfenv_data = load_data("ovf-env.xml")
+        fileutil.write_file(ovfenv_file, ovfenv_data)
+
+        ph.run()
+        ph.report_event.assert_called_once_with(
+            "[ProvisionError] --unit-test--")
+
+
 if __name__ == '__main__':
     unittest.main()
 
diff --git a/tests/tools.py b/tests/tools.py
index 94fab7f5..65e27c6e 100644
--- a/tests/tools.py
+++ b/tests/tools.py
@@ -34,9 +34,9 @@ from azurelinuxagent.common.version import PY_VERSION_MAJOR
 
 #Import mock module for Python2 and Python3
 try:
-    from unittest.mock import Mock, patch, MagicMock, DEFAULT, call
+    from unittest.mock import Mock, patch, MagicMock, DEFAULT, ANY, call
 except ImportError:
-    from mock import Mock, patch, MagicMock, DEFAULT, call
+    from mock import Mock, patch, MagicMock, DEFAULT, ANY, call
 
 test_dir = os.path.dirname(os.path.abspath(__file__))
 data_dir = os.path.join(test_dir, "data")