Repository: Azure/WALinuxAgent
Pull request #1125 (2018-04-19 23:18:40)

Compare file contents in history directory
Test03 flushes twice for the same timestamp and expects the second flush to overwrite the first. This commit adds checks to ensure the contents of the pushed files are different the second time around and makes sure the contents of the history directory are as expected after each flush.


Patch:
diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py
index e3352d60..bcb87601 100644
--- a/azurelinuxagent/common/event.py
+++ b/azurelinuxagent/common/event.py
@@ -251,7 +251,11 @@ class EventLogger(object):
         if not is_success or log_event:
             _log_event(name, op, message, duration, is_success=is_success)
 
-        event = TelemetryEvent(1, "69B669B9-4AF8-4C50-BDC4-6006FA76E975")
+        self._add_event(duration, evt_type, is_internal, is_success, message, name, op, version, eventId=1)
+        self._add_event(duration, evt_type, is_internal, is_success, message, name, op, version, eventId=6)
+
+    def _add_event(self, duration, evt_type, is_internal, is_success, message, name, op, version, eventId):
+        event = TelemetryEvent(eventId, "69B669B9-4AF8-4C50-BDC4-6006FA76E975")
         event.parameters.append(TelemetryEventParam('Name', name))
         event.parameters.append(TelemetryEventParam('Version', str(version)))
         event.parameters.append(TelemetryEventParam('IsInternal', is_internal))
diff --git a/azurelinuxagent/common/protocol/imds.py b/azurelinuxagent/common/protocol/imds.py
new file mode 100644
index 00000000..f6467325
--- /dev/null
+++ b/azurelinuxagent/common/protocol/imds.py
@@ -0,0 +1,89 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+import json
+
+import azurelinuxagent.common.utils.restutil as restutil
+from azurelinuxagent.common.exception import HttpError, ProtocolError
+from azurelinuxagent.common.future import ustr
+from azurelinuxagent.common.protocol.restapi import DataContract, set_properties
+
+IMDS_ENDPOINT = '169.254.169.254'
+APIVERSION = '2017-08-01'
+BASE_URI = "http://{0}/metadata/instance/{1}?api-version={2}"
+
+
+def get_imds_client():
+    return ImdsClient()
+
+
+class ComputeInfo(DataContract):
+    def __init__(self,
+                 location=None,
+                 name=None,
+                 offer=None,
+                 osType=None,
+                 placementGroupId=None,
+                 platformFaultDomain=None,
+                 placementUpdateDomain=None,
+                 publisher=None,
+                 resourceGroupName=None,
+                 sku=None,
+                 subscriptionId=None,
+                 tags=None,
+                 version=None,
+                 vmId=None,
+                 vmSize=None):
+        self.location = location
+        self.name = name
+        self.offer = offer
+        self.osType = osType
+        self.placementGroupId = placementGroupId
+        self.platformFaultDomain = platformFaultDomain
+        self.platformUpdateDomain = placementUpdateDomain
+        self.publisher = publisher
+        self.resourceGroupName = resourceGroupName
+        self.sku = sku
+        self.subscriptionId = subscriptionId
+        self.tags = tags
+        self.version = version
+        self.vmId = vmId
+        self.vmSize = vmSize
+
+    @property
+    def image_info(self):
+        return "{0}:{1}:{2}:{3}".format(self.publisher, self.offer, self.sku, self.version)
+
+
+class ImdsClient(object):
+    def __init__(self, version=APIVERSION):
+        self._api_version = version
+        self._headers = {
+            'User-Agent': restutil.HTTP_USER_AGENT,
+            'Metadata': True,
+        }
+        pass
+
+    @property
+    def compute_url(self):
+        return BASE_URI.format(IMDS_ENDPOINT, 'compute', self._api_version)
+
+    def get_compute(self):
+        """
+        Fetch compute information.
+
+        :return: instance of a ComputeInfo
+        :rtype: ComputeInfo
+        """
+
+        resp = restutil.http_get(self.compute_url, headers=self._headers)
+
+        if restutil.request_failed(resp):
+            raise HttpError("{0} - GET: {1}".format(resp.status, self.compute_url))
+
+        data = resp.read()
+        data = json.loads(ustr(data, encoding="utf-8"))
+
+        compute_info = ComputeInfo()
+        set_properties('compute', compute_info, data)
+
+        return compute_info
diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py
index 07b57d6f..a07d9a35 100644
--- a/azurelinuxagent/ga/monitor.py
+++ b/azurelinuxagent/ga/monitor.py
@@ -28,10 +28,11 @@ import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.logger as logger
 
 from azurelinuxagent.common.event import add_event, WALAEventOperation
-from azurelinuxagent.common.exception import EventError, ProtocolError, OSUtilError
+from azurelinuxagent.common.exception import EventError, ProtocolError, OSUtilError, HttpError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
 from azurelinuxagent.common.protocol import get_protocol_util
+from azurelinuxagent.common.protocol.imds import get_imds_client
 from azurelinuxagent.common.protocol.restapi import TelemetryEventParam, \
                                                     TelemetryEventList, \
                                                     TelemetryEvent, \
@@ -93,6 +94,7 @@ class MonitorHandler(object):
     def __init__(self):
         self.osutil = get_osutil()
         self.protocol_util = get_protocol_util()
+        self.imds_client = get_imds_client()
         self.sysinfo = []
         self.event_thread = None
 
@@ -142,6 +144,19 @@ class MonitorHandler(object):
         except ProtocolError as e:
             logger.warn("Failed to get system info: {0}", e)
 
+        try:
+            vminfo = self.imds_client.get_compute()
+            self.sysinfo.append(TelemetryEventParam('Location',
+                                                    vminfo.location))
+            self.sysinfo.append(TelemetryEventParam('SubscriptionId',
+                                                    vminfo.subscriptionId))
+            self.sysinfo.append(TelemetryEventParam('ResourceGroupName',
+                                                    vminfo.resourceGroupName))
+            self.sysinfo.append(TelemetryEventParam('VMId',
+                                                    vminfo.vmId))
+        except (HttpError, ValueError) as e:
+            logger.warn("failed to get IMDS info: {0}", e)
+
     def collect_event(self, evt_file_name):
         try:
             logger.verbose("Found event file: {0}", evt_file_name)


Test patch:
diff --git a/tests/common/test_event.py b/tests/common/test_event.py
index 7ab4b3a1..3917b37c 100644
--- a/tests/common/test_event.py
+++ b/tests/common/test_event.py
@@ -159,11 +159,11 @@ class TestEvent(AgentTestCase):
 
     def test_save_event(self):
         add_event('test', message='test event')
-        self.assertTrue(len(os.listdir(self.tmp_dir)) == 1)
+        self.assertTrue(len(os.listdir(self.tmp_dir)) == 2)
 
     def test_save_event_rollover(self):
         add_event('test', message='first event')
-        for i in range(0, 999):
+        for i in range(0, 499):
             add_event('test', message='test event {0}'.format(i))
 
         events = os.listdir(self.tmp_dir)
@@ -207,7 +207,7 @@ class TestEvent(AgentTestCase):
         first_event = os.path.join(self.tmp_dir, events[0])
         with open(first_event) as first_fh:
             first_event_text = first_fh.read()
-            self.assertTrue('test event 1001' in first_event_text)
+            self.assertTrue('test event 1002' in first_event_text)
 
         last_event = os.path.join(self.tmp_dir, events[-1])
         with open(last_event) as last_fh:
diff --git a/tests/protocol/test_imds.py b/tests/protocol/test_imds.py
new file mode 100644
index 00000000..2168aae3
--- /dev/null
+++ b/tests/protocol/test_imds.py
@@ -0,0 +1,100 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+import json
+
+from azurelinuxagent.common.exception import HttpError
+from azurelinuxagent.common.protocol.imds import ComputeInfo, ImdsClient
+from azurelinuxagent.common.protocol.restapi import set_properties
+from azurelinuxagent.common.utils import restutil
+from tests.ga.test_update import ResponseMock
+from tests.tools import *
+
+
+class TestImds(AgentTestCase):
+    @patch("azurelinuxagent.ga.update.restutil.http_get")
+    def test_get(self, mock_http_get):
+        mock_http_get.return_value = ResponseMock(response='''{
+        "location": "westcentralus",
+        "name": "unit_test",
+        "offer": "UnitOffer",
+        "osType": "Linux",
+        "placementGroupId": "",
+        "platformFaultDomain": "0",
+        "platformUpdateDomain": "0",
+        "publisher": "UnitPublisher",
+        "resourceGroupName": "UnitResourceGroupName",
+        "sku": "UnitSku",
+        "subscriptionId": "e4402c6c-2804-4a0a-9dee-d61918fc4d28",
+        "tags": "Key1:Value1;Key2:Value2",
+        "vmId": "f62f23fb-69e2-4df0-a20b-cb5c201a3e7a",
+        "version": "UnitVersion",
+        "vmSize": "Standard_D1_v2"
+        }'''.encode('utf-8'))
+
+        test_subject = ImdsClient()
+        test_subject.get_compute()
+
+        self.assertEqual(1, mock_http_get.call_count)
+        positional_args, kw_args = mock_http_get.call_args
+
+        self.assertEqual('http://169.254.169.254/metadata/instance/compute?api-version=2017-08-01', positional_args[0])
+        self.assertTrue('User-Agent' in kw_args['headers'])
+        self.assertTrue('Metadata' in kw_args['headers'])
+        self.assertEqual(True, kw_args['headers']['Metadata'])
+
+    @patch("azurelinuxagent.ga.update.restutil.http_get")
+    def test_get_bad_request(self, mock_http_get):
+        mock_http_get.return_value = ResponseMock(status=restutil.httpclient.BAD_REQUEST)
+
+        test_subject = ImdsClient()
+        self.assertRaises(HttpError, test_subject.get_compute)
+
+    @patch("azurelinuxagent.ga.update.restutil.http_get")
+    def test_get_empty_response(self, mock_http_get):
+        mock_http_get.return_value = ResponseMock(response=''.encode('utf-8'))
+
+        test_subject = ImdsClient()
+        self.assertRaises(ValueError, test_subject.get_compute)
+
+    def test_deserialize_ComputeInfo(self):
+        s = '''{
+        "location": "westcentralus",
+        "name": "unit_test",
+        "offer": "UnitOffer",
+        "osType": "Linux",
+        "placementGroupId": "",
+        "platformFaultDomain": "0",
+        "platformUpdateDomain": "0",
+        "publisher": "UnitPublisher",
+        "resourceGroupName": "UnitResourceGroupName",
+        "sku": "UnitSku",
+        "subscriptionId": "e4402c6c-2804-4a0a-9dee-d61918fc4d28",
+        "tags": "Key1:Value1;Key2:Value2",
+        "vmId": "f62f23fb-69e2-4df0-a20b-cb5c201a3e7a",
+        "version": "UnitVersion",
+        "vmSize": "Standard_D1_v2"
+        }'''
+
+        data = json.loads(s, encoding='utf-8')
+
+        compute_info = ComputeInfo()
+        set_properties("compute", compute_info, data)
+
+        self.assertEqual('westcentralus', compute_info.location)
+        self.assertEqual('unit_test', compute_info.name)
+        self.assertEqual('UnitOffer', compute_info.offer)
+        self.assertEqual('Linux', compute_info.osType)
+        self.assertEqual('', compute_info.placementGroupId)
+        self.assertEqual('0', compute_info.platformFaultDomain)
+        self.assertEqual('0', compute_info.platformUpdateDomain)
+        self.assertEqual('UnitPublisher', compute_info.publisher)
+        self.assertEqual('UnitResourceGroupName', compute_info.resourceGroupName)
+        self.assertEqual('UnitSku', compute_info.sku)
+        self.assertEqual('e4402c6c-2804-4a0a-9dee-d61918fc4d28', compute_info.subscriptionId)
+        self.assertEqual('Key1:Value1;Key2:Value2', compute_info.tags)
+        self.assertEqual('f62f23fb-69e2-4df0-a20b-cb5c201a3e7a', compute_info.vmId)
+        self.assertEqual('UnitVersion', compute_info.version)
+        self.assertEqual('Standard_D1_v2', compute_info.vmSize)
+
+        self.assertEqual('UnitPublisher:UnitOffer:UnitSku:UnitVersion', compute_info.image_info)
+
diff --git a/tests/utils/test_archive.py b/tests/utils/test_archive.py
index f0564c88..1016c9aa 100644
--- a/tests/utils/test_archive.py
+++ b/tests/utils/test_archive.py
@@ -164,9 +164,18 @@ class TestArchive(AgentTestCase):
             'Microsoft.Azure.Extensions.CustomScript.0.xml'
         ]
 
-        def _write_goal_state_files(temp_files):
+        def _write_goal_state_files(temp_files, content=None):
             for f in temp_files:
-                self._write_file(f)
+                self._write_file(f, content)
+
+        def _check_history_files(timestamp_dir, files, content=None):
+            for f in files:
+                history_path = os.path.join(self.history_dir, timestamp_dir, f)
+                msg = "expected the temp file {0} to exist".format(history_path)
+                self.assertTrue(os.path.exists(history_path), msg)
+                expected_content = f if content is None else content
+                actual_content = fileutil.read_file(history_path)
+                self.assertEqual(expected_content, actual_content)
 
         timestamp = datetime.utcnow()
 
@@ -174,24 +183,7 @@ class TestArchive(AgentTestCase):
         test_subject = StateFlusher(self.tmp_dir)
         test_subject.flush(timestamp)
 
-        # Ensure that the file contets are what we expect after a flush.
-        # The file contents are simply the name of the file.
-        fn = os.path.join(self.history_dir, timestamp.isoformat(), 'Prod.0.manifest.xml')
-        self.assertEqual('Prod.0.manifest.xml', fileutil.read_file(fn))
-
-        # Modify the contents of the file on disk, and re-flush().
-        fileutil.write_file(fn, "--this-has-been-changed--")
-        self.assertEqual("--this-has-been-changed--", fileutil.read_file(fn))
-
-        # re-write all of the same files, and flush again. .flush()
-        # should overwrite the existing ones
-
-        _write_goal_state_files(temp_files)
-        test_subject.flush(timestamp)
-
-        # The contents of the file were overwritten as a result of the flush.
-        self.assertEqual('Prod.0.manifest.xml', fileutil.read_file(fn))
-
+        # Ensure history directory exists, has proper timestamped-based name,
         self.assertTrue(os.path.exists(self.history_dir))
         self.assertTrue(os.path.isdir(self.history_dir))
 
@@ -202,10 +194,16 @@ class TestArchive(AgentTestCase):
         ts = self.parse_isoformat(timestamp_dirs[0])
         self.assertDateTimeCloseTo(ts, datetime.utcnow(), timedelta(seconds=30))
 
-        for f in temp_files:
-            history_path = os.path.join(self.history_dir, timestamp_dirs[0], f)
-            msg = "expected the temp file {0} to exist".format(history_path)
-            self.assertTrue(os.path.exists(history_path), msg)
+        # Ensure saved files contain the right content
+        _check_history_files(timestamp_dirs[0], temp_files)
+
+        # re-write all of the same files with different content, and flush again.
+        # .flush() should overwrite the existing ones
+        _write_goal_state_files(temp_files, "--this-has-been-changed--")
+        test_subject.flush(timestamp)
+
+        # The contents of the saved files were overwritten as a result of the flush.
+        _check_history_files(timestamp_dirs[0], temp_files, "--this-has-been-changed--")
 
     def test_archive04(self):
         """