Repository: Azure/WALinuxAgent
Pull request #683 (2017-04-28 21:45:40)

Added auto-update support for Azure Stack (#518)
This addresses #518 by cleaning up the URIs used with Azure Stack and adding auto-update support.

Signed-off-by: Brendan Dixon <brendandixon@me.com>


Patch:
diff --git a/azurelinuxagent/common/protocol/metadata.py b/azurelinuxagent/common/protocol/metadata.py
index c61e373d..c50b3ddc 100644
--- a/azurelinuxagent/common/protocol/metadata.py
+++ b/azurelinuxagent/common/protocol/metadata.py
@@ -21,17 +21,19 @@ import json
 import os
 import shutil
 import re
+
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.shellutil as shellutil
 import azurelinuxagent.common.utils.textutil as textutil
+
 from azurelinuxagent.common.future import httpclient
 from azurelinuxagent.common.protocol.restapi import *
 from azurelinuxagent.common.utils.cryptutil import CryptUtil
 
 METADATA_ENDPOINT = '169.254.169.254'
 APIVERSION = '2015-05-01-preview'
-BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}{3}"
+BASE_URI = "http://{0}/Microsoft.Compute/{1}?api-version={2}"
 
 TRANSPORT_PRV_FILE_NAME = "V2TransportPrivate.pem"
 TRANSPORT_CERT_FILE_NAME = "V2TransportCert.pem"
@@ -39,6 +41,9 @@ P7M_FILE_NAME = "Certificates.p7m"
 P7B_FILE_NAME = "Certificates.p7b"
 PEM_FILE_NAME = "Certificates.pem"
 
+KEY_AGENT_VERSION_URIS = "versionsManifestUris"
+KEY_URI = "uri"
+
 # TODO remote workaround for azure stack
 MAX_PING = 30
 RETRY_PING_INTERVAL = 10
@@ -56,13 +61,13 @@ class MetadataProtocol(Protocol):
         self.apiversion = apiversion
         self.endpoint = endpoint
         self.identity_uri = BASE_URI.format(self.endpoint, "identity",
-                                            self.apiversion, "&$expand=*")
+                                            self.apiversion)
         self.cert_uri = BASE_URI.format(self.endpoint, "certificates",
-                                        self.apiversion, "&$expand=*")
+                                        self.apiversion)
         self.ext_uri = BASE_URI.format(self.endpoint, "extensionHandlers",
-                                       self.apiversion, "&$expand=*")
+                                       self.apiversion)
         self.vmagent_uri = BASE_URI.format(self.endpoint, "vmAgentVersions",
-                                           self.apiversion, "&$expand=*")
+                                           self.apiversion)
         self.provision_status_uri = BASE_URI.format(self.endpoint,
                                                     "provisioningStatus",
                                                     self.apiversion, "")
@@ -74,6 +79,8 @@ class MetadataProtocol(Protocol):
         self.event_uri = BASE_URI.format(self.endpoint, "status/telemetry",
                                          self.apiversion, "")
         self.certs = None
+        self.agent_manifests = None
+        self.agent_etag = None
 
     def _get_data(self, url, headers=None):
         try:
@@ -166,32 +173,55 @@ class MetadataProtocol(Protocol):
             return None
         return self.certs
 
-    def get_vmagent_manifests(self, last_etag=None):
-        manifests = VMAgentManifestList()
+    def get_vmagent_manifests(self):
         self.update_goal_state()
+
         data, etag = self._get_data(self.vmagent_uri)
-        if last_etag is None or last_etag < etag:
-            set_properties("vmAgentManifests",
-                           manifests.vmAgentManifests,
-                           data)
-        return manifests, etag
+        if self.agent_etag is None or self.agent_etag < etag:
+            self.agent_etag = etag
+
+            # Create a list with a single manifest
+            # -- The protocol lacks "family," use the configured family
+            self.agent_manifests = VMAgentManifestList()
+
+            manifest = VMAgentManifest()
+            manifest.family = family=conf.get_autoupdate_gafamily()
+            
+            if not KEY_AGENT_VERSION_URIS in data:
+                raise ProtocolError(
+                    "Agent versions missing '{0}': {1}".format(
+                        KEY_AGENT_VERSION_URIS, data))
+
+            for version in data[KEY_AGENT_VERSION_URIS]:
+                if not KEY_URI in version:
+                    raise ProtocolError(
+                        "Agent versions missing '{0': {1}".format(
+                            KEY_URI, data))
+                manifest_uri = VMAgentManifestUri(uri=version[KEY_URI])
+                manifest.versionsManifestUris.append(manifest_uri)
+        
+            self.agent_manifests.vmAgentManifests.append(manifest)
+        
+        return self.agent_manifests, self.agent_etag
 
     def get_vmagent_pkgs(self, vmagent_manifest):
-        # Agent package is the same with extension handler
-        vmagent_pkgs = ExtHandlerPackageList()
         data = None
+        etag = None
         for manifest_uri in vmagent_manifest.versionsManifestUris:
             try:
-                data = self._get_data(manifest_uri.uri)
+                data, etag = self._get_data(manifest_uri.uri)
                 break
             except ProtocolError as e:
-                logger.warn("Failed to get vmagent versions: {0}", e)
-                logger.info("Retry getting vmagent versions")
+                logger.verbose(
+                    "Error retrieving agent package from {0}: {1}".format(
+                        manifest_uri, e))
+
         if data is None:
-            raise ProtocolError(("Failed to get versions for vm agent: {0}"
-                                 "").format(vmagent_manifest.family))
+            raise ProtocolError(
+                "Failed retrieving agent package from all URIs")
+
+        vmagent_pkgs = ExtHandlerPackageList()
         set_properties("vmAgentVersions", vmagent_pkgs, data)
-        # TODO: What etag should this return?
         return vmagent_pkgs
 
     def get_ext_handlers(self, last_etag=None):
@@ -251,11 +281,9 @@ class MetadataProtocol(Protocol):
         self._put_data(uri, data)
 
     def report_event(self, events):
-        # TODO disable telemetry for azure stack test
-        # validate_param('events', events, TelemetryEventList)
-        # data = get_properties(events)
-        # self._post_data(self.event_uri, data)
-        pass
+        validate_param('events', events, TelemetryEventList)
+        data = get_properties(events)
+        self._post_data(self.event_uri, data)
 
     def update_certs(self):
         certificates = self.get_certs()


Test patch:
diff --git a/tests/data/events/1478123456789000.tld b/tests/data/events/1478123456789000.tld
new file mode 100644
index 00000000..a689f4c0
--- /dev/null
+++ b/tests/data/events/1478123456789000.tld
@@ -0,0 +1,1 @@
+{"eventId": 1, "providerId": "69B669B9-4AF8-4C50-BDC4-6006FA76E975", "parameters": [{"name": "Name", "value": "Test Event"}, {"name": "Version", "value": "2.2.0"}, {"name": "IsInternal", "value": false}, {"name": "Operation", "value": "Some Operation"}, {"name": "OperationSuccess", "value": true}, {"name": "Message", "value": ""}, {"name": "Duration", "value": 0}, {"name": "ExtensionType", "value": ""}]}
\ No newline at end of file
diff --git a/tests/data/events/1478123456789001.tld b/tests/data/events/1478123456789001.tld
new file mode 100644
index 00000000..95460e3f
--- /dev/null
+++ b/tests/data/events/1478123456789001.tld
@@ -0,0 +1,1 @@
+{"eventId": 1, "providerId": "69B669B9-4AF8-4C50-BDC4-6006FA76E975", "parameters": [{"name": "Name", "value": "Linux Event"}, {"name": "Version", "value": "2.2.0"}, {"name": "IsInternal", "value": false}, {"name": "Operation", "value": "Linux Operation"}, {"name": "OperationSuccess", "value": false}, {"name": "Message", "value": "Linux Message"}, {"name": "Duration", "value": 42}, {"name": "ExtensionType", "value": "Linux Event Type"}]}
\ No newline at end of file
diff --git a/tests/data/events/1479766858966718.tld b/tests/data/events/1479766858966718.tld
new file mode 100644
index 00000000..cc7ac67d
--- /dev/null
+++ b/tests/data/events/1479766858966718.tld
@@ -0,0 +1,1 @@
+{"eventId": 1, "providerId": "69B669B9-4AF8-4C50-BDC4-6006FA76E975", "parameters": [{"name": "Name", "value": "WALinuxAgent"}, {"name": "Version", "value": "2.3.0.1"}, {"name": "IsInternal", "value": false}, {"name": "Operation", "value": "Enable"}, {"name": "OperationSuccess", "value": true}, {"name": "Message", "value": "Agent WALinuxAgent-2.3.0.1 launched with command 'python install.py' is successfully running"}, {"name": "Duration", "value": 0}, {"name": "ExtensionType", "value": ""}]}
\ No newline at end of file
diff --git a/tests/data/metadata/vmagent_manifest1.json b/tests/data/metadata/vmagent_manifest1.json
new file mode 100644
index 00000000..544a7082
--- /dev/null
+++ b/tests/data/metadata/vmagent_manifest1.json
@@ -0,0 +1,20 @@
+{
+    "versions": [
+        {
+            "version": "2.2.8",
+            "uris": [
+                {
+                    "uri": "https: //notused.com/ga/WALinuxAgent-2.2.8.zip"
+                }
+            ]
+        },
+        {
+            "version": "2.2.9",
+            "uris": [
+                {
+                    "uri": "https: //notused.com/ga/WALinuxAgent-2.2.9.zip"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/tests/data/metadata/vmagent_manifest2.json b/tests/data/metadata/vmagent_manifest2.json
new file mode 100644
index 00000000..544a7082
--- /dev/null
+++ b/tests/data/metadata/vmagent_manifest2.json
@@ -0,0 +1,20 @@
+{
+    "versions": [
+        {
+            "version": "2.2.8",
+            "uris": [
+                {
+                    "uri": "https: //notused.com/ga/WALinuxAgent-2.2.8.zip"
+                }
+            ]
+        },
+        {
+            "version": "2.2.9",
+            "uris": [
+                {
+                    "uri": "https: //notused.com/ga/WALinuxAgent-2.2.9.zip"
+                }
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/tests/data/metadata/vmagent_manifests.json b/tests/data/metadata/vmagent_manifests.json
new file mode 100644
index 00000000..2628f890
--- /dev/null
+++ b/tests/data/metadata/vmagent_manifests.json
@@ -0,0 +1,7 @@
+{
+    "versionsManifestUris" :
+    [
+        { "uri" : "https://notused.com/vmagent_manifest1.json" },
+        { "uri" : "https://notused.com/vmagent_manifest2.json" }
+    ]
+}
diff --git a/tests/data/metadata/vmagent_manifests_invalid1.json b/tests/data/metadata/vmagent_manifests_invalid1.json
new file mode 100644
index 00000000..55b08d16
--- /dev/null
+++ b/tests/data/metadata/vmagent_manifests_invalid1.json
@@ -0,0 +1,10 @@
+{
+    "notTheRightKey": [
+        {
+            "uri": "https://notused.com/vmagent_manifest1.json"
+        },
+        {
+            "uri": "https://notused.com/vmagent_manifest2.json"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/tests/data/metadata/vmagent_manifests_invalid2.json b/tests/data/metadata/vmagent_manifests_invalid2.json
new file mode 100644
index 00000000..5df42529
--- /dev/null
+++ b/tests/data/metadata/vmagent_manifests_invalid2.json
@@ -0,0 +1,10 @@
+{
+    "notTheRightKey": [
+        {
+            "foo": "https://notused.com/vmagent_manifest1.json"
+        },
+        {
+            "bar": "https://notused.com/vmagent_manifest2.json"
+        }
+    ]
+}
\ No newline at end of file
diff --git a/tests/protocol/test_metadata.py b/tests/protocol/test_metadata.py
index f390f7a7..ee4ba3e6 100644
--- a/tests/protocol/test_metadata.py
+++ b/tests/protocol/test_metadata.py
@@ -15,14 +15,23 @@
 # Requires Python 2.4+ and Openssl 1.0+
 #
 
-from tests.tools import *
-from tests.protocol.mockmetadata import *
+import json
+
+from azurelinuxagent.common.future import ustr
+
 from azurelinuxagent.common.utils.restutil import httpclient
-from azurelinuxagent.common.protocol.metadata import MetadataProtocol
+from azurelinuxagent.common.protocol.metadata import *
+from azurelinuxagent.common.protocol.restapi import *
+
+from tests.protocol.mockmetadata import *
+from tests.tools import *
 
-@patch("time.sleep")
-@patch("azurelinuxagent.common.protocol.metadata.restutil")
 class TestMetadataProtocolGetters(AgentTestCase):
+    def load_json(self, path):
+        return json.loads(ustr(load_data(path)), encoding="utf-8")
+
+    @patch("time.sleep")
+    @patch("azurelinuxagent.common.protocol.metadata.restutil")
     def _test_getters(self, test_data, mock_restutil ,_):
         mock_restutil.http_get.side_effect = test_data.mock_http_get
 
@@ -43,3 +52,108 @@ class TestMetadataProtocolGetters(AgentTestCase):
         self._test_getters(test_data, *args)
 
 
+    @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol.update_goal_state")
+    @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol._get_data")
+    def test_get_vmagents_manifests(self, mock_get, mock_update):
+        data = self.load_json("metadata/vmagent_manifests.json")
+        mock_get.return_value = data, 42
+
+        protocol = MetadataProtocol()
+        manifests, etag = protocol.get_vmagent_manifests()
+
+        self.assertEqual(mock_update.call_count, 1)
+        self.assertEqual(mock_get.call_count, 1)
+
+        manifests_uri = BASE_URI.format(
+            METADATA_ENDPOINT,
+            "vmAgentVersions",
+            APIVERSION)
+        self.assertEqual(mock_get.call_args[0][0], manifests_uri)
+
+        self.assertEqual(etag, 42)
+        self.assertNotEqual(None, manifests)
+        self.assertEqual(len(manifests.vmAgentManifests), 1)
+
+        manifest = manifests.vmAgentManifests[0]
+        self.assertEqual(manifest.family, conf.get_autoupdate_gafamily())
+        self.assertEqual(len(manifest.versionsManifestUris), 2)
+
+        # Same etag returns the same data
+        data = self.load_json("metadata/vmagent_manifests_invalid1.json")
+        mock_get.return_value = data, 42
+        next_manifests, etag = protocol.get_vmagent_manifests()
+
+        self.assertEqual(etag, 42)
+        self.assertEqual(manifests, next_manifests)
+
+        # New etag returns new data
+        mock_get.return_value = data, 43
+        self.assertRaises(ProtocolError, protocol.get_vmagent_manifests)
+
+    @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol.update_goal_state")
+    @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol._get_data")
+    def test_get_vmagents_manifests_raises(self, mock_get, mock_update):
+        data = self.load_json("metadata/vmagent_manifests_invalid1.json")
+        mock_get.return_value = data, 42
+
+        protocol = MetadataProtocol()
+        self.assertRaises(ProtocolError, protocol.get_vmagent_manifests)
+
+        data = self.load_json("metadata/vmagent_manifests_invalid2.json")
+        mock_get.return_value = data, 43
+        self.assertRaises(ProtocolError, protocol.get_vmagent_manifests)
+
+    @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol.update_goal_state")
+    @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol._get_data")
+    def test_get_vmagent_pkgs(self, mock_get, mock_update):
+        data = self.load_json("metadata/vmagent_manifests.json")
+        mock_get.return_value = data, 42
+
+        protocol = MetadataProtocol()
+        manifests, etag = protocol.get_vmagent_manifests()
+        manifest = manifests.vmAgentManifests[0]
+
+        data = self.load_json("metadata/vmagent_manifest1.json")
+        mock_get.return_value = data, 42
+        pkgs = protocol.get_vmagent_pkgs(manifest)
+
+        self.assertNotEqual(None, pkgs)
+        self.assertEqual(len(pkgs.versions), 2)
+
+        for pkg in pkgs.versions:
+            self.assertNotEqual(None, pkg.version)
+            self.assertTrue(len(pkg.uris) > 0)
+
+            for uri in pkg.uris:
+                self.assertTrue(uri.uri.endswith("zip"))
+
+    @patch("azurelinuxagent.common.protocol.metadata.MetadataProtocol._post_data")
+    def test_report_event(self, mock_post):
+        events = TelemetryEventList()
+
+        data = self.load_json("events/1478123456789000.tld")
+        event = TelemetryEvent()
+        set_properties("event", event, data)
+        events.events.append(event)
+
+        data = self.load_json("events/1478123456789001.tld")
+        event = TelemetryEvent()
+        set_properties("event", event, data)
+        events.events.append(event)
+
+        data = self.load_json("events/1479766858966718.tld")
+        event = TelemetryEvent()
+        set_properties("event", event, data)
+        events.events.append(event)
+
+        protocol = MetadataProtocol()
+        protocol.report_event(events)
+
+        events_uri = BASE_URI.format(
+            METADATA_ENDPOINT,
+            "status/telemetry",
+            APIVERSION)
+
+        self.assertEqual(mock_post.call_count, 1)
+        self.assertEqual(mock_post.call_args[0][0], events_uri)
+        self.assertEqual(mock_post.call_args[0][1], get_properties(events))