Repository: RedHatInsights/insights-core

Issue:
Curl command is returning stats in data, command should use --silent switch
The curl commands for specs `aws_instance_id_doc`, `aws_instance_id_pkcs7` and `aws_instance_type` are returning the curl stats in the data.  The commands should use the `-s` switch to prevent this data from being output.

```
% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   483  100   483    0     0   607k      0 --:--:-- --:--:-- --:--:--  471k
{
    "devpayProductCodes" : null,
    ...
}
```


Discussion:

Resolving patch:
diff --git a/insights/parsers/aws_instance_id.py b/insights/parsers/aws_instance_id.py
--- a/insights/parsers/aws_instance_id.py
+++ b/insights/parsers/aws_instance_id.py
@@ -5,8 +5,8 @@
 These parsers read the output of commands to collect identify information
 from AWS instances.
 
-* ``curl http://169.254.169.254/latest/meta-data/instance-identity/document`` and
-* ``curl http://169.254.169.254/latest/meta-data/instance-identity/pkcs7``
+* ``curl -s http://169.254.169.254/latest/dynamic/instance-identity/document`` and
+* ``curl -s http://169.254.169.254/latest/dynamic/instance-identity/pkcs7``
 """
 from __future__ import print_function
 import json
@@ -21,7 +21,7 @@ class AWSInstanceIdDoc(CommandParser, dict):
     """
     Class for parsing the AWS Instance Identity Document returned by the command::
 
-        curl http://169.254.169.254/latest/meta-data/instance-identity/document
+        curl -s http://169.254.169.254/latest/dynamic/instance-identity/document
 
     Typical output of this command is::
 
@@ -65,7 +65,14 @@ def parse_content(self, content):
         if not content or 'curl: ' in content[0]:
             raise SkipException()
 
-        self.json = '\n'.join([l.rstrip() for l in content])
+        # Just in case curl stats are present in data
+        startline = 0
+        for l in content:
+            if l.strip().startswith('{'):
+                break
+            startline += 1
+
+        self.json = '\n'.join([l.rstrip() for l in content[startline:]])
 
         try:
             doc_values = json.loads(self.json)
@@ -79,7 +86,7 @@ class AWSInstanceIdPkcs7(CommandParser):
     """
     Class for parsing the AWS Instance Identity PKCS7 signature returned by the command::
 
-        curl http://169.254.169.254/latest/meta-data/instance-identity/pkcs7
+        curl -s http://169.254.169.254/latest/dynamic/instance-identity/pkcs7
 
     Typical output of this command is::
 
@@ -115,4 +122,11 @@ def parse_content(self, content):
         if not content or 'curl: ' in content[0]:
             raise SkipException()
 
-        self.signature = '-----BEGIN PKCS7-----\n' + '\n'.join([l.rstrip() for l in content]) + "\n-----END PKCS7-----"
+        # Just in case curl stats are present in data
+        startline = 0
+        for l in content:
+            if ' ' not in l.strip() and len(l.strip()) > 0:
+                break
+            startline += 1
+
+        self.signature = '-----BEGIN PKCS7-----\n' + '\n'.join([l.rstrip() for l in content[startline:]]) + "\n-----END PKCS7-----"
diff --git a/insights/parsers/aws_instance_type.py b/insights/parsers/aws_instance_type.py
--- a/insights/parsers/aws_instance_type.py
+++ b/insights/parsers/aws_instance_type.py
@@ -3,7 +3,7 @@
 ===============
 
 This parser simply reads the output of command
-``curl http://169.254.169.254/latest/meta-data/instance-type``,
+``curl -s http://169.254.169.254/latest/meta-data/instance-type``,
 which is used to check the type of the AWS instance on the host.
 
 """
@@ -17,7 +17,7 @@
 class AWSInstanceType(CommandParser):
     """
     Class for parsing the AWS Instance type returned by command
-    ``curl http://169.254.169.254/latest/meta-data/instance-type``
+    ``curl -s http://169.254.169.254/latest/meta-data/instance-type``
 
     Typical output of this command is::
 
@@ -39,16 +39,20 @@ class AWSInstanceType(CommandParser):
     """
 
     def parse_content(self, content):
-        if (not content or len(content) > 1 or 'curl: ' in content[0]):
+        if not content or 'curl: ' in content[0]:
             raise SkipException()
 
         self.raw = self.type = None
-        if '.' in content[0]:
-            self.raw = content[0].strip()
-            self.type = self.raw.split('.')[0].upper()
+
+        # Ignore any curl stats that may be present in data
+        for l in content:
+            if ' ' not in l.strip() and len(l.strip()) > 0:
+                self.raw = l.strip()
+                if '.' in self.raw:
+                    self.type = self.raw.split('.')[0].upper()
 
         if not self.type:
-            raise ParseException('Unrecognized type: "{0}"', content[0])
+            raise ParseException('Unrecognized type: "{0}"'.format(content))
 
     def __repr__(self):
         return "<aws_type: {t}, raw: {r}>".format(t=self.type, r=self.raw)
diff --git a/insights/specs/default.py b/insights/specs/default.py
--- a/insights/specs/default.py
+++ b/insights/specs/default.py
@@ -104,9 +104,9 @@ def is_aws(broker):
             return True
         raise SkipComponent()
 
-    aws_instance_id_doc = simple_command("/usr/bin/curl http://169.254.169.254/latest/dynamic/instance-identity/document --connect-timeout 5", deps=[is_aws])
-    aws_instance_id_pkcs7 = simple_command("/usr/bin/curl http://169.254.169.254/latest/dynamic/instance-identity/pkcs7 --connect-timeout 5", deps=[is_aws])
-    aws_instance_type = simple_command("/usr/bin/curl http://169.254.169.254/latest/meta-data/instance-type --connect-timeout 5", deps=[is_aws])
+    aws_instance_id_doc = simple_command("/usr/bin/curl -s http://169.254.169.254/latest/dynamic/instance-identity/document --connect-timeout 5", deps=[is_aws])
+    aws_instance_id_pkcs7 = simple_command("/usr/bin/curl -s http://169.254.169.254/latest/dynamic/instance-identity/pkcs7 --connect-timeout 5", deps=[is_aws])
+    aws_instance_type = simple_command("/usr/bin/curl -s http://169.254.169.254/latest/meta-data/instance-type --connect-timeout 5", deps=[is_aws])
 
     @datasource(CloudProvider)
     def is_azure(broker):


Test patch:
diff --git a/insights/parsers/tests/test_aws_instance_id.py b/insights/parsers/tests/test_aws_instance_id.py
--- a/insights/parsers/tests/test_aws_instance_id.py
+++ b/insights/parsers/tests/test_aws_instance_id.py
@@ -28,6 +28,30 @@
     "region" : "us-west-2"
 }"""
 
+AWS_ID_DOC_CURL_STATS = """
+% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
+  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+100   483  100   483    0     0   607k      0 --:--:-- --:--:-- --:--:--  471k
+{
+    "devpayProductCodes" : null,
+    "marketplaceProductCodes" : [ "1abc2defghijklm3nopqrs4tu" ],
+    "availabilityZone" : "us-west-2b",
+    "privateIp" : "10.158.112.84",
+    "version" : "2017-09-30",
+    "instanceId" : "i-1234567890abcdef0",
+    "billingProducts" : [ "bp-6ba54002" ],
+    "instanceType" : "t2.micro",
+    "accountId" : "123456789012",
+    "imageId" : "ami-5fb8c835",
+    "pendingTime" : "2016-11-19T16:32:11Z",
+    "architecture" : "x86_64",
+    "kernelId" : null,
+    "ramdiskId" : null,
+    "region" : "us-west-2"
+}"""
+
 AWS_ID_DOC_ERROR = """
 Invalid json
 """
@@ -50,6 +74,27 @@
 FFBjvSfpJIlJ00zbhNYS5f6GuoEDmFJl0ZxBHjJnyp378OD8uTs7fLvjx79LjSTb
 NYiytVbZPQUQ5Yaxu2jXnimvw3rrszlaEXAMPLE"""
 
+AWS_ID_PKCS7_CURL_STATS = """
+ % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                  Dload  Upload   Total   Spent    Left  Speed
+
+   0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+ 100  1126  100  1126    0     0  1374k      0 --:--:-- --:--:-- --:--:-- 1099k
+MIICiTCCAfICCQD6m7oRw0uXOjANBgkqhkiG9w0BAQUFADCBiDELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgTAldBMRAwDgYDVQQHEwdTZWF0dGxlMQ8wDQYDVQQKEwZBbWF6
+b24xFDASBgNVBAsTC0lBTSBDb25zb2xlMRIwEAYDVQQDEwlUZXN0Q2lsYWMxHzAd
+BgkqhkiG9w0BCQEWEG5vb25lQGFtYXpvbi5jb20wHhcNMTEwNDI1MjA0NTIxWhcN
+MTIwNDI0MjA0NTIxWjCBiDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAldBMRAwDgYD
+VQQHEwdTZWF0dGxlMQ8wDQYDVQQKEwZBbWF6b24xFDASBgNVBAsTC0lBTSBDb25z
+b2xlMRIwEAYDVQQDEwlUZXN0Q2lsYWMxHzAdBgkqhkiG9w0BCQEWEG5vb25lQGFt
+YXpvbi5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMaK0dn+a4GmWIWJ
+21uUSfwfEvySWtC2XADZ4nB+BLYgVIk60CpiwsZ3G93vUEIO3IyNoH/f0wYK8m9T
+rDHudUZg3qX4waLG5M43q7Wgc/MbQITxOUSQv7c7ugFFDzQGBzZswY6786m86gpE
+Ibb3OhjZnzcvQAaRHhdlQWIMm2nrAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAtCu4
+nUhVVxYUntneD9+h8Mg9q6q+auNKyExzyLwaxlAoo7TJHidbtS4J5iNmZgXL0Fkb
+FFBjvSfpJIlJ00zbhNYS5f6GuoEDmFJl0ZxBHjJnyp378OD8uTs7fLvjx79LjSTb
+NYiytVbZPQUQ5Yaxu2jXnimvw3rrszlaEXAMPLE"""
+
 
 def test_aws_instance_id_doc():
     with pytest.raises(SkipException):
@@ -83,6 +128,27 @@ def test_aws_instance_id_doc():
     }
     assert "whatchamacallit" not in doc
 
+    doc = AWSInstanceIdDoc(context_wrap(AWS_ID_DOC_CURL_STATS))
+    assert doc is not None
+    assert doc == {
+        "devpayProductCodes": None,
+        "marketplaceProductCodes": ["1abc2defghijklm3nopqrs4tu", ],
+        "availabilityZone": "us-west-2b",
+        "privateIp": "10.158.112.84",
+        "version": "2017-09-30",
+        "instanceId": "i-1234567890abcdef0",
+        "billingProducts": ["bp-6ba54002", ],
+        "instanceType": "t2.micro",
+        "accountId": "123456789012",
+        "imageId": "ami-5fb8c835",
+        "pendingTime": "2016-11-19T16:32:11Z",
+        "architecture": "x86_64",
+        "kernelId": None,
+        "ramdiskId": None,
+        "region": "us-west-2"
+    }
+    assert "whatchamacallit" not in doc
+
 
 def test_aws_instance_id_pkcs7():
     with pytest.raises(SkipException):
@@ -111,6 +177,26 @@ def test_aws_instance_id_pkcs7():
 NYiytVbZPQUQ5Yaxu2jXnimvw3rrszlaEXAMPLE
 -----END PKCS7-----""".strip()
 
+    pkcs7 = AWSInstanceIdPkcs7(context_wrap(AWS_ID_PKCS7_CURL_STATS))
+    assert pkcs7 is not None
+    assert pkcs7.signature == """
+-----BEGIN PKCS7-----
+MIICiTCCAfICCQD6m7oRw0uXOjANBgkqhkiG9w0BAQUFADCBiDELMAkGA1UEBhMC
+VVMxCzAJBgNVBAgTAldBMRAwDgYDVQQHEwdTZWF0dGxlMQ8wDQYDVQQKEwZBbWF6
+b24xFDASBgNVBAsTC0lBTSBDb25zb2xlMRIwEAYDVQQDEwlUZXN0Q2lsYWMxHzAd
+BgkqhkiG9w0BCQEWEG5vb25lQGFtYXpvbi5jb20wHhcNMTEwNDI1MjA0NTIxWhcN
+MTIwNDI0MjA0NTIxWjCBiDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAldBMRAwDgYD
+VQQHEwdTZWF0dGxlMQ8wDQYDVQQKEwZBbWF6b24xFDASBgNVBAsTC0lBTSBDb25z
+b2xlMRIwEAYDVQQDEwlUZXN0Q2lsYWMxHzAdBgkqhkiG9w0BCQEWEG5vb25lQGFt
+YXpvbi5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMaK0dn+a4GmWIWJ
+21uUSfwfEvySWtC2XADZ4nB+BLYgVIk60CpiwsZ3G93vUEIO3IyNoH/f0wYK8m9T
+rDHudUZg3qX4waLG5M43q7Wgc/MbQITxOUSQv7c7ugFFDzQGBzZswY6786m86gpE
+Ibb3OhjZnzcvQAaRHhdlQWIMm2nrAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAtCu4
+nUhVVxYUntneD9+h8Mg9q6q+auNKyExzyLwaxlAoo7TJHidbtS4J5iNmZgXL0Fkb
+FFBjvSfpJIlJ00zbhNYS5f6GuoEDmFJl0ZxBHjJnyp378OD8uTs7fLvjx79LjSTb
+NYiytVbZPQUQ5Yaxu2jXnimvw3rrszlaEXAMPLE
+-----END PKCS7-----""".strip()
+
 
 def test_doc_examples():
     env = {
diff --git a/insights/parsers/tests/test_aws_instance_type.py b/insights/parsers/tests/test_aws_instance_type.py
--- a/insights/parsers/tests/test_aws_instance_type.py
+++ b/insights/parsers/tests/test_aws_instance_type.py
@@ -7,6 +7,13 @@
 from insights.core.plugins import ContentException
 
 AWS_TYPE = "r3.xlarge"
+AWS_TYPE_CURL_STATS = """
+ % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                  Dload  Upload   Total   Spent    Left  Speed
+
+   0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+ 100  1126  100  1126    0     0  1374k      0 --:--:-- --:--:-- --:--:-- 1099k
+r3.xlarge"""
 AWS_TYPE_AB_1 = """
 curl: (7) Failed to connect to 169.254.169.254 port 80: Connection timed out
 """.strip()
@@ -54,6 +61,13 @@ def test_aws_instance_type():
     assert 'large' in str(aws)
 
 
+def test_aws_instance_type_stats():
+    aws = AWSInstanceType(context_wrap(AWS_TYPE_CURL_STATS))
+    assert aws.type == "R3"
+    assert aws.raw == "r3.xlarge"
+    assert 'large' in str(aws)
+
+
 def test_doc_examples():
     env = {
             'aws_inst': AWSInstanceType(context_wrap(AWS_TYPE))