Repository: RedHatInsights/insights-core

Issue:
[RFE] Results to include positive NACK - CVEs to which a system is NOT vulnerable
Use case: VMaaS provides list of CVEs to which a system is vulnerable, based on list of packages' NEVRAs installed on the system.  This is overridden by results from the rules engine evaluation, which can check if a certain port is closed, or a configuration value prevents exploit of a vulnerability.  Unless the rules engine results includes "we evaluated for vulnerability to CVE-x and the system is NOT vulnerable", we (vulnerability-engine) won't know to override the VMaaS results that report the system is vulnerable based on the version of the package installed.

Vulnerability-engine will be able to assume that a system is no longer vulnerable to a rules engine-reported CVE if it was reported as vulnerable in previous evaluation.  However, a new system checking in with a fix already in place... vulnerability-engine won't know that the vulnerability is mitigated by a config setting, closed port, etc. unless rules engine explicitly reports a CVE that was evaluated and found the system NOT vulnerable.


Discussion:
Core provides [make_pass](https://github.com/RedHatInsights/insights-core/blob/master/insights/core/plugins.py#L312) and [make_fail](https://github.com/RedHatInsights/insights-core/blob/master/insights/core/plugins.py#L307) response types that rules can use. However, `make_pass` isn't yet included in the json report consumed by existing systems. If you want to stand up your own service and have the passes included in the response, we'll need to collect them along with the other types in the [evaluator](https://github.com/RedHatInsights/insights-core/blob/master/insights/core/evaluators.py#L89) and include them as part of the [response](https://github.com/RedHatInsights/insights-core/blob/master/insights/core/evaluators.py#L76). Existing systems will ignore that section.

Resolving patch:
diff --git a/insights/core/evaluators.py b/insights/core/evaluators.py
--- a/insights/core/evaluators.py
+++ b/insights/core/evaluators.py
@@ -1,6 +1,9 @@
 import logging
+import six
 import sys
 
+from collections import defaultdict
+
 from ..formats import Formatter
 from ..specs import Specs
 from ..combiners.hostname import hostname as combiner_hostname
@@ -17,9 +20,8 @@ def get_simple_module_name(obj):
 class Evaluator(Formatter):
     def __init__(self, broker=None, stream=sys.stdout, incremental=False):
         super(Evaluator, self).__init__(broker or dr.Broker(), stream)
+        self.results = defaultdict(list)
         self.rule_skips = []
-        self.rule_results = []
-        self.fingerprint_results = []
         self.hostname = None
         self.metadata = {}
         self.metadata_keys = {}
@@ -80,30 +82,32 @@ def get_response(self):
                 "metadata": self.metadata,
                 "hostname": self.hostname
             },
-            "reports": self.rule_results,
-            "fingerprints": self.fingerprint_results,
+            "reports": self.results["rule"],
+            "fingerprints": self.results["fingerprint"],
             "skips": self.rule_skips,
         })
+
+        for k, v in six.iteritems(self.results):
+            if k not in ("rule", "fingerprint"):
+                r[k] = v
+
         return self.format_response(r)
 
     def handle_result(self, plugin, r):
         type_ = r["type"]
-        if type_ == "metadata":
+
+        if type_ == "skip":
+            self.rule_skips.append(r)
+        elif type_ == "metadata":
             self.append_metadata(r)
-        elif type_ == "rule":
-            self.rule_results.append(self.format_result({
-                "rule_id": "{0}|{1}".format(get_simple_module_name(plugin), r["error_key"]),
-                "details": r
-            }))
-        elif type_ == "fingerprint":
-            self.fingerprint_results.append(self.format_result({
-                "fingerprint_id": "{0}|{1}".format(get_simple_module_name(plugin), r["fingerprint_key"]),
+        elif type_ == "metadata_key":
+            self.metadata_keys[r.get_key()] = r["value"]
+        else:
+            response_id = "%s_id" % r.response_type
+            self.results[type_].append(self.format_result({
+                response_id: "{0}|{1}".format(get_simple_module_name(plugin), r.get_key()),
                 "details": r
             }))
-        elif type_ == "skip":
-            self.rule_skips.append(r)
-        elif type_ == "metadata_key":
-            self.metadata_keys[r["key"]] = r["value"]
 
 
 class InsightsEvaluator(SingleEvaluator):


Test patch:
diff --git a/insights/tests/test_evaluators.py b/insights/tests/test_evaluators.py
--- a/insights/tests/test_evaluators.py
+++ b/insights/tests/test_evaluators.py
@@ -1,11 +1,20 @@
-from insights import dr, rule, make_fail, make_pass
-from insights.core.plugins import component
+from insights import dr, rule, make_fail, make_pass, make_fingerprint
+from insights.core.plugins import component, Response
 from insights.core.evaluators import InsightsEvaluator, SingleEvaluator
 from insights.combiners.hostname import hostname
 from insights.specs import Specs
 from insights.tests import context_wrap
 
 
+class make_unsure(Response):
+    """
+    Show that anybody can make a Response subclass for rules to return
+    and have it included in the evaluator results.
+    """
+    response_type = "unsure"
+    key_name = "unsure_key"
+
+
 @component()
 def one():
     return 1
@@ -31,6 +40,16 @@ def report_fail():
     return make_fail("FAIL")
 
 
+@rule()
+def report_fingerprint():
+    return make_fingerprint("FINGERPRINT")
+
+
+@rule()
+def report_unsure():
+    return make_unsure("unsure")
+
+
 @rule(boom)
 def report_boom():
     pass
@@ -50,6 +69,8 @@ def report(a, b):
     boom,
     report_pass,
     report_fail,
+    report_fingerprint,
+    report_unsure,
     report_boom,
     report
 ]
@@ -131,3 +152,59 @@ def test_insights_evaluator_attrs_incremental_process():
     assert result["system"]["hostname"] == "www.example.com"
     assert result["system"]["system_id"] == "12345"
     assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)"
+
+
+def test_insights_evaluator_make_fail():
+    broker = dr.Broker()
+    broker[Specs.hostname] = context_wrap("www.example.com")
+    broker[Specs.machine_id] = context_wrap("12345")
+    broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)")
+    e = InsightsEvaluator(broker)
+    e.process(components)
+    result = e.get_response()
+    assert result["system"]["hostname"] == "www.example.com"
+    assert result["system"]["system_id"] == "12345"
+    assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)"
+    assert len(result["reports"]) == 2
+
+
+def test_insights_evaluator_make_pass():
+    broker = dr.Broker()
+    broker[Specs.hostname] = context_wrap("www.example.com")
+    broker[Specs.machine_id] = context_wrap("12345")
+    broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)")
+    e = InsightsEvaluator(broker)
+    e.process(components)
+    result = e.get_response()
+    assert result["system"]["hostname"] == "www.example.com"
+    assert result["system"]["system_id"] == "12345"
+    assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)"
+    assert len(result["pass"]) == 1
+
+
+def test_insights_evaluator_make_fingerprint():
+    broker = dr.Broker()
+    broker[Specs.hostname] = context_wrap("www.example.com")
+    broker[Specs.machine_id] = context_wrap("12345")
+    broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)")
+    e = InsightsEvaluator(broker)
+    e.process(components)
+    result = e.get_response()
+    assert result["system"]["hostname"] == "www.example.com"
+    assert result["system"]["system_id"] == "12345"
+    assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)"
+    assert len(result["fingerprints"]) == 1
+
+
+def test_insights_evaluator_make_unsure():
+    broker = dr.Broker()
+    broker[Specs.hostname] = context_wrap("www.example.com")
+    broker[Specs.machine_id] = context_wrap("12345")
+    broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)")
+    e = InsightsEvaluator(broker)
+    e.process(components)
+    result = e.get_response()
+    assert result["system"]["hostname"] == "www.example.com"
+    assert result["system"]["system_id"] == "12345"
+    assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)"
+    assert len(result["unsure"]) == 1