Repository: cloud-custodian/cloud-custodian

Issue:
c7n-left - junitxml output
### Describe the feature

to better support additional ci pipelines, it would be nice to have junitxml output which would help cover off on

- gitlab
- circleci
- jenkins


### Extra information or context

_No response_


Discussion:

Resolving patch:
diff --git a/tools/c7n_left/c7n_left/core.py b/tools/c7n_left/c7n_left/core.py
--- a/tools/c7n_left/c7n_left/core.py
+++ b/tools/c7n_left/c7n_left/core.py
@@ -241,7 +241,7 @@ def run(self) -> bool:
                     continue
                 result_set = self.run_policy(p, graph, resources, event, rtype)
                 if result_set:
-                    self.reporter.on_results(result_set)
+                    self.reporter.on_results(p, result_set)
                     found = True
         self.reporter.on_execution_ended()
         return found
@@ -249,6 +249,7 @@ def run(self) -> bool:
     def run_policy(self, policy, graph, resources, event, resource_type):
         event = dict(event)
         event.update({"graph": graph, "resources": resources, "resource_type": resource_type})
+        self.reporter.on_policy_start(policy, event)
         return policy.push(event)
 
     def get_provider(self):
diff --git a/tools/c7n_left/c7n_left/output.py b/tools/c7n_left/c7n_left/output.py
--- a/tools/c7n_left/c7n_left/output.py
+++ b/tools/c7n_left/c7n_left/output.py
@@ -6,8 +6,10 @@
 from datetime import datetime
 from pathlib import Path
 from importlib.metadata import version as pkg_version
+import sys
 import time
 import uuid
+import xml.etree.ElementTree as etree
 
 from rich.console import Console
 from rich.syntax import Syntax
@@ -35,13 +37,16 @@ def __init__(self, ctx, config):
         self.config = config
 
     def on_execution_started(self, policies, graph):
-        pass
+        """called when collection execution is about to start"""
 
     def on_execution_ended(self):
-        pass
+        """called when collection execution has ended."""
 
-    def on_results(self, results):
-        pass
+    def on_policy_start(self, policy, event):
+        """called when a policy is about to be run"""
+
+    def on_results(self, policy, results):
+        """called when a policy matches resources"""
 
 
 class RichCli(Output):
@@ -63,7 +68,7 @@ def on_execution_ended(self):
             "Evaluation complete %0.2f seconds -> %s" % (time.time() - self.started, message)
         )
 
-    def on_results(self, results):
+    def on_results(self, policy, results):
         for r in results:
             self.console.print(RichResult(r))
         self.matches += len(results)
@@ -140,7 +145,7 @@ def on_execution_started(self, policies, graph):
         self.counter_policies_by_type = type_policies
         self.count_total_resources = resource_count
 
-    def on_results(self, results):
+    def on_results(self, policy, results):
         for r in results:
             self.count_policy_matches += 1
             self.resource_name_matches.add(r.resource.name)
@@ -197,8 +202,8 @@ def on_execution_started(self, policies, graph):
         super().on_execution_started(policies, graph)
         self.policies = sorted(map(PolicyMetadata, policies), key=severity_key)
 
-    def on_results(self, results):
-        super().on_results(results)
+    def on_results(self, policy, results):
+        super().on_results(policy, results)
         for r in results:
             self.counter_policy_matches[r.policy.name] += 1
 
@@ -239,8 +244,8 @@ def on_execution_started(self, policies, graph):
         super().on_execution_started(policies, graph)
         self.policies = {p.name: p for p in sorted(map(PolicyMetadata, policies), key=severity_key)}
 
-    def on_results(self, results):
-        super().on_results(results)
+    def on_results(self, policy, results):
+        super().on_results(policy, results)
         for r in results:
             self.resource_policy_matches.setdefault(r.resource.name, []).append(r)
 
@@ -299,15 +304,19 @@ def on_execution_ended(self):
         for o in self.outputs:
             o.on_execution_ended()
 
-    def on_results(self, results):
+    def on_policy_start(self, policy, event):
+        for o in self.outputs:
+            o.on_policy_start(policy, event)
+
+    def on_results(self, policy, results):
         for o in self.outputs:
-            o.on_results(results)
+            o.on_results(policy, results)
 
 
 class GithubFormat(Output):
     # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
 
-    def on_results(self, results):
+    def on_results(self, policy, results):
         for r in results:
             print(self.format_result(r), file=self.config.output_file)
 
@@ -351,7 +360,7 @@ def __init__(self, ctx, config):
         super().__init__(ctx, config)
         self.results = []
 
-    def on_results(self, results):
+    def on_results(self, policy, results):
         self.results.extend(results)
 
     def on_execution_ended(self):
@@ -377,6 +386,141 @@ def format_result(self, result):
         return formatted
 
 
+class JunitReport(Output):
+    """Junit xml output
+
+    Junit is a mess, without a useful canonical schema, and many informal
+    extensions without documentation. Which appears to be leading to a
+    proliferation of additional formats :(
+
+    The 'offical' schema is
+    https://raw.githubusercontent.com/junit-team/junit5/main/platform-tests/src/test/resources/jenkins-junit.xsd
+
+    which is relatively simple, but in practice that's not what tools
+    use, some documentation around common conventions exists at
+
+    https://github.com/testmoapp/junitxml
+
+    For gitlab, which only supports junit output, the parser is
+    https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/parsers/test/junit.rb
+
+    for azure devops, its unclear what they support, but they link to
+    a different schema that's commonly linked
+    https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd
+
+    This schema is also incompatible with the canonical schema, and
+    references several required fields that are not in common use.
+
+    ibm also has a page which also documents its specific non standard
+    handling (filename prefix to message)
+    https://www.ibm.com/docs/en/developer-for-zos/16.0?topic=formats-junit-xml-format
+
+    Looking at pytest's junitxml generator which has fairly wide
+    adoption in the python ecosystem and broad tool integration, and
+    what they generate shows light conformance to yet another spec
+    https://github.com/pytest-dev/pytest/blob/main/src/_pytest/junitxml.py
+
+    which in turn references a different format.
+    https://github.com/jenkinsci/xunit-plugin/blob/master/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd
+    """  # noqa
+
+    suite_name = "c7n-left"
+
+    def __init__(self, ctx, config):
+        super().__init__(ctx, config)
+        self.policy_results = {}
+        self.start_time = None
+
+    def on_results(self, policy, results):
+        self.policy_results[policy.name].extend(results)
+
+    def on_execution_started(self, policies, graph):
+        self.start_time = datetime.utcnow()
+        self.policies = {p.name: p for p in sorted(map(PolicyMetadata, policies), key=severity_key)}
+        self.policy_resources = {pname: [] for pname in self.policies}
+        self.policy_results = {pname: [] for pname in self.policies}
+
+    def on_policy_start(self, policy, event):
+        self.policy_resources[policy.name] = list(event['resources'])
+
+    def on_execution_ended(self):
+        info = self.get_info()
+
+        builder = etree.TreeBuilder()
+        builder.start("testsuites", info)
+        builder.start("testsuite", info)
+
+        for pname in self.policies:
+            p = self.policies[pname]
+            presources = self.policy_resources[pname]
+            matched = {m.resource.id for m in self.policy_results[pname]}
+            for r in presources:
+                if r.id in matched:
+                    continue
+                self.format_test_case(builder, p, r)
+            for m in self.policy_results[pname]:
+                self.format_result(builder, p, m)
+        doc = builder.close()
+        self.config.output_file.write(etree.tostring(doc).decode("utf8"))
+
+    def get_info(self):
+        return {
+            "id": "c7n-left",
+            "name": "IaC Policy Compliance",
+            "time": "%0.2f" % (datetime.utcnow() - self.start_time).total_seconds(),
+            "tests": str(sum(map(len, self.policy_resources.values()))),
+            "failures": str(sum(map(len, self.policy_results.values()))),
+        }
+
+    def _start_case(self, builder, policy_md, resource):
+        file_path = resource.src_dir / resource.filename
+        builder.start(
+            "testcase",
+            {
+                "name": f"[{policy_md.severity}] {policy_md.name}",
+                "file": str(file_path),
+                "classname": "%s.%s" % (str(file_path), resource.id),
+            },
+        )
+
+    def format_test_case(self, builder, policy_md, resource):
+        self._start_case(builder, policy_md, resource)
+        builder.end("testcase")
+
+    def format_result(self, builder, policy_md, result):
+        resource = result.resource
+        self._start_case(builder, policy_md, resource)
+        text_data = [
+            "",
+            "Resource: %s" % resource.id,
+            "File %s %d-%d" % (resource.filename, resource.line_start, resource.line_end),
+            "",
+        ]
+
+        lines = resource.get_source_lines()
+        line_idx = resource.line_start
+        for l in lines:
+            text_data.append("    %d | %s" % (line_idx, l))
+            line_idx += 1
+
+        builder.start(
+            "failure", {"type": "failure", "message": policy_md.description or policy_md.name}
+        )
+        builder.data("\n".join(text_data))
+        builder.end("failure")
+        builder.end("testcase")
+
+
+@report_outputs.register("junit")
+class Junit(MultiOutput):
+    def __init__(self, ctx, config):
+        if config.output_file.isatty() is False:
+            parts = [RichCli(ctx, config.copy(output_file=sys.stdout)), JunitReport(ctx, config)]
+        else:
+            parts = [JunitReport(ctx, config)]
+        super().__init__(parts)
+
+
 @report_outputs.register("gitlab_sast")
 class GitlabSAST(Output):
     SCHEMA_FILE = "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/v15.0.6/dist/sast-report-format.json"  # noqa
@@ -386,7 +530,7 @@ def __init__(self, ctx, config):
         self.results = []
         self.start_time = None
 
-    def on_results(self, results):
+    def on_results(self, policy, results):
         self.results.extend(results)
 
     def on_execution_started(self, *args):
diff --git a/tools/c7n_left/c7n_left/providers/terraform/resource.py b/tools/c7n_left/c7n_left/providers/terraform/resource.py
--- a/tools/c7n_left/c7n_left/providers/terraform/resource.py
+++ b/tools/c7n_left/c7n_left/providers/terraform/resource.py
@@ -17,6 +17,10 @@ def __init__(self, name, data):
             self.location = data["__tfmeta"]
         super().__init__(data)
 
+    @property
+    def id(self):
+        return self.location['path']
+
     @property
     def filename(self):
         return self.location["filename"]


Test patch:
diff --git a/tools/c7n_left/c7n_left/test.py b/tools/c7n_left/c7n_left/test.py
--- a/tools/c7n_left/c7n_left/test.py
+++ b/tools/c7n_left/c7n_left/test.py
@@ -208,6 +208,6 @@ def on_test_result(self, test: Test):
 
 
 class TestChecker(Output):
-    def on_results(self, results):
+    def on_results(self, policy, results):
         for r in results:
             self.ctx.check_execution_result(r)
diff --git a/tools/c7n_left/tests/test_left.py b/tools/c7n_left/tests/test_left.py
--- a/tools/c7n_left/tests/test_left.py
+++ b/tools/c7n_left/tests/test_left.py
@@ -7,6 +7,7 @@
 from pathlib import Path
 from unittest.mock import ANY
 from urllib.request import urlopen
+import xml.etree.ElementTree as etree
 
 import jsonschema
 import pytest
@@ -47,7 +48,10 @@ def on_execution_started(self, policies, graph):
     def on_execution_ended(self):
         pass
 
-    def on_results(self, results):
+    def on_policy_start(self, policy, event):
+        pass
+
+    def on_results(self, policy, results):
         self.results.extend(results)
 
 
@@ -882,6 +886,68 @@ def test_cli_no_policies(tmp_path, caplog):
     assert caplog.record_tuples == [("c7n.iac", 30, "no policies found")]
 
 
+def test_cli_junit_output(policy_env, tmp_path, debug_cli_runner):
+    policy_env.write_tf(
+        """
+resource "aws_cloudwatch_log_group" "yada" {
+  name = "Bar"
+}
+resource "aws_cloudwatch_log_group" "june" {
+  name = "June"
+}
+resource "aws_cloudwatch_log_group" "april" {
+  name = "April"
+  tags = {
+        Env = "Dev"
+  }
+}
+        """
+    )
+    policy_env.write_policy(
+        {
+            "name": "tag-required",
+            "description": "tags are required on log groups",
+            "metadata": {"url": "https://cloudcustodian.io", "severity": "high"},
+            "resource": "terraform.aws_cloudwatch_log_group",
+            "filters": [{"tags": "absent"}],
+        }
+    )
+    runner = CliRunner()
+    result = runner.invoke(
+        cli.cli,
+        [
+            "run",
+            "-p",
+            str(tmp_path),
+            "-d",
+            str(tmp_path),
+            "-o",
+            "junit",
+            "--output-file",
+            str(tmp_path / "output.xml"),
+        ],
+    )
+
+    assert "2 Failure" in result.output
+
+    report_text = (tmp_path / "output.xml").read_text()
+    report = etree.XML(report_text)
+    attrib = dict(report.attrib)
+    attrib.pop('time')
+    assert attrib == {
+        'tests': '3',
+        'failures': '2',
+        'id': 'c7n-left',
+        'name': 'IaC Policy Compliance',
+    }
+    cases = list(report.find('testsuite').findall('testcase'))
+    assert len(cases) == 3
+    assert cases[-1].find('failure').attrib == {
+        'type': 'failure',
+        'message': 'tags are required on log groups',
+    }
+
+
 @pytest.mark.skipif(
     os.environ.get('GITHUB_ACTIONS') is None,
     reason="runs in github actions as it requires network access for schema validation",