Repository: cloud-custodian/cloud-custodian

Issue:
c7n-left - support cli flag filtering for warning and failures
currently we provide for selection filters (`--filters)` on what policies to execute and resources to evaluate. we should additionally provide for allowing users to specify which specific policies and severities should not cause an exit 1, ie a `--warn-on` flag to allow rolling out new policies in warn mode. We also currently default to exit 1 on any resource matches, and we should allow for customization of this via a `--fail-on` filtering. both of these new flags should use the same filtering language as the filters flag. note we currently do implicit comparison on `severity=medium` to automatically pickup levels above, one additional consideration might be syntax for >= as the default comparison mode is inverted for warn-on vs fail-on


Discussion:

Resolving patch:
diff --git a/tools/c7n_left/c7n_left/cli.py b/tools/c7n_left/c7n_left/cli.py
--- a/tools/c7n_left/c7n_left/cli.py
+++ b/tools/c7n_left/c7n_left/cli.py
@@ -66,6 +66,9 @@ def dump(directory, var_file, output_file, output_query):
 @cli.command()
 @click.option("--format", default="terraform")
 @click.option("--filters", help="Filter policies or resources as k=v pairs with globbing")
+@click.option(
+    "--warn-on", help="Select policies to log instead of fail on via k=v pairs with globbing"
+)
 @click.option("-p", "--policy-dir", type=click.Path(), help="Directory with policies")
 @click.option("-d", "--directory", type=click.Path(), help="IaC directory to evaluate")
 @click.option(
@@ -104,6 +107,7 @@ def run(
     output_query,
     summary,
     filters,
+    warn_on,
     reporter=None,
 ):
     """evaluate policies against IaC sources.
@@ -121,6 +125,7 @@ def run(
         output_query=output_query,
         var_file=var_file,
         summary=summary,
+        warn_on=warn_on,
         filters=filters,
     )
     policies = config.exec_filter.filter_policies(load_policies(policy_dir, config))
@@ -158,6 +163,7 @@ def get_config(
     output_query=None,
     summary=None,
     filters=None,
+    warn_on=None,
     format='terraform',
 ):
     config = Config.empty(
@@ -169,10 +175,11 @@ def get_config(
         output_query=output_query,
         summary=summary,
         filters=filters,
+        warn_on=warn_on,
         format=format,
     )
-    exec_filter = ExecutionFilter.parse(config)
-    config["exec_filter"] = exec_filter
+    config["exec_filter"] = ExecutionFilter.parse(config.filters)
+    config["warn_filter"] = ExecutionFilter.parse(config.warn_on)
     return config
 
 
diff --git a/tools/c7n_left/c7n_left/core.py b/tools/c7n_left/c7n_left/core.py
--- a/tools/c7n_left/c7n_left/core.py
+++ b/tools/c7n_left/c7n_left/core.py
@@ -118,16 +118,16 @@ def __len__(self):
         return len(self.filters)
 
     @classmethod
-    def parse(cls, options):
+    def parse(cls, filters_config):
         """cli option filtering support
 
         --filters "type=aws_sqs_queue,aws_rds_* policy=*encryption* severity=high"
         """
-        if not options.filters:
+        if not filters_config:
             return cls(defaultdict(list))
 
         filters = defaultdict(list)
-        for kv in options.filters.split(" "):
+        for kv in filters_config.split(" "):
             if "=" not in kv:
                 raise ValueError("key=value pair missing `=`")
             k, v = kv.split("=")
@@ -256,6 +256,10 @@ def run(self) -> bool:
                 result_set = self.run_policy(p, graph, resources, event, rtype)
                 if result_set:
                     self.reporter.on_results(p, result_set)
+                if result_set and (
+                    not self.options.warn_filter
+                    or not self.options.warn_filter.filter_policies((p,))
+                ):
                     found = True
         self.reporter.on_execution_ended()
         return found


Test patch:
diff --git a/tools/c7n_left/tests/test_left.py b/tools/c7n_left/tests/test_left.py
--- a/tools/c7n_left/tests/test_left.py
+++ b/tools/c7n_left/tests/test_left.py
@@ -62,9 +62,7 @@ def on_results(self, policy, results):
 
 def run_policy(policy, terraform_dir, tmp_path):
     (tmp_path / "policies.json").write_text(json.dumps({"policies": [policy]}, indent=2))
-    config = Config.empty(
-        policy_dir=tmp_path, source_dir=terraform_dir, exec_filter=None, var_files=()
-    )
+    config = cli.get_config(policy_dir=tmp_path, directory=terraform_dir)
     policies = policy_core.load_policies(tmp_path, config)
     reporter = ResultsReporter()
     core.CollectionRunner(policies, config, reporter).run()
@@ -91,7 +89,7 @@ def get_graph(self, root_module=None, config=None):
         return provider.parse(**params)
 
     def get_selection(self, filter_expression):
-        return core.ExecutionFilter.parse(Config.empty(filters=filter_expression))
+        return core.ExecutionFilter.parse(filter_expression)
 
     def write_tf(self, content, path="main.tf"):
         tf_file = self.policy_dir / path
@@ -106,12 +104,7 @@ def write_policy(self, policy, path="policy.json"):
         policy_file.write_text(json.dumps(extant))
 
     def run(self, policy_dir=None, terraform_dir=None):
-        config = Config.empty(
-            policy_dir=policy_dir or self.policy_dir,
-            source_dir=terraform_dir or self.policy_dir,
-            exec_filter=None,
-            var_files=(),
-        )
+        config = cli.get_config(terraform_dir or self.policy_dir, policy_dir or self.policy_dir)
         policies = policy_core.load_policies(config.policy_dir, config)
         reporter = ResultsReporter()
         core.CollectionRunner(policies, config, reporter).run()
@@ -1111,6 +1104,7 @@ def write_output_test_policy(tmp_path, policy=None, policy_path="policy.json"):
                     "name": "check-bucket",
                     "resource": "terraform.aws_s3_bucket",
                     "description": "a description",
+                    "metadata": {"category": ["test"]},
                     "filters": [{"server_side_encryption_configuration": "absent"}],
                 }
             ]
@@ -1285,6 +1279,28 @@ def test_cli_output_rich(tmp_path):
     assert "1 failed 2 passed" in result.output
 
 
+def test_cli_output_rich_warn_on(tmp_path):
+    write_output_test_policy(tmp_path)
+    runner = CliRunner()
+    result = runner.invoke(
+        cli.cli,
+        [
+            "run",
+            "-p",
+            str(tmp_path),
+            "-d",
+            str(terraform_dir / "aws_s3_encryption_audit"),
+            "--warn-on",
+            "category=test",
+            "-o",
+            "cli",
+        ],
+    )
+    assert result.exit_code == 0
+    assert "Reason: a description\n" in result.output
+    assert "1 failed 2 passed" in result.output
+
+
 def test_cli_selection(tmp_path):
     write_output_test_policy(
         tmp_path,
@@ -1368,7 +1384,7 @@ def test_cli_output_github(tmp_path):
     assert result.exit_code == 1
     expected = (
         "::error file=tests/terraform/aws_s3_encryption_audit/main.tf,line=25,lineEnd=28,"
-        "title=terraform.aws_s3_bucket - policy:check-bucket severity:unknown::a description"
+        "title=terraform.aws_s3_bucket - policy:check-bucket category:test severity:unknown::a description"  # noqa
     )
     assert expected in result.output
 
@@ -1437,6 +1453,7 @@ def test_cli_output_json(tmp_path):
             "file_path": "tests/terraform/aws_s3_encryption_audit/main.tf",
             "policy": {
                 "filters": [{"server_side_encryption_configuration": "absent"}],
+                "metadata": {"category": ["test"]},
                 "mode": {"type": "terraform-source"},
                 "name": "check-bucket",
                 "resource": "terraform.aws_s3_bucket",