Repository: bridgecrewio/checkov
Pull request: Add --hard-fail-on and --soft-fail-on flags
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

This PR adds 2 new flags:
- `--soft-fail-on`
- `--hard-fail-on`

Behavior:
- The flags give the user the ability to define a list of check-ids that would hard-fail or soft-fail the build. 
- `--soft-fail`, `--hard-fail-on` and `--soft-fail-on` cannot be used together.

Usage:
```
checkov --hard-fail-on CKV1,CKV2
# Exit code = 1 if CKV1 or CKV2 fail, 0 otherwise

checkov --soft-fail-on CKV1,CKV2
# Exit code = 0 if the only checks failing are CKV1 or CKV2, 1 otherwise
```


Problem statement:
Add --warn option similar to --check
There are many great checks that come with Checkov that allow me to block deployments of insecure, noncompliant resources. The problem is, there are many cases where I want devs to be AWARE of a potential security misconfig without entirely blocking their deployment. For example, I wouldn't want to block a deployment of an ALB for not redirecting to HTTPS, but I would like them to KNOW about it.

A simple --warn flag similar to the --check flag would solve this issue. Provide a list of check IDs that you would like to warn on in the output without causing a failure. This would also be similar to the --soft-fail flag, but only for a subset of checks that are deemed non-essential, but desired. 

Alternatively, the soft-fail flag could be modified to take a list of checks to soft-fail on, with ALL being specified to mimic the option's current function.

Hints from issue discussion:

Commit message: Fix typos

Patch to review:
diff --git a/checkov/common/output/report.py b/checkov/common/output/report.py
index 56879d3561..effd4f9e16 100644
--- a/checkov/common/output/report.py
+++ b/checkov/common/output/report.py
@@ -9,6 +9,7 @@
 
 from checkov.common.models.enums import CheckResult
 from checkov.common.output.record import Record
+from checkov.common.util.type_forcers import convert_csv_string_arg_to_list
 from checkov.version import version
 
 init(autoreset=True)
@@ -72,7 +73,34 @@ def get_dict(self, is_quiet=False) -> dict:
                 "summary": self.get_summary()
             }
 
-    def get_exit_code(self, soft_fail: bool) -> int:
+    def get_exit_code(self, soft_fail: bool, soft_fail_on: Union[list, None] = None,
+                      hard_fail_on: Union[list, None] = None) -> int:
+        """
+        Returns the appropriate exit code depending on the flags that are passed in.
+
+        :param soft_fail: If true, exit code is always 0. (default is false)
+        :param soft_fail_on: A list of checks that will return exit code 0 if they fail. Other failing checks will
+        result exit code 1.
+        :param hard_fail_on: A list of checks that will return exit code 1 if they fail. Other failing checks will
+        result exit code 0.
+        :return: Exit code 0 or 1.
+        """
+        if soft_fail_on:
+            soft_fail_on = convert_csv_string_arg_to_list(soft_fail_on)
+            if all(check_id in soft_fail_on for check_id in
+                   (failed_check.check_id for failed_check in self.failed_checks)):
+                # List of "failed checks" is a subset of the "soft fail on" list.
+                return 0
+            else:
+                return 1
+        if hard_fail_on:
+            hard_fail_on = convert_csv_string_arg_to_list(hard_fail_on)
+            if any(check_id in hard_fail_on for check_id in
+                   (failed_check.check_id for failed_check in self.failed_checks)):
+                # Any check from the list of "failed checks" is in the list of "hard fail on checks".
+                return 1
+            else:
+                return 0
         if soft_fail:
             return 0
         elif len(self.failed_checks) > 0:
diff --git a/checkov/common/runners/runner_registry.py b/checkov/common/runners/runner_registry.py
index 707016358a..306a6a771a 100644
--- a/checkov/common/runners/runner_registry.py
+++ b/checkov/common/runners/runner_registry.py
@@ -68,7 +68,7 @@ def print_reports(self, scan_reports, config, url=None, created_baseline_path=No
                     report.print_console(is_quiet=config.quiet, is_compact=config.compact, created_baseline_path=created_baseline_path,baseline=baseline)
                     if url:
                         print("More details: {}".format(url))
-            exit_codes.append(report.get_exit_code(config.soft_fail))
+            exit_codes.append(report.get_exit_code(config.soft_fail, config.soft_fail_on, config.hard_fail_on))
         if config.output == "junitxml":
             if len(junit_reports) == 1:
                 junit_reports[0].print_junit_xml()
diff --git a/checkov/common/util/ext_argument_parser.py b/checkov/common/util/ext_argument_parser.py
index 87eecb7f57..70f0837975 100644
--- a/checkov/common/util/ext_argument_parser.py
+++ b/checkov/common/util/ext_argument_parser.py
@@ -26,11 +26,15 @@ def write_config_file(self, parsed_namespace, output_file_paths, exit_after=Fals
             # generate the config file contents
             config_items = self.get_items_for_config_file_output(
                 self._source_to_settings, parsed_namespace)
-            # convert check and skip_check to list
+            # convert check, skip_check, soft_fail_on and hard_fail_on to list
             if 'check' in config_items.keys():
                 config_items['check'] = config_items['check'][0].split(",")
             if 'skip-check' in config_items.keys():
                 config_items['skip-check'] = config_items['skip-check'][0].split(",")
+            if 'soft-fail-on' in config_items.keys():
+                config_items['soft-fail-on'] = config_items['soft-fail-on'][0].split(",")
+            if 'hard-fail-on' in config_items.keys():
+                config_items['hard-fail-on'] = config_items['hard-fail-on'][0].split(",")
             # convert strings to booleans
             for k in config_items.keys():
                 config_items[k] = convert_str_to_bool(config_items[k])
diff --git a/checkov/common/util/type_forcers.py b/checkov/common/util/type_forcers.py
index 67776fb857..efc9e98fac 100644
--- a/checkov/common/util/type_forcers.py
+++ b/checkov/common/util/type_forcers.py
@@ -93,4 +93,24 @@ def extract_policy_dict(policy: Union[dict, str]) -> Optional[dict]:
         except JSONDecodeError:
             return None
 
-    return None
\ No newline at end of file
+    return None
+
+
+def convert_csv_string_arg_to_list(csv_string_arg: Union[List[str], str, None]) -> list:
+    """
+    Converts list type arguments that also support comma delimited strings into a list.
+    For instance the --check flag in the CLI:
+        checkov -c CKV_1,CKV2
+        will translate to ['CKV_1', 'CKV_2']
+    :param csv_string_arg: Comma delimited string
+    :return: List of strings or empty list
+    """
+    if csv_string_arg is None:
+        return []
+    if isinstance(csv_string_arg, str):
+        return csv_string_arg.split(',')
+    elif isinstance(csv_string_arg, list) and len(csv_string_arg) == 1:
+        return csv_string_arg[0].split(',')
+    else:
+        return csv_string_arg
+
diff --git a/checkov/main.py b/checkov/main.py
index ac78c9d117..ca4fc6256a 100644
--- a/checkov/main.py
+++ b/checkov/main.py
@@ -139,6 +139,11 @@ def run(banner=checkov_banner, argv=sys.argv[1:]):
     external_checks_dir = get_external_checks_dir(config)
     url = None
     created_baseline_path = None
+
+    if sum(arg is not None for arg in [config.soft_fail, config.soft_fail_on, config.hard_fail_on]) > 1:
+        parser.error("'--soft-fail', '--soft-fail-on' and '--hard-fail-on' cannot be used together.")
+        return
+
     if config.directory:
         exit_codes = []
         for root_folder in config.directory:
@@ -246,6 +251,12 @@ def add_parser_args(parser):
                     'specify multiple checks separated by comma delimiter', action='append', default=None)
     parser.add('-s', '--soft-fail',
                help='Runs checks but suppresses error code', action='store_true')
+    parser.add('--soft-fail-on',
+               help='Exits with a 0 exit code for specified checks. You can '
+                    'specify multiple checks separated by comma delimiter', action='append', default=None)
+    parser.add('--hard-fail-on',
+               help='Exits with a non-zero exit code for specified checks. You can '
+                    'specify multiple checks separated by comma delimiter', action='append', default=None)
     parser.add('--bc-api-key', help='Bridgecrew API key', env_var='BC_API_KEY')
     parser.add('--docker-image', help='Scan docker images by name or ID. Only works with --bc-api-key flag')
     parser.add('--dockerfile-path', help='Path to the Dockerfile of the scanned docker image')
diff --git a/checkov/runner_filter.py b/checkov/runner_filter.py
index af9c967f87..6d5511f0ea 100644
--- a/checkov/runner_filter.py
+++ b/checkov/runner_filter.py
@@ -3,6 +3,7 @@
 from typing import Set, Optional, Union, List
 
 from checkov.common.util.consts import DEFAULT_EXTERNAL_MODULES_DIR
+from checkov.common.util.type_forcers import convert_csv_string_arg_to_list
 
 
 class RunnerFilter(object):
@@ -24,23 +25,8 @@ def __init__(
         excluded_paths: Optional[List[str]] = None
     ) -> None:
 
-        if checks is None:
-            checks = []
-        if isinstance(checks, str):
-            self.checks = checks.split(",")
-        elif isinstance(checks, list) and len(checks) == 1:
-            self.checks = checks[0].split(",")
-        else:
-            self.checks = checks
-
-        if skip_checks is None:
-            skip_checks = []
-        if isinstance(skip_checks, str):
-            self.skip_checks = skip_checks.split(",")
-        elif isinstance(skip_checks, list) and len(skip_checks) == 1:
-            self.skip_checks = skip_checks[0].split(",")
-        else:
-            self.skip_checks = skip_checks
+        self.checks = convert_csv_string_arg_to_list(checks)
+        self.skip_checks = convert_csv_string_arg_to_list(skip_checks)
 
         if skip_framework is None:
             self.framework = framework
diff --git a/tests/common/output/test_get_exit_code.py b/tests/common/output/test_get_exit_code.py
new file mode 100644
index 0000000000..ed3dd79e0e
--- /dev/null
+++ b/tests/common/output/test_get_exit_code.py
@@ -0,0 +1,81 @@
+import unittest
+
+
+from checkov.common.models.enums import CheckResult
+from checkov.common.output.report import Report
+from checkov.common.output.record import Record
+
+
+class TestGetExitCode(unittest.TestCase):
+
+    def test_get_exit_code(self):
+        record1 = Record(check_id='CKV_AWS_157',
+                         check_name="Some RDS check", check_result={"result": CheckResult.FAILED},
+                         code_block=None, file_path="./rds.tf",
+                         file_line_range='1:3',
+                         resource='aws_db_instance.sample', evaluations=None,
+                         check_class=None, file_abs_path=',.',
+                         entity_tags={
+                             'tag1': 'value1'
+                         })
+        record2 = Record(check_id='CKV_AWS_16',
+                         check_name="Another RDS check",
+                         check_result={"result": CheckResult.FAILED},
+                         code_block=None, file_path="./rds.tf",
+                         file_line_range='1:3',
+                         resource='aws_db_instance.sample', evaluations=None,
+                         check_class=None, file_abs_path=',.',
+                         entity_tags={
+                             'tag1': 'value1'
+                         })
+
+        record3 = Record(check_id='CKV_AWS_161',
+                         check_name="Another RDS check",
+                         check_result={"result": CheckResult.PASSED},
+                         code_block=None, file_path="./rds.tf",
+                         file_line_range='1:3',
+                         resource='aws_db_instance.sample', evaluations=None,
+                         check_class=None, file_abs_path=',.',
+                         entity_tags={
+                             'tag1': 'value1'
+                         })
+        record4 = Record(check_id='CKV_AWS_118',
+                         check_name="Another RDS check",
+                         check_result={"result": CheckResult.PASSED},
+                         code_block=None, file_path="./rds.tf",
+                         file_line_range='1:3',
+                         resource='aws_db_instance.sample', evaluations=None,
+                         check_class=None, file_abs_path=',.',
+                         entity_tags={
+                             'tag1': 'value1'
+                         })
+
+        r = Report("terraform")
+        r.add_record(record1)
+        r.add_record(record2)
+        r.add_record(record3)
+        r.add_record(record4)
+
+        # When soft_fail=True, the exit code should always be 0.
+        test_soft_fail = r.get_exit_code(soft_fail=True, soft_fail_on=None, hard_fail_on=None)
+
+        # When soft_fail_on=['check1', 'check2'], exit code should be 0 if the only failing checks are in the
+        # soft_fail_on list
+        positive_test_soft_fail_on_code = r.get_exit_code(None, soft_fail_on=['CKV_AWS_157', 'CKV_AWS_16'],
+                                                          hard_fail_on=None)
+        negative_test_soft_fail_on_code = r.get_exit_code(None, soft_fail_on=['CKV_AWS_157'], hard_fail_on=None)
+
+        # When hard_fail_on=['check1', 'check2'], exit code should be 1 if any checks in the hard_fail_on list fail
+        positive_test_hard_fail_on_code = r.get_exit_code(None, soft_fail_on=None, hard_fail_on=['CKV_AWS_157'])
+        negative_test_hard_fail_on_code = r.get_exit_code(None, soft_fail_on=None,
+                                                          hard_fail_on=['CKV_AWS_161', 'CKV_AWS_118'])
+
+        self.assertEqual(test_soft_fail, 0)
+        self.assertEqual(positive_test_soft_fail_on_code, 0)
+        self.assertEqual(negative_test_soft_fail_on_code, 1)
+        self.assertEqual(positive_test_hard_fail_on_code, 1)
+        self.assertEqual(negative_test_hard_fail_on_code, 0)
+
+
+if __name__ == '__main__':
+    unittest.main()


Code review comments:
diff_hunk: @@ -139,6 +139,11 @@ def run(banner=checkov_banner, argv=sys.argv[1:]):
     external_checks_dir = get_external_checks_dir(config)
     url = None
     created_baseline_path = None
+
+    if sum(arg is not None for arg in [config.soft_fail, config.soft_fail_on, config.hard_fail_on]) > 1:
+        parser.error("'--soft-fail', '--soft-fail-on' and '--hard-fail-on' cannot be used together.")
+        return
path: checkov/main.py
text: @user1:
@author Consider using `argparse`'s `add_mutually_exclusive_group` (inherited in configargparse) instead please 🙏 

@author:
Got it!

diff_hunk: @@ -72,7 +73,34 @@ def get_dict(self, is_quiet=False) -> dict:
                 "summary": self.get_summary()
             }
 
-    def get_exit_code(self, soft_fail: bool) -> int:
+    def get_exit_code(self, soft_fail: bool, soft_fail_on: Union[list, None] = None,
+                      hard_fail_on: Union[list, None] = None) -> int:
path: checkov/common/output/report.py
text: @user1:
@author `Union` is for multiple types, if it's either a list or None, use the `Optional` operator, i.e. `Optional[list]`

@author:
Thanks for pointing that out!