Repository: datreeio/datree
Pull request #927 (2023-04-11 11:51:17)

feat:add configuration for schema_locations, policy_config in config
related issue - #737 

adifayer: Thank you for raising this PR! @Meyazhagan We will look into it in the upcoming days 🤩

Patch:
diff --git a/cmd/config/main.go b/cmd/config/main.go
index 80358e3..2c6a463 100644
--- a/cmd/config/main.go
+++ b/cmd/config/main.go
@@ -10,7 +10,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var ConfigAvailableKeys = []string{"token", "offline"}
+var ConfigAvailableKeys = []string{"token", "offline", "policy_config", "schema_locations"}
 
 type Messager interface {
 	LoadVersionMessages(cliVersion string) chan *messager.VersionMessage
diff --git a/pkg/cliClient/evaluation.go b/pkg/cliClient/evaluation.go
index a023443..952e6d0 100644
--- a/pkg/cliClient/evaluation.go
+++ b/pkg/cliClient/evaluation.go
@@ -140,12 +140,13 @@ type FailureLocation struct {
 }
 
 type Configuration struct {
-	Name             string            `json:"metadataName"`
-	Kind             string            `json:"kind"`
-	Occurrences      int               `json:"occurrences"`
-	IsSkipped        bool              `json:"isSkipped"`
-	SkipMessage      string            `json:"skipMessage"`
-	FailureLocations []FailureLocation `json:"failureLocation"`
+	Name                      string            `json:"metadataName"`
+	Kind                      string            `json:"kind"`
+	Occurrences               int               `json:"occurrences"`
+	IsSkipped                 bool              `json:"isSkipped"`
+	SkipMessage               string            `json:"skipMessage"`
+	FailureLocations          []FailureLocation `json:"failureLocation"`
+	ValidationFailureMessages []string          `json:"validationFailureMessages"`
 }
 
 type FailedRule struct {
diff --git a/pkg/evaluation/evaluator.go b/pkg/evaluation/evaluator.go
index f91ea87..8168868 100644
--- a/pkg/evaluation/evaluator.go
+++ b/pkg/evaluation/evaluator.go
@@ -11,6 +11,7 @@ import (
 	"github.com/datreeio/datree/pkg/cliClient"
 	"github.com/datreeio/datree/pkg/extractor"
 	"github.com/datreeio/datree/pkg/jsonSchemaValidator"
+	extensions "github.com/datreeio/datree/pkg/jsonSchemaValidator/extensions"
 	"github.com/datreeio/datree/pkg/utils"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 
@@ -216,15 +217,20 @@ func (e *Evaluator) evaluateRule(rule policy_factory.RuleWithSchema, configurati
 	}
 
 	configuration := cliClient.Configuration{
-		Name:             configurationName,
-		Kind:             configurationKind,
-		Occurrences:      occurrences,
-		IsSkipped:        false,
-		SkipMessage:      "",
-		FailureLocations: []cliClient.FailureLocation{},
+		Name:                      configurationName,
+		Kind:                      configurationKind,
+		Occurrences:               occurrences,
+		IsSkipped:                 false,
+		SkipMessage:               "",
+		FailureLocations:          []cliClient.FailureLocation{},
+		ValidationFailureMessages: []string{},
 	}
 
+	var validationFailureMessages []string
 	for _, detailedResult := range validationResult {
+		if strings.Contains(detailedResult.KeywordLocation, extensions.CustomKeyValidationErrorKeyPath) {
+			validationFailureMessages = append(validationFailureMessages, detailedResult.Error)
+		}
 		failedErrorLine, failedErrorColumn := e.getFailedRuleLineAndColumn(detailedResult.InstanceLocation, yamlNode)
 
 		failureLocation := cliClient.FailureLocation{
@@ -236,6 +242,8 @@ func (e *Evaluator) evaluateRule(rule policy_factory.RuleWithSchema, configurati
 		configuration.FailureLocations = append(configuration.FailureLocations, failureLocation)
 	}
 
+	configuration.ValidationFailureMessages = validationFailureMessages
+
 	if skipRuleExists {
 		configuration.IsSkipped = true
 		configuration.SkipMessage = skipMessage
@@ -332,12 +340,13 @@ func (e *Evaluator) formatEvaluationResults(evaluationResults FailedRulesByFiles
 				mapper[filePath][ruleIdentifier].OccurrencesDetails = append(
 					mapper[filePath][ruleIdentifier].OccurrencesDetails,
 					OccurrenceDetails{
-						MetadataName:     configuration.Name,
-						Kind:             configuration.Kind,
-						Occurrences:      configuration.Occurrences,
-						IsSkipped:        configuration.IsSkipped,
-						SkipMessage:      configuration.SkipMessage,
-						FailureLocations: configuration.FailureLocations,
+						MetadataName:              configuration.Name,
+						Kind:                      configuration.Kind,
+						Occurrences:               configuration.Occurrences,
+						IsSkipped:                 configuration.IsSkipped,
+						SkipMessage:               configuration.SkipMessage,
+						FailureLocations:          configuration.FailureLocations,
+						ValidationFailureMessages: configuration.ValidationFailureMessages,
 					},
 				)
 			}
diff --git a/pkg/evaluation/printer.go b/pkg/evaluation/printer.go
index 242b8d1..e417db5 100644
--- a/pkg/evaluation/printer.go
+++ b/pkg/evaluation/printer.go
@@ -346,14 +346,17 @@ func parseToPrinterWarnings(results *EvaluationResults, invalidYamlFiles []*extr
 			for _, ruleUniqueName := range rulesUniqueNames {
 				rule := rules[ruleUniqueName]
 				var fixLink string
+				var PACIdentifier string
 				if verbose {
 					fixLink = rule.DocumentationUrl
+					PACIdentifier = rule.Identifier
 				}
 				failedRule := printer.FailedRule{
 					Name:               rule.Name,
 					DocumentationUrl:   fixLink,
 					Suggestion:         rule.MessageOnFailure,
 					Occurrences:        rule.GetFailedOccurrencesCount(),
+					PACIdentifier:      PACIdentifier,
 					OccurrencesDetails: []printer.OccurrenceDetails{},
 				}
 				skippedRule := failedRule
@@ -374,9 +377,10 @@ func parseToPrinterWarnings(results *EvaluationResults, invalidYamlFiles []*extr
 						failedRule.OccurrencesDetails = append(
 							failedRule.OccurrencesDetails,
 							printer.OccurrenceDetails{
-								MetadataName:     occurrenceDetails.MetadataName,
-								Kind:             occurrenceDetails.Kind,
-								FailureLocations: occurrenceDetails.FailureLocations,
+								MetadataName:              occurrenceDetails.MetadataName,
+								Kind:                      occurrenceDetails.Kind,
+								FailureLocations:          occurrenceDetails.FailureLocations,
+								ValidationFailureMessages: occurrenceDetails.ValidationFailureMessages,
 							},
 						)
 					}
diff --git a/pkg/evaluation/rule.go b/pkg/evaluation/rule.go
index 8339f0c..0e6997c 100644
--- a/pkg/evaluation/rule.go
+++ b/pkg/evaluation/rule.go
@@ -21,10 +21,11 @@ func (rp *Rule) GetFailedOccurrencesCount() int {
 }
 
 type OccurrenceDetails struct {
-	MetadataName     string                      `yaml:"metadataName" json:"metadataName" xml:"metadataName"`
-	Kind             string                      `yaml:"kind" json:"kind" xml:"kind"`
-	SkipMessage      string                      `yaml:"skipMessage" json:"skipMessage" xml:"skipMessage"`
-	Occurrences      int                         `yaml:"occurrences" json:"occurrences" xml:"occurrences"`
-	IsSkipped        bool                        `yaml:"isSkipped" json:"isSkipped" xml:"isSkipped"`
-	FailureLocations []cliClient.FailureLocation `yaml:"failureLocations" json:"failureLocations" xml:"failureLocations"`
+	MetadataName              string                      `yaml:"metadataName" json:"metadataName" xml:"metadataName"`
+	Kind                      string                      `yaml:"kind" json:"kind" xml:"kind"`
+	SkipMessage               string                      `yaml:"skipMessage" json:"skipMessage" xml:"skipMessage"`
+	Occurrences               int                         `yaml:"occurrences" json:"occurrences" xml:"occurrences"`
+	IsSkipped                 bool                        `yaml:"isSkipped" json:"isSkipped" xml:"isSkipped"`
+	FailureLocations          []cliClient.FailureLocation `yaml:"failureLocations" json:"failureLocations" xml:"failureLocations"`
+	ValidationFailureMessages []string                    `yaml:"validationFailureMessages" json:"validationFailureMessages" xml:"validationFailureMessages"`
 }
diff --git a/pkg/jsonSchemaValidator/extensions/customKeyRegoDefinition.go b/pkg/jsonSchemaValidator/extensions/customKeyRegoDefinition.go
index 4c8d4c7..2f1cfdc 100644
--- a/pkg/jsonSchemaValidator/extensions/customKeyRegoDefinition.go
+++ b/pkg/jsonSchemaValidator/extensions/customKeyRegoDefinition.go
@@ -55,38 +55,38 @@ type RegoDefinition struct {
 func (customKeyRegoDefinitionSchema CustomKeyRegoDefinitionSchema) Validate(ctx jsonschema.ValidationContext, dataValue interface{}) error {
 	regoDefinitionSchema, err := convertCustomKeyRegoDefinitionSchemaToRegoDefinitionSchema(customKeyRegoDefinitionSchema)
 	if err != nil {
-		return ctx.Error(RegoDefinitionCustomKey, err.Error())
+		return ctx.Error(CustomKeyValidationErrorKeyPath, err.Error())
 	}
 
 	regoCtx := context.Background()
 
 	regoObject, err := retrieveRegoFromSchema(regoDefinitionSchema)
 	if err != nil {
-		return ctx.Error(RegoDefinitionCustomKey, "can't compile rego code, %s", err.Error())
+		return ctx.Error(CustomKeyValidationErrorKeyPath, "can't compile rego code, %s", err.Error())
 	}
 
 	// Create a prepared query that can be evaluated.
 	query, err := regoObject.PrepareForEval(regoCtx)
 	if err != nil {
-		return ctx.Error(RegoDefinitionCustomKey, "can't compile rego code, %s", err.Error())
+		return ctx.Error(CustomKeyValidationErrorKeyPath, "can't compile rego code, %s", err.Error())
 	}
 
 	// Execute the prepared query.
 	rs, err := query.Eval(regoCtx, rego.EvalInput(dataValue))
 
 	if err != nil {
-		return ctx.Error(RegoDefinitionCustomKey, "failed to evaluate rego due to %s", err.Error())
+		return ctx.Error(CustomKeyValidationErrorKeyPath, "failed to evaluate rego due to %s", err.Error())
 	}
 
 	if len(rs) != 1 || len(rs[0].Expressions) != 1 {
-		return ctx.Error(RegoDefinitionCustomKey, "failed to evaluate rego, unexpected results")
+		return ctx.Error(CustomKeyValidationErrorKeyPath, "failed to evaluate rego, unexpected results")
 	}
 
 	resultValues := (rs[0].Expressions[0].Value).([]interface{})
 	for _, resultValue := range resultValues {
 		violationReturnValue, ok := resultValue.(bool)
 		if !ok {
-			return ctx.Error(RegoDefinitionCustomKey, "violation needs to return a boolean")
+			return ctx.Error(CustomKeyValidationErrorKeyPath, "violation needs to return a boolean")
 		}
 		if violationReturnValue {
 			return ctx.Error(RegoDefinitionCustomKey, "values in data value %v do not match", dataValue)
diff --git a/pkg/jsonSchemaValidator/extensions/customKeyRule101.go b/pkg/jsonSchemaValidator/extensions/customKeyRule101.go
index e00d1ee..d7b7942 100644
--- a/pkg/jsonSchemaValidator/extensions/customKeyRule101.go
+++ b/pkg/jsonSchemaValidator/extensions/customKeyRule101.go
@@ -40,8 +40,6 @@ type Rules101 struct {
 }
 
 func (s CustomKeyRule101Schema) Validate(ctx jsonschema.ValidationContext, dataValue interface{}) error {
-	keywordPath := "customKeyRule101"
-
 	b, _ := json.Marshal(dataValue)
 	var ruleArray []Rules101
 	err := json.Unmarshal(b, &ruleArray)
@@ -55,7 +53,7 @@ func (s CustomKeyRule101Schema) Validate(ctx jsonschema.ValidationContext, dataV
 			if resource == "pods" || resource == "*" {
 				for _, verb := range rule.Verbs {
 					if verb == "create" || verb == "*" {
-						return ctx.Error(keywordPath, "invalid verb or resource")
+						return ctx.Error("customKeyRule101", "invalid verb or resource")
 					}
 				}
 			}
diff --git a/pkg/jsonSchemaValidator/extensions/customKeyRule81.go b/pkg/jsonSchemaValidator/extensions/customKeyRule81.go
index 2130009..213fda7 100644
--- a/pkg/jsonSchemaValidator/extensions/customKeyRule81.go
+++ b/pkg/jsonSchemaValidator/extensions/customKeyRule81.go
@@ -48,8 +48,6 @@ type Limits struct {
 }
 
 func (s CustomKeyRule81Schema) Validate(ctx jsonschema.ValidationContext, dataValue interface{}) error {
-	keywordPath := "customKeyRule81"
-
 	b, _ := json.Marshal(dataValue)
 	var resources Resources
 	err := json.Unmarshal(b, &resources)
@@ -59,11 +57,11 @@ func (s CustomKeyRule81Schema) Validate(ctx jsonschema.ValidationContext, dataVa
 	}
 
 	if (resources.Requests.Memory == "" || resources.Limits.Memory == "") && (resources.Requests.Memory != resources.Limits.Memory) {
-		return ctx.Error(keywordPath, "empty value in %v", dataValue)
+		return ctx.Error("customKeyRule81", "empty value in %v", dataValue)
 	}
 
 	if resources.Requests.Memory != resources.Limits.Memory {
-		return ctx.Error(keywordPath, "values in data value %v do not match", dataValue)
+		return ctx.Error("customKeyRule81", "values in data value %v do not match", dataValue)
 	}
 
 	return nil
diff --git a/pkg/jsonSchemaValidator/extensions/sharedVariables.go b/pkg/jsonSchemaValidator/extensions/sharedVariables.go
new file mode 100644
index 0000000..e0dd8da
--- /dev/null
+++ b/pkg/jsonSchemaValidator/extensions/sharedVariables.go
@@ -0,0 +1,3 @@
+package jsonSchemaValidator
+
+var CustomKeyValidationErrorKeyPath = "customKeyValidationErrorKeyPath"
diff --git a/pkg/jsonSchemaValidator/validator.go b/pkg/jsonSchemaValidator/validator.go
index ea6047d..85efcc3 100644
--- a/pkg/jsonSchemaValidator/validator.go
+++ b/pkg/jsonSchemaValidator/validator.go
@@ -116,17 +116,17 @@ func (s resourceMinimumSchema) Validate(ctx jsonschema.ValidationContext, dataVa
 
 	dataValueStr, validStr := dataValue.(string)
 	if !validStr {
-		return ctx.Error(keywordPath, "%s must be a string", dataValueStr)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "%s must be a string", dataValueStr)
 	}
 
 	dataValueParsedQty, err := resource.ParseQuantity(dataValueStr)
 	if err != nil {
-		return ctx.Error(keywordPath, "failed parsing data value %v", dataValue)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "failed parsing data value %v", dataValue)
 	}
 
 	schemaResourceMinParsedQty, err := resource.ParseQuantity(schemaResourceMinStr)
 	if err != nil {
-		return ctx.Error(keywordPath, "failed parsing schema value %v", schemaResourceMinStr)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "failed parsing schema value %v", schemaResourceMinStr)
 	}
 
 	dataValueParsedQtyDecimal := dataValueParsedQty.AsDec().String()
@@ -134,12 +134,12 @@ func (s resourceMinimumSchema) Validate(ctx jsonschema.ValidationContext, dataVa
 
 	dataValueFloat, err := strconv.ParseFloat(dataValueParsedQtyDecimal, 64)
 	if err != nil {
-		return ctx.Error(keywordPath, "failed float parsing value %v", dataValueFloat)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "failed float parsing value %v", dataValueFloat)
 	}
 
 	schemaMinValueFloat, err := strconv.ParseFloat(schemaResourceMinParsedQtyDecimal, 64)
 	if err != nil {
-		return ctx.Error(keywordPath, "failed float parsing value %v", schemaMinValueFloat)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "failed float parsing value %v", schemaMinValueFloat)
 	}
 
 	if schemaMinValueFloat > dataValueFloat {
@@ -154,17 +154,17 @@ func (s resourceMaximumSchema) Validate(ctx jsonschema.ValidationContext, dataVa
 
 	dataValueStr, validStr := dataValue.(string)
 	if !validStr {
-		return ctx.Error(keywordPath, "%s must be a string", dataValueStr)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "%s must be a string", dataValueStr)
 	}
 
 	dataValueParsedQty, err := resource.ParseQuantity(dataValue.(string))
 	if err != nil {
-		return ctx.Error(keywordPath, "failed parsing data value %v", dataValue)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "failed parsing data value %v", dataValue)
 	}
 
 	schemaResourceMaxParsedQty, err := resource.ParseQuantity(schemaResourceMaxStr)
 	if err != nil {
-		return ctx.Error(keywordPath, "failed parsing schema value %v", schemaResourceMaxStr)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "failed parsing schema value %v", schemaResourceMaxStr)
 	}
 
 	dataValueParsedQtyDecimal := dataValueParsedQty.AsDec().String()
@@ -172,12 +172,12 @@ func (s resourceMaximumSchema) Validate(ctx jsonschema.ValidationContext, dataVa
 
 	dataValueFloat, err := strconv.ParseFloat(dataValueParsedQtyDecimal, 64)
 	if err != nil {
-		return ctx.Error(keywordPath, "failed float parsing value %v", dataValueFloat)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "failed float parsing value %v", dataValueFloat)
 	}
 
 	schemaMaxValueFloat, err := strconv.ParseFloat(schemaResourceMaxParsedQtyDecimal, 64)
 	if err != nil {
-		return ctx.Error(keywordPath, "failed float parsing value %v", schemaMaxValueFloat)
+		return ctx.Error(extensions.CustomKeyValidationErrorKeyPath, "failed float parsing value %v", schemaMaxValueFloat)
 	}
 
 	if schemaMaxValueFloat < dataValueFloat {
diff --git a/pkg/localConfig/localConfig.go b/pkg/localConfig/localConfig.go
index de1ed61..bf8bf32 100644
--- a/pkg/localConfig/localConfig.go
+++ b/pkg/localConfig/localConfig.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/datreeio/datree/pkg/networkValidator"
 
@@ -13,10 +14,12 @@ import (
 )
 
 type LocalConfig struct {
-	Token         string
-	ClientId      string
-	SchemaVersion string
-	Offline       string
+	Token           string
+	ClientId        string
+	SchemaVersion   string
+	Offline         string
+	PolicyConfig    string
+	SchemaLocations []string
 }
 
 type TokenClient interface {
@@ -36,10 +39,12 @@ func NewLocalConfigClient(t TokenClient, nv *networkValidator.NetworkValidator)
 }
 
 const (
-	clientIdKey      = "client_id"
-	tokenKey         = "token"
-	schemaVersionKey = "schema_version"
-	offlineKey       = "offline"
+	clientIdKey        = "client_id"
+	tokenKey           = "token"
+	schemaVersionKey   = "schema_version"
+	offlineKey         = "offline"
+	policyConfigKey    = "policy_config"
+	schemaLocationsKey = "schema_locations"
 )
 
 func (lc *LocalConfigClient) GetLocalConfiguration() (*LocalConfig, error) {
@@ -55,6 +60,8 @@ func (lc *LocalConfigClient) GetLocalConfiguration() (*LocalConfig, error) {
 	clientId := viper.GetString(clientIdKey)
 	schemaVersion := viper.GetString(schemaVersionKey)
 	offline := viper.GetString(offlineKey)
+	policyConfig := viper.GetString(policyConfigKey)
+	schemaLocations := viper.GetStringSlice(schemaLocationsKey)
 
 	if offline == "" {
 		offline = "fail"
@@ -87,7 +94,7 @@ func (lc *LocalConfigClient) GetLocalConfiguration() (*LocalConfig, error) {
 		}
 	}
 
-	return &LocalConfig{Token: token, ClientId: clientId, SchemaVersion: schemaVersion, Offline: offline}, nil
+	return &LocalConfig{Token: token, ClientId: clientId, SchemaVersion: schemaVersion, Offline: offline, PolicyConfig: policyConfig, SchemaLocations: schemaLocations}, nil
 }
 
 func (lc *LocalConfigClient) Set(key string, value string) error {
@@ -101,7 +108,15 @@ func (lc *LocalConfigClient) Set(key string, value string) error {
 		return err
 	}
 
-	viper.Set(key, value)
+	if key == policyConfigKey {
+		absPath, _ := filepath.Abs(value)
+		viper.Set(policyConfigKey, absPath)
+	} else if key == schemaLocationsKey {
+		viper.Set(schemaLocationsKey, strings.Split(value, ","))
+	} else {
+		viper.Set(key, value)
+	}
+
 	writeClientIdErr := viper.WriteConfig()
 	if writeClientIdErr != nil {
 		return writeClientIdErr
diff --git a/pkg/printer/printer.go b/pkg/printer/printer.go
index 70e0407..2b1ad7f 100644
--- a/pkg/printer/printer.go
+++ b/pkg/printer/printer.go
@@ -32,14 +32,16 @@ type FailedRule struct {
 	Occurrences        int
 	Suggestion         string
 	DocumentationUrl   string
+	PACIdentifier      string
 	OccurrencesDetails []OccurrenceDetails
 }
 
 type OccurrenceDetails struct {
-	MetadataName     string
-	Kind             string
-	SkipMessage      string
-	FailureLocations []cliClient.FailureLocation
+	MetadataName              string
+	Kind                      string
+	SkipMessage               string
+	FailureLocations          []cliClient.FailureLocation
+	ValidationFailureMessages []string
 }
 
 type InvalidYamlInfo struct {
@@ -176,6 +178,11 @@ func (p *Printer) GetWarningsText(warnings []Warning, quiet bool) string {
 
 					sb.WriteString(fmt.Sprintf("%v %v\n", p.Theme.Emoji.Skip, ruleName))
 
+					if skippedRule.PACIdentifier != "" {
+						PACIdentifier := p.Theme.Colors.Cyan.Sprint(skippedRule.PACIdentifier)
+						sb.WriteString(fmt.Sprintf("    Policy as code identifier: %v\n", PACIdentifier))
+					}
+
 					if skippedRule.DocumentationUrl != "" {
 						howToFix := p.Theme.Colors.Cyan.Sprint(skippedRule.DocumentationUrl)
 						sb.WriteString(fmt.Sprintf("    How to fix: %v\n", howToFix))
@@ -206,6 +213,11 @@ func (p *Printer) GetWarningsText(warnings []Warning, quiet bool) string {
 
 				sb.WriteString(fmt.Sprintf("%v %v %v\n", p.Theme.Emoji.Error, ruleName, occurrences))
 
+				if failedRule.PACIdentifier != "" {
+					PACIdentifier := p.Theme.Colors.Cyan.Sprint(failedRule.PACIdentifier)
+					sb.WriteString(fmt.Sprintf("    Policy as code identifier: %v\n", PACIdentifier))
+				}
+
 				if failedRule.DocumentationUrl != "" {
 					howToFix := p.Theme.Colors.Cyan.Sprint(failedRule.DocumentationUrl)
 					sb.WriteString(fmt.Sprintf("    How to fix: %v\n", howToFix))
@@ -220,6 +232,12 @@ func (p *Printer) GetWarningsText(warnings []Warning, quiet bool) string {
 						}
 					}
 					sb.WriteString("\n")
+					if len(occurrenceDetails.ValidationFailureMessages) > 0 {
+						for _, validationFailureMessage := range occurrenceDetails.ValidationFailureMessages {
+							sb.WriteString(validationFailureMessage)
+						}
+						sb.WriteString("\n")
+					}
 
 				}
 				sb.WriteString(fmt.Sprintf("%v %v\n", p.Theme.Emoji.Suggestion, failedRule.Suggestion))


Test patch:
diff --git a/cmd/test/main.go b/cmd/test/main.go
index 3705804..4fb36d3 100644
--- a/cmd/test/main.go
+++ b/cmd/test/main.go
@@ -278,6 +278,11 @@ func (flags *TestCommandFlags) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(&flags.Quiet, "quiet", "", false, "Don't print skipped rules messages")
 }
 
+const (
+	DatreePolicyConfig    = "DATREE_POLICY_CONFIG"
+	DatreeSchemaLocations = "DATREE_SCHEMA_LOCATION"
+)
+
 func GenerateTestCommandData(testCommandFlags *TestCommandFlags, localConfigContent *localConfig.LocalConfig, evaluationPrerunDataResp *cliClient.EvaluationPrerunDataResponse) (*TestCommandData, error) {
 	k8sVersion := testCommandFlags.K8sVersion
 	if k8sVersion == "" {
@@ -294,12 +299,21 @@ func GenerateTestCommandData(testCommandFlags *TestCommandFlags, localConfigCont
 	var policies *defaultPolicies.EvaluationPrerunPolicies
 	var err error
 
+	var policyConfig string
 	if testCommandFlags.PolicyConfig != "" {
+		policyConfig = testCommandFlags.PolicyConfig
+	} else if policyConfigEnv, ok := os.LookupEnv(DatreePolicyConfig); ok {
+		policyConfig = policyConfigEnv
+	} else if localConfigContent.PolicyConfig != "" {
+		policyConfig = localConfigContent.PolicyConfig
+	}
+
+	if policyConfig != "" {
 		if localConfigContent.Offline != "local" && !evaluationPrerunDataResp.IsPolicyAsCodeMode {
-			return nil, fmt.Errorf("to use --policy-config flag you must first enable policy-as-code mode: https://hub.datree.io/policy-as-code")
+			return nil, fmt.Errorf("to use custom policy-config you must first enable policy-as-code mode: https://hub.datree.io/policy-as-code")
 		}
 
-		policies, err = policy.GetPoliciesFileFromPath(testCommandFlags.PolicyConfig)
+		policies, err = policy.GetPoliciesFileFromPath(policyConfig)
 		if err != nil {
 			return nil, err
 		}
@@ -317,6 +331,15 @@ func GenerateTestCommandData(testCommandFlags *TestCommandFlags, localConfigCont
 		return nil, err
 	}
 
+	var schemaLocations []string
+	if len(testCommandFlags.SchemaLocations) != 0 {
+		schemaLocations = testCommandFlags.SchemaLocations
+	} else if schemaLocationsEnv, ok := os.LookupEnv(DatreeSchemaLocations); ok {
+		schemaLocations = strings.Split(schemaLocationsEnv, ",")
+	} else if len(localConfigContent.SchemaLocations) != 0 {
+		schemaLocations = localConfigContent.SchemaLocations
+	}
+
 	testCommandOptions := &TestCommandData{Output: testCommandFlags.Output,
 		K8sVersion:            k8sVersion,
 		IgnoreMissingSchemas:  testCommandFlags.IgnoreMissingSchemas,
@@ -324,7 +347,7 @@ func GenerateTestCommandData(testCommandFlags *TestCommandFlags, localConfigCont
 		Verbose:               testCommandFlags.Verbose,
 		NoRecord:              testCommandFlags.NoRecord,
 		Policy:                policy,
-		SchemaLocations:       testCommandFlags.SchemaLocations,
+		SchemaLocations:       schemaLocations,
 		Token:                 localConfigContent.Token,
 		ClientId:              localConfigContent.ClientId,
 		RegistrationURL:       evaluationPrerunDataResp.RegistrationURL,
diff --git a/pkg/evaluation/evaluator_test.go b/pkg/evaluation/evaluator_test.go
index 568e617..377e748 100644
--- a/pkg/evaluation/evaluator_test.go
+++ b/pkg/evaluation/evaluator_test.go
@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/datreeio/datree/pkg/defaultPolicies"
+
 	"github.com/datreeio/datree/pkg/defaultRules"
 	"gopkg.in/yaml.v3"
 
@@ -190,6 +192,23 @@ func TestGetFailedRuleLineAndColumn(t *testing.T) {
 	assert.Equal(t, 18, column)
 }
 
+//go:embed test_fixtures/customRuleWithRegoCodeThatCantBeCompiled.yaml
+var customRuleWithRegoCodeThatCantBeCompiledStr string
+
+func TestEvaluateRuleWithCustomKeyThatIsNotValid(t *testing.T) {
+	customRuleWithRegoByteArray := []byte(customRuleWithRegoCodeThatCantBeCompiledStr)
+	var customRegoRule defaultPolicies.CustomRule
+	_ = yaml.Unmarshal(customRuleWithRegoByteArray, &customRegoRule)
+	customRuleWithRegoObj := policy_factory.RuleWithSchema{RuleIdentifier: customRegoRule.Identifier, RuleName: customRegoRule.Name, Schema: customRegoRule.Schema, MessageOnFailure: customRegoRule.DefaultMessageOnFailure}
+
+	mockedCliClient := &mockCliClient{}
+	evaluator := New(mockedCliClient, nil)
+
+	failedRule, _ := evaluator.evaluateRule(customRuleWithRegoObj, []byte(FailureLocationsStr), "test", "Deployment", nil, yaml.Node{})
+	assert.NotEmpty(t, failedRule.Configurations[0].ValidationFailureMessages)
+	assert.Contains(t, failedRule.Configurations[0].ValidationFailureMessages[0], "can't compile rego code")
+}
+
 type evaluateArgs struct {
 	policyCheckData PolicyCheckData
 }
diff --git a/pkg/evaluation/printer_test_expected_outputs/json_output.json b/pkg/evaluation/printer_test_expected_outputs/json_output.json
index b287048..83e847e 100644
--- a/pkg/evaluation/printer_test_expected_outputs/json_output.json
+++ b/pkg/evaluation/printer_test_expected_outputs/json_output.json
@@ -1,1 +1,1 @@
-{"policyValidationResults":[{"fileName":"File1","ruleResults":[{"identifier":"CONTAINERS_MISSING_IMAGE_VALUE_VERSION","name":"Ensure each container image has a pinned (tag) version","messageOnFailure":"Incorrect value for key `image` - specify an image version to avoid unpleasant \"version surprises\" in the future","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0.image","failedErrorLine":10,"failedErrorColumn":20}]}]},{"identifier":"CONTAINERS_MISSING_MEMORY_LIMIT_KEY","name":"Ensure each container has a configured memory limit","messageOnFailure":"Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0.resources.limits","failedErrorLine":95,"failedErrorColumn":15}]}]},{"identifier":"WORKLOAD_INVALID_LABELS_VALUE","name":"Ensure workload has valid label values","messageOnFailure":"Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"metadata.labels.owner","failedErrorLine":7,"failedErrorColumn":12}]}]},{"identifier":"CONTAINERS_MISSING_LIVENESSPROBE_KEY","name":"Ensure each container has a configured liveness probe","messageOnFailure":"Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0","failedErrorLine":22,"failedErrorColumn":11}]}]}]}],"policySummary":{"policyName":"Default","totalRulesInPolicy":21,"totalSkippedRules":0,"totalRulesFailed":4,"totalPassedCount":0},"evaluationSummary":{"configsCount":1,"filesCount":1,"passedYamlValidationCount":1,"k8sValidation":"1/1","passedPolicyValidationCount":0},"yamlValidationResults":null,"k8sValidationResults":null,"loginUrl":"https://app.datree.io/login?t=tDJhAU478UTDeSwxGAy99y"}
+{"policyValidationResults":[{"fileName":"File1","ruleResults":[{"identifier":"CONTAINERS_MISSING_IMAGE_VALUE_VERSION","name":"Ensure each container image has a pinned (tag) version","messageOnFailure":"Incorrect value for key `image` - specify an image version to avoid unpleasant \"version surprises\" in the future","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0.image","failedErrorLine":10,"failedErrorColumn":20}],"validationFailureMessages":null}]},{"identifier":"CONTAINERS_MISSING_MEMORY_LIMIT_KEY","name":"Ensure each container has a configured memory limit","messageOnFailure":"Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0.resources.limits","failedErrorLine":95,"failedErrorColumn":15}],"validationFailureMessages":null}]},{"identifier":"WORKLOAD_INVALID_LABELS_VALUE","name":"Ensure workload has valid label values","messageOnFailure":"Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"metadata.labels.owner","failedErrorLine":7,"failedErrorColumn":12}],"validationFailureMessages":null}]},{"identifier":"CONTAINERS_MISSING_LIVENESSPROBE_KEY","name":"Ensure each container has a configured liveness probe","messageOnFailure":"Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0","failedErrorLine":22,"failedErrorColumn":11}],"validationFailureMessages":null}]}]}],"policySummary":{"policyName":"Default","totalRulesInPolicy":21,"totalSkippedRules":0,"totalRulesFailed":4,"totalPassedCount":0},"evaluationSummary":{"configsCount":1,"filesCount":1,"passedYamlValidationCount":1,"k8sValidation":"1/1","passedPolicyValidationCount":0},"yamlValidationResults":null,"k8sValidationResults":null,"loginUrl":"https://app.datree.io/login?t=tDJhAU478UTDeSwxGAy99y"}
diff --git a/pkg/evaluation/printer_test_expected_outputs/json_output_verbose.json b/pkg/evaluation/printer_test_expected_outputs/json_output_verbose.json
index f7c72e7..7845ec1 100644
--- a/pkg/evaluation/printer_test_expected_outputs/json_output_verbose.json
+++ b/pkg/evaluation/printer_test_expected_outputs/json_output_verbose.json
@@ -1,1 +1,1 @@
-{"policyValidationResults":[{"fileName":"File1","ruleResults":[{"identifier":"CONTAINERS_MISSING_IMAGE_VALUE_VERSION","name":"Ensure each container image has a pinned (tag) version","messageOnFailure":"Incorrect value for key `image` - specify an image version to avoid unpleasant \"version surprises\" in the future","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0.image","failedErrorLine":10,"failedErrorColumn":20}]}],"documentationUrl":"https://hub.datree.io/ensure-image-pinned-version"},{"identifier":"CONTAINERS_MISSING_MEMORY_LIMIT_KEY","name":"Ensure each container has a configured memory limit","messageOnFailure":"Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0.resources.limits","failedErrorLine":95,"failedErrorColumn":15}]}],"documentationUrl":"https://hub.datree.io/ensure-memory-limit"},{"identifier":"WORKLOAD_INVALID_LABELS_VALUE","name":"Ensure workload has valid label values","messageOnFailure":"Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"metadata.labels.owner","failedErrorLine":7,"failedErrorColumn":12}]}],"documentationUrl":"https://hub.datree.io/ensure-labels-value-valid"},{"identifier":"CONTAINERS_MISSING_LIVENESSPROBE_KEY","name":"Ensure each container has a configured liveness probe","messageOnFailure":"Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0","failedErrorLine":22,"failedErrorColumn":11}]}],"documentationUrl":"https://hub.datree.io/ensure-liveness-probe"}]}],"policySummary":{"policyName":"Default","totalRulesInPolicy":21,"totalSkippedRules":0,"totalRulesFailed":4,"totalPassedCount":0},"evaluationSummary":{"configsCount":1,"filesCount":1,"passedYamlValidationCount":1,"k8sValidation":"1/1","passedPolicyValidationCount":0},"yamlValidationResults":null,"k8sValidationResults":null,"loginUrl":"https://app.datree.io/login?t=tDJhAU478UTDeSwxGAy99y"}
+{"policyValidationResults":[{"fileName":"File1","ruleResults":[{"identifier":"CONTAINERS_MISSING_IMAGE_VALUE_VERSION","name":"Ensure each container image has a pinned (tag) version","messageOnFailure":"Incorrect value for key `image` - specify an image version to avoid unpleasant \"version surprises\" in the future","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0.image","failedErrorLine":10,"failedErrorColumn":20}],"validationFailureMessages":null}],"documentationUrl":"https://hub.datree.io/ensure-image-pinned-version"},{"identifier":"CONTAINERS_MISSING_MEMORY_LIMIT_KEY","name":"Ensure each container has a configured memory limit","messageOnFailure":"Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0.resources.limits","failedErrorLine":95,"failedErrorColumn":15}],"validationFailureMessages":null}],"documentationUrl":"https://hub.datree.io/ensure-memory-limit"},{"identifier":"WORKLOAD_INVALID_LABELS_VALUE","name":"Ensure workload has valid label values","messageOnFailure":"Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"metadata.labels.owner","failedErrorLine":7,"failedErrorColumn":12}],"validationFailureMessages":null}],"documentationUrl":"https://hub.datree.io/ensure-labels-value-valid"},{"identifier":"CONTAINERS_MISSING_LIVENESSPROBE_KEY","name":"Ensure each container has a configured liveness probe","messageOnFailure":"Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks","occurrencesDetails":[{"metadataName":"rss-site","kind":"Deployment","skipMessage":"","occurrences":1,"isSkipped":false,"failureLocations":[{"schemaPath":"spec.template.spec.containers.0","failedErrorLine":22,"failedErrorColumn":11}],"validationFailureMessages":null}],"documentationUrl":"https://hub.datree.io/ensure-liveness-probe"}]}],"policySummary":{"policyName":"Default","totalRulesInPolicy":21,"totalSkippedRules":0,"totalRulesFailed":4,"totalPassedCount":0},"evaluationSummary":{"configsCount":1,"filesCount":1,"passedYamlValidationCount":1,"k8sValidation":"1/1","passedPolicyValidationCount":0},"yamlValidationResults":null,"k8sValidationResults":null,"loginUrl":"https://app.datree.io/login?t=tDJhAU478UTDeSwxGAy99y"}
diff --git a/pkg/evaluation/printer_test_expected_outputs/yaml_output.yaml b/pkg/evaluation/printer_test_expected_outputs/yaml_output.yaml
index 13d4668..ce8f900 100644
--- a/pkg/evaluation/printer_test_expected_outputs/yaml_output.yaml
+++ b/pkg/evaluation/printer_test_expected_outputs/yaml_output.yaml
@@ -15,6 +15,7 @@ policyValidationResults:
       - schemapath: spec.template.spec.containers.0.image
         failederrorline: 10
         failederrorcolumn: 20
+      validationFailureMessages: []
   - identifier: CONTAINERS_MISSING_MEMORY_LIMIT_KEY
     name: Ensure each container has a configured memory limit
     messageOnFailure: Missing property object `limits.memory` - value should be within
@@ -29,6 +30,7 @@ policyValidationResults:
       - schemapath: spec.template.spec.containers.0.resources.limits
         failederrorline: 95
         failederrorcolumn: 15
+      validationFailureMessages: []
   - identifier: WORKLOAD_INVALID_LABELS_VALUE
     name: Ensure workload has valid label values
     messageOnFailure: Incorrect value for key(s) under `labels` - the vales syntax
@@ -43,6 +45,7 @@ policyValidationResults:
       - schemapath: metadata.labels.owner
         failederrorline: 7
         failederrorcolumn: 12
+      validationFailureMessages: []
   - identifier: CONTAINERS_MISSING_LIVENESSPROBE_KEY
     name: Ensure each container has a configured liveness probe
     messageOnFailure: Missing property object `livenessProbe` - add a properly configured
@@ -57,6 +60,7 @@ policyValidationResults:
       - schemapath: spec.template.spec.containers.0
         failederrorline: 22
         failederrorcolumn: 11
+      validationFailureMessages: []
 policySummary:
   policyName: Default
   totalRulesInPolicy: 21
diff --git a/pkg/evaluation/test_fixtures/customRuleWithRegoCodeThatCantBeCompiled.yaml b/pkg/evaluation/test_fixtures/customRuleWithRegoCodeThatCantBeCompiled.yaml
new file mode 100644
index 0000000..9849e47
--- /dev/null
+++ b/pkg/evaluation/test_fixtures/customRuleWithRegoCodeThatCantBeCompiled.yaml
@@ -0,0 +1,29 @@
+identifier: CUSTOM_DEPLOYMENT_BILLING_LABEL_EXISTS
+name: Ensure Deployment has billing label [CUSTOM RULE]
+defaultMessageOnFailure: deployment labels should contain billing label
+schema:
+  if:
+    properties:
+      kind:
+        type: string
+        enum:
+          - Deployment
+  then:
+    regoDefinition:
+      libs:
+        - |
+          package lib.helpers
+          check_if_missing(missing) = isMissing {
+                      isMissing := count(missing) > 0
+                    }
+      code: |
+        package foosystemrequiredlabels
+        package foosystemrequiredlabels123
+        import data.lib.helpers
+        
+        violation[labelIsMissing] {
+        provided := {label | input.metadata.labels[label]}
+        required := {"billing"}
+        missing := required - provided
+        labelIsMissing := helpers.check_if_missing(missing)
+        }