Repository: redhat-openshift-ecosystem/openshift-preflight
Pull request #719 (2022-06-28 18:53:17)

adding in PFLT_JUNIT to the configs md for clarity
- `PFLT_JUNIT` environment variable was never documented, so adding it to CONFIG.md, so we do not forget that it exists.

Signed-off-by: Adam D. Cornett <adc@redhat.com>

coveralls: 
[![Coverage Status](https://coveralls.io/builds/50436850/badge)](https://coveralls.io/builds/50436850)

Coverage decreased (-0.05%) to 68.846% when pulling **214c2d814c62c5d4597226bcc57c14a9c822d291 on acornett21:update_config_md** into **2eb92db1b776df2c69674e57c089b36499671a75 on redhat-openshift-ecosystem:main**.

openshift-ci[bot]: @acornett21: The following test **failed**, say `/retest` to rerun all failed tests or `/retest-required` to rerun all mandatory failed tests:

Test name | Commit | Details | Required | Rerun command
--- | --- | --- | --- | ---
ci/prow/4.8-images | 214c2d814c62c5d4597226bcc57c14a9c822d291 | [link](https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/redhat-openshift-ecosystem_openshift-preflight/719/pull-ci-redhat-openshift-ecosystem-openshift-preflight-main-4.8-images/1541857297163096064) | true | `/test 4.8-images`

[Full PR test history](https://prow.ci.openshift.org/pr-history?org=redhat-openshift-ecosystem&repo=openshift-preflight&pr=719). [Your PR dashboard](https://prow.ci.openshift.org/pr?query=is:pr+state:open+author:acornett21).

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->
acornett21: /test 4.8-images

Patch:
diff --git a/certification/engine/engine.go b/certification/engine/engine.go
index bcae42f..97a47a9 100644
--- a/certification/engine/engine.go
+++ b/certification/engine/engine.go
@@ -104,31 +104,31 @@ func makeCheckList(checks []certification.Check) []string {
 }
 
 // checkNamesFor produces a slice of names for checks in the requested policy.
-func checkNamesFor(p policy.Policy) []string {
+func checkNamesFor(ctx context.Context, p policy.Policy) []string {
 	// stub the config. We don't technically need the policy here, but why not.
 	c := &runtime.Config{Policy: p}
-	checks, _ := initializeChecks(context.TODO(), p, c.ReadOnly())
+	checks, _ := initializeChecks(ctx, p, c.ReadOnly())
 	return makeCheckList(checks)
 }
 
 // OperatorPolicy returns the names of checks in the operator policy.
-func OperatorPolicy() []string {
-	return checkNamesFor(policy.PolicyOperator)
+func OperatorPolicy(ctx context.Context) []string {
+	return checkNamesFor(ctx, policy.PolicyOperator)
 }
 
 // ContainerPolicy returns the names of checks in the container policy.
-func ContainerPolicy() []string {
-	return checkNamesFor(policy.PolicyContainer)
+func ContainerPolicy(ctx context.Context) []string {
+	return checkNamesFor(ctx, policy.PolicyContainer)
 }
 
 // ScratchContainerPolicy returns the names of checks in the
 // container policy with scratch exception.
-func ScratchContainerPolicy() []string {
-	return checkNamesFor(policy.PolicyScratch)
+func ScratchContainerPolicy(ctx context.Context) []string {
+	return checkNamesFor(ctx, policy.PolicyScratch)
 }
 
 // RootExceptionContainerPolicy returns the names of checks in the
 // container policy with root exception.
-func RootExceptionContainerPolicy() []string {
-	return checkNamesFor(policy.PolicyRoot)
+func RootExceptionContainerPolicy(ctx context.Context) []string {
+	return checkNamesFor(ctx, policy.PolicyRoot)
 }
diff --git a/certification/formatters/errors.go b/certification/formatters/errors.go
deleted file mode 100644
index 5c138c8..0000000
--- a/certification/formatters/errors.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package formatters
-
-import "errors"
-
-var ErrUnknownFormatter = errors.New("requested format is unknown")
diff --git a/certification/formatters/formatters.go b/certification/formatters/formatters.go
index a32d999..167946f 100644
--- a/certification/formatters/formatters.go
+++ b/certification/formatters/formatters.go
@@ -38,8 +38,8 @@ func NewForConfig(cfg certification.Config) (ResponseFormatter, error) {
 func NewByName(name string) (ResponseFormatter, error) {
 	formatter, defined := availableFormatters[name]
 	if !defined {
-		return nil, fmt.Errorf("%w: %s",
-			ErrUnknownFormatter,
+		return nil, fmt.Errorf("%s: %s",
+			"The requested formatter is unknown",
 			name,
 		)
 	}
diff --git a/cmd/list_checks.go b/cmd/list_checks.go
index 84650d6..cca0bad 100644
--- a/cmd/list_checks.go
+++ b/cmd/list_checks.go
@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"strings"
@@ -25,11 +26,11 @@ func listChecksRunFunc(cmd *cobra.Command, args []string) {
 // printChecks writes the formatted check list output to w.
 func printChecks(w io.Writer) {
 	fmt.Fprintln(w, "These are the available checks for each policy:")
-	fmt.Fprintln(w, formattedPolicyBlock("Operator", engine.OperatorPolicy(), "invoked on operator bundles"))
-	fmt.Fprintln(w, formattedPolicyBlock("Container", engine.ContainerPolicy(), "invoked on container images"))
-	fmt.Fprintln(w, formattedPolicyBlock("Container Root Exception", engine.RootExceptionContainerPolicy(),
+	fmt.Fprintln(w, formattedPolicyBlock("Operator", engine.OperatorPolicy(context.TODO()), "invoked on operator bundles"))
+	fmt.Fprintln(w, formattedPolicyBlock("Container", engine.ContainerPolicy(context.TODO()), "invoked on container images"))
+	fmt.Fprintln(w, formattedPolicyBlock("Container Root Exception", engine.RootExceptionContainerPolicy(context.TODO()),
 		"automatically applied for container images if preflight determines a root exception flag has been added to your Red Hat Connect project"))
-	fmt.Fprintln(w, formattedPolicyBlock("Container Scratch Exception", engine.ScratchContainerPolicy(),
+	fmt.Fprintln(w, formattedPolicyBlock("Container Scratch Exception", engine.ScratchContainerPolicy(context.TODO()),
 		"automatically applied for container checks if preflight determines a scratch exception flag has been added to your Red Hat Connect project"))
 }
 
diff --git a/docs/CONFIG.md b/docs/CONFIG.md
index 82fe691..6853b56 100644
--- a/docs/CONFIG.md
+++ b/docs/CONFIG.md
@@ -9,7 +9,7 @@ The following configurables are available for the `preflight` tool.
 |`PFLT_LOGLEVEL`|env|The verbosity of the preflight tool itself. Ex. warn, debug, trace, info, error|optional|[warn](https://github.com/redhat-openshift-ecosystem/openshift-preflight/blob/main/cmd/defaults.go#L6)|
 |`PFLT_LOGFILE`|env|Where the execution logfile will be written.|optional|[preflight.log](https://github.com/redhat-openshift-ecosystem/openshift-preflight/blob/main/cmd/defaults.go#L5)|
 |`PFLT_ARTIFACTS`|env|Where check-specific artifacts will be written.|optional|[artifacts/](https://github.com/redhat-openshift-ecosystem/openshift-preflight/blob/main/cmd/defaults.go#L7)|
-
+|`PFLT_JUNIT`|env|Will write results as JUnit XML.|optional|false|
 
 ## Operator Policy Configuration
 


Test patch:
diff --git a/certification/engine/engine_test.go b/certification/engine/engine_test.go
index 8059a59..a156456 100644
--- a/certification/engine/engine_test.go
+++ b/certification/engine/engine_test.go
@@ -10,7 +10,7 @@ import (
 )
 
 var _ = Describe("Engine Creation", func() {
-	Describe("When getting a new engine for a configuration", func() {
+	When("getting a new engine for a configuration", func() {
 		Context("with a valid configurations", func() {
 			Context("for the container policy", func() {
 				cfg := runtime.Config{
@@ -24,6 +24,19 @@ var _ = Describe("Engine Creation", func() {
 					Expect(err).ToNot(HaveOccurred())
 					Expect(engine).ToNot(BeNil())
 				})
+				It("should return the correct names", func() {
+					names := ContainerPolicy(context.TODO())
+					Expect(names).To(ContainElements([]string{
+						"HasLicense",
+						"HasUniqueTag",
+						"LayerCountAcceptable",
+						"HasNoProhibitedPackages",
+						"HasRequiredLabel",
+						"RunAsNonRoot",
+						"HasModifiedFiles",
+						"BasedOnUbi",
+					}))
+				})
 			})
 			Context("for the operator policy", func() {
 				cfg := runtime.Config{
@@ -36,6 +49,15 @@ var _ = Describe("Engine Creation", func() {
 					Expect(err).ToNot(HaveOccurred())
 					Expect(engine).ToNot(BeNil())
 				})
+				It("should return the correct names", func() {
+					names := OperatorPolicy(context.TODO())
+					Expect(names).To(ContainElements([]string{
+						"ScorecardBasicSpecCheck",
+						"ScorecardOlmSuiteCheck",
+						"DeployableByOLM",
+						"ValidateOperatorBundle",
+					}))
+				})
 			})
 
 			Context("for the scratch policy", func() {
@@ -50,6 +72,16 @@ var _ = Describe("Engine Creation", func() {
 					Expect(err).ToNot(HaveOccurred())
 					Expect(engine).ToNot(BeNil())
 				})
+				It("should return the correct names", func() {
+					names := ScratchContainerPolicy(context.TODO())
+					Expect(names).To(ContainElements([]string{
+						"HasLicense",
+						"HasUniqueTag",
+						"LayerCountAcceptable",
+						"HasRequiredLabel",
+						"RunAsNonRoot",
+					}))
+				})
 			})
 
 			Context("for the Root policy", func() {
@@ -64,6 +96,31 @@ var _ = Describe("Engine Creation", func() {
 					Expect(err).ToNot(HaveOccurred())
 					Expect(engine).ToNot(BeNil())
 				})
+				It("should return the correct names", func() {
+					names := RootExceptionContainerPolicy(context.TODO())
+					Expect(names).To(ContainElements([]string{
+						"HasLicense",
+						"HasUniqueTag",
+						"LayerCountAcceptable",
+						"HasNoProhibitedPackages",
+						"HasRequiredLabel",
+						"HasModifiedFiles",
+					}))
+				})
+			})
+		})
+
+		Context("with an invalid policy", func() {
+			cfg := runtime.Config{
+				Image:          "dummy/image",
+				Policy:         "invalid",
+				ResponseFormat: "json",
+			}
+
+			It("should return an error and no engine", func() {
+				engine, err := NewForConfig(context.TODO(), cfg.ReadOnly())
+				Expect(err).To(HaveOccurred())
+				Expect(engine).To(BeNil())
 			})
 		})
 	})
diff --git a/certification/formatters/junitxml_test.go b/certification/formatters/junitxml_test.go
new file mode 100644
index 0000000..ac98384
--- /dev/null
+++ b/certification/formatters/junitxml_test.go
@@ -0,0 +1,88 @@
+package formatters
+
+import (
+	"context"
+	"errors"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/redhat-openshift-ecosystem/openshift-preflight/certification"
+	"github.com/redhat-openshift-ecosystem/openshift-preflight/certification/runtime"
+)
+
+var _ = Describe("JUnitXML Formatter", func() {
+	Context("With a valid UserResponse", func() {
+		var response runtime.Results
+		BeforeEach(func() {
+			response = runtime.Results{
+				TestedImage:   "example.com/repo/image:tag",
+				PassedOverall: true,
+				TestedOn: runtime.OpenshiftClusterVersion{
+					Name:    "ClusterName",
+					Version: "Clusterversion",
+				},
+				CertificationHash: "",
+				Passed: []runtime.Result{
+					{
+						Check: certification.NewGenericCheck(
+							"PassedCheck",
+							func(ctx context.Context, ir certification.ImageReference) (bool, error) { return true, nil },
+							certification.Metadata{
+								Description:      "description",
+								KnowledgeBaseURL: "kburl",
+								CheckURL:         "checkurl",
+							},
+							certification.HelpText{
+								Message:    "helptext",
+								Suggestion: "suggestion",
+							}),
+						ElapsedTime: 0,
+					},
+				},
+				Failed: []runtime.Result{
+					{
+						Check: certification.NewGenericCheck(
+							"FailedCheck",
+							func(ctx context.Context, ir certification.ImageReference) (bool, error) { return false, nil },
+							certification.Metadata{
+								Description:      "description",
+								KnowledgeBaseURL: "kburl",
+								CheckURL:         "checkurl",
+							},
+							certification.HelpText{
+								Message:    "helptext",
+								Suggestion: "suggestion",
+							}),
+						ElapsedTime: 0,
+					},
+				},
+				Errors: []runtime.Result{
+					{
+						Check: certification.NewGenericCheck(
+							"ErroredCheck",
+							func(ctx context.Context, ir certification.ImageReference) (bool, error) {
+								return false, errors.New("someerror")
+							},
+							certification.Metadata{
+								Description:      "description",
+								KnowledgeBaseURL: "kburl",
+								CheckURL:         "checkurl",
+							},
+							certification.HelpText{
+								Message:    "helptext",
+								Suggestion: "suggestion",
+							}),
+						ElapsedTime: 0,
+					},
+				},
+			}
+		})
+		It("should format without error", func() {
+			out, err := junitXMLFormatter(context.TODO(), response)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(string(out)).To(ContainSubstring("PassedCheck"))
+			Expect(string(out)).To(ContainSubstring("FailedCheck"))
+			Expect(string(out)).To(ContainSubstring("ErroredCheck"))
+		})
+	})
+})
diff --git a/cmd/list_checks_test.go b/cmd/list_checks_test.go
index aa55d5a..b6d1efe 100644
--- a/cmd/list_checks_test.go
+++ b/cmd/list_checks_test.go
@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -39,7 +40,7 @@ var _ = Describe("list checks subcommand", func() {
 
 	Context("Printing checks", func() {
 		It("should always contain the container policy", func() {
-			expected := formatList(engine.ContainerPolicy())
+			expected := formatList(engine.ContainerPolicy(context.TODO()))
 			buf := strings.Builder{}
 			printChecks(&buf)
 
@@ -47,7 +48,7 @@ var _ = Describe("list checks subcommand", func() {
 		})
 
 		It("should always contain the operator policy", func() {
-			expected := formatList(engine.OperatorPolicy())
+			expected := formatList(engine.OperatorPolicy(context.TODO()))
 			buf := strings.Builder{}
 			printChecks(&buf)
 
@@ -55,7 +56,7 @@ var _ = Describe("list checks subcommand", func() {
 		})
 
 		It("should always contain the root exception policy", func() {
-			expected := formatList(engine.RootExceptionContainerPolicy())
+			expected := formatList(engine.RootExceptionContainerPolicy(context.TODO()))
 			buf := strings.Builder{}
 			printChecks(&buf)
 
@@ -63,7 +64,7 @@ var _ = Describe("list checks subcommand", func() {
 		})
 
 		It("should always contain the scratch exception policy", func() {
-			expected := formatList(engine.ScratchContainerPolicy())
+			expected := formatList(engine.ScratchContainerPolicy(context.TODO()))
 			buf := strings.Builder{}
 			printChecks(&buf)
 
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
index 2efc59b..c997889 100644
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -36,7 +36,7 @@ var _ = Describe("policy validation", func() {
 			results := e.Results(ctx)
 
 			It("should pass all checks", func() {
-				Expect(len(results.Passed)).To(Equal(len(engine.OperatorPolicy())))
+				Expect(len(results.Passed)).To(Equal(len(engine.OperatorPolicy(context.TODO()))))
 			})
 		})
 
@@ -82,7 +82,7 @@ var _ = Describe("policy validation", func() {
 			results := e.Results(ctx)
 
 			It("should pass all checks", func() {
-				Expect(len(results.Passed)).To(Equal(len(engine.ContainerPolicy())))
+				Expect(len(results.Passed)).To(Equal(len(engine.ContainerPolicy(context.TODO()))))
 				Expect(len(results.Errors)).To(BeZero())
 				Expect(len(results.Failed)).To(BeZero())
 			})