Repository: kyverno/chainsaw
Pull request #77 (2023-10-16 18:42:52)

feat: more runner flags
More runner flags.

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/kyverno/chainsaw/pull/77?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) Report
> Merging [#77](https://app.codecov.io/gh/kyverno/chainsaw/pull/77?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) (fb43f23) into [main](https://app.codecov.io/gh/kyverno/chainsaw/commit/9b20d0fea5720e294759d0f0e65f7344ac667761?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) (9b20d0f) will **decrease** coverage by `0.29%`.
> The diff coverage is `33.33%`.

```diff
@@            Coverage Diff             @@
##             main      #77      +/-   ##
==========================================
- Coverage   73.71%   73.42%   -0.29%     
==========================================
  Files           9        9              
  Lines         369      365       -4     
==========================================
- Hits          272      268       -4     
  Misses         68       68              
  Partials       29       29              
```


| [Files](https://app.codecov.io/gh/kyverno/chainsaw/pull/77?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) | Coverage Δ | |
|---|---|---|
| [pkg/commands/test/command.go](https://app.codecov.io/gh/kyverno/chainsaw/pull/77?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno#diff-cGtnL2NvbW1hbmRzL3Rlc3QvY29tbWFuZC5nbw==) | `58.51% <33.33%> (-1.70%)` | :arrow_down: |

:mega: We’re building smart automated test selection to slash your CI/CD build times. [Learn more](https://about.codecov.io/iterative-testing/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno)


Patch:
diff --git a/config/crds/chainsaw.kyverno.io_configurations.yaml b/config/crds/chainsaw.kyverno.io_configurations.yaml
index 4a3dc9d4..32cd3ff7 100644
--- a/config/crds/chainsaw.kyverno.io_configurations.yaml
+++ b/config/crds/chainsaw.kyverno.io_configurations.yaml
@@ -35,6 +35,10 @@ spec:
           spec:
             description: Configuration spec.
             properties:
+              failFast:
+                description: FailFast determines whether the test should stop upon
+                  encountering the first failure.
+                type: boolean
               fullName:
                 description: FullName makes use of the full test case folder path
                   instead of the folder name.
@@ -68,10 +72,6 @@ spec:
                 description: SkipTestRegex is used to skip tests based on a regular
                   expression.
                 type: string
-              stopOnFirstFailure:
-                description: StopOnFirstFailure determines whether the test should
-                  stop upon encountering the first failure.
-                type: boolean
               suppress:
                 description: Suppress is used to suppress logs.
                 items:
diff --git a/pkg/apis/v1alpha1/configuration_spec.go b/pkg/apis/v1alpha1/configuration_spec.go
index 315ab623..8bd210a2 100644
--- a/pkg/apis/v1alpha1/configuration_spec.go
+++ b/pkg/apis/v1alpha1/configuration_spec.go
@@ -27,9 +27,9 @@ type ConfigurationSpec struct {
 	// +optional
 	SkipDelete bool `json:"skipDelete,omitempty"`
 
-	// StopOnFirstFailure determines whether the test should stop upon encountering the first failure.
+	// FailFast determines whether the test should stop upon encountering the first failure.
 	// +optional
-	StopOnFirstFailure bool `json:"stopOnFirstFailure,omitempty"`
+	FailFast bool `json:"failFast,omitempty"`
 
 	// The maximum number of tests to run at once.
 	// +kubebuilder:default:=8
diff --git a/pkg/data/crds/chainsaw.kyverno.io_configurations.yaml b/pkg/data/crds/chainsaw.kyverno.io_configurations.yaml
index 4a3dc9d4..32cd3ff7 100644
--- a/pkg/data/crds/chainsaw.kyverno.io_configurations.yaml
+++ b/pkg/data/crds/chainsaw.kyverno.io_configurations.yaml
@@ -35,6 +35,10 @@ spec:
           spec:
             description: Configuration spec.
             properties:
+              failFast:
+                description: FailFast determines whether the test should stop upon
+                  encountering the first failure.
+                type: boolean
               fullName:
                 description: FullName makes use of the full test case folder path
                   instead of the folder name.
@@ -68,10 +72,6 @@ spec:
                 description: SkipTestRegex is used to skip tests based on a regular
                   expression.
                 type: string
-              stopOnFirstFailure:
-                description: StopOnFirstFailure determines whether the test should
-                  stop upon encountering the first failure.
-                type: boolean
               suppress:
                 description: Suppress is used to suppress logs.
                 items:
diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go
index 159cd043..a5f3005c 100644
--- a/pkg/runner/runner.go
+++ b/pkg/runner/runner.go
@@ -3,10 +3,10 @@ package runner
 import (
 	"context"
 	"flag"
+	"fmt"
 	"path/filepath"
 	"strconv"
 	"testing"
-	"time"
 
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
 	"github.com/kyverno/chainsaw/pkg/client"
@@ -16,17 +16,12 @@ import (
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-type Options struct {
-	Timeout  time.Duration
-	Parallel int
-}
-
-func Run(cfg *rest.Config, options Options, tests ...discovery.Test) (int, error) {
+func Run(cfg *rest.Config, options v1alpha1.ConfigurationSpec, tests ...discovery.Test) (int, error) {
 	if len(tests) == 0 {
 		return 0, nil
 	}
-	flag.Parse()
 	testing.Init()
+	flag.Parse()
 	// Set the verbose test flag to true since we are not using the regular go test CLI.
 	if err := flag.Set("test.v", "true"); err != nil {
 		return 0, err
@@ -37,7 +32,16 @@ func Run(cfg *rest.Config, options Options, tests ...discovery.Test) (int, error
 	if err := flag.Set("test.timeout", options.Timeout.String()); err != nil {
 		return 0, err
 	}
-	// TODO: flags
+	if err := flag.Set("test.failfast", fmt.Sprint(options.FailFast)); err != nil {
+		return 0, err
+	}
+	if err := flag.Set("test.paniconexit0", "true"); err != nil {
+		return 0, err
+	}
+	if err := flag.Set("test.fullpath", "false"); err != nil {
+		return 0, err
+	}
+	// regex related flags
 	var testDeps testDeps
 	m := testing.MainStart(
 		&testDeps,
diff --git a/testdata/step/invalid-testStep.yaml b/testdata/step/invalid.yaml
similarity index 100%
rename from testdata/step/invalid-testStep.yaml
rename to testdata/step/invalid.yaml
diff --git a/testdata/step/multiple-testStep.yaml b/testdata/step/multiple.yaml
similarity index 100%
rename from testdata/step/multiple-testStep.yaml
rename to testdata/step/multiple.yaml
diff --git a/website/docs/apis/chainsaw.v1alpha1.md b/website/docs/apis/chainsaw.v1alpha1.md
index 0a3f78f8..a203c29d 100644
--- a/website/docs/apis/chainsaw.v1alpha1.md
+++ b/website/docs/apis/chainsaw.v1alpha1.md
@@ -90,7 +90,7 @@ auto_generated: true
 | `timeout` | [`meta/v1.Duration`](https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Duration) |  |  | <p>Timeout per test step.</p> |
 | `testDirs` | `[]string` |  |  | <p>Directories containing test cases to run.</p> |
 | `skipDelete` | `bool` |  |  | <p>If set, do not delete the resources after running the tests (implies SkipClusterDelete).</p> |
-| `stopOnFirstFailure` | `bool` |  |  | <p>StopOnFirstFailure determines whether the test should stop upon encountering the first failure.</p> |
+| `failFast` | `bool` |  |  | <p>FailFast determines whether the test should stop upon encountering the first failure.</p> |
 | `parallel` | `int` | :white_check_mark: |  | <p>The maximum number of tests to run at once.</p> |
 | `reportFormat` | [`ReportFormatType`](#chainsaw-kyverno-io-v1alpha1-ReportFormatType) |  |  | <p>ReportFormat determines test report format (JSON|XML|nil) nil == no report. maps to report.Type, however we don't want generated.deepcopy to have reference to it.</p> |
 | `reportName` | `string` |  |  | <p>ReportName defines the name of report to create. It defaults to "chainsaw-report".</p> |


Test patch:
diff --git a/pkg/commands/test/command.go b/pkg/commands/test/command.go
index ca004cf8..c37115eb 100644
--- a/pkg/commands/test/command.go
+++ b/pkg/commands/test/command.go
@@ -21,7 +21,7 @@ type options struct {
 	timeout             metav1.Duration
 	testDirs            []string
 	skipDelete          bool
-	stopOnFirstFailure  bool
+	failFast            bool
 	parallel            int
 	reportFormat        string
 	reportName          string
@@ -71,7 +71,7 @@ func Command() *cobra.Command {
 				configuration.Spec.SkipDelete = options.skipDelete
 			}
 			if flagutils.IsSet(flags, "stop-on-first-failure") {
-				configuration.Spec.StopOnFirstFailure = options.stopOnFirstFailure
+				configuration.Spec.FailFast = options.failFast
 			}
 			if flagutils.IsSet(flags, "parallel") {
 				configuration.Spec.Parallel = options.parallel
@@ -109,11 +109,7 @@ func Command() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			runnerOptions := runner.Options{
-				Timeout:  options.timeout.Duration,
-				Parallel: options.parallel,
-			}
-			if _, err := runner.Run(cfg, runnerOptions, tests...); err != nil {
+			if _, err := runner.Run(cfg, configuration.Spec, tests...); err != nil {
 				return err
 			}
 			// done
@@ -125,7 +121,7 @@ func Command() *cobra.Command {
 	cmd.Flags().StringVar(&options.config, "config", "", "Chainsaw configuration file.")
 	cmd.Flags().StringArrayVar(&options.testDirs, "test-dir", []string{}, "Directories containing test cases to run.")
 	cmd.Flags().BoolVar(&options.skipDelete, "skip-delete", false, "If set, do not delete the resources after running the tests.")
-	cmd.Flags().BoolVar(&options.stopOnFirstFailure, "stop-on-first-failure", false, "Stop the test upon encountering the first failure.")
+	cmd.Flags().BoolVar(&options.failFast, "stop-on-first-failure", false, "Stop the test upon encountering the first failure.")
 	cmd.Flags().IntVar(&options.parallel, "parallel", 8, "The maximum number of tests to run at once.")
 	cmd.Flags().StringVar(&options.reportFormat, "report-format", "", "Test report format (JSON|XML|nil).")
 	cmd.Flags().StringVar(&options.reportName, "report-name", "chainsaw-report", "The name of the report to create.")
diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go
index 4524a274..6f749faa 100644
--- a/pkg/config/load_test.go
+++ b/pkg/config/load_test.go
@@ -46,13 +46,13 @@ func TestLoad(t *testing.T) {
 					Timeout: &metav1.Duration{
 						Duration: 30 * time.Second,
 					},
-					SkipDelete:         false,
-					StopOnFirstFailure: false,
-					Parallel:           8,
-					ReportFormat:       "",
-					ReportName:         "chainsaw-report",
-					FullName:           false,
-					SkipTestRegex:      "",
+					SkipDelete:    false,
+					FailFast:      false,
+					Parallel:      8,
+					ReportFormat:  "",
+					ReportName:    "chainsaw-report",
+					FullName:      false,
+					SkipTestRegex: "",
 				},
 			},
 		},
@@ -71,13 +71,13 @@ func TestLoad(t *testing.T) {
 					Timeout: &metav1.Duration{
 						Duration: 45 * time.Second,
 					},
-					SkipDelete:         true,
-					StopOnFirstFailure: true,
-					Parallel:           4,
-					ReportFormat:       "JSON",
-					ReportName:         "custom-report",
-					FullName:           true,
-					SkipTestRegex:      "skip-*",
+					SkipDelete:    true,
+					FailFast:      true,
+					Parallel:      4,
+					ReportFormat:  "JSON",
+					ReportName:    "custom-report",
+					FullName:      true,
+					SkipTestRegex: "skip-*",
 				},
 			},
 		},
diff --git a/pkg/step/load_test.go b/pkg/step/load_test.go
index a91c1b08..def3687f 100644
--- a/pkg/step/load_test.go
+++ b/pkg/step/load_test.go
@@ -39,7 +39,7 @@ func TestLoad(t *testing.T) {
 		},
 		{
 			name:    "invalid testStep",
-			path:    filepath.Join(basePath, "invalid-testStep.yaml"),
+			path:    filepath.Join(basePath, "invalid.yaml"),
 			wantErr: true,
 		},
 		{
@@ -71,7 +71,7 @@ func TestLoad(t *testing.T) {
 		},
 		{
 			name: "mlutiple testStep",
-			path: filepath.Join(basePath, "multiple-testStep.yaml"),
+			path: filepath.Join(basePath, "multiple.yaml"),
 			want: []*v1alpha1.TestStep{
 				{
 					TypeMeta: metav1.TypeMeta{
diff --git a/testdata/commands/test/config/config_all_fields.yaml b/testdata/commands/test/config/config_all_fields.yaml
index caaa4a56..b52d4b2e 100644
--- a/testdata/commands/test/config/config_all_fields.yaml
+++ b/testdata/commands/test/config/config_all_fields.yaml
@@ -9,7 +9,7 @@ spec:
     - "/path/to/testdir1"
     - "/path/to/testdir2"
   skipDelete: true
-  stopOnFirstFailure: true
+  failFast: true
   parallel: 5
   reportFormat: "JSON"
   reportName: "custom-chainsaw-report"
diff --git a/testdata/config/custom-config.yaml b/testdata/config/custom-config.yaml
index 1ddb7fb5..e2e67f34 100644
--- a/testdata/config/custom-config.yaml
+++ b/testdata/config/custom-config.yaml
@@ -5,7 +5,7 @@ metadata:
 spec:
   timeout: 45s
   skipDelete: true
-  stopOnFirstFailure: true
+  failFast: true
   parallel: 4
   reportFormat: "JSON"
   reportName: "custom-report"