Repository: kyverno/chainsaw
Pull request #365 (2023-11-03 16:32:02)

test : 100% Coverage pkg/test & pkg/step
Fix : https://github.com/kyverno/chainsaw/issues/337
Fix : https://github.com/kyverno/chainsaw/issues/336

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/kyverno/chainsaw/pull/365?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) Report
> Merging [#365](https://app.codecov.io/gh/kyverno/chainsaw/pull/365?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) (63f1e8c) into [main](https://app.codecov.io/gh/kyverno/chainsaw/commit/6d5ffbee63f9631eb890fa16e234230c4fc58854?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) (6d5ffbe) will **increase** coverage by `0.57%`.
> Report is 1 commits behind head on main.
> The diff coverage is `100.00%`.

```diff
@@            Coverage Diff             @@
##             main     #365      +/-   ##
==========================================
+ Coverage   78.78%   79.35%   +0.57%     
==========================================
  Files          49       50       +1     
  Lines        1607     1671      +64     
==========================================
+ Hits         1266     1326      +60     
- Misses        267      274       +7     
+ Partials       74       71       -3     
```


| [Files](https://app.codecov.io/gh/kyverno/chainsaw/pull/365?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) | Coverage Δ | |
|---|---|---|
| [pkg/step/load.go](https://app.codecov.io/gh/kyverno/chainsaw/pull/365?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno#diff-cGtnL3N0ZXAvbG9hZC5nbw==) | `100.00% <100.00%> (+28.20%)` | :arrow_up: |
| [pkg/test/load.go](https://app.codecov.io/gh/kyverno/chainsaw/pull/365?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno#diff-cGtnL3Rlc3QvbG9hZC5nbw==) | `100.00% <100.00%> (+28.20%)` | :arrow_up: |

... and [2 files with indirect coverage changes](https://app.codecov.io/gh/kyverno/chainsaw/pull/365/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno)

:mega: Codecov offers a browser extension for seamless coverage viewing on GitHub. Try it in [Chrome](https://chrome.google.com/webstore/detail/codecov/gedikamndpbemklijjkncpnolildpbgo) or [Firefox](https://addons.mozilla.org/en-US/firefox/addon/codecov/) today!

shubham-cmyk: @eddycharly codecov report failed here

Patch:
diff --git a/pkg/step/load.go b/pkg/step/load.go
index 652191ce..4ab36a08 100644
--- a/pkg/step/load.go
+++ b/pkg/step/load.go
@@ -10,9 +10,17 @@ import (
 	"github.com/kyverno/kyverno/ext/resource/convert"
 	"github.com/kyverno/kyverno/ext/resource/loader"
 	"github.com/kyverno/kyverno/ext/yaml"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/openapi"
 	"sigs.k8s.io/kubectl-validate/pkg/openapiclient"
 )
 
+type (
+	splitter      = func([]byte) ([][]byte, error)
+	loaderFactory = func(openapi.Client) (loader.Loader, error)
+	converter     = func(unstructured.Unstructured) (*v1alpha1.TestStep, error)
+)
+
 var testStep_v1alpha1 = v1alpha1.SchemeGroupVersion.WithKind("TestStep")
 
 func Load(path string) ([]*v1alpha1.TestStep, error) {
@@ -31,13 +39,26 @@ func Load(path string) ([]*v1alpha1.TestStep, error) {
 }
 
 func Parse(content []byte) ([]*v1alpha1.TestStep, error) {
-	documents, err := yaml.SplitDocuments(content)
+	return parse(content, nil, nil, nil)
+}
+
+func parse(content []byte, splitter splitter, loaderFactory loaderFactory, converter converter) ([]*v1alpha1.TestStep, error) {
+	if splitter == nil {
+		splitter = yaml.SplitDocuments
+	}
+	if loaderFactory == nil {
+		loaderFactory = loader.New
+	}
+	if converter == nil {
+		converter = convert.To[v1alpha1.TestStep]
+	}
+	documents, err := splitter(content)
 	if err != nil {
 		return nil, err
 	}
 	var testSteps []*v1alpha1.TestStep
 	// TODO: no need to allocate a validator every time
-	loader, err := loader.New(openapiclient.NewLocalCRDFiles(data.Crds(), data.CrdsFolder))
+	loader, err := loaderFactory(openapiclient.NewLocalCRDFiles(data.Crds(), data.CrdsFolder))
 	if err != nil {
 		return nil, err
 	}
@@ -48,7 +69,7 @@ func Parse(content []byte) ([]*v1alpha1.TestStep, error) {
 		}
 		switch gvk {
 		case testStep_v1alpha1:
-			testStep, err := convert.To[v1alpha1.TestStep](untyped)
+			testStep, err := converter(untyped)
 			if err != nil {
 				return nil, err
 			}


Test patch:
diff --git a/pkg/step/load_test.go b/pkg/step/load_test.go
index 0cc5b62d..ff14cea0 100644
--- a/pkg/step/load_test.go
+++ b/pkg/step/load_test.go
@@ -1,12 +1,19 @@
 package step
 
 import (
+	"errors"
+	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
+	"github.com/kyverno/kyverno/ext/resource/loader"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/openapi"
 )
 
 func TestLoad(t *testing.T) {
@@ -244,3 +251,69 @@ func TestLoad(t *testing.T) {
 		})
 	}
 }
+
+type fakeLoader struct{}
+
+func (fakeLoader) Load([]byte) (schema.GroupVersionKind, unstructured.Unstructured, error) {
+	return schema.GroupVersionKind{Group: "v1", Kind: "Something"}, unstructured.Unstructured{}, nil
+}
+
+func Test_parse(t *testing.T) {
+	content, err := os.ReadFile("../../testdata/step/custom-step.yaml")
+	assert.NoError(t, err)
+	tests := []struct {
+		name          string
+		splitter      splitter
+		loaderFactory loaderFactory
+		converter     converter
+		wantErr       bool
+	}{{
+		name:          "default",
+		splitter:      nil,
+		loaderFactory: nil,
+		converter:     nil,
+		wantErr:       false,
+	}, {
+		name: "splitter error",
+		splitter: func([]byte) ([][]byte, error) {
+			return nil, errors.New("splitter")
+		},
+		loaderFactory: nil,
+		converter:     nil,
+		wantErr:       true,
+	}, {
+		name:     "loader factory error",
+		splitter: nil,
+		loaderFactory: func(openapi.Client) (loader.Loader, error) {
+			return nil, errors.New("loader factory")
+		},
+		converter: nil,
+		wantErr:   true,
+	}, {
+		name:     "loader error",
+		splitter: nil,
+		loaderFactory: func(openapi.Client) (loader.Loader, error) {
+			return fakeLoader{}, nil
+		},
+		converter: nil,
+		wantErr:   true,
+	}, {
+		name:          "converter error",
+		splitter:      nil,
+		loaderFactory: nil,
+		converter: func(unstructured.Unstructured) (*v1alpha1.TestStep, error) {
+			return nil, errors.New("converter")
+		},
+		wantErr: true,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := parse(content, tt.splitter, tt.loaderFactory, tt.converter)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
diff --git a/pkg/test/load.go b/pkg/test/load.go
index 318188fc..3bf16315 100644
--- a/pkg/test/load.go
+++ b/pkg/test/load.go
@@ -10,9 +10,17 @@ import (
 	"github.com/kyverno/kyverno/ext/resource/convert"
 	"github.com/kyverno/kyverno/ext/resource/loader"
 	"github.com/kyverno/kyverno/ext/yaml"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/openapi"
 	"sigs.k8s.io/kubectl-validate/pkg/openapiclient"
 )
 
+type (
+	splitter      = func([]byte) ([][]byte, error)
+	loaderFactory = func(openapi.Client) (loader.Loader, error)
+	converter     = func(unstructured.Unstructured) (*v1alpha1.Test, error)
+)
+
 var test_v1alpha1 = v1alpha1.SchemeGroupVersion.WithKind("Test")
 
 func Load(path string) ([]*v1alpha1.Test, error) {
@@ -31,13 +39,26 @@ func Load(path string) ([]*v1alpha1.Test, error) {
 }
 
 func Parse(content []byte) ([]*v1alpha1.Test, error) {
-	documents, err := yaml.SplitDocuments(content)
+	return parse(content, nil, nil, nil)
+}
+
+func parse(content []byte, splitter splitter, loaderFactory loaderFactory, converter converter) ([]*v1alpha1.Test, error) {
+	if splitter == nil {
+		splitter = yaml.SplitDocuments
+	}
+	if loaderFactory == nil {
+		loaderFactory = loader.New
+	}
+	if converter == nil {
+		converter = convert.To[v1alpha1.Test]
+	}
+	documents, err := splitter(content)
 	if err != nil {
 		return nil, err
 	}
 	var tests []*v1alpha1.Test
 	// TODO: no need to allocate a validator every time
-	loader, err := loader.New(openapiclient.NewLocalCRDFiles(data.Crds(), data.CrdsFolder))
+	loader, err := loaderFactory(openapiclient.NewLocalCRDFiles(data.Crds(), data.CrdsFolder))
 	if err != nil {
 		return nil, err
 	}
@@ -48,7 +69,7 @@ func Parse(content []byte) ([]*v1alpha1.Test, error) {
 		}
 		switch gvk {
 		case test_v1alpha1:
-			test, err := convert.To[v1alpha1.Test](untyped)
+			test, err := converter(untyped)
 			if err != nil {
 				return nil, err
 			}
diff --git a/pkg/test/load_test.go b/pkg/test/load_test.go
index c50de075..88dcbe19 100644
--- a/pkg/test/load_test.go
+++ b/pkg/test/load_test.go
@@ -1,12 +1,19 @@
 package test
 
 import (
+	"errors"
+	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
+	"github.com/kyverno/kyverno/ext/resource/loader"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/openapi"
 )
 
 func TestLoad(t *testing.T) {
@@ -168,3 +175,69 @@ func TestLoad(t *testing.T) {
 		})
 	}
 }
+
+type fakeLoader struct{}
+
+func (fakeLoader) Load([]byte) (schema.GroupVersionKind, unstructured.Unstructured, error) {
+	return schema.GroupVersionKind{Group: "v1", Kind: "Something"}, unstructured.Unstructured{}, nil
+}
+
+func Test_parse(t *testing.T) {
+	content, err := os.ReadFile("../../testdata/test/custom-test.yaml")
+	assert.NoError(t, err)
+	tests := []struct {
+		name          string
+		splitter      splitter
+		loaderFactory loaderFactory
+		converter     converter
+		wantErr       bool
+	}{{
+		name:          "default",
+		splitter:      nil,
+		loaderFactory: nil,
+		converter:     nil,
+		wantErr:       false,
+	}, {
+		name: "splitter error",
+		splitter: func([]byte) ([][]byte, error) {
+			return nil, errors.New("splitter")
+		},
+		loaderFactory: nil,
+		converter:     nil,
+		wantErr:       true,
+	}, {
+		name:     "loader factory error",
+		splitter: nil,
+		loaderFactory: func(openapi.Client) (loader.Loader, error) {
+			return nil, errors.New("loader factory")
+		},
+		converter: nil,
+		wantErr:   true,
+	}, {
+		name:     "loader error",
+		splitter: nil,
+		loaderFactory: func(openapi.Client) (loader.Loader, error) {
+			return fakeLoader{}, nil
+		},
+		converter: nil,
+		wantErr:   true,
+	}, {
+		name:          "converter error",
+		splitter:      nil,
+		loaderFactory: nil,
+		converter: func(unstructured.Unstructured) (*v1alpha1.Test, error) {
+			return nil, errors.New("converter")
+		},
+		wantErr: true,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := parse(content, tt.splitter, tt.loaderFactory, tt.converter)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
diff --git a/testdata/step/custom-step.yaml b/testdata/step/custom-step.yaml
new file mode 100644
index 00000000..88de6f6b
--- /dev/null
+++ b/testdata/step/custom-step.yaml
@@ -0,0 +1,34 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: TestStep
+metadata:
+  name: example-test-step
+spec:
+  skipDelete: false
+  try:
+    - assert:
+        - file: /assertions/verify-service.yaml
+          timeout: 30s
+          continueOnError: false
+    - apply:
+        - file: /configs/service-config.yaml
+          timeout: 45s
+          shouldFail: false
+          continueOnError: true
+    - error:
+        - file: /errors/expected-timeout-error.yaml
+          timeout: 15s
+          continueOnError: true
+    - delete:
+        - apiVersion: v1
+          kind: Pod
+          name: example-pod
+          namespace: test
+          timeout: 10s
+          continueOnError: false
+    - exec:
+        - command:
+            entrypoint: /bin/sh
+            args:
+            - echo "Executing script..."
+          skipLogOutput: false
+          continueOnError: true
diff --git a/testdata/test/custom-test.yaml b/testdata/test/custom-test.yaml
new file mode 100644
index 00000000..2acdb4e3
--- /dev/null
+++ b/testdata/test/custom-test.yaml
@@ -0,0 +1,31 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: custom-test
+spec:
+  skip: false
+  concurrent: true
+  skipDelete: true
+  namespace: "custom-namespace-for-test"
+  steps:
+    - name: "test-step-initial-setup"
+      timeouts:
+        apply: "5s"
+      skipDelete: false
+      try:
+        - apply:
+          - file: /configs/service-config.yaml
+            timeout: 45s
+            shouldFail: false
+            continueOnError: true
+    - name: "test-step-apply-changes"
+      try:
+        - apply:
+          - file: /configs/service-config.yaml
+            timeout: 45s
+            shouldFail: false
+            continueOnError: true
+          assert:
+            - file: /assertions/verify-service.yaml
+              timeout: 30s
+              continueOnError: false