Repository: kyverno/chainsaw
Pull request #481 (2023-11-24 11:18:21)

fix: discovery of manifests in the wrong order
Fixed discovery of manifests in the wrong order.

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/kyverno/chainsaw/pull/481?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Comparison is base [(`2ec8aa0`)](https://app.codecov.io/gh/kyverno/chainsaw/commit/2ec8aa06a1071ba8dad7a4a4b6ea5df4810d5cbc?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) 80.65% compared to head [(`22e6fc0`)](https://app.codecov.io/gh/kyverno/chainsaw/pull/481?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) 80.82%.


<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #481      +/-   ##
==========================================
+ Coverage   80.65%   80.82%   +0.17%     
==========================================
  Files          49       49              
  Lines        1783     1799      +16     
==========================================
+ Hits         1438     1454      +16     
  Misses        274      274              
  Partials       71       71              
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/kyverno/chainsaw/pull/481?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno).


Patch:
diff --git a/.release-notes/main.md b/.release-notes/main.md
index 76882320..31ef2538 100644
--- a/.release-notes/main.md
+++ b/.release-notes/main.md
@@ -24,3 +24,4 @@ Release notes for `TODO`.
 
 - Fixed a kuttl migration failure in case of unsupported file name
 - Fixed a potential invalid name when migrating a kuttl test step
+- Fixed a manifest discovery issue where manifests could be loaded in the wrong order
diff --git a/pkg/discovery/load.go b/pkg/discovery/load.go
index cabb1b60..817c6981 100644
--- a/pkg/discovery/load.go
+++ b/pkg/discovery/load.go
@@ -103,6 +103,24 @@ func LoadTest(fileName string, path string) (*Test, error) {
 					}
 				}
 			}
+			var assertFiles []string
+			var errorFiles []string
+			var otherFiles []string
+			for _, file := range manifestFiles {
+				groups := StepFileName.FindStringSubmatch(file)
+				switch groups[2] {
+				case "assert":
+					assertFiles = append(assertFiles, file)
+				case "error":
+					errorFiles = append(errorFiles, file)
+				default:
+					otherFiles = append(otherFiles, file)
+				}
+			}
+			manifestFiles = []string{}
+			manifestFiles = append(manifestFiles, otherFiles...)
+			manifestFiles = append(manifestFiles, assertFiles...)
+			manifestFiles = append(manifestFiles, errorFiles...)
 			for _, file := range manifestFiles {
 				groups := StepFileName.FindStringSubmatch(file)
 				step, ok := stepsMap[groups[1]]


Test patch:
diff --git a/pkg/discovery/discovery_test.go b/pkg/discovery/discovery_test.go
index 54fcecda..9e507d0b 100644
--- a/pkg/discovery/discovery_test.go
+++ b/pkg/discovery/discovery_test.go
@@ -75,15 +75,9 @@ func TestDiscoverTests(t *testing.T) {
 				},
 				Spec: v1alpha1.TestSpec{
 					Steps: []v1alpha1.TestSpecStep{{
-						Name: "assert",
+						Name: "configmap",
 						Spec: v1alpha1.TestStepSpec{
 							Try: []v1alpha1.Operation{{
-								Assert: &v1alpha1.Assert{
-									FileRef: v1alpha1.FileRef{
-										File: "01-assert.yaml",
-									},
-								},
-							}, {
 								Apply: &v1alpha1.Apply{
 									FileRefOrResource: v1alpha1.FileRefOrResource{
 										FileRef: v1alpha1.FileRef{
@@ -91,6 +85,12 @@ func TestDiscoverTests(t *testing.T) {
 										},
 									},
 								},
+							}, {
+								Assert: &v1alpha1.Assert{
+									FileRef: v1alpha1.FileRef{
+										File: "01-assert.yaml",
+									},
+								},
 							}, {
 								Error: &v1alpha1.Error{
 									FileRef: v1alpha1.FileRef{
diff --git a/pkg/discovery/load_test.go b/pkg/discovery/load_test.go
index a60ebba0..46e6857b 100644
--- a/pkg/discovery/load_test.go
+++ b/pkg/discovery/load_test.go
@@ -236,12 +236,6 @@ func TestLoadTest(t *testing.T) {
 										File: "bar.yaml",
 									},
 								},
-							}, {
-								Assert: &v1alpha1.Assert{
-									FileRef: v1alpha1.FileRef{
-										File: "01-assert.yaml",
-									},
-								},
 							}, {
 								Apply: &v1alpha1.Apply{
 									FileRefOrResource: v1alpha1.FileRefOrResource{
@@ -250,6 +244,12 @@ func TestLoadTest(t *testing.T) {
 										},
 									},
 								},
+							}, {
+								Assert: &v1alpha1.Assert{
+									FileRef: v1alpha1.FileRef{
+										File: "01-assert.yaml",
+									},
+								},
 							}, {
 								Error: &v1alpha1.Error{
 									FileRef: v1alpha1.FileRef{
@@ -279,15 +279,9 @@ func TestLoadTest(t *testing.T) {
 				},
 				Spec: v1alpha1.TestSpec{
 					Steps: []v1alpha1.TestSpecStep{{
-						Name: "assert",
+						Name: "configmap",
 						Spec: v1alpha1.TestStepSpec{
 							Try: []v1alpha1.Operation{{
-								Assert: &v1alpha1.Assert{
-									FileRef: v1alpha1.FileRef{
-										File: "01-assert.yaml",
-									},
-								},
-							}, {
 								Apply: &v1alpha1.Apply{
 									FileRefOrResource: v1alpha1.FileRefOrResource{
 										FileRef: v1alpha1.FileRef{
@@ -295,6 +289,12 @@ func TestLoadTest(t *testing.T) {
 										},
 									},
 								},
+							}, {
+								Assert: &v1alpha1.Assert{
+									FileRef: v1alpha1.FileRef{
+										File: "01-assert.yaml",
+									},
+								},
 							}, {
 								Error: &v1alpha1.Error{
 									FileRef: v1alpha1.FileRef{