Repository: kyverno/chainsaw
Pull request #61 (2023-10-14 12:30:07)

feat: add discovery of tests based on file names
Add discovery of tests based on file names.

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/kyverno/chainsaw/pull/61?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) Report
> Merging [#61](https://app.codecov.io/gh/kyverno/chainsaw/pull/61?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) (d024858) into [main](https://app.codecov.io/gh/kyverno/chainsaw/commit/9d0159906ef705d1ecd036e870adcee465810d5d?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) (9d01599) will **decrease** coverage by `10.29%`.
> The diff coverage is `0.00%`.

```diff
@@             Coverage Diff             @@
##             main      #61       +/-   ##
===========================================
- Coverage   69.17%   58.89%   -10.29%     
===========================================
  Files           9        9               
  Lines         292      343       +51     
===========================================
  Hits          202      202               
- Misses         62      113       +51     
  Partials       28       28               
```


| [Files](https://app.codecov.io/gh/kyverno/chainsaw/pull/61?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno) | Coverage Δ | |
|---|---|---|
| [pkg/discovery/discovery.go](https://app.codecov.io/gh/kyverno/chainsaw/pull/61?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno#diff-cGtnL2Rpc2NvdmVyeS9kaXNjb3ZlcnkuZ28=) | `26.92% <0.00%> (-50.86%)` | :arrow_down: |

:mega: We’re building smart automated test selection to slash your CI/CD build times. [Learn more](https://about.codecov.io/iterative-testing/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kyverno)


Patch:
diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go
index 3a9d2def..13757eca 100644
--- a/pkg/discovery/discovery.go
+++ b/pkg/discovery/discovery.go
@@ -4,11 +4,16 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"regexp"
 
 	"github.com/kyverno/chainsaw/pkg/apis/v1alpha1"
 	"github.com/kyverno/chainsaw/pkg/test"
+	"golang.org/x/exp/slices"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+var stepFileName = regexp.MustCompile(`^(\d\d)-(.*)\.(?:yaml|yml)$`)
+
 type Test struct {
 	*v1alpha1.Test
 	BasePath string
@@ -47,3 +52,92 @@ func DiscoverTests(fileName string, paths ...string) ([]Test, error) {
 	}
 	return tests, nil
 }
+
+func DiscoverTests2(paths ...string) ([]Test, error) {
+	var folders []string
+	for _, path := range paths {
+		if _, err := os.Lstat(path); err == nil {
+			err := filepath.Walk(path, func(file string, info fs.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.IsDir() {
+					folders = append(folders, file)
+				}
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	var tests []Test
+	for _, folder := range folders {
+		// list files in folder
+		files, err := os.ReadDir(folder)
+		if err == nil {
+			var stepFiles []string
+			for _, file := range files {
+				fileName := file.Name()
+				if !file.IsDir() {
+					if stepFileName.MatchString(fileName) {
+						stepFiles = append(stepFiles, fileName)
+					}
+				}
+			}
+			// we found some steps
+			if len(stepFiles) != 0 {
+				slices.Sort(stepFiles)
+				test := &v1alpha1.Test{
+					TypeMeta: metav1.TypeMeta{
+						APIVersion: "chainsaw.kyverno.io/v1alpha1",
+						Kind:       "Test",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name: filepath.Base(folder),
+					},
+				}
+				stepsMap := map[string]v1alpha1.TestStepSpec{}
+				for _, stepFile := range stepFiles {
+					// TODO: consider case the file contains a test step
+					fileRef := v1alpha1.FileRef{
+						File: stepFile,
+					}
+					groups := stepFileName.FindStringSubmatch(stepFile)
+					step, ok := stepsMap[groups[1]]
+					if !ok {
+						step = v1alpha1.TestStepSpec{}
+					}
+					switch groups[2] {
+					case "assert":
+						step.Assert = append(step.Assert, v1alpha1.Assert{
+							FileRef: fileRef,
+						})
+					case "error":
+						step.Error = append(step.Error, v1alpha1.Error{
+							FileRef: fileRef,
+						})
+					default:
+						step.Apply = append(step.Apply, v1alpha1.Apply{
+							FileRef: fileRef,
+						})
+					}
+					stepsMap[groups[1]] = step
+				}
+				keys := make([]string, 0, len(stepsMap))
+				for k := range stepsMap {
+					keys = append(keys, k)
+				}
+				slices.Sort(keys)
+				for _, key := range keys {
+					test.Spec.Steps = append(test.Spec.Steps, stepsMap[key])
+				}
+				tests = append(tests, Test{
+					Test:     test,
+					BasePath: folder,
+				})
+			}
+		}
+	}
+	return tests, nil
+}


Test patch:
diff --git a/pkg/discovery/discovery_test.go b/pkg/discovery/discovery_test.go
index 170b62b8..728096e3 100644
--- a/pkg/discovery/discovery_test.go
+++ b/pkg/discovery/discovery_test.go
@@ -60,3 +60,58 @@ func TestDiscoverTests(t *testing.T) {
 		})
 	}
 }
+
+func TestDiscoverTests2(t *testing.T) {
+	tests := []struct {
+		name    string
+		paths   []string
+		want    []Test
+		wantErr bool
+	}{{
+		name:    "quick start",
+		paths:   []string{"../../testdata/tests/not-a-test"},
+		want:    nil,
+		wantErr: false,
+	}, {
+		name:  "quick start",
+		paths: []string{"../../testdata/tests/manifests-based"},
+		want: []Test{{
+			BasePath: "../../testdata/tests/manifests-based",
+			Test: &v1alpha1.Test{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: "chainsaw.kyverno.io/v1alpha1",
+					Kind:       "Test",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "manifests-based",
+				},
+				Spec: v1alpha1.TestSpec{
+					Steps: []v1alpha1.TestStepSpec{{
+						Apply: []v1alpha1.Apply{{
+							FileRef: v1alpha1.FileRef{
+								File: "01-configmap.yaml",
+							},
+						}},
+						Assert: []v1alpha1.Assert{{
+							FileRef: v1alpha1.FileRef{
+								File: "01-assert.yaml",
+							},
+						}},
+					}},
+				},
+			},
+		}},
+		wantErr: false,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := DiscoverTests2(tt.paths...)
+			if tt.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.Equal(t, tt.want, got)
+		})
+	}
+}
diff --git a/testdata/tests/manifests-based/01-assert.yaml b/testdata/tests/manifests-based/01-assert.yaml
new file mode 100644
index 00000000..38194185
--- /dev/null
+++ b/testdata/tests/manifests-based/01-assert.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: chainsaw-quick-start
+data:
+  foo: bar
diff --git a/testdata/tests/manifests-based/01-configmap.yaml b/testdata/tests/manifests-based/01-configmap.yaml
new file mode 100644
index 00000000..38194185
--- /dev/null
+++ b/testdata/tests/manifests-based/01-configmap.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: chainsaw-quick-start
+data:
+  foo: bar
diff --git a/testdata/tests/not-a-test/configmap.yaml b/testdata/tests/not-a-test/configmap.yaml
new file mode 100644
index 00000000..38194185
--- /dev/null
+++ b/testdata/tests/not-a-test/configmap.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: chainsaw-quick-start
+data:
+  foo: bar