Repository: ysugimoto/falco (go)

Issue:
[Feature] Filter test suites by tag
### Kind of proposals

You can turn on multiple checkboxes.

- [] Parser
- [] Linter
- [] Simulator
- [x] Testing

### Describe the feature

Currently falco does not have filters what the user would like to run.
In most case, user wants to switch test suites by each environment (dev or prod) because occasionally VCL has environment specific logic, for example, need acl firewall for development environment.

So, falco should be able to filter the tests suites.


### Give us your considerations

I would propose that filter tests by tagging, is much inspired by [Playwright](https://playwright.dev/docs/test-annotations#tag-tests).

Example testing VCL is the following:

```vcl
// @suite: skip tests by tagging
// @scope: recv
// @tag: prod
sub test_recv {
  ...some assertions
}

// @suite: skip tests by tagging 2
// @scope: recv
// @tag: !prod
sub test_recv2 {
  ...some assertions
}
```

and then falco would accept `tag` cli option like `--tag prod`, and then the first test will be run and the second one **will not run**.

Additionally, the tag filter control should have `!` prefix operator - the inverse operator - in order to be useful for filtering tests.

To summarize, the following table describes whether test suite will run or not:

| tag specification | tag cli option | run test |
|:----------------:|:-------------:|:---------| 
| prod                    | N/A                  | **NO**   |
| prod                    | prod                | YES         |
| prod                    | dev                  | **NO**    |
| !prod                   | N/A                  | YES        |
| !prod                   | prod                | **NO**    |
| !prod                   | dev                  | YES         |
| N/A                      | N/A                  | YES         |
| N/A                      | prod                 | YES         |
| N/A                      | dev                  | YES         |


Pull request:
Implement testing tags
Fixes #457 

This PR implements testing tag related features:

- Can specify tags on test subroutine by applying `@tag` annotation
- Can provide test-running tag by provided `-t,--tag` cli option on `falco test` command

So the user would be able to control whether run specific test suite or not by modified tags.

Patch:
diff --git a/cmd/falco/help.go b/cmd/falco/help.go
index c08c7f4..36fc849 100644
--- a/cmd/falco/help.go
+++ b/cmd/falco/help.go
@@ -159,13 +159,14 @@ Flags:
     -I, --include_path : Add include path
     -h, --help         : Show this help
     -r, --remote       : Connect with Fastly API
-    -t, --timeout      : Set timeout to running test
     -f, --filter       : Override glob filter to find test files
+    -w, --watch        : Watch VCL file changes and run test
+    -t, --tag          : Provide tag for testing
     -json              : Output results as JSON
     -request           : Override request config
+    --timeout          : Set timeout to running test
     --max_backends     : Override max backends limitation
     --max_acls         : Override max acls limitation
-    -w, --watch        : Watch VCL file changes and run test
     --coverage         : Report code coverage
 
 Local testing example:
diff --git a/config/config.go b/config/config.go
index 0f297b6..de7d1da 100644
--- a/config/config.go
+++ b/config/config.go
@@ -63,8 +63,9 @@ type SimulatorConfig struct {
 
 // Testing configuration
 type TestConfig struct {
-	Timeout      int      `cli:"t,timeout" yaml:"timeout"`
+	Timeout      int      `cli:"timeout" yaml:"timeout"`
 	Filter       string   `cli:"f,filter" default:"*.test.vcl"`
+	Tags         []string `cli:"t,tag"`
 	IncludePaths []string // Copy from root field
 	OverrideHost string   `yaml:"host" cli:"host"`
 	Watch        bool     `cli:"w,watch"`      // Enable only in CLI option


Test patch:
diff --git a/cmd/falco/runner_test.go b/cmd/falco/runner_test.go
index d384ae8..59af047 100644
--- a/cmd/falco/runner_test.go
+++ b/cmd/falco/runner_test.go
@@ -539,3 +539,72 @@ func TestInjectDictionaryTesting(t *testing.T) {
 		})
 	}
 }
+
+func TestSkippingTests(t *testing.T) {
+	tests := []struct {
+		name    string
+		main    string
+		tags    []string
+		skipped int
+	}{
+		{
+			name:    "no tags are provided",
+			main:    "../../examples/testing/skipping_tests/default.vcl",
+			tags:    []string{},
+			skipped: 2,
+		},
+		{
+			name:    "provide prod tag",
+			main:    "../../examples/testing/skipping_tests/default.vcl",
+			tags:    []string{"prod"},
+			skipped: 2,
+		},
+		{
+			name:    "provide dev tag",
+			main:    "../../examples/testing/skipping_tests/default.vcl",
+			tags:    []string{"dev"},
+			skipped: 2,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			main, err := filepath.Abs(tt.main)
+			if err != nil {
+				t.Errorf("Unexpected making absolute path error: %s", err)
+				return
+			}
+			c := &config.Config{
+				Linter: &config.LinterConfig{
+					VerboseWarning: true,
+				},
+				Testing: &config.TestConfig{
+					Filter: "*/skipping_tests/*.test.vcl",
+					Tags:   tt.tags,
+				},
+				Commands: config.Commands{"test", main},
+			}
+			resolvers, err := resolver.NewFileResolvers(main, c.IncludePaths)
+			if err != nil {
+				t.Errorf("Unexpected runner creation error: %s", err)
+				return
+			}
+			ret, err := NewRunner(c, nil).Test(resolvers[0])
+			if err != nil {
+				t.Errorf("Unexpected runner creation error: %s", err)
+			}
+			for _, v := range ret.Results {
+				for _, c := range v.Cases {
+					if c.Error != nil {
+						t.Errorf(`Test case "%s" raises error: %s`, c.Name, c.Error)
+						return
+					}
+				}
+			}
+			if ret.Statistics.Skips != tt.skipped {
+				t.Errorf("Testing skipped count should be %d, got: %d", tt.skipped, ret.Statistics.Skips)
+				return
+			}
+		})
+	}
+}
diff --git a/config/config_test.go b/config/config_test.go
index 04be44e..cbb0007 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -32,6 +32,10 @@ func TestConfigFromCLI(t *testing.T) {
 		"-r",
 		"-V",
 		"--json",
+		"-t",
+		"foo",
+		"-t",
+		"bar",
 		"lint",
 	}
 	c, err := New(args)
@@ -62,6 +66,7 @@ func TestConfigFromCLI(t *testing.T) {
 		Testing: &TestConfig{
 			Filter:          "*.test.vcl",
 			IncludePaths:    []string{"."},
+			Tags:            []string{"foo", "bar"},
 			OverrideRequest: &RequestConfig{},
 		},
 		Console: &ConsoleConfig{
diff --git a/docs/testing.md b/docs/testing.md
index 7e02d7d..df637cc 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -193,7 +193,11 @@ sub some_test_suite {
 
 ### Skipping Test
 
-You can skip test case by adding `@skip` annotation comment. falco recognizes this annotation and skip testing.
+You can skip test case by a couple of ways.
+
+#### Adding `@skip` annotation comment
+
+When you write `@skip` annotation comment in leading comment of testing subroutine, falco skips this testing subroutine.
 
 ```vcl
 // @skip
@@ -202,6 +206,55 @@ sub some_test_suite {
 }
 ```
 
+> [!NOTE]
+> Testing subroutines which is applied `@skip` annotation comment are always skipped.
+
+#### Specify `@tag` annotation and match against `-t,--tag` cli option
+
+When you write `@tag: [tag1],[tag2],...` annotation comment in leading comment of testing subroutine, falco evaluates tag maching and run if matched..
+
+```vcl
+// @tag: prod
+sub some_test_suite {
+    ...
+}
+```
+
+And you can provide matcher tags via `-t,--tag` option:
+
+```shell
+falco test -t prod /path/to/vcl/default.vcl
+```
+
+Then, falco evaluates whether `prod` tag matches against `@tag` values (on the above case, test will be run).
+And the `@tag` annotation could appect inverse flag like `!prod`:
+
+```vcl
+// @tag: !prod
+sub some_test_suite {
+    ...
+}
+```
+
+Then this test suite will be run if `prod` tag is NOT provided.
+We describes the falco treats and evaluates tag specification and providing cli option as the following table:
+
+
+| Tag Specification | Tag CLI Option | Run Test |
+|:-----------------:|:--------------:|:---------|
+| prod              | N/A            | **NO**   |
+| prod              | prod           | YES      |
+| prod              | dev            | **NO**   |
+| !prod             | N/A            | YES      |
+| !prod             | prod           | **NO**   |
+| !prod             | dev            | YES      |
+| N/A               | N/A            | YES      |
+| N/A               | prod           | YES      |
+| N/A               | dev            | YES      |
+
+> [!IMPORTANT]
+> Above table describes significant thing that if you specify some tag annotation, the test suite only runs when some tag option is provided.
+
 ### Testing preparation
 
 When the test suite runs on a specific scope like `FETCH`, you need to set up a pre-condition to run target VCL.
diff --git a/examples/testing/skipping_tests/default.test.vcl b/examples/testing/skipping_tests/default.test.vcl
new file mode 100644
index 0000000..617df75
--- /dev/null
+++ b/examples/testing/skipping_tests/default.test.vcl
@@ -0,0 +1,10 @@
+// @skip
+sub skip_by_annotation {}
+
+sub always_run {}
+
+// @tag: prod
+sub tagging {}
+
+// @tag: !prod
+sub inversed_tagging {}
diff --git a/examples/testing/skipping_tests/default.vcl b/examples/testing/skipping_tests/default.vcl
new file mode 100644
index 0000000..e69de29
diff --git a/tester/metadata.go b/tester/metadata.go
index f841344..054114a 100644
--- a/tester/metadata.go
+++ b/tester/metadata.go
@@ -1,18 +1,62 @@
 package tester
 
 import (
+	"bytes"
+	"io"
 	"strings"
 
 	"github.com/ysugimoto/falco/ast"
 	"github.com/ysugimoto/falco/interpreter/context"
 )
 
+// Testing tag struct.
+// The Inverse fields indicates inverse tag match like !prod.
+type Tag struct {
+	Name    string
+	Inverse bool
+}
+
 // Define test metadata.
 // This struct fields are filled from annotation comments
 type Metadata struct {
 	Name   string
 	Scopes []context.Scope
 	Skip   bool
+	Tags   []Tag
+}
+
+func (m *Metadata) MatchTags(tags []string) bool {
+	// If any tags are not specified in test suite, always run
+	if len(m.Tags) == 0 {
+		return false
+	}
+
+	// If any tags are not provided via cli option, check non-inversed tag is specified
+	if len(tags) == 0 {
+		for _, v := range m.Tags {
+			if !v.Inverse {
+				return false
+			}
+		}
+		return true
+	}
+
+	// Otherwise, compare both tags
+	for i := range tags {
+		for _, v := range m.Tags {
+			var matched bool
+			if v.Name == tags[i] {
+				matched = !v.Inverse
+			} else {
+				matched = v.Inverse
+			}
+			if matched {
+				return true
+			}
+		}
+	}
+
+	return false
 }
 
 // Find test metadata from annotation comment
@@ -21,6 +65,7 @@ func getTestMetadata(sub *ast.SubroutineDeclaration) *Metadata {
 		Name:   sub.Name.Value,
 		Scopes: []context.Scope{},
 		Skip:   false,
+		Tags:   []Tag{},
 	}
 
 	comments := sub.GetMeta().Leading
@@ -40,6 +85,13 @@ func getTestMetadata(sub *ast.SubroutineDeclaration) *Metadata {
 			metadata.Skip = true
 		}
 
+		// Parse testing tags
+		if strings.HasPrefix(l, "@tag:") {
+			metadata.Tags = parseTestingTags(
+				strings.TrimSpace(strings.TrimPrefix(l, "@tag:")),
+			)
+		}
+
 		var Scopes []string
 		if strings.HasPrefix(l, "@scope:") {
 			Scopes = strings.Split(strings.TrimPrefix(l, "@scope:"), ",")
@@ -84,3 +136,46 @@ func getTestMetadata(sub *ast.SubroutineDeclaration) *Metadata {
 
 	return metadata
 }
+
+func parseTestingTags(tagValues string) []Tag {
+	var tags []Tag
+
+	buf := bytes.Buffer{}
+	var inverse bool
+	r := strings.NewReader(tagValues)
+	for {
+		b, err := r.ReadByte()
+		if err != nil {
+			// The err must be EOF
+			if err != io.EOF {
+				goto END
+			}
+			if buf.Len() > 0 {
+				tags = append(tags, Tag{
+					Name:    buf.String(),
+					Inverse: inverse,
+				})
+			}
+			goto END
+		}
+
+		switch b {
+		case 0x21: // "!"
+			inverse = true
+		case 0x2C: // ","
+			tags = append(tags, Tag{
+				Name:    buf.String(),
+				Inverse: inverse,
+			})
+			buf.Reset()
+			inverse = false
+		case 0x20: // " "
+			continue
+		default:
+			buf.WriteByte(b)
+		}
+	}
+END:
+
+	return tags
+}
diff --git a/tester/metadata_test.go b/tester/metadata_test.go
index 011f8ff..65023d9 100644
--- a/tester/metadata_test.go
+++ b/tester/metadata_test.go
@@ -26,6 +26,7 @@ sub test_subroutine {}
 			expect: &Metadata{
 				Name:   "basic metadata test",
 				Scopes: []context.Scope{context.RecvScope},
+				Tags:   []Tag{},
 			},
 		},
 		{
@@ -37,6 +38,7 @@ sub test_subroutine {}
 			expect: &Metadata{
 				Name:   "test_subroutine",
 				Scopes: []context.Scope{context.RecvScope},
+				Tags:   []Tag{},
 			},
 		},
 		{
@@ -48,6 +50,7 @@ sub test_subroutine {}
 			expect: &Metadata{
 				Name:   "metadata test",
 				Scopes: []context.Scope{context.RecvScope},
+				Tags:   []Tag{},
 			},
 		},
 		{
@@ -60,6 +63,7 @@ sub test_subroutine {}
 			expect: &Metadata{
 				Name:   "metadata test",
 				Scopes: []context.Scope{context.FetchScope, context.PassScope, context.LogScope},
+				Tags:   []Tag{},
 			},
 		},
 		{
@@ -74,6 +78,24 @@ sub test_subroutine {}
 				Name:   "metadata test",
 				Scopes: []context.Scope{context.RecvScope},
 				Skip:   true,
+				Tags:   []Tag{},
+			},
+		},
+		{
+			name: "tagged",
+			vcl: `
+// @suite: metadata test
+// @scope: recv
+// @tag: prod,!stg
+sub test_subroutine {}
+`,
+			expect: &Metadata{
+				Name:   "metadata test",
+				Scopes: []context.Scope{context.RecvScope},
+				Tags: []Tag{
+					{Name: "prod"},
+					{Name: "stg", Inverse: true},
+				},
 			},
 		},
 	}
@@ -93,3 +115,90 @@ sub test_subroutine {}
 		})
 	}
 }
+
+// https://github.com/ysugimoto/falco/issues/457
+func TestTagsMatch(t *testing.T) {
+	tests := []struct {
+		name   string
+		tag    []string
+		tags   []Tag
+		expect bool
+	}{
+		{
+			name:   "no tags",
+			tag:    []string{"prod"},
+			tags:   []Tag{},
+			expect: false,
+		},
+		{
+			name: "singular tag match",
+			tag:  []string{"prod"},
+			tags: []Tag{
+				{Name: "prod"},
+			},
+			expect: true,
+		},
+		{
+			name: "singular inversed tag unmatch",
+			tag:  []string{"prod"},
+			tags: []Tag{
+				{Name: "prod", Inverse: true},
+			},
+			expect: false,
+		},
+		{
+			name: "singular inversed tag match",
+			tag:  []string{"dev"},
+			tags: []Tag{
+				{Name: "prod", Inverse: true},
+			},
+			expect: true,
+		},
+		{
+			name: "multiple tag match",
+			tag:  []string{"stg"},
+			tags: []Tag{
+				{Name: "prod"},
+				{Name: "stg"},
+			},
+			expect: true,
+		},
+		{
+			name: "multiple inversed tag unmatch",
+			tag:  []string{"prod"},
+			tags: []Tag{
+				{Name: "prod", Inverse: true},
+				{Name: "stg"},
+			},
+			expect: false,
+		},
+		{
+			name: "multiple inversed tag match",
+			tag:  []string{"stg"},
+			tags: []Tag{
+				{Name: "prod", Inverse: true},
+				{Name: "stg", Inverse: true},
+			},
+			expect: true,
+		},
+		{
+			name: "always match",
+			tag:  []string{"prod"},
+			tags: []Tag{
+				{Name: "prod", Inverse: true},
+				{Name: "prod"},
+			},
+			expect: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := &Metadata{Tags: tt.tags}
+			actual := m.MatchTags(tt.tag)
+			if diff := cmp.Diff(tt.expect, actual); diff != "" {
+				t.Errorf("Tags.Match result unmatch, diff=%s", diff)
+			}
+		})
+	}
+}
diff --git a/tester/tester.go b/tester/tester.go
index 4dd6d1f..1e1c9b0 100644
--- a/tester/tester.go
+++ b/tester/tester.go
@@ -155,8 +155,8 @@ func (t *Tester) run(testFile string) (*TestResult, error) {
 				}
 				metadata := getTestMetadata(st)
 				for _, s := range metadata.Scopes {
-					// Skip this testsuite when marked as @skip
-					if metadata.Skip {
+					// Skip this testsuite when marked as @skip or @tag matched
+					if metadata.Skip || metadata.MatchTags(t.config.Tags) {
 						cases = append(cases, &TestCase{
 							Name:  metadata.Name,
 							Scope: s.String(),
@@ -229,8 +229,8 @@ func (t *Tester) runDescribedTests(
 	for _, sub := range d.Subroutines {
 		metadata := getTestMetadata(sub)
 		for _, s := range metadata.Scopes {
-			// Skip this testsuite when marked as @skip
-			if metadata.Skip {
+			// Skip this testsuite when marked as @skip or @tag matched
+			if metadata.Skip || metadata.MatchTags(t.config.Tags) {
 				cases = append(cases, &TestCase{
 					Name:  metadata.Name,
 					Scope: s.String(),