Repository: ysugimoto/falco
Pull request #22 (2021-07-12 17:56:57)

fix add statement linter
This PR has a couple of changes:

1. Change severity from `WARNING` to `ERROR` when type is different between left and right on `=` operator in `set` statement
2. Raise `ERROR` when `add` statement is used not for HTTP headers (refers [add statement](https://developer.fastly.com/reference/vcl/statements/add/)
3. Check validity of regex string inside if expression


Patch:
diff --git a/.gitignore b/.gitignore
index a67f79e..b75848c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,4 @@
 # vendor/
 falco
 dist/*
+playground
diff --git a/README.md b/README.md
index c342e98..472dac3 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,7 @@ Flags:
     -I, --include_path : Add include path
     -t, --transformer  : Specify transformer
     -h, --help         : Show this help
+    -r, --remote       : Communicate with Fastly API
     -V, --version      : Display build version
     -v,                : Verbose warning lint result
     -vv,               : Verbose all lint result
@@ -124,15 +125,7 @@ Following table describes annotation name and recognizing scope:
 
 ## Fastly related features
 
-Currently, we don't support snippets which are managed in Fastly:
-
-- Edge Dictionary
-- VCL Snippets
-- Log definitions
-- Etc
-
-Above snippets will be injected to your VCL top or extracting `FASTLY XXX` macro, but this tool aims to run locally, not communicating with Fastly service.
-However, we're planning to solve them using the Fastly API.
+Partially supports fetching Fastly managed VCL snippets. See [remote.md](https://github.com/ysugimoto/falco/blob/master/docs/remote.md) in detail.
 
 ## Lint error
 
diff --git a/cli/main.go b/cli/main.go
index d9215ff..26e64d1 100644
--- a/cli/main.go
+++ b/cli/main.go
@@ -43,6 +43,7 @@ type Config struct {
 	V            bool
 	VV           bool
 	Version      bool
+	Remote       bool
 }
 
 func write(c *color.Color, format string, args ...interface{}) {
@@ -65,9 +66,10 @@ Flags:
     -I, --include_path : Add include path
     -t, --transformer  : Specify transformer
     -h, --help         : Show this help
+    -r, --remote       : Communicate with Fastly API
     -V, --version      : Display build version
-    -v,                : Verbose warning lint result
-    -vv,               : Varbose all lint result
+    -v                 : Verbose warning lint result
+    -vv                : Varbose all lint result
 
 Example:
     falco -I . -vv /path/to/vcl/main.vcl
@@ -90,6 +92,8 @@ func main() {
 	fs.BoolVar(&c.VV, "vv", false, "Verbose info")
 	fs.BoolVar(&c.Version, "V", false, "Print Version")
 	fs.BoolVar(&c.Version, "version", false, "Print Version")
+	fs.BoolVar(&c.Remote, "r", false, "Use Remote")
+	fs.BoolVar(&c.Remote, "remote", false, "Use Remote")
 
 	if err := fs.Parse(os.Args[1:]); err != nil {
 		if err == flag.ErrHelp {
diff --git a/cli/runner.go b/cli/runner.go
index f30a232..60dd346 100644
--- a/cli/runner.go
+++ b/cli/runner.go
@@ -2,9 +2,11 @@ package main
 
 import (
 	"bytes"
+	_context "context"
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"path/filepath"
 
@@ -63,6 +65,33 @@ func NewRunner(mainVcl string, c *Config) (*Runner, error) {
 		overrides:   make(map[string]linter.Severity),
 	}
 
+	if c.Remote {
+		writeln(cyan, "Remote option supplied. Fetch snippets from Fastly.")
+		// If remote flag is provided, fetch predefined data from Fastly.
+		// Currently, we only support Edge Dictionary.
+		//
+		// We communicate Fastly API with service id and api key,
+		// lookup fixed environment variable, FASTLY_SERVICE_ID and FASTLY_API_KEY
+		// So user needs to set them with "-r" argument.
+		serviceId := os.Getenv("FASTLY_SERVICE_ID")
+		apiKey := os.Getenv("FASTLY_API_KEY")
+		if serviceId == "" || apiKey == "" {
+			return nil, errors.New("Both FASTLY_SERVICE_ID and FASTLY_API_KEY environment variables must be specified")
+		}
+		func() {
+			ctx, timeout := _context.WithTimeout(_context.Background(), 20*time.Second)
+			defer timeout()
+
+			snippet := NewSnippet(serviceId, apiKey)
+			// Remote communication is optional so we keep processing even if remote communication is failed
+			if err := snippet.Fetch(ctx); err != nil {
+				writeln(red, err.Error())
+			} else if err := snippet.Compile(r.context); err != nil {
+				writeln(red, err.Error())
+			}
+		}()
+	}
+
 	// Directory which placed main VCL adds to include path
 	r.includePaths = append(r.includePaths, filepath.Dir(mainVcl))
 
diff --git a/cli/snippet.go b/cli/snippet.go
new file mode 100644
index 0000000..ff56b16
--- /dev/null
+++ b/cli/snippet.go
@@ -0,0 +1,86 @@
+package main
+
+import (
+	"bytes"
+	_context "context"
+	"fmt"
+	"strings"
+
+	"net/http"
+	"text/template"
+
+	"github.com/ysugimoto/falco/context"
+	"github.com/ysugimoto/falco/lexer"
+	"github.com/ysugimoto/falco/linter"
+	"github.com/ysugimoto/falco/parser"
+	"github.com/ysugimoto/falco/remote"
+)
+
+var tableTemplate = `
+table {{ .Name }} {
+	{{- range .Items }}
+	"{{ .Key }}": "{{ .Value }}",
+	{{- end }}
+}
+`
+
+type Snippet struct {
+	client   *remote.FastlyClient
+	snippets []string
+}
+
+func NewSnippet(serviceId, apiKey string) *Snippet {
+	return &Snippet{
+		client: remote.NewFastlyClient(http.DefaultClient, serviceId, apiKey),
+	}
+}
+
+func (s *Snippet) Compile(ctx *context.Context) error {
+	vcl, err := parser.New(lexer.NewFromString(strings.Join(s.snippets, "\n"))).ParseVCL()
+	if err != nil {
+		return err
+	}
+	l := linter.New()
+	l.Lint(vcl, ctx)
+	return nil
+}
+
+func (s *Snippet) Fetch(c _context.Context) error {
+	write(white, "Fetching Edge Dictionaries...")
+	dicts, err := s.fetchEdgeDictionary(c)
+	if err != nil {
+		return err
+	}
+	writeln(white, "Done")
+	s.snippets = append(s.snippets, dicts...)
+	return nil
+}
+
+// Fetch remote Edge dictionary items
+func (s *Snippet) fetchEdgeDictionary(c _context.Context) ([]string, error) {
+	// Fetch latest version
+	version, err := s.client.LatestVersion(c)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get latest version %w", err)
+	}
+
+	dicts, err := s.client.ListEdgeDictionaries(c, version)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get latest version %w", err)
+	}
+
+	tmpl, err := template.New("table").Parse(tableTemplate)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to compilte table template: %w", err)
+	}
+
+	var snippets []string
+	for _, dict := range dicts {
+		buf := new(bytes.Buffer)
+		if err := tmpl.Execute(buf, dict); err != nil {
+			return nil, fmt.Errorf("Failed to render table template: %w", err)
+		}
+		snippets = append(snippets, buf.String())
+	}
+	return snippets, nil
+}
diff --git a/docs/remote.md b/docs/remote.md
new file mode 100644
index 0000000..7bdef3f
--- /dev/null
+++ b/docs/remote.md
@@ -0,0 +1,59 @@
+# Use Fastly managed snippets
+
+`falco` aims to run locally, but probably vcl has dependency of Fastly managed VCL -- Edge Dictionary, VCL snippets --- and linter may cause error if these are not specified,
+
+In order to use them, falco provides remote option which prefetch Fastly managed VCL and do parse and lint.
+
+## Prefetching managed VCL
+
+### Provide remote flag in CLI
+
+By supplying `-r, --remote` flag in command, falco communicates to Fastly API and retrieve VCL snippets.
+
+For example:
+
+```shell
+falco -r -v /path/to/example.vcl
+```
+
+### Environment variables
+
+Fastly API requires `API key` to authenticate, and `Service ID` to distinguish service, therefore you need to specift these values in your environment variable.
+The environment varialbe name is fixed:
+
+| variable name     | usage |
+|:------------------|:----  |
+| FASTLY_SERVICE_ID | Service ID |
+| FASTLY_API_KEY    | API Key, you can create via [Personal API Tokens](https://manage.fastly.com/account/personal/tokens) |
+
+**Note: We recommend the Fastly API Key has `global:read` scope. falco only just call _read_ related API.**
+
+
+### Edge Dictionary
+
+Prefetch [Edge Dictionary](https://docs.fastly.com/en/guides/about-edge-dictionaries) from Fastly and parse as `Table`.
+Note that Edge Dictionary always treats as `STRING` type in VCL:
+
+If you defined Edge Dictionary named `my_dictionary`, falco deals with as table:
+
+```
+table my_dictionary STRING {
+  "[item_key01": "[item_value01]",
+  "[item_key02": "[item_value02]",
+  ...
+}
+```
+
+You can access `my_dictionary` table in your custom VCL.
+
+### Logging
+
+Currently not supported.
+
+### VCL snippets
+
+Currently not supported.
+
+### Access control lists
+
+Currently not supported.
diff --git a/linter/errors.go b/linter/errors.go
index 6bcfdf9..cdb83e6 100644
--- a/linter/errors.go
+++ b/linter/errors.go
@@ -73,9 +73,11 @@ func InvalidValue(m *ast.Meta, tt, val string) *LintError {
 	}
 }
 
+// InvalidType raises ERROR due to strict type assertion failed.
+// Actually, it cause compile error for that VCL.
 func InvalidType(m *ast.Meta, name string, expect, actual types.Type) *LintError {
 	return &LintError{
-		Severity: WARNING,
+		Severity: ERROR,
 		Token:    m.Token,
 		Message:  fmt.Sprintf("%s wants type %s but assign %s", name, expect.String(), actual.String()),
 	}
diff --git a/linter/linter.go b/linter/linter.go
index 9c8000f..fa9b5fd 100644
--- a/linter/linter.go
+++ b/linter/linter.go
@@ -3,6 +3,7 @@ package linter
 import (
 	"fmt"
 	"net"
+	"regexp"
 	"strings"
 
 	"github.com/ysugimoto/falco/ast"
@@ -892,6 +893,22 @@ func (l *Linter) lintAddStatement(stmt *ast.AddStatement, ctx *context.Context)
 		l.Error(InvalidName(stmt.Ident.GetMeta(), stmt.Ident.Value, "add").Match(ADD_STATEMENT_SYNTAX))
 	}
 
+	// Add statement could use only for HTTP headers.
+	// https://developer.fastly.com/reference/vcl/statements/add/
+	if !strings.Contains(stmt.Ident.Value, "req.http.") &&
+		!strings.Contains(stmt.Ident.Value, "bereq.http.") &&
+		!strings.Contains(stmt.Ident.Value, "beresp.http.") &&
+		!strings.Contains(stmt.Ident.Value, "obj.http.") &&
+		!strings.Contains(stmt.Ident.Value, "resp.http.") {
+
+		err := &LintError{
+			Severity: ERROR,
+			Token:    stmt.Ident.GetMeta().Token,
+			Message:  "Add statement could not use for " + stmt.Ident.Value,
+		}
+		l.Error(err.Match(ADD_STATEMENT_SYNTAX))
+	}
+
 	left, err := ctx.Get(stmt.Ident.Value)
 	if err != nil {
 		l.Error(&LintError{
@@ -1179,6 +1196,17 @@ func (l *Linter) lintInfixExpression(exp *ast.InfixExpression, ctx *context.Cont
 		} else if !expectType(right, types.StringType, types.IPType, types.AclType) {
 			l.Error(InvalidTypeExpression(exp.GetMeta(), right, types.StringType, types.IPType, types.AclType).Match(OPERATOR_CONDITIONAL))
 		}
+		// And, if right expression is STRING, regex must be valid
+		if v, ok := exp.Right.(*ast.String); ok {
+			if _, err := regexp.Compile(v.Value); err != nil {
+				err := &LintError{
+					Severity: ERROR,
+					Token:    exp.Right.GetMeta().Token,
+					Message:  "regex string is invalid, " + err.Error(),
+				}
+				l.Error(err)
+			}
+		}
 		return types.BoolType
 	case "+":
 		// Plus operator behaves string concatenation.
diff --git a/remote/client.go b/remote/client.go
new file mode 100644
index 0000000..f028bb6
--- /dev/null
+++ b/remote/client.go
@@ -0,0 +1,113 @@
+package remote
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"encoding/json"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	fastlyApiBaseUrl = "https://api.fastly.com"
+)
+
+type FastlyClient struct {
+	serviceId string
+	apiKey    string
+	client    *http.Client
+}
+
+func NewFastlyClient(c *http.Client, serviceId, apiKey string) *FastlyClient {
+	return &FastlyClient{
+		serviceId: serviceId,
+		apiKey:    apiKey,
+		client:    c,
+	}
+}
+
+func (c *FastlyClient) request(ctx context.Context, method, url string, body io.Reader, v interface{}) error {
+	req, err := http.NewRequestWithContext(ctx, method, fastlyApiBaseUrl+url, body)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	req.Header.Set("Fastly-Key", c.apiKey)
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return errors.WithStack(err)
+	} else if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("API respond not 200 code: %d", resp.StatusCode)
+	}
+	defer resp.Body.Close()
+	if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+func (c *FastlyClient) LatestVersion(ctx context.Context) (int64, error) {
+	ctx, timeout := context.WithTimeout(ctx, 5*time.Second)
+	defer timeout()
+
+	endpoint := fmt.Sprintf("/service/%s/version/active", c.serviceId)
+	var v Version
+	if err := c.request(ctx, http.MethodGet, endpoint, nil, &v); err != nil {
+		return 0, errors.WithStack(err)
+	}
+
+	return v.Number, nil
+}
+
+func (c *FastlyClient) ListEdgeDictionaries(ctx context.Context, version int64) ([]*EdgeDictionary, error) {
+	endpoint := fmt.Sprintf("/service/%s/version/%d/dictionary", c.serviceId, version)
+	var dicts []*EdgeDictionary
+	if err := c.request(ctx, http.MethodGet, endpoint, nil, &dicts); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	var wg sync.WaitGroup
+	var once sync.Once
+	errch := make(chan error)
+	for _, d := range dicts {
+		wg.Add(1)
+		go func(d *EdgeDictionary) {
+			defer wg.Done()
+			var err error
+			if d.Items, err = c.ListEdgeDictionaryItems(ctx, d.Id); err != nil {
+				once.Do(func() {
+					errch <- err
+				})
+			}
+		}(d)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		return dicts, nil
+	case err := <-errch:
+		return nil, err
+	}
+}
+
+func (c *FastlyClient) ListEdgeDictionaryItems(ctx context.Context, dictId string) ([]*EdgeDictionaryItem, error) {
+	endpoint := fmt.Sprintf("/service/%s/dictionary/%s/items", c.serviceId, dictId)
+	var items []*EdgeDictionaryItem
+	if err := c.request(ctx, http.MethodGet, endpoint, nil, &items); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return items, nil
+}
diff --git a/remote/entity.go b/remote/entity.go
new file mode 100644
index 0000000..55c86ce
--- /dev/null
+++ b/remote/entity.go
@@ -0,0 +1,16 @@
+package remote
+
+type Version struct {
+	Number int64 `json:"number"`
+}
+
+type EdgeDictionary struct {
+	Id    string `json:"id"`
+	Name  string `json:"name"`
+	Items []*EdgeDictionaryItem
+}
+
+type EdgeDictionaryItem struct {
+	Key   string `json:"item_key"`
+	Value string `json:"item_value"`
+}


Test patch:
diff --git a/linter/linter_test.go b/linter/linter_test.go
index b687ede..37039b6 100644
--- a/linter/linter_test.go
+++ b/linter/linter_test.go
@@ -504,6 +504,15 @@ sub foo {
 		assertError(t, input)
 	})
 
+	t.Run("only can use for HTTP headers", func(t *testing.T) {
+		input := `
+sub foo {
+	declare local var.FOO STRING;
+	add var.FOO = "bar";
+}`
+
+		assertError(t, input)
+	})
 }
 
 func TestLintCallStatement(t *testing.T) {
@@ -1229,3 +1238,13 @@ sub vcl_recv {
 }`
 	assertNoError(t, input)
 }
+
+func TestRegexExpressionIsInvalid(t *testing.T) {
+	input := `
+sub vcl_recv {
+	#Fastly recv
+	if (req.url ~ "/foo/(.+") {
+	}
+}`
+	assertError(t, input)
+}
diff --git a/remote/client_test.go b/remote/client_test.go
new file mode 100644
index 0000000..cf74a3a
--- /dev/null
+++ b/remote/client_test.go
@@ -0,0 +1,138 @@
+package remote
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"io/ioutil"
+	"net/http"
+)
+
+type TestRoundTripper struct {
+	StatusCode int
+	Body       string
+}
+
+func (t *TestRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	if r.Header.Get("Fastly-Key") == "" {
+		return nil, errors.New("Fastly-Key header is required")
+	}
+
+	return &http.Response{
+		StatusCode: t.StatusCode,
+		Body:       ioutil.NopCloser(strings.NewReader(t.Body)),
+	}, nil
+}
+
+func TestLatestVersion(t *testing.T) {
+	c := NewFastlyClient(&http.Client{
+		Transport: &TestRoundTripper{
+			StatusCode: 200,
+			Body: `
+{
+	"created_at": "2020-04-09T18:14:30.000Z",
+	"updated_at": "2020-04-09T18:15:30.000Z",
+	"deleted_at": null,
+	"active": true,
+	"comment": "",
+	"deployed": true,
+	"locked": false,
+	"number": 1,
+	"staging": false,
+	"testing": false,
+	"service_id": "SU1Z0isxPaozGVKXdv0eY"
+}`,
+		},
+	}, "dummy", "dummy")
+
+	version, err := c.LatestVersion(context.Background())
+	if err != nil {
+		t.Errorf("Unexpected error: %s", err)
+		t.FailNow()
+	}
+	if version != 1 {
+		t.Errorf("Version assertion error, expects=1 but got=%d", version)
+		t.FailNow()
+	}
+}
+
+func TestListDictionaries(t *testing.T) {
+	c := NewFastlyClient(&http.Client{
+		Transport: &TestRoundTripper{
+			StatusCode: 200,
+			Body: `
+[
+  {
+	"created_at": "2020-04-29T22:16:23.000Z",
+	"deleted_at": null,
+	"id": "3vjTN8v1O7nOAY7aNDGOL",
+	"name": "my_dictionary",
+	"service_id": "SU1Z0isxPaozGVKXdv0eY",
+	"updated_at": "2020-04-29T22:16:23.000Z",
+	"version": 1,
+	"write_only": false
+  }
+]`,
+		},
+	}, "dummy", "dummy")
+
+	dicts, err := c.ListEdgeDictionaries(context.Background(), 10)
+	if err != nil {
+		t.Errorf("Unexpected error: %s", err)
+		t.FailNow()
+	}
+	if len(dicts) != 1 {
+		t.Errorf("dictionaries should have 1 but got %d", len(dicts))
+		t.FailNow()
+	}
+	d := dicts[0]
+	if d.Id != "3vjTN8v1O7nOAY7aNDGOL" {
+		t.Errorf("dict id assertion error, expects=3vjTN8v1O7nOAY7aNDGOL but got=%s", d.Id)
+		t.FailNow()
+	}
+	if d.Name != "my_dictionary" {
+		t.Errorf("dict name assertion error, expects=my_dictionary but got=%s", d.Name)
+		t.FailNow()
+	}
+}
+
+func TestListDictionaryItems(t *testing.T) {
+	c := NewFastlyClient(&http.Client{
+		Transport: &TestRoundTripper{
+			StatusCode: 200,
+			Body: `
+[
+  {
+	"dictionary_id": "3vjTN8v1O7nOAY7aNDGOL",
+	"service_id": "SU1Z0isxPaozGVKXdv0eY",
+	"item_key": "some_key",
+	"item_value": "some_value",
+	"created_at": "2020-04-21T18:14:32.000Z",
+	"deleted_at": null,
+	"updated_at": "2020-04-21T18:14:32.000Z"
+  }
+]`,
+		},
+	}, "dummy", "dummy")
+
+	items, err := c.ListEdgeDictionaryItems(context.Background(), "3vjTN8v1O7nOAY7aNDGOL")
+	if err != nil {
+		t.Errorf("Unexpected error: %s", err)
+		t.FailNow()
+	}
+	if len(items) != 1 {
+		t.Errorf("dictionaries should have 1 items but got %d", len(items))
+		t.FailNow()
+	}
+	i := items[0]
+	if i.Key != "some_key" {
+		t.Errorf("item key assertion error, expects=some_key but got=%s", i.Key)
+		t.FailNow()
+	}
+	if i.Value != "some_value" {
+		t.Errorf("item value assertion error, expects=some_value but got=%s", i.Value)
+		t.FailNow()
+	}
+}