Repository: kythe/kythe
Pull request #4346 (2020-02-03 22:15:53)

test(extraction): check VNameRewriteRules can roundtrip through JSON


Patch:
diff --git a/kythe/go/util/vnameutil/BUILD b/kythe/go/util/vnameutil/BUILD
index 951a22a17..4fe349e0c 100644
--- a/kythe/go/util/vnameutil/BUILD
+++ b/kythe/go/util/vnameutil/BUILD
@@ -24,6 +24,7 @@ go_test(
     deps = [
         "//kythe/proto:storage_go_proto",
         "@com_github_golang_protobuf//proto:go_default_library",
+        "@com_github_google_go_cmp//cmp:go_default_library",
     ],
 )
 
diff --git a/kythe/go/util/vnameutil/rewrite.go b/kythe/go/util/vnameutil/rewrite.go
index de673391e..2c4e34bb6 100644
--- a/kythe/go/util/vnameutil/rewrite.go
+++ b/kythe/go/util/vnameutil/rewrite.go
@@ -32,6 +32,8 @@ import (
 	spb "kythe.io/kythe/proto/storage_go_proto"
 )
 
+var jsonMarshaler = &jsonpb.Marshaler{OrigName: true}
+
 // A Rule associates a regular expression pattern with a VName template.  A
 // Rule can be applied to a string to produce a VName.
 type Rule struct {
@@ -58,6 +60,44 @@ func (r Rule) Apply(input string) (*spb.VName, bool) {
 	}, true
 }
 
+// ToProto returns an equivalent VNameRewriteRule proto.
+func (r Rule) ToProto() *spb.VNameRewriteRule {
+	return &spb.VNameRewriteRule{
+		Pattern: r.Regexp.String(),
+		VName: &spb.VName{
+			Corpus:    unfixTemplate(r.VName.Corpus),
+			Root:      unfixTemplate(r.VName.Root),
+			Path:      unfixTemplate(r.VName.Path),
+			Language:  unfixTemplate(r.VName.Language),
+			Signature: unfixTemplate(r.VName.Signature),
+		},
+	}
+}
+
+// String returns a debug string of r.
+func (r Rule) String() string { return r.ToProto().String() }
+
+// MarshalJSON implements the json.Marshaler interface.
+func (r Rule) MarshalJSON() ([]byte, error) {
+	var buf bytes.Buffer
+	err := jsonMarshaler.Marshal(&buf, r.ToProto())
+	return buf.Bytes(), err
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (r *Rule) UnmarshalJSON(rec []byte) error {
+	var p spb.VNameRewriteRule
+	if err := jsonpb.Unmarshal(bytes.NewReader(rec), &p); err != nil {
+		return err
+	}
+	rule, err := ConvertRule(&p)
+	if err != nil {
+		return err
+	}
+	*r = rule
+	return nil
+}
+
 func (r Rule) expand(match []int, input, template string) string {
 	return string(r.ExpandString(nil, template, input, match))
 }
@@ -105,7 +145,10 @@ func ConvertRule(r *spb.VNameRewriteRule) (Rule, error) {
 	}, nil
 }
 
-var fieldRE = regexp.MustCompile(`@(\w+)@`)
+var (
+	fieldRE  = regexp.MustCompile(`@(\w+)@`)
+	markerRE = regexp.MustCompile(`([^$]|^)(\$\$)*\${\w+}`)
+)
 
 // fixTemplate rewrites @x@ markers in the template to the ${x} markers used by
 // the regexp.Expand function, to simplify rewriting.
@@ -119,16 +162,25 @@ func fixTemplate(s string) string {
 		})
 }
 
+func unfixTemplate(s string) string {
+	return strings.Replace(markerRE.ReplaceAllStringFunc(s, func(s string) string {
+		var prefix int
+		for ; !strings.HasPrefix(s[prefix:], "${"); prefix++ {
+		}
+		return s[:prefix] + "@" + strings.TrimPrefix(strings.TrimSuffix(s[prefix:], "}"), "${") + "@"
+	}), "$$", "$", -1)
+}
+
 func expectDelim(de *json.Decoder, expected json.Delim) error {
 	if tok, err := de.Token(); err != nil {
 		return err
 	} else if delim, ok := tok.(json.Delim); !ok || delim != expected {
-		return fmt.Errorf("expected %s; found %s", expected, tok)
+		return fmt.Errorf("expected %s; found %v", expected, tok)
 	}
 	return nil
 }
 
-// ParseRules reads Rules data from a byte array.
+// ParseRules reads Rules from JSON-encoded data in a byte array.
 func ParseRules(data []byte) (Rules, error) { return ReadRules(bytes.NewReader(data)) }
 
 // ReadRules parses Rules from JSON-encoded data in the following format:
@@ -156,7 +208,7 @@ func ReadRules(r io.Reader) (Rules, error) {
 	}
 
 	// Parse each element of the array as a VNameRewriteRule.
-	var rules Rules
+	rules := Rules{}
 	for de.More() {
 		var pb spb.VNameRewriteRule
 		if err := jsonpb.UnmarshalNext(de, &pb); err != nil {


Test patch:
diff --git a/kythe/go/util/vnameutil/rewrite_test.go b/kythe/go/util/vnameutil/rewrite_test.go
index 7b38effd6..939874c97 100644
--- a/kythe/go/util/vnameutil/rewrite_test.go
+++ b/kythe/go/util/vnameutil/rewrite_test.go
@@ -17,11 +17,14 @@
 package vnameutil
 
 import (
+	"encoding/json"
 	"regexp"
+	"strconv"
 	"strings"
 	"testing"
 
 	"github.com/golang/protobuf/proto"
+	"github.com/google/go-cmp/cmp"
 
 	spb "kythe.io/kythe/proto/storage_go_proto"
 )
@@ -97,6 +100,41 @@ func TestParseConsistency(t *testing.T) {
 	}
 }
 
+func TestRoundtripJSON(t *testing.T) {
+	tests := []string{
+		"[]",
+		`[{"pattern": "p(.)", "vname": {"corpus": "$@1@"}}]`,
+		`[{
+  "pattern": "(?P<corpus>.+)/(?P<root>.+)::(?P<path>.+)",
+  "vname": {"corpus": "@corpus@", "root": "@root@", "path": "@path@"}
+}]`,
+		testConfig,
+	}
+
+	for i, test := range tests {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			expected, err := ReadRules(strings.NewReader(test))
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("Rules: %s", expected)
+			rec, err := json.Marshal(expected)
+			if err != nil {
+				t.Fatalf("Error marshaling rules %+v: %v", expected, err)
+			}
+
+			r, err := ParseRules(rec)
+			if err != nil {
+				t.Fatalf("Error parsing rules %q: %v", rec, err)
+			}
+
+			if diff := cmp.Diff(r, expected, transformRegexp); diff != "" {
+				t.Errorf("Unexpected diff (- found; + expected):\n%s", diff)
+			}
+		})
+	}
+}
+
 func TestCornerCases(t *testing.T) {
 	testRule1 := Rule{regexp.MustCompile(`(?P<first>\w+)(?:/(?P<second>\w+))?`), V{Corpus: "${first}", Path: "${second}"}.pb()}
 	testRule2 := Rule{regexp.MustCompile(`x/(?P<sig>\w+)/y/(?P<tail>.+)$`), V{Path: "${tail}", Sig: "|${sig}|"}.pb()}
@@ -172,3 +210,7 @@ func (v V) pb() *spb.VName {
 		Language:  v.Lang,
 	}
 }
+
+var (
+	transformRegexp = cmp.Transformer("Regexp", func(r *regexp.Regexp) string { return r.String() })
+)