Repository: openconfig/goyang
Pull request #122 (2020-05-15 06:32:28)

Add Support for posix-pattern Extension within YangType
Added global variable `UsePosixPatternExt` to allow a `posix-pattern` extension from `openconfig-extensions.yang` to be used to replace pattern statements. This variable must be flipped prior to using goyang for consistent behaviour.

Restriction when `UsePosixPatternExt` is set:
- The number of `posix-pattern` statements must match the number of `pattern` statements within each type statement. Otherwise `goyang` parsing will fail.


Patch:
diff --git a/pkg/yang/node.go b/pkg/yang/node.go
index 39f08e4..5bedf58 100644
--- a/pkg/yang/node.go
+++ b/pkg/yang/node.go
@@ -124,6 +124,23 @@ func FindModuleByPrefix(n Node, prefix string) *Module {
 	return nil
 }
 
+// MatchingExtensions returns the subset of the given node's extensions
+// that match the given module and identifier.
+func MatchingExtensions(n Node, module, identifier string) ([]*Statement, error) {
+	var matchingExtensions []*Statement
+	for _, ext := range n.Exts() {
+		names := strings.SplitN(ext.Keyword, ":", 2)
+		mod := FindModuleByPrefix(n, names[0])
+		if mod == nil {
+			return nil, fmt.Errorf("MatchingExtensions: module prefix %q not found", names[0])
+		}
+		if len(names) == 2 && names[1] == identifier && mod.Name == module {
+			matchingExtensions = append(matchingExtensions, ext)
+		}
+	}
+	return matchingExtensions, nil
+}
+
 // RootNode returns the submodule or module that n was defined in.
 func RootNode(n Node) *Module {
 	for ; n.ParentNode() != nil; n = n.ParentNode() {
diff --git a/pkg/yang/types.go b/pkg/yang/types.go
index 4c71016..57470b6 100644
--- a/pkg/yang/types.go
+++ b/pkg/yang/types.go
@@ -351,23 +351,46 @@ check:
 	// Patterns are ANDed according to section 9.4.6.  If all the patterns
 	// declared by t were also declared by the type t is based on, then
 	// no patterns are added.
-	patterns := map[string]bool{}
+	seenPatterns := map[string]bool{}
 	for _, p := range y.Pattern {
-		patterns[p] = true
+		seenPatterns[p] = true
 	}
+	seenPOSIXPatterns := map[string]bool{}
+	for _, p := range y.POSIXPattern {
+		seenPOSIXPatterns[p] = true
+	}
+
+	// First parse out the pattern statements.
+	// These patterns are not checked because there is no support for W3C regexes by Go.
 	for _, pv := range t.Pattern {
-		p := pv.Name
-		if _, err := syntax.Parse(p, syntax.Perl); err != nil {
+		if !seenPatterns[pv.Name] {
+			seenPatterns[pv.Name] = true
+			y.Pattern = append(y.Pattern, pv.Name)
+		}
+	}
+
+	// Then, parse out the posix-pattern statements, if they exist.
+	// A YANG module could make use of either or both, so we deal with each separately.
+	posixPatterns, err := MatchingExtensions(t, "openconfig-extensions", "posix-pattern")
+	if err != nil {
+		return []error{err}
+	}
+
+	checkPattern := func(n Node, p string, flags syntax.Flags) {
+		if _, err := syntax.Parse(p, flags); err != nil {
 			if re, ok := err.(*syntax.Error); ok {
 				// Error adds "error parsing regexp" to
 				// the error, re.Code is the real error.
 				err = errors.New(re.Code.String())
 			}
-			errs = append(errs, fmt.Errorf("%s: bad pattern: %v: %s", Source(pv), err, p))
+			errs = append(errs, fmt.Errorf("%s: bad pattern: %v: %s", Source(n), err, p))
 		}
-		if !patterns[p] {
-			patterns[p] = true
-			y.Pattern = append(y.Pattern, p)
+	}
+	for _, ext := range posixPatterns {
+		checkPattern(ext, ext.Argument, syntax.POSIX)
+		if !seenPOSIXPatterns[ext.Argument] {
+			seenPOSIXPatterns[ext.Argument] = true
+			y.POSIXPattern = append(y.POSIXPattern, ext.Argument)
 		}
 	}
 
diff --git a/pkg/yang/types_builtin.go b/pkg/yang/types_builtin.go
index 3f10471..62267a2 100644
--- a/pkg/yang/types_builtin.go
+++ b/pkg/yang/types_builtin.go
@@ -285,6 +285,7 @@ type YangType struct {
 	OptionalInstance bool        `json:",omitempty"` // !require-instances which defaults to true
 	Path             string      `json:",omitempty"` // the path in a leafref
 	Pattern          []string    `json:",omitempty"` // limiting XSD-TYPES expressions on strings
+	POSIXPattern     []string    `json:",omitempty"` // limiting POSIX ERE on strings (specified by openconfig-extensions:posix-pattern)
 	Range            YangRange   `json:",omitempty"` // range for integers
 	Type             []*YangType `json:",omitempty"` // for unions
 }


Test patch:
diff --git a/pkg/yang/node_test.go b/pkg/yang/node_test.go
index ac70aca..a89989c 100644
--- a/pkg/yang/node_test.go
+++ b/pkg/yang/node_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/openconfig/gnmi/errdiff"
 )
 
 func TestNodePath(t *testing.T) {
@@ -69,10 +70,11 @@ func TestNodePath(t *testing.T) {
 // the YANG library.
 func TestNode(t *testing.T) {
 	tests := []struct {
-		desc      string
-		inFn      func(*Modules) (Node, error)
-		inModules map[string]string
-		wantNode  func(Node) error
+		desc          string
+		inFn          func(*Modules) (Node, error)
+		inModules     map[string]string
+		wantNode      func(Node) error
+		wantErrSubstr string
 	}{{
 		desc: "import reference statement",
 		inFn: func(ms *Modules) (Node, error) {
@@ -171,6 +173,190 @@ func TestNode(t *testing.T) {
 
 			return nil
 		},
+	}, {
+		desc: "Test MatchingExtensions",
+		inFn: func(ms *Modules) (Node, error) {
+
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+
+			_, err = ms.FindModuleByPrefix("f")
+			if err != nil {
+				return nil, err
+			}
+
+			return m.Leaf[0].Type, nil
+		},
+		inModules: map[string]string{
+			"test": `
+				module test {
+					prefix "t";
+					namespace "urn:t";
+
+					import foo {
+						prefix "f";
+						description "foo module";
+					}
+
+					import foo2 {
+						prefix "f2";
+						description "foo2 module";
+					}
+
+					leaf test-leaf {
+						type string {
+							pattern 'alpha';
+							// Test different modules and different ext names.
+							f:bar 'boo';
+							f2:bar 'boo2';
+
+							f:bar 'coo';
+							f2:bar 'coo2';
+
+							f:far 'doo';
+							f2:far 'doo2';
+
+							f:bar 'foo';
+							f2:bar 'foo2';
+
+							f:far 'goo';
+							f2:far 'goo2';
+						}
+					}
+				}
+			`,
+			"foo": `
+				module foo {
+					prefix "f";
+					namespace "urn:f";
+
+					extension bar {
+						argument "baz";
+					}
+
+					extension far {
+						argument "baz";
+					}
+				}
+			`,
+			"foo2": `
+				module foo2 {
+					prefix "f2";
+					namespace "urn:f2";
+
+					extension bar {
+						argument "baz";
+					}
+
+					extension far {
+						argument "baz";
+					}
+				}
+			`,
+		},
+		wantNode: func(n Node) error {
+			n, ok := n.(*Type)
+			if !ok {
+				return fmt.Errorf("got node: %v, want type: Leaf", n)
+			}
+
+			var bars []string
+			matches, err := MatchingExtensions(n, "foo", "bar")
+			if err != nil {
+				return err
+			}
+			for _, ext := range matches {
+				bars = append(bars, ext.Argument)
+			}
+
+			if diff := cmp.Diff(bars, []string{"boo", "coo", "foo"}); diff != "" {
+				return fmt.Errorf("MatchingExtensions (-got, +want):\n%s", diff)
+			}
+
+			return nil
+		},
+	}, {
+		desc: "Test MatchingExtensions when module is not found",
+		inFn: func(ms *Modules) (Node, error) {
+
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+
+			_, err = ms.FindModuleByPrefix("f")
+			if err != nil {
+				return nil, err
+			}
+
+			return m.Leaf[0].Type, nil
+		},
+		inModules: map[string]string{
+			"test": `
+				module test {
+					prefix "t";
+					namespace "urn:t";
+
+					import foo {
+						prefix "f";
+						description "foo module";
+					}
+
+					leaf test-leaf {
+						type string {
+							pattern 'alpha';
+							not-found:bar 'foo';
+						}
+					}
+				}
+			`,
+			"foo": `
+				module foo {
+					prefix "f";
+					namespace "urn:f";
+
+					extension bar {
+						argument "baz";
+					}
+
+					extension far {
+						argument "baz";
+					}
+				}
+			`,
+		},
+		wantNode: func(n Node) error {
+			n, ok := n.(*Type)
+			if !ok {
+				return fmt.Errorf("got node: %v, want type: Leaf", n)
+			}
+
+			var bars []string
+			matches, err := MatchingExtensions(n, "foo", "bar")
+			if err != nil {
+				return err
+			}
+			for _, ext := range matches {
+				bars = append(bars, ext.Argument)
+			}
+
+			if diff := cmp.Diff(bars, []string{"boo", "coo", "foo"}); diff != "" {
+				return fmt.Errorf("MatchingExtensions (-got, +want):\n%s", diff)
+			}
+
+			return nil
+		},
+		wantErrSubstr: `module prefix "not-found" not found`,
 	}}
 
 	for _, tt := range tests {
@@ -183,6 +369,20 @@ func TestNode(t *testing.T) {
 				}
 			}
 
+			errs := ms.Process()
+			var err error
+			if len(errs) > 1 {
+				t.Fatalf("Got more than 1 error: %v", errs)
+			} else if len(errs) == 1 {
+				err = errs[0]
+			}
+			if diff := errdiff.Substring(err, tt.wantErrSubstr); diff != "" {
+				t.Errorf("Did not get expected error: %s", diff)
+			}
+			if err != nil {
+				return
+			}
+
 			node, err := tt.inFn(ms)
 			if err != nil {
 				t.Fatalf("cannot run in function, %v", err)
diff --git a/pkg/yang/types_test.go b/pkg/yang/types_test.go
index 3f76105..58f6cc4 100644
--- a/pkg/yang/types_test.go
+++ b/pkg/yang/types_test.go
@@ -15,8 +15,14 @@
 package yang
 
 import (
+	"fmt"
 	"reflect"
+	"sort"
 	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/openconfig/gnmi/errdiff"
 )
 
 func TestTypeResolve(t *testing.T) {
@@ -100,3 +106,327 @@ func TestTypeResolve(t *testing.T) {
 		}
 	}
 }
+
+func TestPattern(t *testing.T) {
+	tests := []struct {
+		desc                string
+		inGetFn             func(*Modules) (*YangType, error)
+		leafNode            string
+		wantPatternsRegular []string
+		wantPatternsPOSIX   []string
+		wantErrSubstr       string
+	}{{
+		desc: "Only normal patterns",
+		leafNode: `
+			leaf test-leaf {
+				type string {
+					o:bar 'coo';
+					o:bar 'foo';
+					pattern 'charlie';
+					o:bar 'goo';
+				}
+			}
+		} // end module`,
+		inGetFn: func(ms *Modules) (*YangType, error) {
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+			e := ToEntry(m)
+			return e.Dir["test-leaf"].Type, nil
+		},
+		wantPatternsRegular: []string{"charlie"},
+	}, {
+		desc: "Only posix patterns",
+		leafNode: `
+			leaf test-leaf {
+				type string {
+					o:bar 'coo';
+					o:posix-pattern 'bravo';
+					o:bar 'foo';
+					o:posix-pattern 'charlie';
+					o:bar 'goo';
+				}
+			}
+		} // end module`,
+		inGetFn: func(ms *Modules) (*YangType, error) {
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+			e := ToEntry(m)
+			return e.Dir["test-leaf"].Type, nil
+		},
+		wantPatternsPOSIX: []string{"bravo", "charlie"},
+	}, {
+		desc: "No patterns",
+		leafNode: `
+			leaf test-leaf {
+				type string;
+			}
+		}`,
+		inGetFn: func(ms *Modules) (*YangType, error) {
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+			e := ToEntry(m)
+			return e.Dir["test-leaf"].Type, nil
+		},
+		wantPatternsRegular: nil,
+		wantPatternsPOSIX:   nil,
+	}, {
+		desc: "Both patterns",
+		leafNode: `
+			leaf test-leaf {
+				type string {
+					pattern 'alpha';
+					o:posix-pattern 'bravo';
+					o:posix-pattern 'charlie';
+					o:bar 'coo';
+					o:posix-pattern 'delta';
+				}
+			}
+		} // end module`,
+		inGetFn: func(ms *Modules) (*YangType, error) {
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+			e := ToEntry(m)
+			return e.Dir["test-leaf"].Type, nil
+		},
+		wantPatternsRegular: []string{"alpha"},
+		wantPatternsPOSIX:   []string{"bravo", "charlie", "delta"},
+	}, {
+		desc: "Both patterns, but with non-openconfig-extensions pretenders",
+		leafNode: `
+			leaf test-leaf {
+				type string {
+					pattern 'alpha';
+					o:bar 'coo';
+					o:posix-pattern 'delta';
+
+					n:posix-pattern 'golf';
+
+					pattern 'bravo';
+					o:bar 'foo';
+					o:posix-pattern 'echo';
+
+					pattern 'charlie';
+					o:bar 'goo';
+					o:posix-pattern 'foxtrot';
+
+					n:posix-pattern 'hotel';
+				}
+			}
+		} // end module`,
+		inGetFn: func(ms *Modules) (*YangType, error) {
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+			e := ToEntry(m)
+			return e.Dir["test-leaf"].Type, nil
+		},
+		wantPatternsRegular: []string{"alpha", "bravo", "charlie"},
+		wantPatternsPOSIX:   []string{"delta", "echo", "foxtrot"},
+	}, {
+		desc: "Union type",
+		leafNode: `
+			leaf test-leaf {
+				type union {
+					type string {
+						pattern 'alpha';
+						o:bar 'coo';
+						o:posix-pattern 'delta';
+
+						pattern 'bravo';
+						o:bar 'foo';
+						o:posix-pattern 'echo';
+						n:posix-pattern 'echo2';
+
+						pattern 'charlie';
+						o:bar 'goo';
+						o:posix-pattern 'foxtrot';
+					}
+					type uint64;
+				}
+			}
+		} // end module`,
+		inGetFn: func(ms *Modules) (*YangType, error) {
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+			e := ToEntry(m)
+			return e.Dir["test-leaf"].Type.Type[0], nil
+		},
+		wantPatternsRegular: []string{"alpha", "bravo", "charlie"},
+		wantPatternsPOSIX:   []string{"delta", "echo", "foxtrot"},
+	}, {
+		desc: "typedef",
+		leafNode: `
+			leaf test-leaf {
+				type leaf-type;
+			}
+
+			typedef leaf-type {
+				type string {
+					pattern 'alpha';
+					o:bar 'coo';
+					o:posix-pattern 'delta';
+
+					pattern 'bravo';
+					o:bar 'foo';
+					o:posix-pattern 'echo';
+
+					pattern 'charlie';
+					o:bar 'goo';
+					o:posix-pattern 'foxtrot';
+				}
+			}
+		} // end module`,
+		inGetFn: func(ms *Modules) (*YangType, error) {
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+			e := ToEntry(m)
+			return e.Dir["test-leaf"].Type, nil
+		},
+		wantPatternsRegular: []string{"alpha", "bravo", "charlie"},
+		wantPatternsPOSIX:   []string{"delta", "echo", "foxtrot"},
+	}, {
+		desc: "invalid POSIX pattern",
+		leafNode: `
+			leaf test-leaf {
+				type leaf-type;
+			}
+
+			typedef leaf-type {
+				type string {
+					o:posix-pattern '?';
+				}
+			}
+		} // end module`,
+		inGetFn: func(ms *Modules) (*YangType, error) {
+			m, err := ms.FindModuleByPrefix("t")
+			if err != nil {
+				return nil, fmt.Errorf("can't find module in %v", ms)
+			}
+			if len(m.Leaf) == 0 {
+				return nil, fmt.Errorf("node %v is missing imports", m)
+			}
+			e := ToEntry(m)
+			return e.Dir["test-leaf"].Type, nil
+		},
+		wantErrSubstr: "bad pattern",
+	}}
+
+	for _, tt := range tests {
+		inModules := map[string]string{
+			"test": `
+				module test {
+					prefix "t";
+					namespace "urn:t";
+
+					import non-openconfig-extensions {
+						prefix "n";
+						description "non-openconfig-extensions module";
+					}
+					import openconfig-extensions {
+						prefix "o";
+						description "openconfig-extensions module";
+					}` + tt.leafNode,
+			"openconfig-extensions": `
+				module openconfig-extensions {
+					prefix "o";
+					namespace "urn:o";
+
+					extension bar {
+						argument "baz";
+					}
+
+					extension posix-pattern {
+						argument "pattern";
+					}
+				}
+			`,
+			"non-openconfig-extensions": `
+				module non-openconfig-extensions {
+					prefix "n";
+					namespace "urn:n";
+
+					extension bar {
+						argument "baz";
+					}
+
+					extension posix-pattern {
+						argument "pattern";
+					}
+				}
+			`,
+		}
+
+		t.Run(tt.desc, func(t *testing.T) {
+			ms := NewModules()
+			for n, m := range inModules {
+				if err := ms.Parse(m, n); err != nil {
+					t.Fatalf("error parsing module %s, got: %v, want: nil", n, err)
+				}
+			}
+			errs := ms.Process()
+			var err error
+			if len(errs) > 1 {
+				t.Fatalf("Got more than 1 error: %v", errs)
+			} else if len(errs) == 1 {
+				err = errs[0]
+			}
+			if diff := errdiff.Substring(err, tt.wantErrSubstr); diff != "" {
+				t.Errorf("Did not get expected error: %s", diff)
+			}
+			if err != nil {
+				return
+			}
+
+			yangType, err := tt.inGetFn(ms)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			sort.Strings(yangType.Pattern)
+			sort.Strings(tt.wantPatternsRegular)
+			if diff := cmp.Diff(yangType.Pattern, tt.wantPatternsRegular, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("Type.resolve() pattern test (-got, +want):\n%s", diff)
+			}
+
+			sort.Strings(yangType.POSIXPattern)
+			sort.Strings(tt.wantPatternsPOSIX)
+			if diff := cmp.Diff(yangType.POSIXPattern, tt.wantPatternsPOSIX, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("Type.resolve() posix-pattern test (-got, +want):\n%s", diff)
+			}
+		})
+	}
+}