Repository: goccy/go-yaml
Pull request #252 (2021-09-07 08:38:06)

Support getting YAMLPath from ast.Node
Support `GetPath() string` API to the `ast.Node` .
This API allows you to get the YAMLPath from `ast.Node` .



Patch:
diff --git a/ast/ast.go b/ast/ast.go
index 6a1aea7..83b1ee6 100644
--- a/ast/ast.go
+++ b/ast/ast.go
@@ -179,6 +179,10 @@ type Node interface {
 	SetComment(*CommentGroupNode) error
 	// Comment returns comment token instance
 	GetComment() *CommentGroupNode
+	// GetPath returns YAMLPath for the current node
+	GetPath() string
+	// SetPath set YAMLPath for the current node
+	SetPath(string)
 	// MarshalYAML
 	MarshalYAML() ([]byte, error)
 	// already read length
@@ -198,6 +202,7 @@ type ScalarNode interface {
 }
 
 type BaseNode struct {
+	Path    string
 	Comment *CommentGroupNode
 	read    int
 }
@@ -218,6 +223,22 @@ func (n *BaseNode) addReadLen(len int) {
 	n.read += len
 }
 
+// GetPath returns YAMLPath for the current node.
+func (n *BaseNode) GetPath() string {
+	if n == nil {
+		return ""
+	}
+	return n.Path
+}
+
+// SetPath set YAMLPath for the current node.
+func (n *BaseNode) SetPath(path string) {
+	if n == nil {
+		return
+	}
+	n.Path = path
+}
+
 // GetComment returns comment token instance
 func (n *BaseNode) GetComment() *CommentGroupNode {
 	return n.Comment
@@ -1916,42 +1937,70 @@ func Walk(v Visitor, node Node) {
 	switch n := node.(type) {
 	case *CommentNode:
 	case *NullNode:
+		walkComment(v, n.BaseNode)
 	case *IntegerNode:
+		walkComment(v, n.BaseNode)
 	case *FloatNode:
+		walkComment(v, n.BaseNode)
 	case *StringNode:
+		walkComment(v, n.BaseNode)
 	case *MergeKeyNode:
+		walkComment(v, n.BaseNode)
 	case *BoolNode:
+		walkComment(v, n.BaseNode)
 	case *InfinityNode:
+		walkComment(v, n.BaseNode)
 	case *NanNode:
+		walkComment(v, n.BaseNode)
 	case *LiteralNode:
+		walkComment(v, n.BaseNode)
 		Walk(v, n.Value)
 	case *DirectiveNode:
+		walkComment(v, n.BaseNode)
 		Walk(v, n.Value)
 	case *TagNode:
+		walkComment(v, n.BaseNode)
 		Walk(v, n.Value)
 	case *DocumentNode:
+		walkComment(v, n.BaseNode)
 		Walk(v, n.Body)
 	case *MappingNode:
+		walkComment(v, n.BaseNode)
 		for _, value := range n.Values {
 			Walk(v, value)
 		}
 	case *MappingKeyNode:
+		walkComment(v, n.BaseNode)
 		Walk(v, n.Value)
 	case *MappingValueNode:
+		walkComment(v, n.BaseNode)
 		Walk(v, n.Key)
 		Walk(v, n.Value)
 	case *SequenceNode:
+		walkComment(v, n.BaseNode)
 		for _, value := range n.Values {
 			Walk(v, value)
 		}
 	case *AnchorNode:
+		walkComment(v, n.BaseNode)
 		Walk(v, n.Name)
 		Walk(v, n.Value)
 	case *AliasNode:
+		walkComment(v, n.BaseNode)
 		Walk(v, n.Value)
 	}
 }
 
+func walkComment(v Visitor, base *BaseNode) {
+	if base == nil {
+		return
+	}
+	if base.Comment == nil {
+		return
+	}
+	Walk(v, base.Comment)
+}
+
 type filterWalker struct {
 	typ     NodeType
 	results []Node
diff --git a/parser/context.go b/parser/context.go
index 5749e43..9e93fd4 100644
--- a/parser/context.go
+++ b/parser/context.go
@@ -1,13 +1,64 @@
 package parser
 
-import "github.com/goccy/go-yaml/token"
+import (
+	"fmt"
+	"strings"
+
+	"github.com/goccy/go-yaml/token"
+)
 
 // context context at parsing
 type context struct {
+	parent *context
 	idx    int
 	size   int
 	tokens token.Tokens
 	mode   Mode
+	path   string
+}
+
+var pathSpecialChars = []string{
+	"$", "*", ".", "[", "]",
+}
+
+func containsPathSpecialChar(path string) bool {
+	for _, char := range pathSpecialChars {
+		if strings.Contains(path, char) {
+			return true
+		}
+	}
+	return false
+}
+
+func normalizePath(path string) string {
+	if containsPathSpecialChar(path) {
+		return fmt.Sprintf("'%s'", path)
+	}
+	return path
+}
+
+func (c *context) withChild(path string) *context {
+	ctx := c.copy()
+	path = normalizePath(path)
+	ctx.path += fmt.Sprintf(".%s", path)
+	return ctx
+}
+
+func (c *context) withIndex(idx uint) *context {
+	ctx := c.copy()
+	ctx.path += fmt.Sprintf("[%d]", idx)
+	return ctx
+}
+
+func (c *context) copy() *context {
+	return &context{
+		parent: c,
+		idx:    c.idx,
+		size:   c.size,
+		tokens: append(token.Tokens{}, c.tokens...),
+		mode:   c.mode,
+		path:   c.path,
+	}
 }
 
 func (c *context) next() bool {
@@ -22,6 +73,9 @@ func (c *context) previousToken() *token.Token {
 }
 
 func (c *context) insertToken(idx int, tk *token.Token) {
+	if c.parent != nil {
+		c.parent.insertToken(idx, tk)
+	}
 	if c.size < idx {
 		return
 	}
@@ -104,6 +158,9 @@ func (c *context) isCurrentCommentToken() bool {
 }
 
 func (c *context) progressIgnoreComment(num int) {
+	if c.parent != nil {
+		c.parent.progressIgnoreComment(num)
+	}
 	if c.size <= c.idx+num {
 		c.idx = c.size
 	} else {
@@ -135,5 +192,6 @@ func newContext(tokens token.Tokens, mode Mode) *context {
 		size:   len(filteredTokens),
 		tokens: filteredTokens,
 		mode:   mode,
+		path:   "$",
 	}
 }
diff --git a/parser/parser.go b/parser/parser.go
index 70f6e5c..fcbd3bd 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -15,7 +15,9 @@ import (
 type parser struct{}
 
 func (p *parser) parseMapping(ctx *context) (ast.Node, error) {
-	node := ast.Mapping(ctx.currentToken(), true)
+	mapTk := ctx.currentToken()
+	node := ast.Mapping(mapTk, true)
+	node.SetPath(ctx.path)
 	ctx.progress(1) // skip MappingStart token
 	for ctx.next() {
 		tk := ctx.currentToken()
@@ -43,6 +45,7 @@ func (p *parser) parseMapping(ctx *context) (ast.Node, error) {
 
 func (p *parser) parseSequence(ctx *context) (ast.Node, error) {
 	node := ast.Sequence(ctx.currentToken(), true)
+	node.SetPath(ctx.path)
 	ctx.progress(1) // skip SequenceStart token
 	for ctx.next() {
 		tk := ctx.currentToken()
@@ -54,7 +57,7 @@ func (p *parser) parseSequence(ctx *context) (ast.Node, error) {
 			continue
 		}
 
-		value, err := p.parseToken(ctx, tk)
+		value, err := p.parseToken(ctx.withIndex(uint(len(node.Values))), tk)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to parse sequence value in flow sequence node")
 		}
@@ -67,6 +70,7 @@ func (p *parser) parseSequence(ctx *context) (ast.Node, error) {
 func (p *parser) parseTag(ctx *context) (ast.Node, error) {
 	tagToken := ctx.currentToken()
 	node := ast.Tag(tagToken)
+	node.SetPath(ctx.path)
 	ctx.progress(1) // skip tag token
 	var (
 		value ast.Node
@@ -135,6 +139,17 @@ func (p *parser) createNullToken(base *token.Token) *token.Token {
 }
 
 func (p *parser) parseMapValue(ctx *context, key ast.Node, colonToken *token.Token) (ast.Node, error) {
+	node, err := p.createMapValueNode(ctx, key, colonToken)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create map value node")
+	}
+	if node != nil && node.GetPath() == "" {
+		node.SetPath(ctx.path)
+	}
+	return node, nil
+}
+
+func (p *parser) createMapValueNode(ctx *context, key ast.Node, colonToken *token.Token) (ast.Node, error) {
 	tk := ctx.currentToken()
 	if tk == nil {
 		nullToken := p.createNullToken(colonToken)
@@ -190,6 +205,8 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse map key")
 	}
+	keyText := key.GetToken().Value
+	key.SetPath(ctx.withChild(keyText).path)
 	if err := p.validateMapKey(key.GetToken()); err != nil {
 		return nil, errors.Wrapf(err, "validate mapping key error")
 	}
@@ -199,7 +216,7 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
 		return nil, errors.ErrSyntax("unexpected map", key.GetToken())
 	}
 	ctx.progress(1) // progress to value token
-	if err := p.setSameLineCommentIfExists(ctx, key); err != nil {
+	if err := p.setSameLineCommentIfExists(ctx.withChild(keyText), key); err != nil {
 		return nil, errors.Wrapf(err, "failed to set same line comment to node")
 	}
 	if key.GetComment() != nil {
@@ -208,7 +225,7 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
 		ctx.progressIgnoreComment(1)
 	}
 
-	value, err := p.parseMapValue(ctx, key, tk)
+	value, err := p.parseMapValue(ctx.withChild(keyText), key, tk)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse map value")
 	}
@@ -217,7 +234,9 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
 	}
 
 	mvnode := ast.MappingValue(tk, key, value)
+	mvnode.SetPath(ctx.withChild(keyText).path)
 	node := ast.Mapping(tk, false, mvnode)
+	node.SetPath(ctx.withChild(keyText).path)
 
 	ntk := ctx.nextNotCommentToken()
 	antk := ctx.afterNextNotCommentToken()
@@ -257,6 +276,7 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
 func (p *parser) parseSequenceEntry(ctx *context) (ast.Node, error) {
 	tk := ctx.currentToken()
 	sequenceNode := ast.Sequence(tk, false)
+	sequenceNode.SetPath(ctx.path)
 	curColumn := tk.Position.Column
 	for tk.Type == token.SequenceEntryType {
 		ctx.progress(1) // skip sequence token
@@ -270,11 +290,12 @@ func (p *parser) parseSequenceEntry(ctx *context) (ast.Node, error) {
 			}
 			ctx.progress(1) // skip sequence token
 		}
-		value, err := p.parseToken(ctx, ctx.currentToken())
+		value, err := p.parseToken(ctx.withIndex(uint(len(sequenceNode.Values))), ctx.currentToken())
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to parse sequence")
 		}
 		if comment != nil {
+			comment.SetPath(ctx.withIndex(uint(len(sequenceNode.Values))).path)
 			sequenceNode.ValueComments = append(sequenceNode.ValueComments, comment)
 		} else {
 			sequenceNode.ValueComments = append(sequenceNode.ValueComments, nil)
@@ -298,6 +319,7 @@ func (p *parser) parseSequenceEntry(ctx *context) (ast.Node, error) {
 func (p *parser) parseAnchor(ctx *context) (ast.Node, error) {
 	tk := ctx.currentToken()
 	anchor := ast.Anchor(tk)
+	anchor.SetPath(ctx.path)
 	ntk := ctx.nextToken()
 	if ntk == nil {
 		return nil, errors.ErrSyntax("unexpected anchor. anchor name is undefined", tk)
@@ -324,6 +346,7 @@ func (p *parser) parseAnchor(ctx *context) (ast.Node, error) {
 func (p *parser) parseAlias(ctx *context) (ast.Node, error) {
 	tk := ctx.currentToken()
 	alias := ast.Alias(tk)
+	alias.SetPath(ctx.path)
 	ntk := ctx.nextToken()
 	if ntk == nil {
 		return nil, errors.ErrSyntax("unexpected alias. alias name is undefined", tk)
@@ -366,6 +389,7 @@ func (p *parser) parseScalarValueWithComment(ctx *context, tk *token.Token) (ast
 	if node == nil {
 		return nil, nil
 	}
+	node.SetPath(ctx.path)
 	if p.isSameLineComment(ctx.nextToken(), node) {
 		ctx.progress(1)
 		if err := p.setSameLineCommentIfExists(ctx, node); err != nil {
@@ -428,6 +452,7 @@ func (p *parser) parseLiteral(ctx *context) (ast.Node, error) {
 	var comment *ast.CommentGroupNode
 	if tk.Type == token.CommentType {
 		comment = p.parseCommentOnly(ctx)
+		comment.SetPath(ctx.path)
 		if err := node.SetComment(comment); err != nil {
 			return nil, errors.Wrapf(err, "failed to set comment to literal")
 		}
@@ -460,7 +485,9 @@ func (p *parser) setSameLineCommentIfExists(ctx *context, node ast.Node) error {
 	if !p.isSameLineComment(tk, node) {
 		return nil
 	}
-	if err := node.SetComment(ast.CommentGroup([]*token.Token{tk})); err != nil {
+	comment := ast.CommentGroup([]*token.Token{tk})
+	comment.SetPath(ctx.path)
+	if err := node.SetComment(comment); err != nil {
 		return errors.Wrapf(err, "failed to set comment token to ast.Node")
 	}
 	return nil
@@ -506,6 +533,7 @@ func (p *parser) parseComment(ctx *context) (ast.Node, error) {
 	if node == nil {
 		return group, nil
 	}
+	group.SetPath(node.GetPath())
 	if err := node.SetComment(group); err != nil {
 		return nil, errors.Wrapf(err, "failed to set comment token to node")
 	}
@@ -513,9 +541,11 @@ func (p *parser) parseComment(ctx *context) (ast.Node, error) {
 }
 
 func (p *parser) parseMappingKey(ctx *context) (ast.Node, error) {
-	node := ast.MappingKey(ctx.currentToken())
+	keyTk := ctx.currentToken()
+	node := ast.MappingKey(keyTk)
+	node.SetPath(ctx.path)
 	ctx.progress(1) // skip mapping key token
-	value, err := p.parseToken(ctx, ctx.currentToken())
+	value, err := p.parseToken(ctx.withChild(keyTk.Value), ctx.currentToken())
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse map key")
 	}
@@ -524,6 +554,17 @@ func (p *parser) parseMappingKey(ctx *context) (ast.Node, error) {
 }
 
 func (p *parser) parseToken(ctx *context, tk *token.Token) (ast.Node, error) {
+	node, err := p.createNodeFromToken(ctx, tk)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to create node from token")
+	}
+	if node != nil && node.GetPath() == "" {
+		node.SetPath(ctx.path)
+	}
+	return node, nil
+}
+
+func (p *parser) createNodeFromToken(ctx *context, tk *token.Token) (ast.Node, error) {
 	if tk == nil {
 		return nil, nil
 	}


Test patch:
diff --git a/parser/parser_test.go b/parser/parser_test.go
index f6a97e6..9d86b05 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -3,12 +3,14 @@ package parser_test
 import (
 	"fmt"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 
 	"github.com/goccy/go-yaml/ast"
 	"github.com/goccy/go-yaml/lexer"
 	"github.com/goccy/go-yaml/parser"
+	"github.com/goccy/go-yaml/token"
 )
 
 func TestParser(t *testing.T) {
@@ -758,6 +760,73 @@ foo: > # comment
 	}
 }
 
+func TestNodePath(t *testing.T) {
+	yml := `
+a: # commentA
+  b: # commentB
+    c: foo # commentC
+    d: bar # commentD
+    e: baz # commentE
+  f: # commentF
+    g: hoge # commentG
+  h: # commentH
+   - list1 # comment list1
+   - list2 # comment list2
+   - list3 # comment list3
+  i: fuga # commentI
+j: piyo # commentJ
+k.l.m.n: moge # commentKLMN
+`
+	f, err := parser.ParseBytes([]byte(yml), parser.ParseComments)
+	if err != nil {
+		t.Fatalf("%+v", err)
+	}
+	var capturer pathCapturer
+	for _, doc := range f.Docs {
+		ast.Walk(&capturer, doc.Body)
+	}
+	commentPaths := []string{}
+	for i := 0; i < capturer.capturedNum; i++ {
+		if capturer.orderedTypes[i] == ast.CommentType {
+			commentPaths = append(commentPaths, capturer.orderedPaths[i])
+		}
+	}
+	expectedPaths := []string{
+		"$.a",
+		"$.a.b",
+		"$.a.b.c",
+		"$.a.b.d",
+		"$.a.b.e",
+		"$.a.f",
+		"$.a.f.g",
+		"$.a.h",
+		"$.a.h[0]",
+		"$.a.h[1]",
+		"$.a.h[2]",
+		"$.a.i",
+		"$.j",
+		"$.'k.l.m.n'",
+	}
+	if !reflect.DeepEqual(expectedPaths, commentPaths) {
+		t.Fatalf("failed to get YAMLPath to the comment node:\nexpected[%s]\ngot     [%s]", expectedPaths, commentPaths)
+	}
+}
+
+type pathCapturer struct {
+	capturedNum   int
+	orderedPaths  []string
+	orderedTypes  []ast.NodeType
+	orderedTokens []*token.Token
+}
+
+func (c *pathCapturer) Visit(node ast.Node) ast.Visitor {
+	c.capturedNum++
+	c.orderedPaths = append(c.orderedPaths, node.GetPath())
+	c.orderedTypes = append(c.orderedTypes, node.Type())
+	c.orderedTokens = append(c.orderedTokens, node.GetToken())
+	return c
+}
+
 type Visitor struct {
 }