Repository: goccy/go-yaml
Pull request #19 (2019-11-05 08:14:33)

Fix parser API
### BEFORE

```
var p parser.Parser
doc, err := p.Parse(tokens)
```

### AFTER

```
file, err := parser.Parse(tokens, parser.Mode(0))
// file contains multiple documents
```



Patch:
diff --git a/ast/ast.go b/ast/ast.go
index cefcddf..8831b3b 100644
--- a/ast/ast.go
+++ b/ast/ast.go
@@ -104,10 +104,31 @@ type Node interface {
 	Type() NodeType
 }
 
+// File contains all documents in YAML file
+type File struct {
+	Name string
+	Docs []*Document
+}
+
+// String all documents to text
+func (f *File) String() string {
+	docs := []string{}
+	for _, doc := range f.Docs {
+		docs = append(docs, doc.String())
+	}
+	return strings.Join(docs, "\n")
+}
+
 // Document type of Document
 type Document struct {
-	// Nodes all nodes in document
-	Nodes []Node
+	Start *token.Token // position of DocumentHeader ( `---` )
+	End   *token.Token // position of DocumentEnd ( `...` )
+	Body  Node
+}
+
+// GetToken returns token instance
+func (d *Document) GetToken() *token.Token {
+	return d.Body.GetToken()
 }
 
 // Type returns DocumentType
@@ -115,11 +136,15 @@ func (d *Document) Type() NodeType { return DocumentType }
 
 // String document to text
 func (d *Document) String() string {
-	values := []string{}
-	for _, node := range d.Nodes {
-		values = append(values, strings.TrimLeft(node.String(), " "))
+	doc := []string{}
+	if d.Start != nil {
+		doc = append(doc, d.Start.Value)
 	}
-	return strings.Join(values, "\n")
+	doc = append(doc, d.Body.String())
+	if d.End != nil {
+		doc = append(doc, d.End.Value)
+	}
+	return strings.Join(doc, "\n")
 }
 
 // ScalarNode type for scalar node
diff --git a/decode.go b/decode.go
index 7b1db4a..f5233f9 100644
--- a/decode.go
+++ b/decode.go
@@ -14,7 +14,6 @@ import (
 
 	"github.com/goccy/go-yaml/ast"
 	"github.com/goccy/go-yaml/internal/errors"
-	"github.com/goccy/go-yaml/lexer"
 	"github.com/goccy/go-yaml/parser"
 	"github.com/goccy/go-yaml/token"
 	"golang.org/x/xerrors"
@@ -205,10 +204,10 @@ func (d *Decoder) getArrayNode(node ast.Node) (ast.ArrayNode, error) {
 	return arrayNode, nil
 }
 
-func (d *Decoder) docToNode(doc *ast.Document) ast.Node {
-	for _, node := range doc.Nodes {
-		if v := d.nodeToValue(node); v != nil {
-			return node
+func (d *Decoder) fileToNode(f *ast.File) ast.Node {
+	for _, doc := range f.Docs {
+		if v := d.nodeToValue(doc.Body); v != nil {
+			return doc.Body
 		}
 	}
 	return nil
@@ -768,15 +767,11 @@ func (d *Decoder) resolveReference() error {
 }
 
 func (d *Decoder) decode(bytes []byte) (ast.Node, error) {
-	var (
-		parser parser.Parser
-	)
-	tokens := lexer.Tokenize(string(bytes))
-	doc, err := parser.Parse(tokens)
+	f, err := parser.ParseBytes(bytes, 0)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse yaml")
 	}
-	return d.docToNode(doc), nil
+	return d.fileToNode(f), nil
 }
 
 // Decode reads the next YAML-encoded value from its input
diff --git a/encode.go b/encode.go
index d6f0535..97b8063 100644
--- a/encode.go
+++ b/encode.go
@@ -11,7 +11,6 @@ import (
 
 	"github.com/goccy/go-yaml/ast"
 	"github.com/goccy/go-yaml/internal/errors"
-	"github.com/goccy/go-yaml/lexer"
 	"github.com/goccy/go-yaml/parser"
 	"github.com/goccy/go-yaml/printer"
 	"github.com/goccy/go-yaml/token"
@@ -80,17 +79,13 @@ func (e *Encoder) Encode(v interface{}) error {
 }
 
 func (e *Encoder) encodeDocument(doc []byte) (ast.Node, error) {
-	var (
-		parser parser.Parser
-	)
-	tokens := lexer.Tokenize(string(doc))
-	docNode, err := parser.Parse(tokens)
+	f, err := parser.ParseBytes(doc, 0)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse yaml")
 	}
-	for _, node := range docNode.Nodes {
-		if node != nil {
-			return node, nil
+	for _, docNode := range f.Docs {
+		if docNode.Body != nil {
+			return docNode.Body, nil
 		}
 	}
 	return nil, nil
diff --git a/parser/context.go b/parser/context.go
new file mode 100644
index 0000000..f208d10
--- /dev/null
+++ b/parser/context.go
@@ -0,0 +1,75 @@
+package parser
+
+import "github.com/goccy/go-yaml/token"
+
+// context context at parsing
+type context struct {
+	idx    int
+	size   int
+	tokens token.Tokens
+	mode   Mode
+}
+
+func (c *context) next() bool {
+	return c.idx < c.size
+}
+
+func (c *context) previousToken() *token.Token {
+	if c.idx > 0 {
+		return c.tokens[c.idx-1]
+	}
+	return nil
+}
+
+func (c *context) currentToken() *token.Token {
+	if c.idx >= c.size {
+		return nil
+	}
+	return c.tokens[c.idx]
+}
+
+func (c *context) nextToken() *token.Token {
+	if c.size > c.idx+1 {
+		return c.tokens[c.idx+1]
+	}
+	return nil
+}
+
+func (c *context) afterNextToken() *token.Token {
+	if c.size > c.idx+2 {
+		return c.tokens[c.idx+2]
+	}
+	return nil
+}
+
+func (c *context) enabledComment() bool {
+	return c.mode&ParseComments != 0
+}
+
+func (c *context) progress(num int) {
+	if c.size <= c.idx+num {
+		c.idx = c.size
+	} else {
+		c.idx += num
+	}
+}
+
+func newContext(tokens token.Tokens, mode Mode) *context {
+	filteredTokens := token.Tokens{}
+	if mode&ParseComments != 0 {
+		filteredTokens = tokens
+	} else {
+		for _, tk := range tokens {
+			if tk.Type == token.CommentType {
+				continue
+			}
+			filteredTokens.Add(tk)
+		}
+	}
+	return &context{
+		idx:    0,
+		size:   len(filteredTokens),
+		tokens: filteredTokens,
+		mode:   mode,
+	}
+}
diff --git a/parser/parser.go b/parser/parser.go
index 216d788..83c0887 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -1,81 +1,19 @@
 package parser
 
 import (
+	"io/ioutil"
 	"strings"
 
 	"github.com/goccy/go-yaml/ast"
 	"github.com/goccy/go-yaml/internal/errors"
+	"github.com/goccy/go-yaml/lexer"
 	"github.com/goccy/go-yaml/token"
 	"golang.org/x/xerrors"
 )
 
-// Parser convert from token instances to ast
-type Parser struct {
-}
-
-// Context context at parsing
-type Context struct {
-	idx    int
-	size   int
-	tokens token.Tokens
-}
-
-func (ctx *Context) next() bool {
-	return ctx.idx < ctx.size
-}
-
-func (ctx *Context) previousToken() *token.Token {
-	if ctx.idx > 0 {
-		return ctx.tokens[ctx.idx-1]
-	}
-	return nil
-}
-
-func (ctx *Context) currentToken() *token.Token {
-	if ctx.idx >= ctx.size {
-		return nil
-	}
-	return ctx.tokens[ctx.idx]
-}
-
-func (ctx *Context) nextToken() *token.Token {
-	if ctx.size > ctx.idx+1 {
-		return ctx.tokens[ctx.idx+1]
-	}
-	return nil
-}
-
-func (ctx *Context) afterNextToken() *token.Token {
-	if ctx.size > ctx.idx+2 {
-		return ctx.tokens[ctx.idx+2]
-	}
-	return nil
-}
-
-func (ctx *Context) progress(num int) {
-	if ctx.size <= ctx.idx+num {
-		ctx.idx = ctx.size
-	} else {
-		ctx.idx += num
-	}
-}
-
-func newContext(tokens token.Tokens) *Context {
-	removedCommentTokens := token.Tokens{}
-	for _, tk := range tokens {
-		if tk.Type == token.CommentType {
-			continue
-		}
-		removedCommentTokens.Add(tk)
-	}
-	return &Context{
-		idx:    0,
-		size:   len(removedCommentTokens),
-		tokens: removedCommentTokens,
-	}
-}
+type parser struct{}
 
-func (p *Parser) parseMapping(ctx *Context) (ast.Node, error) {
+func (p *parser) parseMapping(ctx *context) (ast.Node, error) {
 	node := ast.Mapping(ctx.currentToken(), true)
 	ctx.progress(1) // skip MappingStart token
 	for ctx.next() {
@@ -102,7 +40,7 @@ func (p *Parser) parseMapping(ctx *Context) (ast.Node, error) {
 	return node, nil
 }
 
-func (p *Parser) parseSequence(ctx *Context) (ast.Node, error) {
+func (p *parser) parseSequence(ctx *context) (ast.Node, error) {
 	node := ast.Sequence(ctx.currentToken(), true)
 	ctx.progress(1) // skip SequenceStart token
 	for ctx.next() {
@@ -125,7 +63,7 @@ func (p *Parser) parseSequence(ctx *Context) (ast.Node, error) {
 	return node, nil
 }
 
-func (p *Parser) parseTag(ctx *Context) (ast.Node, error) {
+func (p *parser) parseTag(ctx *context) (ast.Node, error) {
 	node := &ast.TagNode{Start: ctx.currentToken()}
 	ctx.progress(1) // skip tag token
 	value, err := p.parseToken(ctx, ctx.currentToken())
@@ -136,7 +74,7 @@ func (p *Parser) parseTag(ctx *Context) (ast.Node, error) {
 	return node, nil
 }
 
-func (p *Parser) validateMapKey(tk *token.Token) error {
+func (p *parser) validateMapKey(tk *token.Token) error {
 	if tk.Type != token.StringType {
 		return nil
 	}
@@ -147,7 +85,7 @@ func (p *Parser) validateMapKey(tk *token.Token) error {
 	return nil
 }
 
-func (p *Parser) parseMappingValue(ctx *Context) (ast.Node, error) {
+func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
 	key := p.parseMapKey(ctx.currentToken())
 	if key == nil {
 		return nil, errors.ErrSyntax("unexpected mapping 'key'. key is undefined", ctx.currentToken())
@@ -219,7 +157,7 @@ func (p *Parser) parseMappingValue(ctx *Context) (ast.Node, error) {
 	return node, nil
 }
 
-func (p *Parser) parseSequenceEntry(ctx *Context) (ast.Node, error) {
+func (p *parser) parseSequenceEntry(ctx *context) (ast.Node, error) {
 	tk := ctx.currentToken()
 	sequenceNode := &ast.SequenceNode{
 		Start:  tk,
@@ -248,7 +186,7 @@ func (p *Parser) parseSequenceEntry(ctx *Context) (ast.Node, error) {
 	return sequenceNode, nil
 }
 
-func (p *Parser) parseAnchor(ctx *Context) (ast.Node, error) {
+func (p *parser) parseAnchor(ctx *context) (ast.Node, error) {
 	tk := ctx.currentToken()
 	anchor := &ast.AnchorNode{Start: tk}
 	ntk := ctx.nextToken()
@@ -274,7 +212,7 @@ func (p *Parser) parseAnchor(ctx *Context) (ast.Node, error) {
 	return anchor, nil
 }
 
-func (p *Parser) parseAlias(ctx *Context) (ast.Node, error) {
+func (p *parser) parseAlias(ctx *context) (ast.Node, error) {
 	tk := ctx.currentToken()
 	alias := &ast.AliasNode{Start: tk}
 	ntk := ctx.nextToken()
@@ -290,7 +228,7 @@ func (p *Parser) parseAlias(ctx *Context) (ast.Node, error) {
 	return alias, nil
 }
 
-func (p *Parser) parseMapKey(tk *token.Token) ast.Node {
+func (p *parser) parseMapKey(tk *token.Token) ast.Node {
 	if node := p.parseStringValue(tk); node != nil {
 		return node
 	}
@@ -303,7 +241,7 @@ func (p *Parser) parseMapKey(tk *token.Token) ast.Node {
 	return nil
 }
 
-func (p *Parser) parseStringValue(tk *token.Token) ast.Node {
+func (p *parser) parseStringValue(tk *token.Token) ast.Node {
 	switch tk.Type {
 	case token.StringType,
 		token.SingleQuoteType,
@@ -313,7 +251,7 @@ func (p *Parser) parseStringValue(tk *token.Token) ast.Node {
 	return nil
 }
 
-func (p *Parser) parseScalarValue(tk *token.Token) ast.Node {
+func (p *parser) parseScalarValue(tk *token.Token) ast.Node {
 	if node := p.parseStringValue(tk); node != nil {
 		return node
 	}
@@ -337,7 +275,7 @@ func (p *Parser) parseScalarValue(tk *token.Token) ast.Node {
 	return nil
 }
 
-func (p *Parser) parseDirective(ctx *Context) (ast.Node, error) {
+func (p *parser) parseDirective(ctx *context) (ast.Node, error) {
 	node := &ast.DirectiveNode{Start: ctx.currentToken()}
 	ctx.progress(1) // skip directive token
 	value, err := p.parseToken(ctx, ctx.currentToken())
@@ -352,7 +290,7 @@ func (p *Parser) parseDirective(ctx *Context) (ast.Node, error) {
 	return node, nil
 }
 
-func (p *Parser) parseLiteral(ctx *Context) (ast.Node, error) {
+func (p *parser) parseLiteral(ctx *context) (ast.Node, error) {
 	node := &ast.LiteralNode{Start: ctx.currentToken()}
 	ctx.progress(1) // skip literal/folded token
 	value, err := p.parseToken(ctx, ctx.currentToken())
@@ -367,7 +305,22 @@ func (p *Parser) parseLiteral(ctx *Context) (ast.Node, error) {
 	return node, nil
 }
 
-func (p *Parser) parseToken(ctx *Context, tk *token.Token) (ast.Node, error) {
+func (p *parser) parseDocument(ctx *context) (*ast.Document, error) {
+	node := &ast.Document{Start: ctx.currentToken()}
+	ctx.progress(1) // skip document header token
+	body, err := p.parseToken(ctx, ctx.currentToken())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse document body")
+	}
+	node.Body = body
+	if ntk := ctx.nextToken(); ntk != nil && ntk.Type == token.DocumentEndType {
+		node.End = ntk
+		ctx.progress(1)
+	}
+	return node, nil
+}
+
+func (p *parser) parseToken(ctx *context, tk *token.Token) (ast.Node, error) {
 	if tk.NextType() == token.MappingValueType {
 		return p.parseMappingValue(ctx)
 	}
@@ -375,6 +328,8 @@ func (p *Parser) parseToken(ctx *Context, tk *token.Token) (ast.Node, error) {
 		return node, nil
 	}
 	switch tk.Type {
+	case token.DocumentHeaderType:
+		return p.parseDocument(ctx)
 	case token.MappingStartType:
 		return p.parseMapping(ctx)
 	case token.SequenceStartType:
@@ -395,10 +350,9 @@ func (p *Parser) parseToken(ctx *Context, tk *token.Token) (ast.Node, error) {
 	return nil, nil
 }
 
-// Parse parse from token instances, and returns ast.Document
-func (p *Parser) Parse(tokens token.Tokens) (*ast.Document, error) {
-	ctx := newContext(tokens)
-	doc := &ast.Document{Nodes: []ast.Node{}}
+func (p *parser) parse(tokens token.Tokens, mode Mode) (*ast.File, error) {
+	ctx := newContext(tokens, mode)
+	file := &ast.File{Docs: []*ast.Document{}}
 	for ctx.next() {
 		node, err := p.parseToken(ctx, ctx.currentToken())
 		if err != nil {
@@ -408,27 +362,51 @@ func (p *Parser) Parse(tokens token.Tokens) (*ast.Document, error) {
 		if node == nil {
 			continue
 		}
-		if len(doc.Nodes) == 0 {
-			doc.Nodes = append(doc.Nodes, node)
-			continue
-		}
-		mvnode, ok := node.(*ast.MappingValueNode)
-		if !ok {
-			doc.Nodes = append(doc.Nodes, node)
-			continue
-		}
-		lastNode := doc.Nodes[len(doc.Nodes)-1]
-		switch lastNode.Type() {
-		case ast.MappingValueType:
-			n := lastNode.(*ast.MappingValueNode)
-			doc.Nodes[len(doc.Nodes)-1] = &ast.MappingNode{
-				Start:  n.GetToken(),
-				Values: []*ast.MappingValueNode{n, mvnode},
-			}
-		case ast.MappingType:
-			n := lastNode.(*ast.MappingNode)
-			n.Values = append(n.Values, mvnode)
+		if doc, ok := node.(*ast.Document); ok {
+			file.Docs = append(file.Docs, doc)
+		} else {
+			file.Docs = append(file.Docs, &ast.Document{Body: node})
 		}
 	}
-	return doc, nil
+	return file, nil
+}
+
+type Mode uint
+
+const (
+	ParseComments Mode = 1 << iota // parse comments and add them to AST
+)
+
+// ParseBytes parse from byte slice, and returns ast.File
+func ParseBytes(bytes []byte, mode Mode) (*ast.File, error) {
+	tokens := lexer.Tokenize(string(bytes))
+	f, err := Parse(tokens, mode)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse")
+	}
+	return f, nil
+}
+
+// Parse parse from token instances, and returns ast.File
+func Parse(tokens token.Tokens, mode Mode) (*ast.File, error) {
+	var p parser
+	f, err := p.parse(tokens, mode)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse")
+	}
+	return f, nil
+}
+
+// Parse parse from filename, and returns ast.File
+func ParseFile(filename string, mode Mode) (*ast.File, error) {
+	file, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read file: %s", filename)
+	}
+	f, err := ParseBytes(file, mode)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse")
+	}
+	f.Name = filename
+	return f, nil
 }


Test patch:
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 9345754..f7026f4 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -66,15 +66,12 @@ func TestParser(t *testing.T) {
 		"v:\n- A\n- |-\n  B\n  C\n",
 		"v:\n- A\n- >-\n  B\n  C\n",
 	}
-	var (
-		p parser.Parser
-	)
 	for _, src := range sources {
 		fmt.Printf(src)
 		tokens := lexer.Tokenize(src)
 		var printer printer.Printer
 		fmt.Println(printer.PrintTokens(tokens))
-		ast, err := p.Parse(tokens)
+		ast, err := parser.Parse(tokens, 0)
 		if err != nil {
 			t.Fatalf("%+v", err)
 		}
@@ -305,6 +302,7 @@ aliased: *anchor
   label: center/big
 `,
 			`
+---
 - &CENTER {x: 1, y: 2}
 - &LEFT {x: 0, y: 2}
 - &BIG {r: 10}
@@ -355,24 +353,43 @@ a:
     f: g
     h: i
 j: k
+`,
+		},
+		{
+			`
+---
+a: 1
+b: 2
+...
+---
+c: 3
+d: 4
+...
+`,
+			`
+---
+a: 1
+b: 2
+...
+---
+c: 3
+d: 4
+...
 `,
 		},
 	}
 	for _, test := range tests {
-		var (
-			p parser.Parser
-		)
 		tokens := lexer.Tokenize(test.source)
 		tokens.Dump()
-		doc, err := p.Parse(tokens)
+		f, err := parser.Parse(tokens, 0)
 		if err != nil {
 			t.Fatalf("%+v", err)
 		}
 		var v Visitor
-		for _, node := range doc.Nodes {
-			ast.Walk(&v, node)
+		for _, doc := range f.Docs {
+			ast.Walk(&v, doc.Body)
 		}
-		expect := fmt.Sprintf("\n%+v\n", doc)
+		expect := fmt.Sprintf("\n%+v\n", f)
 		if test.expect != expect {
 			t.Fatalf("unexpected output: [%s] != [%s]", test.expect, expect)
 		}
@@ -384,11 +401,7 @@ func TestSyntaxError(t *testing.T) {
 		"a:\n- b\n  c: d\n  e: f\n  g: h",
 	}
 	for _, source := range sources {
-		var (
-			p parser.Parser
-		)
-		tokens := lexer.Tokenize(source)
-		_, err := p.Parse(tokens)
+		_, err := parser.ParseBytes([]byte(source), 0)
 		if err == nil {
 			t.Fatal("cannot catch syntax error")
 		}