Repository: goccy/go-yaml
Pull request #590 (2024-12-14 11:02:06)

Fix parsing of multi line text
Current YAML Test Suite Status: `total:[402] passed:[342] failure:[60] passedRate:[85.074631%]`
There are only 4 test cases left that succeed exclusively with `gopkg.in/yaml.v3`

codecov-commenter: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/goccy/go-yaml/pull/590?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) Report
Attention: Patch coverage is `96.95122%` with `5 lines` in your changes missing coverage. Please review.
> Project coverage is 77.43%. Comparing base [(`7601ad3`)](https://app.codecov.io/gh/goccy/go-yaml/commit/7601ad3fe80aefac9b6ba06f08ff2288e29dfd24?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) to head [(`025dbe6`)](https://app.codecov.io/gh/goccy/go-yaml/commit/025dbe61790727a1d9fe4ab0bcf797ae6fefac7f?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima).

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #590      +/-   ##
==========================================
+ Coverage   77.37%   77.43%   +0.06%     
==========================================
  Files          21       21              
  Lines        7704     7738      +34     
==========================================
+ Hits         5961     5992      +31     
- Misses       1333     1335       +2     
- Partials      410      411       +1     
```

</details>

Patch:
diff --git a/parser/parser.go b/parser/parser.go
index 7e2370b..e32b976 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -1016,6 +1016,9 @@ func (p *parser) parseFlowSequence(ctx *context) (*ast.SequenceNode, error) {
 		}
 
 		if tk.Type() == token.CollectEntryType {
+			if isFirst {
+				return nil, errors.ErrSyntax("expected sequence element, but found ','", tk.RawToken())
+			}
 			ctx.goNext()
 		} else if !isFirst {
 			return nil, errors.ErrSyntax("',' or ']' must be specified", tk.RawToken())
diff --git a/scanner/context.go b/scanner/context.go
index 2ca1a13..06e1fc4 100644
--- a/scanner/context.go
+++ b/scanner/context.go
@@ -11,23 +11,28 @@ import (
 
 // Context context at scanning
 type Context struct {
-	idx                      int
-	size                     int
-	notSpaceCharPos          int
-	notSpaceOrgCharPos       int
-	src                      []rune
-	buf                      []rune
-	obuf                     []rune
-	tokens                   token.Tokens
-	isRawFolded              bool
-	isLiteral                bool
-	isFolded                 bool
-	docOpt                   string
-	docFirstLineIndentColumn int
-	docPrevLineIndentColumn  int
-	docLineIndentColumn      int
-	docSpaceOnlyIndentColumn int
-	docFoldedNewLine         bool
+	idx                int
+	size               int
+	notSpaceCharPos    int
+	notSpaceOrgCharPos int
+	src                []rune
+	buf                []rune
+	obuf               []rune
+	tokens             token.Tokens
+	mstate             *MultiLineState
+}
+
+type MultiLineState struct {
+	opt                              string
+	firstLineIndentColumn            int
+	prevLineIndentColumn             int
+	lineIndentColumn                 int
+	lastNotSpaceOnlyLineIndentColumn int
+	spaceOnlyIndentColumn            int
+	foldedNewLine                    bool
+	isRawFolded                      bool
+	isLiteral                        bool
+	isFolded                         bool
 }
 
 var (
@@ -57,15 +62,7 @@ func (c *Context) release() {
 
 func (c *Context) clear() {
 	c.resetBuffer()
-	c.isRawFolded = false
-	c.isLiteral = false
-	c.isFolded = false
-	c.docOpt = ""
-	c.docFirstLineIndentColumn = 0
-	c.docLineIndentColumn = 0
-	c.docPrevLineIndentColumn = 0
-	c.docSpaceOnlyIndentColumn = 0
-	c.docFoldedNewLine = false
+	c.mstate = nil
 }
 
 func (c *Context) reset(src []rune) {
@@ -74,10 +71,7 @@ func (c *Context) reset(src []rune) {
 	c.src = src
 	c.tokens = c.tokens[:0]
 	c.resetBuffer()
-	c.isRawFolded = false
-	c.isLiteral = false
-	c.isFolded = false
-	c.docOpt = ""
+	c.mstate = nil
 }
 
 func (c *Context) resetBuffer() {
@@ -87,27 +81,47 @@ func (c *Context) resetBuffer() {
 	c.notSpaceOrgCharPos = 0
 }
 
-func (c *Context) breakDocument() {
-	c.isLiteral = false
-	c.isRawFolded = false
-	c.isFolded = false
-	c.docOpt = ""
-	c.docFirstLineIndentColumn = 0
-	c.docLineIndentColumn = 0
-	c.docPrevLineIndentColumn = 0
-	c.docSpaceOnlyIndentColumn = 0
-	c.docFoldedNewLine = false
+func (c *Context) breakMultiLine() {
+	c.mstate = nil
 }
 
-func (c *Context) updateDocumentIndentColumn() {
-	indent := c.docFirstLineIndentColumnByDocOpt()
+func (c *Context) getMultiLineState() *MultiLineState {
+	return c.mstate
+}
+
+func (c *Context) setLiteral(lastDelimColumn int, opt string) {
+	mstate := &MultiLineState{
+		isLiteral: true,
+		opt:       opt,
+	}
+	indent := firstLineIndentColumnByOpt(opt)
 	if indent > 0 {
-		c.docFirstLineIndentColumn = indent + 1
+		mstate.firstLineIndentColumn = lastDelimColumn + indent
 	}
+	c.mstate = mstate
 }
 
-func (c *Context) docFirstLineIndentColumnByDocOpt() int {
-	opt := c.docOpt
+func (c *Context) setFolded(lastDelimColumn int, opt string) {
+	mstate := &MultiLineState{
+		isFolded: true,
+		opt:      opt,
+	}
+	indent := firstLineIndentColumnByOpt(opt)
+	if indent > 0 {
+		mstate.firstLineIndentColumn = lastDelimColumn + indent
+	}
+	c.mstate = mstate
+}
+
+func (c *Context) setRawFolded(column int) {
+	mstate := &MultiLineState{
+		isRawFolded: true,
+	}
+	mstate.updateIndentColumn(column)
+	c.mstate = mstate
+}
+
+func firstLineIndentColumnByOpt(opt string) int {
 	opt = strings.TrimPrefix(opt, "-")
 	opt = strings.TrimPrefix(opt, "+")
 	opt = strings.TrimSuffix(opt, "-")
@@ -116,90 +130,136 @@ func (c *Context) docFirstLineIndentColumnByDocOpt() int {
 	return int(i)
 }
 
-func (c *Context) updateDocumentLineIndentColumn(column int) {
-	if c.docFirstLineIndentColumn == 0 {
-		c.docFirstLineIndentColumn = column
+func (s *MultiLineState) lastDelimColumn() int {
+	if s.firstLineIndentColumn == 0 {
+		return 0
+	}
+	return s.firstLineIndentColumn - 1
+}
+
+func (s *MultiLineState) updateIndentColumn(column int) {
+	if s.firstLineIndentColumn == 0 {
+		s.firstLineIndentColumn = column
 	}
-	if c.docLineIndentColumn == 0 {
-		c.docLineIndentColumn = column
+	if s.lineIndentColumn == 0 {
+		s.lineIndentColumn = column
 	}
 }
 
-func (c *Context) updateSpaceOnlyIndentColumn(column int) {
-	if c.docFirstLineIndentColumn != 0 {
+func (s *MultiLineState) updateSpaceOnlyIndentColumn(column int) {
+	if s.firstLineIndentColumn != 0 {
 		return
 	}
-	c.docSpaceOnlyIndentColumn = column
+	s.spaceOnlyIndentColumn = column
 }
 
-func (c *Context) validateDocumentLineIndentAfterSpaceOnly(column int) error {
-	if c.docFirstLineIndentColumn != 0 {
+func (s *MultiLineState) validateIndentAfterSpaceOnly(column int) error {
+	if s.firstLineIndentColumn != 0 {
 		return nil
 	}
-	if c.docSpaceOnlyIndentColumn > column {
+	if s.spaceOnlyIndentColumn > column {
 		return errors.New("invalid number of indent is specified after space only")
 	}
 	return nil
 }
 
-func (c *Context) validateDocumentLineIndentColumn() error {
-	if c.docFirstLineIndentColumnByDocOpt() == 0 {
+func (s *MultiLineState) validateIndentColumn() error {
+	if firstLineIndentColumnByOpt(s.opt) == 0 {
 		return nil
 	}
-	if c.docFirstLineIndentColumn > c.docLineIndentColumn {
-		return errors.New("invalid number of indent is specified in the document header")
+	if s.firstLineIndentColumn > s.lineIndentColumn {
+		return errors.New("invalid number of indent is specified in the multi line header")
 	}
 	return nil
 }
 
-func (c *Context) updateDocumentNewLineState() {
-	c.docPrevLineIndentColumn = c.docLineIndentColumn
-	c.docFoldedNewLine = true
-	c.docLineIndentColumn = 0
+func (s *MultiLineState) updateNewLineState() {
+	s.prevLineIndentColumn = s.lineIndentColumn
+	if s.lineIndentColumn != 0 {
+		s.lastNotSpaceOnlyLineIndentColumn = s.lineIndentColumn
+	}
+	s.foldedNewLine = true
+	s.lineIndentColumn = 0
 }
 
-func (c *Context) isIndentColumn(column int) bool {
-	if c.docFirstLineIndentColumn == 0 {
+func (s *MultiLineState) isIndentColumn(column int) bool {
+	if s.firstLineIndentColumn == 0 {
 		return column == 1
 	}
-	return c.docFirstLineIndentColumn > column
+	return s.firstLineIndentColumn > column
 }
 
-func (c *Context) addDocumentIndent(column int) {
-	if c.docFirstLineIndentColumn == 0 {
+func (s *MultiLineState) addIndent(ctx *Context, column int) {
+	if s.firstLineIndentColumn == 0 {
 		return
 	}
 
-	// If the first line of the document has already been evaluated, the number is treated as the threshold, since the `docFirstLineIndentColumn` is a positive number.
-	if c.docFirstLineIndentColumn <= column {
-		// `c.docFoldedNewLine` is a variable that is set to true for every newline.
-		if (c.isFolded || c.isRawFolded) && c.docFoldedNewLine {
-			c.docFoldedNewLine = false
-		}
-		// Since addBuf ignore space character, add to the buffer directly.
-		c.buf = append(c.buf, ' ')
-		c.notSpaceCharPos = len(c.buf)
+	// If the first line of the document has already been evaluated, the number is treated as the threshold, since the `firstLineIndentColumn` is a positive number.
+	if column < s.firstLineIndentColumn {
+		return
+	}
+
+	// `c.foldedNewLine` is a variable that is set to true for every newline.
+	if !s.isLiteral && s.foldedNewLine {
+		s.foldedNewLine = false
 	}
+	// Since addBuf ignore space character, add to the buffer directly.
+	ctx.buf = append(ctx.buf, ' ')
+	ctx.notSpaceCharPos = len(ctx.buf)
 }
 
-// updateDocumentNewLineInFolded if Folded or RawFolded context and the content on the current line starts at the same column as the previous line,
+// updateNewLineInFolded if Folded or RawFolded context and the content on the current line starts at the same column as the previous line,
 // treat the new-line-char as a space.
-func (c *Context) updateDocumentNewLineInFolded(column int) {
-	if c.isLiteral {
+func (s *MultiLineState) updateNewLineInFolded(ctx *Context, column int) {
+	if s.isLiteral {
 		return
 	}
 
 	// Folded or RawFolded.
 
-	if !c.docFoldedNewLine {
+	if !s.foldedNewLine {
 		return
 	}
-	if c.docLineIndentColumn == c.docPrevLineIndentColumn {
-		if len(c.buf) != 0 && c.buf[len(c.buf)-1] == '\n' {
-			c.buf[len(c.buf)-1] = ' '
+	var (
+		lastChar     rune
+		prevLastChar rune
+	)
+	if len(ctx.buf) != 0 {
+		lastChar = ctx.buf[len(ctx.buf)-1]
+	}
+	if len(ctx.buf) > 1 {
+		prevLastChar = ctx.buf[len(ctx.buf)-2]
+	}
+	if s.lineIndentColumn == s.prevLineIndentColumn {
+		// ---
+		// >
+		//  a
+		//  b
+		if lastChar == '\n' {
+			ctx.buf[len(ctx.buf)-1] = ' '
+		}
+	} else if s.prevLineIndentColumn == 0 && s.lastNotSpaceOnlyLineIndentColumn == column {
+		// if previous line is indent-space and new-line-char only, prevLineIndentColumn is zero.
+		// In this case, last new-line-char is removed.
+		// ---
+		// >
+		//  a
+		//
+		//  b
+		if lastChar == '\n' && prevLastChar == '\n' {
+			ctx.buf = ctx.buf[:len(ctx.buf)-1]
+			ctx.notSpaceCharPos = len(ctx.buf)
 		}
 	}
-	c.docFoldedNewLine = false
+	s.foldedNewLine = false
+}
+
+func (s *MultiLineState) hasTrimAllEndNewlineOpt() bool {
+	return strings.HasPrefix(s.opt, "-") || strings.HasSuffix(s.opt, "-") || s.isRawFolded
+}
+
+func (s *MultiLineState) hasKeepAllEndNewlineOpt() bool {
+	return strings.HasPrefix(s.opt, "+") || strings.HasSuffix(s.opt, "+")
 }
 
 func (c *Context) addToken(tk *token.Token) {
@@ -246,10 +306,6 @@ func (c *Context) removeRightSpaceFromBuf() {
 	}
 }
 
-func (c *Context) isDocument() bool {
-	return c.isLiteral || c.isFolded || c.isRawFolded
-}
-
 func (c *Context) isEOS() bool {
 	return len(c.src)-1 <= c.idx
 }
@@ -307,15 +363,20 @@ func (c *Context) existsBuffer() bool {
 	return len(c.bufferedSrc()) != 0
 }
 
+func (c *Context) isMultiLine() bool {
+	return c.mstate != nil
+}
+
 func (c *Context) bufferedSrc() []rune {
 	src := c.buf[:c.notSpaceCharPos]
-	if c.isDocument() {
+	if c.isMultiLine() {
+		mstate := c.getMultiLineState()
 		// remove end '\n' character and trailing empty lines.
 		// https://yaml.org/spec/1.2.2/#8112-block-chomping-indicator
-		if c.hasTrimAllEndNewlineOpt() {
+		if mstate.hasTrimAllEndNewlineOpt() {
 			// If the '-' flag is specified, all trailing newline characters will be removed.
 			src = []rune(strings.TrimRight(string(src), "\n"))
-		} else if !c.hasKeepAllEndNewlineOpt() {
+		} else if !mstate.hasKeepAllEndNewlineOpt() {
 			// Normally, all but one of the trailing newline characters are removed.
 			var newLineCharCount int
 			for i := len(src) - 1; i >= 0; i-- {
@@ -333,7 +394,7 @@ func (c *Context) bufferedSrc() []rune {
 		}
 
 		// If the text ends with a space character, remove all of them.
-		if c.hasTrimAllEndNewlineOpt() {
+		if mstate.hasTrimAllEndNewlineOpt() {
 			src = []rune(strings.TrimRight(string(src), " "))
 		}
 		if string(src) == "\n" {
@@ -342,21 +403,13 @@ func (c *Context) bufferedSrc() []rune {
 			// so it is treated as an empty string.
 			src = []rune{}
 		}
-		if c.hasKeepAllEndNewlineOpt() && len(src) == 0 {
+		if mstate.hasKeepAllEndNewlineOpt() && len(src) == 0 {
 			src = []rune{'\n'}
 		}
 	}
 	return src
 }
 
-func (c *Context) hasTrimAllEndNewlineOpt() bool {
-	return strings.HasPrefix(c.docOpt, "-") || strings.HasSuffix(c.docOpt, "-") || c.isRawFolded
-}
-
-func (c *Context) hasKeepAllEndNewlineOpt() bool {
-	return strings.HasPrefix(c.docOpt, "+") || strings.HasSuffix(c.docOpt, "+")
-}
-
 func (c *Context) bufferedToken(pos *token.Position) *token.Token {
 	if c.idx == 0 {
 		return nil
@@ -367,7 +420,7 @@ func (c *Context) bufferedToken(pos *token.Position) *token.Token {
 		return nil
 	}
 	var tk *token.Token
-	if c.isDocument() {
+	if c.isMultiLine() {
 		tk = token.String(string(source), string(c.obuf), pos)
 	} else {
 		tk = token.New(string(source), string(c.obuf), pos)
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 796d0ba..4cd07e9 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -73,7 +73,7 @@ func (s *Scanner) bufferedToken(ctx *Context) *token.Token {
 	line := s.line
 	column := s.column - len(ctx.buf)
 	level := s.indentLevel
-	if ctx.isDocument() {
+	if ctx.isMultiLine() {
 		line -= s.newLineCount(ctx.buf)
 		column = strings.Index(string(ctx.obuf), string(ctx.buf)) + 1
 		// Since we are in a literal, folded or raw folded
@@ -155,27 +155,16 @@ func (s *Scanner) updateIndentLevel() {
 }
 
 func (s *Scanner) updateIndentState(ctx *Context) {
-	if s.lastDelimColumn > 0 {
-		if s.lastDelimColumn < s.column {
-			s.indentState = IndentStateUp
-		} else {
-			// If lastDelimColumn and s.column are the same,
-			// treat as Down state since it is the same column as delimiter.
-			s.indentState = IndentStateDown
-		}
-	} else {
-		s.indentState = s.indentStateFromIndentNumDifference()
+	if s.lastDelimColumn == 0 {
+		return
 	}
-}
 
-func (s *Scanner) indentStateFromIndentNumDifference() IndentState {
-	switch {
-	case s.prevLineIndentNum < s.indentNum:
-		return IndentStateUp
-	case s.prevLineIndentNum == s.indentNum:
-		return IndentStateEqual
-	default:
-		return IndentStateDown
+	if s.lastDelimColumn < s.column {
+		s.indentState = IndentStateUp
+	} else {
+		// If lastDelimColumn and s.column are the same,
+		// treat as Down state since it is the same column as delimiter.
+		s.indentState = IndentStateDown
 	}
 }
 
@@ -213,8 +202,8 @@ func (s *Scanner) addBufferedTokenIfExists(ctx *Context) {
 	ctx.addToken(s.bufferedToken(ctx))
 }
 
-func (s *Scanner) breakDocument(ctx *Context) {
-	ctx.breakDocument()
+func (s *Scanner) breakMultiLine(ctx *Context) {
+	ctx.breakMultiLine()
 }
 
 func (s *Scanner) scanSingleQuote(ctx *Context) (*token.Token, error) {
@@ -612,7 +601,7 @@ func (s *Scanner) scanQuote(ctx *Context, ch rune) (bool, error) {
 }
 
 func (s *Scanner) scanWhiteSpace(ctx *Context) bool {
-	if ctx.isDocument() {
+	if ctx.isMultiLine() {
 		return false
 	}
 	if !s.isAnchor && !s.isDirective && !s.isAlias && !s.isFirstCharAtLine {
@@ -744,16 +733,17 @@ func (s *Scanner) scanComment(ctx *Context) bool {
 	return true
 }
 
-func (s *Scanner) scanDocument(ctx *Context, c rune) error {
+func (s *Scanner) scanMultiLine(ctx *Context, c rune) error {
+	state := ctx.getMultiLineState()
 	ctx.addOriginBuf(c)
 	if ctx.isEOS() {
 		if s.isFirstCharAtLine && c == ' ' {
-			ctx.addDocumentIndent(s.column)
+			state.addIndent(ctx, s.column)
 		} else {
 			ctx.addBuf(c)
 		}
-		ctx.updateDocumentLineIndentColumn(s.column)
-		if err := ctx.validateDocumentLineIndentColumn(); err != nil {
+		state.updateIndentColumn(s.column)
+		if err := state.validateIndentColumn(); err != nil {
 			invalidTk := token.Invalid(err.Error(), string(ctx.obuf), s.pos())
 			s.progressColumn(ctx, 1)
 			return ErrInvalidToken(invalidTk)
@@ -764,21 +754,21 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error {
 		s.progressColumn(ctx, 1)
 	} else if s.isNewLineChar(c) {
 		ctx.addBuf(c)
-		ctx.updateSpaceOnlyIndentColumn(s.column - 1)
-		ctx.updateDocumentNewLineState()
+		state.updateSpaceOnlyIndentColumn(s.column - 1)
+		state.updateNewLineState()
 		s.progressLine(ctx)
 		if ctx.next() {
 			if s.foundDocumentSeparatorMarker(ctx.src[ctx.idx:]) {
 				value := ctx.bufferedSrc()
 				ctx.addToken(token.String(string(value), string(ctx.obuf), s.pos()))
 				ctx.clear()
-				s.breakDocument(ctx)
+				s.breakMultiLine(ctx)
 			}
 		}
 	} else if s.isFirstCharAtLine && c == ' ' {
-		ctx.addDocumentIndent(s.column)
+		state.addIndent(ctx, s.column)
 		s.progressColumn(ctx, 1)
-	} else if s.isFirstCharAtLine && c == '\t' && ctx.isIndentColumn(s.column) {
+	} else if s.isFirstCharAtLine && c == '\t' && state.isIndentColumn(s.column) {
 		err := ErrInvalidToken(
 			token.Invalid(
 				"found a tab character where an indentation space is expected",
@@ -787,22 +777,25 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error {
 		)
 		s.progressColumn(ctx, 1)
 		return err
+	} else if c == '\t' && !state.isIndentColumn(s.column) {
+		ctx.addBufWithTab(c)
+		s.progressColumn(ctx, 1)
 	} else {
-		if err := ctx.validateDocumentLineIndentAfterSpaceOnly(s.column); err != nil {
+		if err := state.validateIndentAfterSpaceOnly(s.column); err != nil {
 			invalidTk := token.Invalid(err.Error(), string(ctx.obuf), s.pos())
 			s.progressColumn(ctx, 1)
 			return ErrInvalidToken(invalidTk)
 		}
-		ctx.updateDocumentLineIndentColumn(s.column)
-		if ctx.docFirstLineIndentColumn > 0 {
-			s.lastDelimColumn = ctx.docFirstLineIndentColumn - 1
-		}
-		if err := ctx.validateDocumentLineIndentColumn(); err != nil {
+		state.updateIndentColumn(s.column)
+		if err := state.validateIndentColumn(); err != nil {
 			invalidTk := token.Invalid(err.Error(), string(ctx.obuf), s.pos())
 			s.progressColumn(ctx, 1)
 			return ErrInvalidToken(invalidTk)
 		}
-		ctx.updateDocumentNewLineInFolded(s.column)
+		if col := state.lastDelimColumn(); col > 0 {
+			s.lastDelimColumn = col
+		}
+		state.updateNewLineInFolded(ctx, s.column)
 		ctx.addBufWithTab(c)
 		s.progressColumn(ctx, 1)
 	}
@@ -1032,8 +1025,7 @@ func (s *Scanner) scanRawFoldedChar(ctx *Context) bool {
 		return false
 	}
 
-	ctx.updateDocumentLineIndentColumn(s.column)
-	ctx.isRawFolded = true
+	ctx.setRawFolded(s.column)
 	ctx.addBuf('-')
 	ctx.addOriginBuf('-')
 	s.progressColumn(ctx, 1)
@@ -1066,20 +1058,19 @@ func (s *Scanner) scanSequence(ctx *Context) (bool, error) {
 	return true, nil
 }
 
-func (s *Scanner) scanDocumentHeader(ctx *Context) (bool, error) {
+func (s *Scanner) scanMultiLineHeader(ctx *Context) (bool, error) {
 	if ctx.existsBuffer() {
 		return false, nil
 	}
 
-	if err := s.scanDocumentHeaderOption(ctx); err != nil {
+	if err := s.scanMultiLineHeaderOption(ctx); err != nil {
 		return false, err
 	}
-	ctx.updateDocumentIndentColumn()
 	s.progressLine(ctx)
 	return true, nil
 }
 
-func (s *Scanner) validateDocumentHeaderOption(opt string) error {
+func (s *Scanner) validateMultiLineHeaderOption(opt string) error {
 	if len(opt) == 0 {
 		return nil
 	}
@@ -1104,7 +1095,7 @@ func (s *Scanner) validateDocumentHeaderOption(opt string) error {
 	return nil
 }
 
-func (s *Scanner) scanDocumentHeaderOption(ctx *Context) error {
+func (s *Scanner) scanMultiLineHeaderOption(ctx *Context) error {
 	header := ctx.currentChar()
 	ctx.addOriginBuf(header)
 	s.progress(ctx, 1) // skip '|' or '>' character
@@ -1127,7 +1118,7 @@ func (s *Scanner) scanDocumentHeaderOption(ctx *Context) error {
 		return r == ' ' || r == '\t'
 	})
 	if len(opt) != 0 {
-		if err := s.validateDocumentHeaderOption(opt); err != nil {
+		if err := s.validateMultiLineHeaderOption(opt); err != nil {
 			invalidTk := token.Invalid(err.Error(), string(ctx.obuf), s.pos())
 			s.progressColumn(ctx, progress)
 			return ErrInvalidToken(invalidTk)
@@ -1145,10 +1136,10 @@ func (s *Scanner) scanDocumentHeaderOption(ctx *Context) error {
 	switch header {
 	case '|':
 		ctx.addToken(token.Literal("|"+opt, headerBuf, s.pos()))
-		ctx.isLiteral = true
+		ctx.setLiteral(s.lastDelimColumn, opt)
 	case '>':
 		ctx.addToken(token.Folded(">"+opt, headerBuf, s.pos()))
-		ctx.isFolded = true
+		ctx.setFolded(s.lastDelimColumn, opt)
 	}
 	if commentIndex > 0 {
 		comment := string(value[commentValueIndex+1:])
@@ -1158,7 +1149,6 @@ func (s *Scanner) scanDocumentHeaderOption(ctx *Context) error {
 	}
 	s.indentState = IndentStateKeep
 	ctx.resetBuffer()
-	ctx.docOpt = opt
 	s.progressColumn(ctx, progress)
 	return nil
 }
@@ -1278,7 +1268,7 @@ func (s *Scanner) scan(ctx *Context) error {
 		if s.isChangedToIndentStateDown() {
 			s.addBufferedTokenIfExists(ctx)
 		}
-		if ctx.isDocument() {
+		if ctx.isMultiLine() {
 			if s.isChangedToIndentStateDown() {
 				if tk := ctx.lastToken(); tk != nil {
 					// If literal/folded content is empty, no string token is added.
@@ -1287,7 +1277,7 @@ func (s *Scanner) scan(ctx *Context) error {
 					if tk.Position.Column == 1 {
 						return ErrInvalidToken(
 							token.Invalid(
-								"could not find document",
+								"could not find multi line content",
 								string(ctx.obuf), s.pos(),
 							),
 						)
@@ -1296,9 +1286,9 @@ func (s *Scanner) scan(ctx *Context) error {
 						ctx.addToken(token.String("", "", s.pos()))
 					}
 				}
-				s.breakDocument(ctx)
+				s.breakMultiLine(ctx)
 			} else {
-				if err := s.scanDocument(ctx, c); err != nil {
+				if err := s.scanMultiLine(ctx, c); err != nil {
 					return err
 				}
 				continue
@@ -1356,7 +1346,7 @@ func (s *Scanner) scan(ctx *Context) error {
 				continue
 			}
 		case '|', '>':
-			scanned, err := s.scanDocumentHeader(ctx)
+			scanned, err := s.scanMultiLineHeader(ctx)
 			if err != nil {
 				return err
 			}


Test patch:
diff --git a/parser/parser_test.go b/parser/parser_test.go
index c82adb7..85c9482 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -1236,7 +1236,7 @@ b: - 2
 		{
 			">\n>",
 			`
-[2:1] could not find document
+[2:1] could not find multi line content
    1 | >
 >  2 | >
        ^
@@ -1245,7 +1245,7 @@ b: - 2
 		{
 			">\n1",
 			`
-[2:1] could not find document
+[2:1] could not find multi line content
    1 | >
 >  2 | 1
        ^
@@ -1254,7 +1254,7 @@ b: - 2
 		{
 			"|\n1",
 			`
-[2:1] could not find document
+[2:1] could not find multi line content
    1 | |
 >  2 | 1
        ^
@@ -1263,7 +1263,7 @@ b: - 2
 		{
 			"a: >3\n  1",
 			`
-[2:3] invalid number of indent is specified in the document header
+[2:3] invalid number of indent is specified in the multi line header
    1 | a: >3
 >  2 |   1
          ^
diff --git a/yaml_test_suite_test.go b/yaml_test_suite_test.go
index 346ecc9..8bccd88 100644
--- a/yaml_test_suite_test.go
+++ b/yaml_test_suite_test.go
@@ -30,35 +30,26 @@ var failureTestNames = []string{
 	"flow-collections-over-many-lines/01",
 	"flow-mapping-colon-on-line-after-key/02",
 	"flow-mapping-edge-cases",
-	"flow-sequence-with-invalid-comma-at-the-beginning", // pass yamlv3.
-	"folded-block-scalar",                               // pass yamlv3.
-	"folded-block-scalar-1-3",                           // pass yamlv3.
-	"implicit-flow-mapping-key-on-one-line",             // no json.
+	"implicit-flow-mapping-key-on-one-line", // no json.
 	"invalid-comment-after-comma",
 	"invalid-comment-after-end-of-flow-sequence",
 	"invalid-comma-in-tag",
 	"invalid-tag",                                    // pass yamlv3.
 	"legal-tab-after-indentation",                    // pass yamlv3.
-	"literal-scalars",                                // pass yamlv3.
 	"mapping-key-and-flow-sequence-item-anchors",     // no json.
 	"multiline-plain-value-with-tabs-on-empty-lines", // pass yamlv3.
-	"multiline-scalar-at-top-level",                  // pass yamlv3.
-	"multiline-scalar-at-top-level-1-3",              // pass yamlv3.
 	"nested-implicit-complex-keys",                   // no json.
 	"plain-dashes-in-flow-sequence",
-	"question-mark-edge-cases/00",                           // no json.
-	"question-mark-edge-cases/01",                           // no json.
-	"single-character-streams/01",                           // no json.
-	"single-pair-implicit-entries",                          // no json.
-	"spec-example-2-11-mapping-between-sequences",           // no json.
-	"spec-example-6-12-separation-spaces",                   // no json.
-	"spec-example-7-16-flow-mapping-entries",                // no json.
-	"spec-example-7-3-completely-empty-flow-nodes",          // no json.
-	"spec-example-8-18-implicit-block-mapping-entries",      // no json.
-	"spec-example-8-19-compact-block-mappings",              // no json.
-	"spec-example-6-6-line-folding",                         // pass yamlv3.
-	"spec-example-6-6-line-folding-1-3",                     // pass yamlv3.
-	"spec-example-8-10-folded-lines-8-13-final-empty-lines", // pass yamlv3.
+	"question-mark-edge-cases/00",                      // no json.
+	"question-mark-edge-cases/01",                      // no json.
+	"single-character-streams/01",                      // no json.
+	"single-pair-implicit-entries",                     // no json.
+	"spec-example-2-11-mapping-between-sequences",      // no json.
+	"spec-example-6-12-separation-spaces",              // no json.
+	"spec-example-7-16-flow-mapping-entries",           // no json.
+	"spec-example-7-3-completely-empty-flow-nodes",     // no json.
+	"spec-example-8-18-implicit-block-mapping-entries", // no json.
+	"spec-example-8-19-compact-block-mappings",         // no json.
 	"spec-example-8-2-block-indentation-indicator",
 	"spec-example-9-3-bare-documents",
 	"spec-example-9-4-explicit-documents",