Repository: goccy/go-yaml
Pull request #536 (2024-11-14 15:17:03)

Fix invalid token error
Improve parser's error message when scanner returns invalid token.

codecov-commenter: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/goccy/go-yaml/pull/536?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) Report
Attention: Patch coverage is `85.41667%` with `7 lines` in your changes missing coverage. Please review.
> Project coverage is 79.23%. Comparing base [(`271213a`)](https://app.codecov.io/gh/goccy/go-yaml/commit/271213ad2349e8613e3cbcba74cc74cfb1a4f778?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) to head [(`1b3ca30`)](https://app.codecov.io/gh/goccy/go-yaml/commit/1b3ca30363b25d8120cafd623c3d8e7c15ad9b2e?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima).

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #536      +/-   ##
==========================================
+ Coverage   79.19%   79.23%   +0.04%     
==========================================
  Files          18       18              
  Lines        6235     6257      +22     
==========================================
+ Hits         4938     4958      +20     
- Misses        981      983       +2     
  Partials      316      316              
```

</details>

Patch:
diff --git a/parser/parser.go b/parser/parser.go
index eee8232..55db6b9 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -1007,7 +1007,7 @@ func ParseBytes(bytes []byte, mode Mode, opts ...Option) (*ast.File, error) {
 // Parse parse from token instances, and returns ast.File
 func Parse(tokens token.Tokens, mode Mode, opts ...Option) (*ast.File, error) {
 	if tk := tokens.InvalidToken(); tk != nil {
-		return nil, errors.ErrSyntax("found invalid token", tk)
+		return nil, errors.ErrSyntax(tk.Error, tk)
 	}
 	f, err := newParser(tokens, mode, opts).parse(newContext())
 	if err != nil {
diff --git a/scanner/error.go b/scanner/error.go
index 4298be4..3f5419a 100644
--- a/scanner/error.go
+++ b/scanner/error.go
@@ -3,17 +3,15 @@ package scanner
 import "github.com/goccy/go-yaml/token"
 
 type InvalidTokenError struct {
-	Message string
-	Token   *token.Token
+	Token *token.Token
 }
 
 func (e *InvalidTokenError) Error() string {
-	return e.Message
+	return e.Token.Error
 }
 
-func ErrInvalidToken(msg string, tk *token.Token) *InvalidTokenError {
+func ErrInvalidToken(tk *token.Token) *InvalidTokenError {
 	return &InvalidTokenError{
-		Message: msg,
-		Token:   tk,
+		Token: tk,
 	}
 }
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 50ae83b..fbddf7d 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -257,8 +257,10 @@ func (s *Scanner) scanSingleQuote(ctx *Context) (*token.Token, error) {
 	}
 	s.progressColumn(ctx, 1)
 	return nil, ErrInvalidToken(
-		"could not find end character of single-quotated text",
-		token.Invalid(string(ctx.obuf), srcpos),
+		token.Invalid(
+			"could not find end character of single-quoted text",
+			string(ctx.obuf), srcpos,
+		),
 	)
 }
 
@@ -427,8 +429,10 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (*token.Token, error) {
 	}
 	s.progressColumn(ctx, 1)
 	return nil, ErrInvalidToken(
-		"could not find end character of double-quotated text",
-		token.Invalid(string(ctx.obuf), srcpos),
+		token.Invalid(
+			"could not find end character of double-quoted text",
+			string(ctx.obuf), srcpos,
+		),
 	)
 }
 
@@ -570,8 +574,10 @@ func (s *Scanner) trimCommentFromDocumentOpt(text string, header rune) (string,
 	}
 	if idx == 0 {
 		return "", ErrInvalidToken(
-			fmt.Sprintf("invalid document header %s", text),
-			token.Invalid(string(header)+text, s.pos()),
+			token.Invalid(
+				fmt.Sprintf("invalid document header %s", text),
+				string(header)+text, s.pos(),
+			),
 		)
 	}
 	return text[:idx-1], nil
@@ -582,9 +588,9 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error {
 	if ctx.isEOS() {
 		ctx.updateDocumentLineIndentColumn(s.column)
 		if err := ctx.validateDocumentLineIndentColumn(); err != nil {
-			invalidTk := token.Invalid(string(ctx.obuf), s.pos())
+			invalidTk := token.Invalid(err.Error(), string(ctx.obuf), s.pos())
 			s.progressColumn(ctx, 1)
-			return ErrInvalidToken(err.Error(), invalidTk)
+			return ErrInvalidToken(invalidTk)
 		}
 		ctx.addBuf(c)
 		value := ctx.bufferedSrc()
@@ -600,8 +606,10 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error {
 		s.progressColumn(ctx, 1)
 	} else if s.isFirstCharAtLine && c == '\t' {
 		err := ErrInvalidToken(
-			"found a tab character where an indentation space is expected",
-			token.Invalid(string(ctx.obuf), s.pos()),
+			token.Invalid(
+				"found a tab character where an indentation space is expected",
+				string(ctx.obuf), s.pos(),
+			),
 		)
 		s.progressColumn(ctx, 1)
 		return err
@@ -611,9 +619,9 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error {
 			s.lastDelimColumn = ctx.docFirstLineIndentColumn - 1
 		}
 		if err := ctx.validateDocumentLineIndentColumn(); err != nil {
-			invalidTk := token.Invalid(string(ctx.obuf), s.pos())
+			invalidTk := token.Invalid(err.Error(), string(ctx.obuf), s.pos())
 			s.progressColumn(ctx, 1)
-			return ErrInvalidToken(err.Error(), invalidTk)
+			return ErrInvalidToken(invalidTk)
 		}
 		ctx.updateDocumentNewLineInFolded(s.column)
 		ctx.addBuf(c)
@@ -899,9 +907,9 @@ func (s *Scanner) scanDocumentHeaderOption(ctx *Context) error {
 				return err
 			}
 			if err := s.validateDocumentHeaderOption(opt); err != nil {
-				invalidTk := token.Invalid(string(ctx.obuf), s.pos())
+				invalidTk := token.Invalid(err.Error(), string(ctx.obuf), s.pos())
 				s.progressColumn(ctx, progress)
-				return ErrInvalidToken(err.Error(), invalidTk)
+				return ErrInvalidToken(invalidTk)
 			}
 			hasComment := len(opt) < orgOptLen
 			if s.column == 1 {
@@ -946,9 +954,12 @@ func (s *Scanner) scanDocumentHeaderOption(ctx *Context) error {
 		}
 	}
 	text := string(ctx.src[ctx.idx:])
-	invalidTk := token.Invalid(string(ctx.obuf), s.pos())
+	invalidTk := token.Invalid(
+		fmt.Sprintf("invalid document header: %q", text),
+		string(ctx.obuf), s.pos(),
+	)
 	s.progressColumn(ctx, len(text))
-	return ErrInvalidToken(fmt.Sprintf("invalid document header: %q", text), invalidTk)
+	return ErrInvalidToken(invalidTk)
 }
 
 func (s *Scanner) scanMapKey(ctx *Context) bool {
@@ -1015,7 +1026,12 @@ func (s *Scanner) scanReservedChar(ctx *Context, c rune) error {
 
 	ctx.addBuf(c)
 	ctx.addOriginBuf(c)
-	err := ErrInvalidToken("%q is a reserved character", token.Invalid(string(ctx.obuf), s.pos()))
+	err := ErrInvalidToken(
+		token.Invalid(
+			fmt.Sprintf("%q is a reserved character", c),
+			string(ctx.obuf), s.pos(),
+		),
+	)
 	s.progressColumn(ctx, 1)
 	ctx.clear()
 	return err
@@ -1028,7 +1044,11 @@ func (s *Scanner) scanTab(ctx *Context, c rune) error {
 
 	ctx.addBuf(c)
 	ctx.addOriginBuf(c)
-	err := ErrInvalidToken("found character '\t' that cannot start any token", token.Invalid(string(ctx.obuf), s.pos()))
+	err := ErrInvalidToken(
+		token.Invalid("found character '\t' that cannot start any token",
+			string(ctx.obuf), s.pos(),
+		),
+	)
 	s.progressColumn(ctx, 1)
 	ctx.clear()
 	return err
@@ -1054,8 +1074,10 @@ func (s *Scanner) scan(ctx *Context) error {
 					// But if literal/folded token column is 1, it is invalid at down state.
 					if tk.Position.Column == 1 {
 						return ErrInvalidToken(
-							"could not find document",
-							token.Invalid(string(ctx.obuf), s.pos()),
+							token.Invalid(
+								"could not find document",
+								string(ctx.obuf), s.pos(),
+							),
 						)
 					}
 					if tk.Type != token.StringType {
diff --git a/token/token.go b/token/token.go
index a90d0df..208ebff 100644
--- a/token/token.go
+++ b/token/token.go
@@ -727,14 +727,24 @@ func (p *Position) String() string {
 
 // Token type for token
 type Token struct {
-	Type          Type
+	// Type is a token type.
+	Type Type
+	// CharacterType is a character type.
 	CharacterType CharacterType
-	Indicator     Indicator
-	Value         string
-	Origin        string
-	Position      *Position
-	Next          *Token
-	Prev          *Token
+	// Indicator is a indicator type.
+	Indicator Indicator
+	// Value is a string extracted with only meaningful characters, with spaces and such removed.
+	Value string
+	// Origin is a string that stores the original text as-is.
+	Origin string
+	// Error keeps error message for InvalidToken.
+	Error string
+	// Position is a token position.
+	Position *Position
+	// Next is a next token reference.
+	Next *Token
+	// Prev is a previous token reference.
+	Prev *Token
 }
 
 // PreviousType previous token type
@@ -1090,13 +1100,14 @@ func DocumentEnd(org string, pos *Position) *Token {
 	}
 }
 
-func Invalid(org string, pos *Position) *Token {
+func Invalid(err string, org string, pos *Position) *Token {
 	return &Token{
 		Type:          InvalidType,
 		CharacterType: CharacterTypeInvalid,
 		Indicator:     NotIndicator,
 		Value:         org,
 		Origin:        org,
+		Error:         err,
 		Position:      pos,
 	}
 }


Test patch:
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 6221aaf..7d0eb10 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -1041,7 +1041,7 @@ a: |invalidopt
   foo
 `,
 			`
-[2:4] found invalid token
+[2:4] invalid header option: "invalidopt"
 >  2 | a: |invalidopt
           ^
    3 |   foo`,
@@ -1160,7 +1160,7 @@ b: - 2
 		{
 			`a: 'foobarbaz`,
 			`
-[1:4] found invalid token
+[1:4] could not find end character of single-quoted text
 >  1 | a: 'foobarbaz
           ^
 `,
@@ -1168,7 +1168,7 @@ b: - 2
 		{
 			`a: "\"key\": \"value:\"`,
 			`
-[1:4] found invalid token
+[1:4] could not find end character of double-quoted text
 >  1 | a: "\"key\": \"value:\"
           ^
 `,
@@ -1192,7 +1192,7 @@ b: - 2
 		{
 			">\n>",
 			`
-[2:1] found invalid token
+[2:1] could not find document
    1 | >
 >  2 | >
        ^
@@ -1201,7 +1201,7 @@ b: - 2
 		{
 			">\n1",
 			`
-[2:1] found invalid token
+[2:1] could not find document
    1 | >
 >  2 | 1
        ^
@@ -1210,7 +1210,7 @@ b: - 2
 		{
 			"|\n1",
 			`
-[2:1] found invalid token
+[2:1] could not find document
    1 | |
 >  2 | 1
        ^
@@ -1219,7 +1219,7 @@ b: - 2
 		{
 			"a: >3\n  1",
 			`
-[2:3] found invalid token
+[2:3] invalid number of indent is specified in the document header
    1 | a: >3
 >  2 |   1
          ^
@@ -1261,14 +1261,14 @@ a:
 		{
 			"key: [@val]",
 			`
-[1:7] found invalid token
+[1:7] '@' is a reserved character
 >  1 | key: [@val]
              ^
 `,
 		},
 		{
 			"key: [`val]",
-			"\n[1:7] found invalid token\n>  1 | key: [`val]\n             ^\n",
+			"\n[1:7] '`' is a reserved character\n>  1 | key: [`val]\n             ^\n",
 		},
 		{
 			`{a: b}: v`,