Repository: goccy/go-yaml
Pull request #524 (2024-11-12 07:40:58)

Fix token offset
fix https://github.com/goccy/go-yaml/issues/303

codecov-commenter: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/goccy/go-yaml/pull/524?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 79.22%. Comparing base [(`3fa49bf`)](https://app.codecov.io/gh/goccy/go-yaml/commit/3fa49bfdb5ab8a2c53671f8363014523d859744d?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) to head [(`7c09e13`)](https://app.codecov.io/gh/goccy/go-yaml/commit/7c09e1306401b6393c827a63d2f467fe85567b12?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima).

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #524      +/-   ##
==========================================
+ Coverage   79.21%   79.22%   +0.01%     
==========================================
  Files          17       17              
  Lines        6144     6147       +3     
==========================================
+ Hits         4867     4870       +3     
  Misses        965      965              
  Partials      312      312              
```

</details>

Patch:
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 8739bf5..913c878 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -606,8 +606,10 @@ func (s *Scanner) scanDocument(ctx *Context, c rune) error {
 
 func (s *Scanner) scanNewLine(ctx *Context, c rune) {
 	if len(ctx.buf) > 0 && s.savedPos == nil {
+		bufLen := len(ctx.bufferedSrc())
 		s.savedPos = s.pos()
-		s.savedPos.Column -= len(ctx.bufferedSrc())
+		s.savedPos.Column -= bufLen
+		s.savedPos.Offset -= bufLen
 	}
 
 	// if the following case, origin buffer has unnecessary two spaces.
@@ -631,6 +633,7 @@ func (s *Scanner) scanNewLine(ctx *Context, c rune) {
 	if c == '\r' && ctx.nextChar() == '\n' {
 		ctx.addOriginBuf('\r')
 		s.progress(ctx, 1)
+		s.offset++
 		c = '\n'
 	}
 
diff --git a/token/token.go b/token/token.go
index 2b3d99d..8eaf974 100644
--- a/token/token.go
+++ b/token/token.go
@@ -765,8 +765,8 @@ func (t *Token) Clone() *Token {
 // Dump outputs token information to stdout for debugging.
 func (t *Token) Dump() {
 	fmt.Printf(
-		"[TYPE]:%q [CHARTYPE]:%q [INDICATOR]:%q [VALUE]:%q [ORG]:%q [POS(line:column:level)]: %d:%d:%d\n",
-		t.Type, t.CharacterType, t.Indicator, t.Value, t.Origin, t.Position.Line, t.Position.Column, t.Position.IndentLevel,
+		"[TYPE]:%q [CHARTYPE]:%q [INDICATOR]:%q [VALUE]:%q [ORG]:%q [POS(line:column:level:offset)]: %d:%d:%d:%d\n",
+		t.Type, t.CharacterType, t.Indicator, t.Value, t.Origin, t.Position.Line, t.Position.Column, t.Position.IndentLevel, t.Position.Offset,
 	)
 }
 


Test patch:
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index b57fff4..b7392d2 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -3186,3 +3186,32 @@ a: |invalid`,
 		})
 	}
 }
+
+func TestTokenOffset(t *testing.T) {
+	t.Run("crlf", func(t *testing.T) {
+		content := "project:\r\n  version: 1.2.3\r\n"
+		tokens := lexer.Tokenize(content)
+		if len(tokens) != 5 {
+			t.Fatalf("invalid token num. got %d", len(tokens))
+		}
+		if tokens[4].Value != "1.2.3" {
+			t.Fatalf("unexpected value. got %q", tokens[4].Value)
+		}
+		if tokens[4].Position.Offset != 22 {
+			t.Fatalf("unexpected offset. got %d", tokens[4].Position.Offset)
+		}
+	})
+	t.Run("lf", func(t *testing.T) {
+		content := "project:\n  version: 1.2.3\n"
+		tokens := lexer.Tokenize(content)
+		if len(tokens) != 5 {
+			t.Fatalf("invalid token num. got %d", len(tokens))
+		}
+		if tokens[4].Value != "1.2.3" {
+			t.Fatalf("unexpected value. got %q", tokens[4].Value)
+		}
+		if tokens[4].Position.Offset != 21 {
+			t.Fatalf("unexpected offset. got %d", tokens[4].Position.Offset)
+		}
+	})
+}