Repository: goccy/go-yaml
Pull request #200 (2021-02-19 19:41:31)

Fixing Issue 185, incorrect column annotation when single or double quotes are used

When single or double quotes are in scanner.go:

case ''', '"':
if !ctx.existsBuffer() {
token, progress := s.scanQuote(ctx, c)
ctx.addToken(token)
s.progressColumn(ctx, progress)
...
progress does not account for the initial position advancement form the start quote in the scanQuote() call. i.e. a token with the 3 character:
"1"
only progresses the column 2 positions (scanQuote() returns progress=2), instead of 3, causing a loss in synchronization.




Patch:
diff --git a/scanner/scanner.go b/scanner/scanner.go
index f1d19c7..465c3a7 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -195,8 +195,9 @@ func (s *Scanner) breakLiteral(ctx *Context) {
 
 func (s *Scanner) scanSingleQuote(ctx *Context) (tk *token.Token, pos int) {
 	ctx.addOriginBuf('\'')
+	srcpos := s.pos()
 	startIndex := ctx.idx + 1
-	ctx.progress(1)
+	s.progressColumn(ctx, 1)
 	src := ctx.src
 	size := len(src)
 	value := []rune{}
@@ -223,7 +224,7 @@ func (s *Scanner) scanSingleQuote(ctx *Context) (tk *token.Token, pos int) {
 			idx++
 			continue
 		}
-		tk = token.SingleQuote(string(value), string(ctx.obuf), s.pos())
+		tk = token.SingleQuote(string(value), string(ctx.obuf), srcpos)
 		pos = idx - startIndex + 1
 		return
 	}
@@ -250,8 +251,9 @@ func hexRunesToInt(b []rune) int {
 
 func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
 	ctx.addOriginBuf('"')
+	srcpos := s.pos()
 	startIndex := ctx.idx + 1
-	ctx.progress(1)
+	s.progressColumn(ctx, 1)
 	src := ctx.src
 	size := len(src)
 	value := []rune{}
@@ -363,7 +365,7 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (tk *token.Token, pos int) {
 			isFirstLineChar = false
 			continue
 		}
-		tk = token.DoubleQuote(string(value), string(ctx.obuf), s.pos())
+		tk = token.DoubleQuote(string(value), string(ctx.obuf), srcpos)
 		pos = idx - startIndex + 1
 		return
 	}


Test patch:
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index b54cdcc..ab7f863 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -1,10 +1,12 @@
 package lexer_test
 
 import (
+	"sort"
 	"strings"
 	"testing"
 
 	"github.com/goccy/go-yaml/lexer"
+	"github.com/goccy/go-yaml/token"
 )
 
 func TestTokenize(t *testing.T) {
@@ -59,3 +61,211 @@ func TestTokenize(t *testing.T) {
 		lexer.Tokenize(src).Dump()
 	}
 }
+
+type testToken struct {
+	line   int
+	column int
+	value  string
+}
+
+func TestTokenValueAndLineColumnPosition(t *testing.T) {
+	tests := []struct {
+		name   string
+		src    string
+		expect map[int]string // Column -> Value map.
+	}{
+		{
+			name: "single quote, single value array",
+			src:  "test: ['test']",
+			expect: map[int]string{
+				1:  "test",
+				5:  ":",
+				7:  "[",
+				8:  "test",
+				14: "]",
+			},
+		},
+		{
+			name: "double quote, single value array",
+			src:  `test: ["test"]`,
+			expect: map[int]string{
+				1:  "test",
+				5:  ":",
+				7:  "[",
+				8:  "test",
+				14: "]",
+			},
+		},
+		{
+			name: "no quotes, single value array",
+			src:  "test: [somevalue]",
+			expect: map[int]string{
+				1:  "test",
+				5:  ":",
+				7:  "[",
+				8:  "somevalue",
+				17: "]",
+			},
+		},
+		{
+			name: "single quote, multi value array",
+			src:  "myarr: ['1','2','3', '444' , '55','66' ,  '77'  ]",
+			expect: map[int]string{
+				1:  "myarr",
+				6:  ":",
+				8:  "[",
+				9:  "1",
+				12: ",",
+				13: "2",
+				16: ",",
+				17: "3",
+				20: ",",
+				22: "444",
+				28: ",",
+				30: "55",
+				34: ",",
+				35: "66",
+				40: ",",
+				43: "77",
+				49: "]",
+			},
+		},
+		{
+			name: "double quote, multi value array",
+			src:  `myarr: ["1","2","3", "444" , "55","66" ,  "77"  ]`,
+			expect: map[int]string{
+				1:  "myarr",
+				6:  ":",
+				8:  "[",
+				9:  "1",
+				12: ",",
+				13: "2",
+				16: ",",
+				17: "3",
+				20: ",",
+				22: "444",
+				28: ",",
+				30: "55",
+				34: ",",
+				35: "66",
+				40: ",",
+				43: "77",
+				49: "]",
+			},
+		},
+		{
+			name: "no quote, multi value array",
+			src:  "numbers: [1, 5, 99,100, 3, 7 ]",
+			expect: map[int]string{
+				1:  "numbers",
+				8:  ":",
+				10: "[",
+				11: "1",
+				12: ",",
+				14: "5",
+				15: ",",
+				17: "99",
+				19: ",",
+				20: "100",
+				23: ",",
+				25: "3",
+				26: ",",
+				28: "7",
+				30: "]",
+			},
+		},
+		{
+			name: "double quotes, nested arrays",
+			src:  `Strings: ["1",["2",["3"]]]`,
+			expect: map[int]string{
+				1:  "Strings",
+				8:  ":",
+				10: "[",
+				11: "1",
+				14: ",",
+				15: "[",
+				16: "2",
+				19: ",",
+				20: "[",
+				21: "3",
+				24: "]",
+				25: "]",
+				26: "]",
+			},
+		},
+		{
+			name: "mixed quotes, nested arrays",
+			src:  `Values: [1,['2',"3",4,["5",6]]]`,
+			expect: map[int]string{
+				1:  "Values",
+				7:  ":",
+				9:  "[",
+				10: "1",
+				11: ",",
+				12: "[",
+				13: "2",
+				16: ",",
+				17: "3",
+				20: ",",
+				21: "4",
+				22: ",",
+				23: "[",
+				24: "5",
+				27: ",",
+				28: "6",
+				29: "]",
+				30: "]",
+				31: "]",
+			},
+		},
+		{
+			name: "double quote, empty array",
+			src:  `Empty: ["", ""]`,
+			expect: map[int]string{
+				1:  "Empty",
+				6:  ":",
+				8:  "[",
+				9:  "",
+				11: ",",
+				13: "",
+				15: "]",
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := lexer.Tokenize(tc.src)
+			sort.Slice(got, func(i, j int) bool {
+				return got[i].Position.Column < got[j].Position.Column
+			})
+			var expected []testToken
+			for k, v := range tc.expect {
+				tt := testToken{
+					line:   1,
+					column: k,
+					value:  v,
+				}
+				expected = append(expected, tt)
+			}
+			sort.Slice(expected, func(i, j int) bool {
+				return expected[i].column < expected[j].column
+			})
+			if len(got) != len(expected) {
+				t.Errorf("Tokenize(%s) token count mismatch, expected:%d got:%d", tc.src, len(expected), len(got))
+			}
+			for i, tok := range got {
+				if !tokenMatches(tok, expected[i]) {
+					t.Errorf("Tokenize(%s) expected:%+v got line:%d column:%d value:%s", tc.src, expected[i], tok.Position.Line, tok.Position.Column, tok.Value)
+				}
+			}
+		})
+	}
+}
+
+func tokenMatches(t *token.Token, e testToken) bool {
+	return t != nil && t.Position != nil &&
+		t.Value == e.value &&
+		t.Position.Line == e.line &&
+		t.Position.Column == e.column
+}