Repository: goccy/go-yaml
Pull request #564 (2024-12-02 02:23:40)

Support UTF-16 surrogate pair
FYI: https://github.com/goccy/go-yaml/pull/445

codecov-commenter: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/goccy/go-yaml/pull/564?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) Report
Attention: Patch coverage is `85.18519%` with `8 lines` in your changes missing coverage. Please review.
> Project coverage is 77.45%. Comparing base [(`bf03d4d`)](https://app.codecov.io/gh/goccy/go-yaml/commit/bf03d4d7b18d09c4ccbc0f2e0511ea8c2cefd50a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) to head [(`5473e23`)](https://app.codecov.io/gh/goccy/go-yaml/commit/5473e230e7bdd881d9521742ce7d998a42f3f434?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima).

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #564      +/-   ##
==========================================
+ Coverage   77.31%   77.45%   +0.14%     
==========================================
  Files          21       21              
  Lines        7440     7479      +39     
==========================================
+ Hits         5752     5793      +41     
- Misses       1290     1291       +1     
+ Partials      398      395       -3     
```

</details>

Patch:
diff --git a/scanner/scanner.go b/scanner/scanner.go
index a379a8c..a82ee6b 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -439,25 +439,67 @@ func (s *Scanner) scanDoubleQuote(ctx *Context) (*token.Token, error) {
 					value = append(value, rune(codeNum))
 				}
 			case 'u':
+				// \u0000 style must have 5 characters at least.
 				if idx+5 >= size {
-					progress = 1
-					ctx.addOriginBuf(nextChar)
-					value = append(value, nextChar)
-				} else {
-					progress = 5
-					codeNum := hexRunesToInt(src[idx+2 : idx+progress+1])
-					value = append(value, rune(codeNum))
+					return nil, ErrInvalidToken(
+						token.Invalid(
+							"not enough length for escaped UTF-16 character",
+							string(ctx.obuf), s.pos(),
+						),
+					)
+				}
+				progress = 5
+				codeNum := hexRunesToInt(src[idx+2 : idx+6])
+
+				// handle surrogate pairs.
+				if codeNum >= 0xD800 && codeNum <= 0xDBFF {
+					high := codeNum
+
+					// \u0000\u0000 style must have 11 characters at least.
+					if idx+11 >= size {
+						return nil, ErrInvalidToken(
+							token.Invalid(
+								"not enough length for escaped UTF-16 surrogate pair",
+								string(ctx.obuf), s.pos(),
+							),
+						)
+					}
+
+					if src[idx+6] != '\\' || src[idx+7] != 'u' {
+						return nil, ErrInvalidToken(
+							token.Invalid(
+								"found unexpected character after high surrogate for UTF-16 surrogate pair",
+								string(ctx.obuf), s.pos(),
+							),
+						)
+					}
+
+					low := hexRunesToInt(src[idx+8 : idx+12])
+					if low < 0xDC00 || low > 0xDFFF {
+						return nil, ErrInvalidToken(
+							token.Invalid(
+								"found unexpected low surrogate after high surrogate",
+								string(ctx.obuf), s.pos(),
+							),
+						)
+					}
+					codeNum = ((high - 0xD800) * 0x400) + (low - 0xDC00) + 0x10000
+					progress += 6
 				}
+				value = append(value, rune(codeNum))
 			case 'U':
+				// \U00000000 style must have 9 characters at least.
 				if idx+9 >= size {
-					progress = 1
-					ctx.addOriginBuf(nextChar)
-					value = append(value, nextChar)
-				} else {
-					progress = 9
-					codeNum := hexRunesToInt(src[idx+2 : idx+progress+1])
-					value = append(value, rune(codeNum))
+					return nil, ErrInvalidToken(
+						token.Invalid(
+							"not enough length for escaped UTF-32 character",
+							string(ctx.obuf), s.pos(),
+						),
+					)
 				}
+				progress = 9
+				codeNum := hexRunesToInt(src[idx+2 : idx+10])
+				value = append(value, rune(codeNum))
 			case '\n':
 				isFirstLineChar = true
 				isNewLine = true


Test patch:
diff --git a/decode_test.go b/decode_test.go
index b95cc0d..3b39819 100644
--- a/decode_test.go
+++ b/decode_test.go
@@ -1117,6 +1117,18 @@ c:
 `,
 			[]string{"Fun with \\", "\" \u0007 \b \u001b \f", "\n \r \t \u000b \u0000", "\u0020 \u00a0 \u0085 \u2028 \u2029 A A A"},
 		},
+		{
+			`"\ud83e\udd23"`,
+			"🤣",
+		},
+		{
+			`"\uD83D\uDE00\uD83D\uDE01"`,
+			"😀😁",
+		},
+		{
+			`"\uD83D\uDE00a\uD83D\uDE01"`,
+			"😀a😁",
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.source, func(t *testing.T) {
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index 052c00a..8e94c2f 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -3233,6 +3233,26 @@ a: |
 	c
 `,
 		},
+		{
+			name: "invalid UTF-16 character",
+			src:  `"\u00"`,
+		},
+		{
+			name: "invalid UTF-16 surrogate pair length",
+			src:  `"\ud800"`,
+		},
+		{
+			name: "invalid UTF-16 low surrogate prefix",
+			src:  `"\ud800\v"`,
+		},
+		{
+			name: "invalid UTF-16 low surrogate",
+			src:  `"\ud800\u0000"`,
+		},
+		{
+			name: "invalid UTF-32 character",
+			src:  `"\U0000"`,
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {