Repository: goccy/go-yaml
Pull request #509 (2024-11-06 06:54:44)

Fix number parsing
fix https://github.com/goccy/go-yaml/issues/435


codecov-commenter: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/goccy/go-yaml/pull/509?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) Report
Attention: Patch coverage is `83.01887%` with `9 lines` in your changes missing coverage. Please review.
> Project coverage is 76.64%. Comparing base [(`29b57b4`)](https://app.codecov.io/gh/goccy/go-yaml/commit/29b57b4cc6d61b4652c4a389ab4e4726a2a7faf6?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) to head [(`9da7086`)](https://app.codecov.io/gh/goccy/go-yaml/commit/9da7086c7aff3017e38d825d8dc221523b76f67a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima).

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #509      +/-   ##
==========================================
- Coverage   76.86%   76.64%   -0.22%     
==========================================
  Files          18       18              
  Lines        6206     6208       +2     
==========================================
- Hits         4770     4758      -12     
- Misses       1123     1132       +9     
- Partials      313      318       +5     
```

</details>

Patch:
diff --git a/token/token.go b/token/token.go
index c2d9a4b..fce8903 100644
--- a/token/token.go
+++ b/token/token.go
@@ -523,86 +523,91 @@ type numStat struct {
 	typ   numType
 }
 
-func getNumberStat(str string) *numStat {
+func getNumberStat(value string) *numStat {
 	stat := &numStat{}
-	if str == "" {
+	if value == "" {
 		return stat
 	}
-	if str == "-" || str == "." || str == "+" || str == "_" {
+	dotCount := strings.Count(value, ".")
+	if dotCount > 1 {
 		return stat
 	}
-	if str[0] == '_' {
+
+	trimmed := strings.TrimPrefix(strings.TrimPrefix(value, "+"), "-")
+
+	var typ numType
+	switch {
+	case strings.HasPrefix(trimmed, "0x"):
+		trimmed = strings.TrimPrefix(trimmed, "0x")
+		typ = numTypeHex
+	case strings.HasPrefix(trimmed, "0o"):
+		trimmed = strings.TrimPrefix(trimmed, "0o")
+		typ = numTypeOctet
+	case strings.HasPrefix(trimmed, "0b"):
+		trimmed = strings.TrimPrefix(trimmed, "0b")
+		typ = numTypeBinary
+	case dotCount == 1:
+		typ = numTypeFloat
+	}
+
+	if trimmed == "" {
 		return stat
 	}
-	dotFound := false
-	isNegative := false
-	isExponent := false
-	if str[0] == '-' {
-		isNegative = true
-	}
-	for idx, c := range str {
+
+	var numCount int
+	for idx, c := range trimmed {
+		if isNumber(c) {
+			numCount++
+			continue
+		}
 		switch c {
-		case 'x':
-			if (isNegative && idx == 2) || (!isNegative && idx == 1) {
-				continue
-			}
-		case 'o':
-			if (isNegative && idx == 2) || (!isNegative && idx == 1) {
-				continue
-			}
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+		case '_', '.':
 			continue
-		case 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F':
-			if (len(str) > 2 && str[0] == '0' && str[1] == 'x') ||
-				(len(str) > 3 && isNegative && str[1] == '0' && str[2] == 'x') {
-				// hex number
-				continue
+		case 'a', 'b', 'c', 'd', 'f', 'A', 'B', 'C', 'D', 'F':
+			if typ != numTypeHex && typ != numTypeBinary {
+				return stat
 			}
-			if c == 'b' && ((isNegative && idx == 2) || (!isNegative && idx == 1)) {
-				// binary number
+		case 'e', 'E':
+			if typ == numTypeHex || typ == numTypeBinary {
 				continue
 			}
-			if (c == 'e' || c == 'E') && dotFound {
-				// exponent
-				isExponent = true
-				continue
+			if typ != numTypeFloat {
+				return stat
 			}
-		case '.':
-			if dotFound {
-				// multiple dot
+
+			// looks like exponent number.
+			if len(trimmed) <= idx+2 {
 				return stat
 			}
-			dotFound = true
-			continue
-		case '-':
-			if idx == 0 || isExponent {
-				continue
+			sign := trimmed[idx+1]
+			if sign != '+' && sign != '-' {
+				return stat
 			}
-		case '+':
-			if idx == 0 || isExponent {
-				continue
+			for _, c := range trimmed[idx+2:] {
+				if !isNumber(c) {
+					return stat
+				}
 			}
-		case '_':
-			continue
+			stat.isNum = true
+			stat.typ = typ
+			return stat
+		default:
+			return stat
 		}
-		return stat
 	}
-	stat.isNum = true
-	switch {
-	case dotFound:
-		stat.typ = numTypeFloat
-	case strings.HasPrefix(str, "0b") || strings.HasPrefix(str, "-0b"):
-		stat.typ = numTypeBinary
-	case strings.HasPrefix(str, "0x") || strings.HasPrefix(str, "-0x"):
-		stat.typ = numTypeHex
-	case strings.HasPrefix(str, "0o") || strings.HasPrefix(str, "-0o"):
-		stat.typ = numTypeOctet
-	case (len(str) > 1 && str[0] == '0') || (len(str) > 1 && str[0] == '-' && str[1] == '0'):
-		stat.typ = numTypeOctet
+	if numCount > 1 && trimmed[0] == '0' && typ == numTypeNone {
+		// YAML 1.1 Spec ?
+		typ = numTypeOctet
 	}
+	stat.isNum = true
+	stat.typ = typ
 	return stat
 }
 
+func isNumber(c rune) bool {
+	return c >= '0' && c <= '9'
+}
+
 func looksLikeTimeValue(value string) bool {
 	for i, c := range value {
 		switch c {
@@ -672,7 +677,7 @@ func LiteralBlockHeader(value string) string {
 	}
 }
 
-// New create reserved keyword token or number token and other string token
+// New create reserved keyword token or number token and other string token.
 func New(value string, org string, pos *Position) *Token {
 	fn := reservedKeywordMap[value]
 	if fn != nil {


Test patch:
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index 810d2d2..001e886 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -30,7 +30,7 @@ func TestTokenize(t *testing.T) {
 			YAML: `0_`,
 			Tokens: token.Tokens{
 				{
-					Type:          token.OctetIntegerType,
+					Type:          token.IntegerType,
 					CharacterType: token.CharacterTypeMiscellaneous,
 					Indicator:     token.NotIndicator,
 					Value:         "0_",
@@ -38,6 +38,54 @@ func TestTokenize(t *testing.T) {
 				},
 			},
 		},
+		{
+			YAML: `0x_1A_2B_3C`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.HexIntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0x_1A_2B_3C",
+					Origin:        "0x_1A_2B_3C",
+				},
+			},
+		},
+		{
+			YAML: `+0b1010`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.BinaryIntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "+0b1010",
+					Origin:        "+0b1010",
+				},
+			},
+		},
+		{
+			YAML: `0100`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.OctetIntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0100",
+					Origin:        "0100",
+				},
+			},
+		},
+		{
+			YAML: `0o10`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.OctetIntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0o10",
+					Origin:        "0o10",
+				},
+			},
+		},
 		{
 			YAML: `{}
 		`,
@@ -2197,6 +2245,18 @@ s: >-3
 				},
 			},
 		},
+		{
+			YAML: `1x0`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.StringType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "1x0",
+					Origin:        "1x0",
+				},
+			},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.YAML, func(t *testing.T) {