Repository: goccy/go-yaml
Pull request #498 (2024-11-02 09:41:18)

Fix parsing of number
ref: https://github.com/goccy/go-yaml/issues/191

codecov-commenter: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/goccy/go-yaml/pull/498?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 76.55%. Comparing base [(`e27add4`)](https://app.codecov.io/gh/goccy/go-yaml/commit/e27add428c9c2f5b120ad55c7a1abff73a2635c4?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima) to head [(`4ee9a27`)](https://app.codecov.io/gh/goccy/go-yaml/commit/4ee9a279eaf0f6ffe97c5d97887d331c8def05d9?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Masaaki+Goshima).

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##           master     #498   +/-   ##
=======================================
  Coverage   76.55%   76.55%           
=======================================
  Files          18       18           
  Lines        6027     6027           
=======================================
  Hits         4614     4614           
  Misses       1106     1106           
  Partials      307      307           
```

</details>

Patch:
diff --git a/ast/ast.go b/ast/ast.go
index 29eb6b2..47039ec 100644
--- a/ast/ast.go
+++ b/ast/ast.go
@@ -341,25 +341,19 @@ func Bool(tk *token.Token) *BoolNode {
 
 // Integer create node for integer value
 func Integer(tk *token.Token) *IntegerNode {
-	value := removeUnderScoreFromNumber(tk.Value)
 	switch tk.Type {
 	case token.BinaryIntegerType:
 		// skip two characters because binary token starts with '0b'
-		skipCharacterNum := 2
-		negativePrefix := ""
-		if value[0] == '-' {
-			skipCharacterNum++
-			negativePrefix = "-"
-		}
-		if len(negativePrefix) > 0 {
-			i, _ := strconv.ParseInt(negativePrefix+value[skipCharacterNum:], 2, 64)
+		parsedNum := parseNumber("0b", tk.Value)
+		if parsedNum.isNegative {
+			i, _ := strconv.ParseInt(parsedNum.String(), 2, 64)
 			return &IntegerNode{
 				BaseNode: &BaseNode{},
 				Token:    tk,
 				Value:    i,
 			}
 		}
-		i, _ := strconv.ParseUint(negativePrefix+value[skipCharacterNum:], 2, 64)
+		i, _ := strconv.ParseUint(parsedNum.String(), 2, 64)
 		return &IntegerNode{
 			BaseNode: &BaseNode{},
 			Token:    tk,
@@ -367,28 +361,16 @@ func Integer(tk *token.Token) *IntegerNode {
 		}
 	case token.OctetIntegerType:
 		// octet token starts with '0o' or '-0o' or '0' or '-0'
-		skipCharacterNum := 1
-		negativePrefix := ""
-		if value[0] == '-' {
-			skipCharacterNum++
-			if len(value) > 2 && value[2] == 'o' {
-				skipCharacterNum++
-			}
-			negativePrefix = "-"
-		} else {
-			if value[1] == 'o' {
-				skipCharacterNum++
-			}
-		}
-		if len(negativePrefix) > 0 {
-			i, _ := strconv.ParseInt(negativePrefix+value[skipCharacterNum:], 8, 64)
+		parsedNum := parseNumber("0o", tk.Value)
+		if parsedNum.isNegative {
+			i, _ := strconv.ParseInt(parsedNum.String(), 8, 64)
 			return &IntegerNode{
 				BaseNode: &BaseNode{},
 				Token:    tk,
 				Value:    i,
 			}
 		}
-		i, _ := strconv.ParseUint(value[skipCharacterNum:], 8, 64)
+		i, _ := strconv.ParseUint(parsedNum.String(), 8, 64)
 		return &IntegerNode{
 			BaseNode: &BaseNode{},
 			Token:    tk,
@@ -396,36 +378,32 @@ func Integer(tk *token.Token) *IntegerNode {
 		}
 	case token.HexIntegerType:
 		// hex token starts with '0x' or '-0x'
-		skipCharacterNum := 2
-		negativePrefix := ""
-		if value[0] == '-' {
-			skipCharacterNum++
-			negativePrefix = "-"
-		}
-		if len(negativePrefix) > 0 {
-			i, _ := strconv.ParseInt(negativePrefix+value[skipCharacterNum:], 16, 64)
+		parsedNum := parseNumber("0x", tk.Value)
+		if parsedNum.isNegative {
+			i, _ := strconv.ParseInt(parsedNum.String(), 16, 64)
 			return &IntegerNode{
 				BaseNode: &BaseNode{},
 				Token:    tk,
 				Value:    i,
 			}
 		}
-		i, _ := strconv.ParseUint(value[skipCharacterNum:], 16, 64)
+		i, _ := strconv.ParseUint(parsedNum.String(), 16, 64)
 		return &IntegerNode{
 			BaseNode: &BaseNode{},
 			Token:    tk,
 			Value:    i,
 		}
 	}
-	if value[0] == '-' || value[0] == '+' {
-		i, _ := strconv.ParseInt(value, 10, 64)
+	parsedNum := parseNumber("", tk.Value)
+	if parsedNum.isNegative {
+		i, _ := strconv.ParseInt(parsedNum.String(), 10, 64)
 		return &IntegerNode{
 			BaseNode: &BaseNode{},
 			Token:    tk,
 			Value:    i,
 		}
 	}
-	i, _ := strconv.ParseUint(value, 10, 64)
+	i, _ := strconv.ParseUint(parsedNum.String(), 10, 64)
 	return &IntegerNode{
 		BaseNode: &BaseNode{},
 		Token:    tk,
@@ -433,9 +411,44 @@ func Integer(tk *token.Token) *IntegerNode {
 	}
 }
 
+type parsedNumber struct {
+	isNegative bool
+	num        string
+}
+
+func (n *parsedNumber) String() string {
+	if n.isNegative {
+		return "-" + n.num
+	}
+	return n.num
+}
+
+func parseNumber(prefix, value string) *parsedNumber {
+	isNegative := value[0] == '-'
+	trimmed := strings.TrimPrefix(value, "+")
+	trimmed = strings.TrimPrefix(trimmed, "-")
+	trimmed = strings.TrimPrefix(trimmed, prefix)
+
+	num := make([]rune, 0, len(trimmed))
+	for _, v := range trimmed {
+		if v == '_' {
+			continue
+		}
+		num = append(num, v)
+	}
+	if len(num) == 0 {
+		num = append(num, '0')
+	}
+	return &parsedNumber{
+		isNegative: isNegative,
+		num:        string(num),
+	}
+}
+
 // Float create node for float value
 func Float(tk *token.Token) *FloatNode {
-	f, _ := strconv.ParseFloat(removeUnderScoreFromNumber(tk.Value), 64)
+	parsedNum := parseNumber("", tk.Value)
+	f, _ := strconv.ParseFloat(parsedNum.String(), 64)
 	return &FloatNode{
 		BaseNode: &BaseNode{},
 		Token:    tk,
@@ -662,10 +675,6 @@ func (d *DocumentNode) MarshalYAML() ([]byte, error) {
 	return []byte(d.String()), nil
 }
 
-func removeUnderScoreFromNumber(num string) string {
-	return strings.ReplaceAll(num, "_", "")
-}
-
 // NullNode type of null node
 type NullNode struct {
 	*BaseNode


Test patch:
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index 2a49239..b833221 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -26,6 +26,18 @@ func TestTokenize(t *testing.T) {
 				},
 			},
 		},
+		{
+			YAML: `0_`,
+			Tokens: token.Tokens{
+				{
+					Type:          token.OctetIntegerType,
+					CharacterType: token.CharacterTypeMiscellaneous,
+					Indicator:     token.NotIndicator,
+					Value:         "0_",
+					Origin:        "0_",
+				},
+			},
+		},
 		{
 			YAML: `{}
 		`,
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 09bf8cb..3c85e49 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -17,6 +17,7 @@ import (
 func TestParser(t *testing.T) {
 	sources := []string{
 		"null\n",
+		"0_",
 		"{}\n",
 		"v: hi\n",
 		"v: \"true\"\n",