Repository: goccy/go-yaml

Issue:
Int map keys incorrectly marshaled as string
**Describe the bug**
Whenever go-yaml encounters a map with int key, it marshals it like it had a string key. This makes it incompatible with itself.

**To Reproduce**

This is very easy to reproduce with even a simple map[int]string.
```
import (
	"fmt"

	"github.com/goccy/go-yaml"
)

func main() {
	f := make(map[int]string)
	f[1] = "one"
	f[3] = "three"
	enc, _ := yaml.Marshal(f)
	fmt.Printf("%s\n", enc)
	var g map[int]string
	err := yaml.Unmarshal(enc, &g)
	fmt.Printf("%v\n", err)
	fmt.Printf("%v\n", g)
}
```
Playground link is here: https://go.dev/play/p/OWQPOj_ECBP


**Expected behavior**
I expect that map[int]string would be encoded as `1: "one"` not as `"1":"one"`
I also expect that yaml.Unmarshal can handle without errors whatever output yaml.Marshal has created.

If you do nothing else than replace the import from `github.com/goccy/go-yaml` to `gopkg.in/yaml.v3`, the example code would work as expected.


**Screenshots**
Please run the code on Go Playground :-)

**Version Variables**
 - Go version: 1.24.1
 - go-yaml's Version: v1.15.13
But I think it affects all available versions.

**Additional context**
I tried to workaround this by using a MarshalYAML()([]byte,error) method on the map key type, but even if I return the byte slice directly, it is converted to string.

Discussion:
@ChielTimmermans: @shuheiktgw @goccy What needs to be done to get this merged?@shuheiktgw: Thanks for flagging it! I've taken over the PR from where it's left and created a new one.
https://github.com/goccy/go-yaml/pull/756@herczegzsolt: @shuheiktgw @goccy Is there any work I can do to a fix for this merged?

It should be straightforward to include the fix for basic types. Even if we push the fix for structs to a later date, this being partially fixed would be a big improvement.

We're currently using an internal fork just because of this issue, so I'm willing to do a lot of work to get it fixed upstream. Let me know how I could help.

Solution patch:
diff --git a/decode.go b/decode.go
index 19414dd..43c317f 100644
--- a/decode.go
+++ b/decode.go
@@ -40,6 +40,7 @@ type Decoder struct {
 	isResolvedReference  bool
 	validator            StructValidator
 	disallowUnknownField bool
+	allowedFieldPrefixes []string
 	allowDuplicateMapKey bool
 	useOrderedMap        bool
 	useJSONUnmarshaler   bool
@@ -1446,12 +1447,21 @@ func (d *Decoder) decodeStruct(ctx context.Context, dst reflect.Value, src ast.N
 	// Unknown fields are expected (they could be fields from the parent struct).
 	if len(unknownFields) != 0 && d.disallowUnknownField && src.GetToken() != nil {
 		for key, node := range unknownFields {
-			return errors.ErrUnknownField(fmt.Sprintf(`unknown field "%s"`, key), node.GetToken())
+			var ok bool
+			for _, prefix := range d.allowedFieldPrefixes {
+				if strings.HasPrefix(key, prefix) {
+					ok = true
+					break
+				}
+			}
+			if !ok {
+				return errors.ErrUnknownField(fmt.Sprintf(`unknown field "%s"`, key), node.GetToken())
+			}
 		}
 	}
 
 	if d.validator != nil {
-		if err := d.validator.Struct(dst.Addr().Interface()); err != nil {
+		if err := d.validator.Struct(dst.Interface()); err != nil {
 			ev := reflect.ValueOf(err)
 			if ev.Type().Kind() == reflect.Slice {
 				for i := 0; i < ev.Len(); i++ {
diff --git a/encode.go b/encode.go
index a686f28..e810608 100644
--- a/encode.go
+++ b/encode.go
@@ -712,9 +712,15 @@ func (e *Encoder) encodeMap(ctx context.Context, value reflect.Value, column int
 			anchorNode.Value = encoded
 			encoded = anchorNode
 		}
+
+		kn, err := e.encodeValue(ctx, reflect.ValueOf(key), column)
+		keyNode, ok := kn.(ast.MapKeyNode)
+		if !ok || err != nil {
+			keyNode = e.encodeString(fmt.Sprint(key), column)
+		}
 		node.Values = append(node.Values, ast.MappingValue(
 			nil,
-			e.encodeString(keyText, column),
+			keyNode,
 			encoded,
 		))
 		e.setSmartAnchor(vRef, keyText)
diff --git a/option.go b/option.go
index 32dbe23..12b8f27 100644
--- a/option.go
+++ b/option.go
@@ -69,6 +69,15 @@ func DisallowUnknownField() DecodeOption {
 	}
 }
 
+// AllowFieldPrefixes, when paired with [DisallowUnknownField], allows fields
+// with the specified prefixes to bypass the unknown field check.
+func AllowFieldPrefixes(prefixes ...string) DecodeOption {
+	return func(d *Decoder) error {
+		d.allowedFieldPrefixes = append(d.allowedFieldPrefixes, prefixes...)
+		return nil
+	}
+}
+
 // AllowDuplicateMapKey ignore syntax error when mapping keys that are duplicates.
 func AllowDuplicateMapKey() DecodeOption {
 	return func(d *Decoder) error {
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 13a7ecc..799f469 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -777,6 +777,15 @@ func (s *Scanner) scanComment(ctx *Context) bool {
 func (s *Scanner) scanMultiLine(ctx *Context, c rune) error {
 	state := ctx.getMultiLineState()
 	ctx.addOriginBuf(c)
+	// normalize CR and CRLF to LF
+	if c == '\r' {
+		if ctx.nextChar() == '\n' {
+			ctx.addOriginBuf('\n')
+			s.progress(ctx, 1)
+			s.offset++
+		}
+		c = '\n'
+	}
 	if ctx.isEOS() {
 		if s.isFirstCharAtLine && c == ' ' {
 			state.addIndent(ctx, s.column)
@@ -1148,14 +1157,25 @@ func (s *Scanner) scanMultiLineHeaderOption(ctx *Context) error {
 	s.progress(ctx, 1) // skip '|' or '>' character
 
 	var progress int
+	var crlf bool
 	for idx, c := range ctx.src[ctx.idx:] {
 		progress = idx
 		ctx.addOriginBuf(c)
 		if s.isNewLineChar(c) {
+			nextIdx := ctx.idx + idx + 1
+			if c == '\r' && nextIdx < len(ctx.src) && ctx.src[nextIdx] == '\n' {
+				crlf = true
+				continue // process \n in the next iteration
+			}
 			break
 		}
 	}
-	value := strings.TrimRight(ctx.source(ctx.idx, ctx.idx+progress), " ")
+	endPos := ctx.idx + progress
+	if crlf {
+		// Exclude \r
+		endPos = endPos - 1
+	}
+	value := strings.TrimRight(ctx.source(ctx.idx, endPos), " ")
 	commentValueIndex := strings.Index(value, "#")
 	opt := value
 	if commentValueIndex > 0 {
@@ -1189,7 +1209,7 @@ func (s *Scanner) scanMultiLineHeaderOption(ctx *Context) error {
 		ctx.setFolded(s.lastDelimColumn, opt)
 	}
 	if commentIndex > 0 {
-		comment := string(value[commentValueIndex+1:])
+		comment := value[commentValueIndex+1:]
 		s.offset += len(headerBuf)
 		s.column += len(headerBuf)
 		ctx.addToken(token.Comment(comment, string(ctx.obuf[len(headerBuf):]), s.pos()))



Test patch:
diff --git a/decode_test.go b/decode_test.go
index 798e5e7..2d56942 100644
--- a/decode_test.go
+++ b/decode_test.go
@@ -494,6 +494,24 @@ func TestDecoder(t *testing.T) {
 			value:  map[string]interface{}{"a": "50cent_of_dollar"},
 		},
 
+		// Unconventional keys
+		{
+			source: "1: v\n",
+			value:  map[int]string{1: "v"},
+		},
+		{
+			source: "1.1: v\n",
+			value:  map[float64]string{1.1: "v"},
+		},
+		{
+			source: "true: v\n",
+			value:  map[bool]string{true: "v"},
+		},
+		{
+			source: "2015-01-01T00:00:00Z: v\n",
+			value:  map[time.Time]string{time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC): "v"},
+		},
+
 		// Nulls
 		{
 			source: "null",
@@ -954,6 +972,14 @@ merge:
 			source: "v: |\n  hello\n  ...\n  world\n",
 			value:  map[string]string{"v": "hello\n...\nworld\n"},
 		},
+		{
+			source: "v: |\r\n  hello\r\n  ...\r\n  world\r\n",
+			value:  map[string]string{"v": "hello\n...\nworld\n"},
+		},
+		{
+			source: "v: |\r  hello\r  ...\r  world\r",
+			value:  map[string]string{"v": "hello\n...\nworld\n"},
+		},
 		{
 			source: "a: !!binary gIGC\n",
 			value:  map[string]string{"a": "\x80\x81\x82"},
@@ -970,6 +996,22 @@ merge:
 				},
 			},
 		},
+		{
+			source: "v:\r\n- A\r\n- |-\r\n  B\r\n  C\r\n",
+			value: map[string][]string{
+				"v": {
+					"A", "B\nC",
+				},
+			},
+		},
+		{
+			source: "v:\r- A\r- |-\r  B\r  C\r",
+			value: map[string][]string{
+				"v": {
+					"A", "B\nC",
+				},
+			},
+		},
 		{
 			source: "v:\n- A\n- |-\n  B\n  C\n\n\n",
 			value: map[string][]string{
@@ -986,6 +1028,22 @@ merge:
 				},
 			},
 		},
+		{
+			source: "v:\r\n- A\r\n- >-\r\n  B\r\n  C\r\n",
+			value: map[string][]string{
+				"v": {
+					"A", "B C",
+				},
+			},
+		},
+		{
+			source: "v:\r- A\r- >-\r  B\r  C\r",
+			value: map[string][]string{
+				"v": {
+					"A", "B C",
+				},
+			},
+		},
 		{
 			source: "v:\n- A\n- >-\n  B\n  C\n\n\n",
 			value: map[string][]string{
@@ -1933,6 +1991,32 @@ children:
 			t.Fatalf(`v.Children[1].B should be 2, got %d`, v.Children[1].B)
 		}
 	})
+
+	t.Run("with allowed prefixes", func(t *testing.T) {
+		var v struct {
+			A string `yaml:"a"`
+		}
+		yml := `---
+a: a_value
+x-some-extra-thing: b_value
+`
+
+		if err := yaml.NewDecoder(strings.NewReader(yml), yaml.DisallowUnknownField(), yaml.AllowFieldPrefixes("x-")).Decode(&v); err != nil {
+			t.Fatalf(`parsing should succeed: %s`, err)
+		}
+
+		yml = `---
+a: a_value
+b: b_value
+x-some-extra-thing: b_value
+`
+
+		err := yaml.NewDecoder(strings.NewReader(yml), yaml.DisallowUnknownField(), yaml.AllowFieldPrefixes("x-")).Decode(&v)
+		if err == nil {
+			t.Fatal("expected error")
+		}
+	})
+
 }
 
 func TestDecoder_AllowDuplicateMapKey(t *testing.T) {
diff --git a/encode_test.go b/encode_test.go
index 85a33dd..8d91996 100644
--- a/encode_test.go
+++ b/encode_test.go
@@ -187,6 +187,26 @@ func TestEncoder(t *testing.T) {
 			map[string]string{"a": "-"},
 			nil,
 		},
+		{
+			"1: v\n",
+			map[int]string{1: "v"},
+			nil,
+		},
+		{
+			"1.1: v\n",
+			map[float64]string{1.1: "v"},
+			nil,
+		},
+		{
+			"true: v\n",
+			map[bool]string{true: "v"},
+			nil,
+		},
+		{
+			"2015-01-01T00:00:00Z: v\n",
+			map[time.Time]string{time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC): "v"},
+			nil,
+		},
 		{
 			"123\n",
 			123,