Repository: goccy/go-json
Pull request #425 (2023-02-01 19:37:44)

Fix mapDecoder.DecodeStream() for empty objects containing whitespace
Current master breaks when encountering JSON containing an empty object containing whitespace like `{"obj":{ }}` when using the stream decoder - `Unmarshal()` behaves as expected.

This PR fixed that problem, and `NewDecoder().Decode()` will behave like `Unmarshal()`.



Patch:
diff --git a/internal/decoder/map.go b/internal/decoder/map.go
index 7a6eea3..07a9cae 100644
--- a/internal/decoder/map.go
+++ b/internal/decoder/map.go
@@ -88,7 +88,7 @@ func (d *mapDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) erro
 		mapValue = makemap(d.mapType, 0)
 	}
 	s.cursor++
-	if s.equalChar('}') {
+	if s.skipWhiteSpace() == '}' {
 		*(*unsafe.Pointer)(p) = mapValue
 		s.cursor++
 		return nil
diff --git a/internal/decoder/struct.go b/internal/decoder/struct.go
index 6d32654..313da15 100644
--- a/internal/decoder/struct.go
+++ b/internal/decoder/struct.go
@@ -51,6 +51,14 @@ func init() {
 	}
 }
 
+func toASCIILower(s string) string {
+	b := []byte(s)
+	for i := range b {
+		b[i] = largeToSmallTable[b[i]]
+	}
+	return string(b)
+}
+
 func newStructDecoder(structName, fieldName string, fieldMap map[string]*structFieldSet) *structDecoder {
 	return &structDecoder{
 		fieldMap:         fieldMap,
@@ -91,6 +99,10 @@ func (d *structDecoder) tryOptimize() {
 	for k, v := range d.fieldMap {
 		key := strings.ToLower(k)
 		if key != k {
+			if key != toASCIILower(k) {
+				d.isTriedOptimize = true
+				return
+			}
 			// already exists same key (e.g. Hello and HELLO has same lower case key
 			if _, exists := conflicted[key]; exists {
 				d.isTriedOptimize = true
@@ -158,49 +170,53 @@ func (d *structDecoder) tryOptimize() {
 }
 
 // decode from '\uXXXX'
-func decodeKeyCharByUnicodeRune(buf []byte, cursor int64) ([]byte, int64) {
+func decodeKeyCharByUnicodeRune(buf []byte, cursor int64) ([]byte, int64, error) {
 	const defaultOffset = 4
 	const surrogateOffset = 6
 
+	if cursor+defaultOffset >= int64(len(buf)) {
+		return nil, 0, errors.ErrUnexpectedEndOfJSON("escaped string", cursor)
+	}
+
 	r := unicodeToRune(buf[cursor : cursor+defaultOffset])
 	if utf16.IsSurrogate(r) {
 		cursor += defaultOffset
 		if cursor+surrogateOffset >= int64(len(buf)) || buf[cursor] != '\\' || buf[cursor+1] != 'u' {
-			return []byte(string(unicode.ReplacementChar)), cursor + defaultOffset - 1
+			return []byte(string(unicode.ReplacementChar)), cursor + defaultOffset - 1, nil
 		}
 		cursor += 2
 		r2 := unicodeToRune(buf[cursor : cursor+defaultOffset])
 		if r := utf16.DecodeRune(r, r2); r != unicode.ReplacementChar {
-			return []byte(string(r)), cursor + defaultOffset - 1
+			return []byte(string(r)), cursor + defaultOffset - 1, nil
 		}
 	}
-	return []byte(string(r)), cursor + defaultOffset - 1
+	return []byte(string(r)), cursor + defaultOffset - 1, nil
 }
 
-func decodeKeyCharByEscapedChar(buf []byte, cursor int64) ([]byte, int64) {
+func decodeKeyCharByEscapedChar(buf []byte, cursor int64) ([]byte, int64, error) {
 	c := buf[cursor]
 	cursor++
 	switch c {
 	case '"':
-		return []byte{'"'}, cursor
+		return []byte{'"'}, cursor, nil
 	case '\\':
-		return []byte{'\\'}, cursor
+		return []byte{'\\'}, cursor, nil
 	case '/':
-		return []byte{'/'}, cursor
+		return []byte{'/'}, cursor, nil
 	case 'b':
-		return []byte{'\b'}, cursor
+		return []byte{'\b'}, cursor, nil
 	case 'f':
-		return []byte{'\f'}, cursor
+		return []byte{'\f'}, cursor, nil
 	case 'n':
-		return []byte{'\n'}, cursor
+		return []byte{'\n'}, cursor, nil
 	case 'r':
-		return []byte{'\r'}, cursor
+		return []byte{'\r'}, cursor, nil
 	case 't':
-		return []byte{'\t'}, cursor
+		return []byte{'\t'}, cursor, nil
 	case 'u':
 		return decodeKeyCharByUnicodeRune(buf, cursor)
 	}
-	return nil, cursor
+	return nil, cursor, nil
 }
 
 func decodeKeyByBitmapUint8(d *structDecoder, buf []byte, cursor int64) (int64, *structFieldSet, error) {
@@ -242,7 +258,10 @@ func decodeKeyByBitmapUint8(d *structDecoder, buf []byte, cursor int64) (int64,
 					return 0, nil, errors.ErrUnexpectedEndOfJSON("string", cursor)
 				case '\\':
 					cursor++
-					chars, nextCursor := decodeKeyCharByEscapedChar(buf, cursor)
+					chars, nextCursor, err := decodeKeyCharByEscapedChar(buf, cursor)
+					if err != nil {
+						return 0, nil, err
+					}
 					for _, c := range chars {
 						curBit &= bitmap[keyIdx][largeToSmallTable[c]]
 						if curBit == 0 {
@@ -305,7 +324,10 @@ func decodeKeyByBitmapUint16(d *structDecoder, buf []byte, cursor int64) (int64,
 					return 0, nil, errors.ErrUnexpectedEndOfJSON("string", cursor)
 				case '\\':
 					cursor++
-					chars, nextCursor := decodeKeyCharByEscapedChar(buf, cursor)
+					chars, nextCursor, err := decodeKeyCharByEscapedChar(buf, cursor)
+					if err != nil {
+						return 0, nil, err
+					}
 					for _, c := range chars {
 						curBit &= bitmap[keyIdx][largeToSmallTable[c]]
 						if curBit == 0 {
diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go
index 3b3ff3f..3ae39ba 100644
--- a/internal/encoder/compiler.go
+++ b/internal/encoder/compiler.go
@@ -617,6 +617,13 @@ func (c *Compiler) structCode(typ *runtime.Type, isPtr bool) (*StructCode, error
 	return code, nil
 }
 
+func toElemType(t *runtime.Type) *runtime.Type {
+	for t.Kind() == reflect.Ptr {
+		t = t.Elem()
+	}
+	return t
+}
+
 func (c *Compiler) structFieldCode(structCode *StructCode, tag *runtime.StructTag, isPtr, isOnlyOneFirstField bool) (*StructFieldCode, error) {
 	field := tag.Field
 	fieldType := runtime.Type2RType(field.Type)
@@ -626,7 +633,7 @@ func (c *Compiler) structFieldCode(structCode *StructCode, tag *runtime.StructTa
 		key:           tag.Key,
 		tag:           tag,
 		offset:        field.Offset,
-		isAnonymous:   field.Anonymous && !tag.IsTaggedKey,
+		isAnonymous:   field.Anonymous && !tag.IsTaggedKey && toElemType(fieldType).Kind() == reflect.Struct,
 		isTaggedKey:   tag.IsTaggedKey,
 		isNilableType: c.isNilableType(fieldType),
 		isNilCheck:    true,
diff --git a/internal/encoder/vm_color_indent/util.go b/internal/encoder/vm_color_indent/util.go
index 60e4a8e..2395abe 100644
--- a/internal/encoder/vm_color_indent/util.go
+++ b/internal/encoder/vm_color_indent/util.go
@@ -189,7 +189,7 @@ func appendNullComma(ctx *encoder.RuntimeContext, b []byte) []byte {
 }
 
 func appendColon(_ *encoder.RuntimeContext, b []byte) []byte {
-	return append(b, ':', ' ')
+	return append(b[:len(b)-2], ':', ' ')
 }
 
 func appendMapKeyValue(ctx *encoder.RuntimeContext, code *encoder.Opcode, b, key, value []byte) []byte {
@@ -229,8 +229,9 @@ func appendEmptyObject(_ *encoder.RuntimeContext, b []byte) []byte {
 
 func appendObjectEnd(ctx *encoder.RuntimeContext, code *encoder.Opcode, b []byte) []byte {
 	last := len(b) - 1
-	b[last] = '\n'
-	b = appendIndent(ctx, b, code.Indent-1)
+	// replace comma to newline
+	b[last-1] = '\n'
+	b = appendIndent(ctx, b[:last], code.Indent)
 	return append(b, '}', ',', '\n')
 }
 
diff --git a/internal/encoder/vm_indent/util.go b/internal/encoder/vm_indent/util.go
index fca8f18..6cb745e 100644
--- a/internal/encoder/vm_indent/util.go
+++ b/internal/encoder/vm_indent/util.go
@@ -133,7 +133,7 @@ func appendNullComma(_ *encoder.RuntimeContext, b []byte) []byte {
 }
 
 func appendColon(_ *encoder.RuntimeContext, b []byte) []byte {
-	return append(b, ':', ' ')
+	return append(b[:len(b)-2], ':', ' ')
 }
 
 func appendMapKeyValue(ctx *encoder.RuntimeContext, code *encoder.Opcode, b, key, value []byte) []byte {
@@ -173,8 +173,9 @@ func appendEmptyObject(_ *encoder.RuntimeContext, b []byte) []byte {
 
 func appendObjectEnd(ctx *encoder.RuntimeContext, code *encoder.Opcode, b []byte) []byte {
 	last := len(b) - 1
-	b[last] = '\n'
-	b = appendIndent(ctx, b, code.Indent-1)
+	// replace comma to newline
+	b[last-1] = '\n'
+	b = appendIndent(ctx, b[:last], code.Indent)
 	return append(b, '}', ',', '\n')
 }
 


Test patch:
diff --git a/decode_test.go b/decode_test.go
index f139eae..df914ed 100644
--- a/decode_test.go
+++ b/decode_test.go
@@ -282,6 +282,14 @@ func Test_Decoder_DisallowUnknownFields(t *testing.T) {
 	}
 }
 
+func Test_Decoder_EmptyObjectWithSpace(t *testing.T) {
+	dec := json.NewDecoder(strings.NewReader(`{"obj":{ }}`))
+	var v struct {
+		Obj map[string]int `json:"obj"`
+	}
+	assertErr(t, dec.Decode(&v))
+}
+
 type unmarshalJSON struct {
 	v int
 }
@@ -4015,3 +4023,30 @@ func TestIssue408(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestIssue416(t *testing.T) {
+	b := []byte(`{"Сообщение":"Текст"}`)
+
+	type T struct {
+		Msg string `json:"Сообщение"`
+	}
+	var x T
+	err := json.Unmarshal(b, &x)
+	assertErr(t, err)
+	assertEq(t, "unexpected result", "Текст", x.Msg)
+}
+
+func TestIssue429(t *testing.T) {
+	var x struct {
+		N int32
+	}
+	for _, b := range []string{
+		`{"\u"`,
+		`{"\u0"`,
+		`{"\u00"`,
+	} {
+		if err := json.Unmarshal([]byte(b), &x); err == nil {
+			t.Errorf("unexpected success")
+		}
+	}
+}
diff --git a/encode_test.go b/encode_test.go
index 63740f3..f3b57bf 100644
--- a/encode_test.go
+++ b/encode_test.go
@@ -2629,3 +2629,34 @@ func TestCustomMarshalForMapKey(t *testing.T) {
 	assertErr(t, err)
 	assertEq(t, "custom map key", string(expected), string(got))
 }
+
+func TestIssue417(t *testing.T) {
+	x := map[string]string{
+		"b": "b",
+		"a": "a",
+	}
+	b, err := json.MarshalIndentWithOption(x, "", " ", json.UnorderedMap())
+	assertErr(t, err)
+
+	var y map[string]string
+	err = json.Unmarshal(b, &y)
+	assertErr(t, err)
+
+	assertEq(t, "key b", "b", y["b"])
+	assertEq(t, "key a", "a", y["a"])
+}
+
+func TestIssue426(t *testing.T) {
+	type I interface {
+		Foo()
+	}
+	type A struct {
+		I
+		Val string
+	}
+	var s A
+	s.Val = "456"
+
+	b, _ := json.Marshal(s)
+	assertEq(t, "unexpected result", `{"I":null,"Val":"456"}`, string(b))
+}