Repository: bytedance/sonic
Pull request #218 (2022-04-19 10:07:53)

feat (encoder): add encoder option `NoNullSliceOrMap`
### Benchmark
- before
```
goos: darwin
goarch: amd64
pkg: github.com/bytedance/sonic/encoder
cpu: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz
BenchmarkOptionSliceOrMapNoNull/true-16                 10000000               246.6 ns/op           286 B/op          5 allocs/op
BenchmarkOptionSliceOrMapNoNull/false-16                10000000               247.4 ns/op           288 B/op          5 allocs/op
```
- after
```
goos: darwin
goarch: amd64
pkg: github.com/bytedance/sonic/encoder
cpu: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz
BenchmarkOptionSliceOrMapNoNull/true-16                 10000000               247.4 ns/op           282 B/op          5 allocs/op
BenchmarkOptionSliceOrMapNoNull/false-16                10000000               236.5 ns/op           283 B/op          5 allocs/op
```
- **conclusion**
**no significant effects on performance**

- interesting discovery
writing 4 chars (instruction `MOVL`) is faster than writing 2 chars (instruction `MOVW`)?


### More Benchmarks
```
go: added golang.org/x/perf v0.0.0-20220411212318-84e58bfe0a7e
name                                     old time/op    new time/op    delta
MarshalConcrete/CanadaGeometry_Sonic-12     724µs ±10%     712µs ± 7%     ~     (p=0.842 n=10+9)
MarshalConcrete/CitmCatalog_Sonic-12        450µs ± 8%     462µs ±21%     ~     (p=0.968 n=9+10)
MarshalConcrete/StringEscaped_Sonic-12     4.53µs ±23%    4.84µs ±12%     ~     (p=0.095 n=9+10)
MarshalConcrete/StringUnicode_Sonic-12     4.95µs ±30%    5.77µs ±40%     ~     (p=0.063 n=10+10)
MarshalConcrete/TwitterStatus_Sonic-12      309µs ±19%     281µs ±20%     ~     (p=0.075 n=10+10)
MarshalConcrete/SyntheaFhir_Sonic-12       8.96ms ± 3%    8.82ms ± 2%   -1.55%  (p=0.036 n=8+9)
MarshalConcrete/GolangSource_Sonic-12      2.29ms ± 7%    2.06ms ± 9%  -10.06%  (p=0.000 n=9+10)

name                                     old speed      new speed      delta
MarshalConcrete/CanadaGeometry_Sonic-12   375MB/s ± 9%   380MB/s ± 6%     ~     (p=0.842 n=10+9)
MarshalConcrete/CitmCatalog_Sonic-12     3.78GB/s ±16%  3.78GB/s ±18%     ~     (p=0.853 n=10+10)
MarshalConcrete/StringEscaped_Sonic-12   9.10GB/s ±29%  8.74GB/s ±12%     ~     (p=0.247 n=10+10)
MarshalConcrete/StringUnicode_Sonic-12   3.74GB/s ±25%  3.24GB/s ±31%     ~     (p=0.063 n=10+10)
MarshalConcrete/TwitterStatus_Sonic-12   2.07GB/s ±17%  2.26GB/s ±17%     ~     (p=0.075 n=10+10)
MarshalConcrete/SyntheaFhir_Sonic-12      224MB/s ± 3%   228MB/s ± 2%   +1.57%  (p=0.036 n=8+9)
MarshalConcrete/GolangSource_Sonic-12     849MB/s ± 8%   945MB/s ± 9%  +11.26%  (p=0.000 n=9+10)
```


Patch:
diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml
index 60f1c99..3cd204f 100644
--- a/.github/workflows/license-check.yml
+++ b/.github/workflows/license-check.yml
@@ -4,11 +4,11 @@ on: push
 
 jobs:
   build:
-    runs-on: self-hosted
+    runs-on: [self-hosted, X64]
     steps:
       - uses: actions/checkout@v2
 
       - name: Check License Header
-        uses: apache/skywalking-eyes@main
+        uses: apache/skywalking-eyes/header@main
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/api.go b/api.go
index 8b119cc..c694119 100644
--- a/api.go
+++ b/api.go
@@ -31,6 +31,7 @@ import (
     UseUnicodeErrors              bool
     DisallowUnknownFields         bool
     CopyString                    bool
+    NoNullSliceOrMap              bool
  }
  
  var (
diff --git a/encoder/assembler_amd64_go116.go b/encoder/assembler_amd64_go116.go
index ac7598e..803796f 100644
--- a/encoder/assembler_amd64_go116.go
+++ b/encoder/assembler_amd64_go116.go
@@ -89,11 +89,13 @@ const (
 )
 
 const (
-    _IM_null = 0x6c6c756e           // 'null'
-    _IM_true = 0x65757274           // 'true'
-    _IM_fals = 0x736c6166           // 'fals' ('false' without the 'e')
-    _IM_open = 0x00225c22           // '"\"∅'
-    _IM_mulv = -0x5555555555555555
+    _IM_null   = 0x6c6c756e           // 'null'
+    _IM_true   = 0x65757274           // 'true'
+    _IM_fals   = 0x736c6166           // 'fals' ('false' without the 'e')
+    _IM_open   = 0x00225c22           // '"\"∅'
+    _IM_array  = 0x5d5b               // '[]'
+    _IM_object = 0x7d7b               // '{}'
+    _IM_mulv   = -0x5555555555555555
 )
 
 const (
@@ -204,6 +206,8 @@ func (self *_Assembler) compile() {
 
 var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
     _OP_null           : (*_Assembler)._asm_OP_null,
+    _OP_empty_arr      : (*_Assembler)._asm_OP_empty_arr,
+    _OP_empty_obj      : (*_Assembler)._asm_OP_empty_obj, 
     _OP_bool           : (*_Assembler)._asm_OP_bool,
     _OP_i8             : (*_Assembler)._asm_OP_i8,
     _OP_i16            : (*_Assembler)._asm_OP_i16,
@@ -767,6 +771,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) {
     self.Emit("ADDQ", jit.Imm(4), _RL)                             // ADDQ $4, RL
 }
 
+func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
+    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
+    self.Sjmp("JC", "_empty_arr_{n}")
+    self._asm_OP_null(nil)
+    self.Sjmp("JMP", "_empty_arr_end_{n}")
+    self.Link("_empty_arr_{n}")
+    self.check_size(2)
+    self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0)) 
+    self.Emit("ADDQ", jit.Imm(2), _RL)    
+    self.Link("_empty_arr_end_{n}")                  
+}
+
+func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
+    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
+    self.Sjmp("JC", "_empty_obj_{n}")
+    self._asm_OP_null(nil)
+    self.Sjmp("JMP", "_empty_obj_end_{n}")
+    self.Link("_empty_obj_{n}")
+    self.check_size(2)
+    self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))  
+    self.Emit("ADDQ", jit.Imm(2), _RL) 
+    self.Link("_empty_obj_end_{n}")                                             
+}
+
 func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
     self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0))                // CMPB (SP.p), $0
     self.Sjmp("JE"  , "_false_{n}")                                 // JE   _false_{n}
diff --git a/encoder/assembler_amd64_go117.go b/encoder/assembler_amd64_go117.go
index fee3aa0..1677927 100644
--- a/encoder/assembler_amd64_go117.go
+++ b/encoder/assembler_amd64_go117.go
@@ -91,11 +91,13 @@ const (
 )
 
 const (
-    _IM_null = 0x6c6c756e           // 'null'
-    _IM_true = 0x65757274           // 'true'
-    _IM_fals = 0x736c6166           // 'fals' ('false' without the 'e')
-    _IM_open = 0x00225c22           // '"\"∅'
-    _IM_mulv = -0x5555555555555555
+    _IM_null   = 0x6c6c756e           // 'null'
+    _IM_true   = 0x65757274           // 'true'
+    _IM_fals   = 0x736c6166           // 'fals' ('false' without the 'e')
+    _IM_open   = 0x00225c22           // '"\"∅'
+    _IM_array  = 0x5d5b               // '[]'
+    _IM_object = 0x7d7b               // '{}'
+    _IM_mulv   = -0x5555555555555555
 )
 
 const (
@@ -130,16 +132,16 @@ var (
 )
 
 var (
-	_ST = jit.Reg("R15")     // can't use R14 since it's always scratched by Go...
-	_RP = jit.Reg("DI")
-	_RL = jit.Reg("SI")
-	_RC = jit.Reg("DX")
+    _ST = jit.Reg("R15")     // can't use R14 since it's always scratched by Go...
+    _RP = jit.Reg("DI")
+    _RL = jit.Reg("SI")
+    _RC = jit.Reg("DX")
 )
 
 var (
     _LR = jit.Reg("R9")
-	_ET = jit.Reg("AX")
-	_EP = jit.Reg("BX")
+    _ET = jit.Reg("AX")
+    _EP = jit.Reg("BX")
 )
 
 var (
@@ -209,6 +211,8 @@ func (self *_Assembler) compile() {
 
 var _OpFuncTab = [256]func(*_Assembler, *_Instr) {
     _OP_null           : (*_Assembler)._asm_OP_null,
+    _OP_empty_arr      : (*_Assembler)._asm_OP_empty_arr,
+    _OP_empty_obj      : (*_Assembler)._asm_OP_empty_obj, 
     _OP_bool           : (*_Assembler)._asm_OP_bool,
     _OP_i8             : (*_Assembler)._asm_OP_i8,
     _OP_i16            : (*_Assembler)._asm_OP_i16,
@@ -780,6 +784,30 @@ func (self *_Assembler) _asm_OP_null(_ *_Instr) {
     self.Emit("ADDQ", jit.Imm(4), _RL)                             // ADDQ $4, RL
 }
 
+func (self *_Assembler) _asm_OP_empty_arr(_ *_Instr) {
+    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
+    self.Sjmp("JC", "_empty_arr_{n}")
+    self._asm_OP_null(nil)
+    self.Sjmp("JMP", "_empty_arr_end_{n}")
+    self.Link("_empty_arr_{n}")
+    self.check_size(2)
+    self.Emit("MOVW", jit.Imm(_IM_array), jit.Sib(_RP, _RL, 1, 0)) 
+    self.Emit("ADDQ", jit.Imm(2), _RL)    
+    self.Link("_empty_arr_end_{n}")                  
+}
+
+func (self *_Assembler) _asm_OP_empty_obj(_ *_Instr) {
+    self.Emit("BTQ", jit.Imm(int64(bitNoNullSliceOrMap)), _ARG_fv)
+    self.Sjmp("JC", "_empty_obj_{n}")
+    self._asm_OP_null(nil)
+    self.Sjmp("JMP", "_empty_obj_end_{n}")
+    self.Link("_empty_obj_{n}")
+    self.check_size(2)
+    self.Emit("MOVW", jit.Imm(_IM_object), jit.Sib(_RP, _RL, 1, 0))  
+    self.Emit("ADDQ", jit.Imm(2), _RL) 
+    self.Link("_empty_obj_end_{n}")                                             
+}
+
 func (self *_Assembler) _asm_OP_bool(_ *_Instr) {
     self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0))                // CMPB (SP.p), $0
     self.Sjmp("JE"  , "_false_{n}")                                 // JE   _false_{n}
diff --git a/encoder/compiler.go b/encoder/compiler.go
index a84b658..41727b3 100644
--- a/encoder/compiler.go
+++ b/encoder/compiler.go
@@ -32,6 +32,8 @@ type _Op uint8
 
 const (
     _OP_null _Op = iota + 1
+    _OP_empty_arr
+    _OP_empty_obj
     _OP_bool
     _OP_i8
     _OP_i16
@@ -93,6 +95,8 @@ const (
 
 var _OpNames = [256]string {
     _OP_null           : "null",
+    _OP_empty_arr      : "empty_arr",
+    _OP_empty_obj      : "empty_obj",
     _OP_bool           : "bool",
     _OP_i8             : "i8",
     _OP_i16            : "i16",
@@ -481,19 +485,19 @@ func (self *_Compiler) compileOps(p *_Program, sp int, vt reflect.Type) {
     }
 }
 
-func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, fn func(*_Program, int, reflect.Type)) {
+func (self *_Compiler) compileNil(p *_Program, sp int, vt reflect.Type, nil_op _Op, fn func(*_Program, int, reflect.Type)) {
     x := p.pc()
     p.add(_OP_is_nil)
     fn(p, sp, vt)
     e := p.pc()
     p.add(_OP_goto)
     p.pin(x)
-    p.add(_OP_null)
+    p.add(nil_op)
     p.pin(e)
 }
 
 func (self *_Compiler) compilePtr(p *_Program, sp int, vt reflect.Type) {
-    self.compileNil(p, sp, vt, self.compilePtrBody)
+    self.compileNil(p, sp, vt, _OP_null, self.compilePtrBody)
 }
 
 func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
@@ -505,7 +509,7 @@ func (self *_Compiler) compilePtrBody(p *_Program, sp int, vt reflect.Type) {
 }
 
 func (self *_Compiler) compileMap(p *_Program, sp int, vt reflect.Type) {
-    self.compileNil(p, sp, vt, self.compileMapBody)
+    self.compileNil(p, sp, vt, _OP_empty_obj, self.compileMapBody)
 }
 
 func (self *_Compiler) compileMapBody(p *_Program, sp int, vt reflect.Type) {
@@ -591,7 +595,7 @@ func (self *_Compiler) compileMapBodyUtextPtr(p *_Program, vk reflect.Type) {
 }
 
 func (self *_Compiler) compileSlice(p *_Program, sp int, vt reflect.Type) {
-    self.compileNil(p, sp, vt, self.compileSliceBody)
+    self.compileNil(p, sp, vt, _OP_empty_arr, self.compileSliceBody)
 }
 
 func (self *_Compiler) compileSliceBody(p *_Program, sp int, vt reflect.Type) {
diff --git a/encoder/encoder.go b/encoder/encoder.go
index 28b79b8..043b8a7 100644
--- a/encoder/encoder.go
+++ b/encoder/encoder.go
@@ -37,6 +37,7 @@ const (
     bitEscapeHTML          
     bitCompactMarshaler
     bitNoQuoteTextMarshaler
+    bitNoNullSliceOrMap
 )
 
 const (
@@ -52,12 +53,17 @@ const (
 
     // CompactMarshaler indicates that the output JSON from json.Marshaler 
     // is always compact and needs no validation 
-    CompactMarshaler   Options = 1 << bitCompactMarshaler
+    CompactMarshaler     Options = 1 << bitCompactMarshaler
 
     // NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler 
     // is always escaped string and needs no quoting
     NoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler
 
+    // NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',
+    // instead of 'null'
+    NoNullSliceOrMap     Options = 1 << bitNoNullSliceOrMap
+  
+    // CompatibleWithStd is used to be compatible with std encoder.
     CompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler
 )
 
diff --git a/sonic.go b/sonic.go
index d087768..86b9e8a 100644
--- a/sonic.go
+++ b/sonic.go
@@ -85,6 +85,9 @@ func (cfg Config) Froze() API {
     if cfg.CopyString {
         api.decoderOpts |= decoder.OptionCopyString
     }
+    if cfg.NoNullSliceOrMap {
+        api.encoderOpts |= encoder.NoNullSliceOrMap
+    }
     return api
 }
 


Test patch:
diff --git a/encode_test.go b/encode_test.go
index a0efc56..b19b96d 100644
--- a/encode_test.go
+++ b/encode_test.go
@@ -19,13 +19,13 @@
 package sonic
 
 import (
-    `os`
     `bytes`
     `encoding`
     `encoding/json`
     `fmt`
     `log`
     `math`
+    `os`
     `reflect`
     `regexp`
     `runtime`
@@ -36,6 +36,7 @@ import (
     `unsafe`
 
     `github.com/bytedance/sonic/encoder`
+    `github.com/stretchr/testify/assert`
 )
 
 var (
@@ -1153,3 +1154,18 @@ func TestMarshalerError(t *testing.T) {
         }
     }
 }
+
+func TestMarshalNullNil(t *testing.T) {
+    var v = struct {
+        A []int
+        B map[string]int
+    }{}
+    o, e := Marshal(v)
+    assert.Nil(t, e)
+    assert.Equal(t, `{"A":null,"B":null}`, string(o))
+    o, e = Config{
+        NoNullSliceOrMap: true,
+    }.Froze().Marshal(v)
+    assert.Nil(t, e)
+    assert.Equal(t, `{"A":[],"B":{}}`, string(o))
+}
\ No newline at end of file
diff --git a/encoder/encoder_test.go b/encoder/encoder_test.go
index 6931250..c055c76 100644
--- a/encoder/encoder_test.go
+++ b/encoder/encoder_test.go
@@ -76,6 +76,56 @@ func TestGC(t *testing.T) {
     wg.Wait()
 }
 
+type sample struct {
+    M map[string]interface{}
+    S []interface{}
+    A [0]interface{}
+    MP *map[string]interface{}
+    SP *[]interface{}
+    AP *[0]interface{}
+}
+
+func TestOptionSliceOrMapNoNull(t *testing.T) {
+    obj := sample{}
+    out, err := Encode(obj, NoNullSliceOrMap)
+    if err != nil {
+        t.Fatal(err)
+    }
+    require.Equal(t, `{"M":{},"S":[],"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
+   
+    obj2 := sample{}
+    out, err = Encode(obj2, 0)
+    if err != nil {
+        t.Fatal(err)
+    }
+    require.Equal(t, `{"M":null,"S":null,"A":[],"MP":null,"SP":null,"AP":null}`, string(out))
+}
+
+func BenchmarkOptionSliceOrMapNoNull(b *testing.B) {
+    b.Run("true", func (b *testing.B) {
+        obj := sample{}
+        _, err := Encode(obj, NoNullSliceOrMap)
+        if err != nil {
+            b.Fatal(err)
+        }
+        b.ResetTimer()
+        for i:=0;i<b.N;i++{
+            _, _ = Encode(obj, NoNullSliceOrMap)
+        }
+    })
+   
+    b.Run("false", func (b *testing.B) {
+        obj2 := sample{}
+        _, err := Encode(obj2, 0)
+        if err != nil {
+            b.Fatal(err)
+        }
+        for i:=0;i<b.N;i++{
+            _, _ = Encode(obj2, 0)
+        }
+    })
+}
+
 func runEncoderTest(t *testing.T, fn func(string)string, exp string, arg string) {
     require.Equal(t, exp, fn(arg))
 }