Repository: segmentio/encoding
Pull request #75 (2021-03-22 02:25:58)

Optimize decoders using json.Kind
Follow up to our discussion in #72, this PR adds new APIs to the tokenizer to leverage the decoder flags and avoid doing multiple passes over string values.

I ported a program to use the new APIs, it is both simpler and faster:
```
name       old time/op    new time/op    delta
Unmarshal     916ns ± 1%     799ns ± 3%  -12.81%  (p=0.008 n=5+5)

name       old speed      new speed      delta
Unmarshal  68.8MB/s ± 1%  78.9MB/s ± 3%  +14.72%  (p=0.008 n=5+5)

name       old alloc/op   new alloc/op   delta
Unmarshal      410B ± 0%      408B ± 0%   -0.49%  (p=0.008 n=5+5)

name       old allocs/op  new allocs/op  delta
Unmarshal      5.00 ± 0%      4.00 ± 0%  -20.00%  (p=0.008 n=5+5)
```

Also ran a quick comparison to the master branch to verify that we aren't introducing any obvious regressions:
```
name         old time/op    new time/op    delta
CodeDecoder    10.0ms ± 4%    10.0ms ± 1%   ~     (p=1.000 n=5+5)

name         old speed      new speed      delta
CodeDecoder   193MB/s ± 4%   194MB/s ± 1%   ~     (p=1.000 n=5+5)

name         old alloc/op   new alloc/op   delta
CodeDecoder     443kB ± 0%     443kB ± 0%   ~     (p=0.810 n=5+5)

name         old allocs/op  new allocs/op  delta
CodeDecoder     12.8k ± 0%     12.8k ± 0%   ~     (p=0.714 n=5+5)
```

```
name                           old time/op    new time/op    delta
Unmarshal/*json.codeResponse2    5.98ms ± 4%    5.60ms ± 3%  -6.39%  (p=0.016 n=5+5)

name                           old speed      new speed      delta
Unmarshal/*json.codeResponse2   325MB/s ± 4%   347MB/s ± 3%  +6.81%  (p=0.016 n=5+5)

name                           old alloc/op   new alloc/op   delta
Unmarshal/*json.codeResponse2    5.41kB ± 5%    5.06kB ± 4%  -6.54%  (p=0.016 n=5+5)

name                           old allocs/op  new allocs/op  delta
Unmarshal/*json.codeResponse2      22.8 ± 5%      21.0 ± 5%  -7.89%  (p=0.024 n=5+5)
```

It looks like the change also has a positive impact on decoding in general (not sure if it's the switch to `hasNullPrefix` or the use of `Kind` in various places).


Patch:
diff --git a/json/decode.go b/json/decode.go
index 01f9201..8d934f5 100644
--- a/json/decode.go
+++ b/json/decode.go
@@ -221,7 +221,7 @@ func (d decoder) decodeFloat32(b []byte, p unsafe.Pointer) ([]byte, error) {
 		return b[4:], nil
 	}
 
-	v, r, err := d.parseNumber(b)
+	v, r, _, err := d.parseNumber(b)
 	if err != nil {
 		return d.inputError(b, float32Type)
 	}
@@ -240,7 +240,7 @@ func (d decoder) decodeFloat64(b []byte, p unsafe.Pointer) ([]byte, error) {
 		return b[4:], nil
 	}
 
-	v, r, err := d.parseNumber(b)
+	v, r, _, err := d.parseNumber(b)
 	if err != nil {
 		return d.inputError(b, float64Type)
 	}
@@ -259,7 +259,7 @@ func (d decoder) decodeNumber(b []byte, p unsafe.Pointer) ([]byte, error) {
 		return b[4:], nil
 	}
 
-	v, r, err := d.parseNumber(b)
+	v, r, _, err := d.parseNumber(b)
 	if err != nil {
 		return d.inputError(b, numberType)
 	}
@@ -317,25 +317,17 @@ func (d decoder) decodeFromString(b []byte, p unsafe.Pointer, decode decodeFunc)
 }
 
 func (d decoder) decodeFromStringToInt(b []byte, p unsafe.Pointer, t reflect.Type, decode decodeFunc) ([]byte, error) {
-	if hasPrefix(b, "null") {
+	if hasNullPrefix(b) {
 		return decode(d, b, p)
 	}
 
 	if len(b) > 0 && b[0] != '"' {
-		v, r, err := d.parseNumber(b)
+		v, r, k, err := d.parseNumber(b)
 		if err == nil {
 			// The encoding/json package will return a *json.UnmarshalTypeError if
 			// the input was a floating point number representation, even tho a
 			// string is expected here.
-			isFloat := true
-			switch {
-			case bytes.IndexByte(v, '.') >= 0:
-			case bytes.IndexByte(v, 'e') >= 0:
-			case bytes.IndexByte(v, 'E') >= 0:
-			default:
-				isFloat = false
-			}
-			if isFloat {
+			if k == Float {
 				_, err := strconv.ParseFloat(*(*string)(unsafe.Pointer(&v)), 64)
 				if err != nil {
 					return r, unmarshalTypeError(v, t)
@@ -384,10 +376,10 @@ func (d decoder) decodeFromStringToInt(b []byte, p unsafe.Pointer, t reflect.Typ
 		}
 		// When the input value was a valid number representation we retain the
 		// error returned by the decoder.
-		if _, _, err := d.parseNumber(v); err != nil {
+		if _, _, _, err := d.parseNumber(v); err != nil {
 			// When the input value valid JSON we mirror the behavior of the
 			// encoding/json package and return a generic error.
-			if _, _, err := d.parseValue(v); err == nil {
+			if _, _, _, err := d.parseValue(v); err == nil {
 				return b, fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into int", prefix(v))
 			}
 		}
@@ -557,7 +549,7 @@ func (d decoder) decodeArray(b []byte, p unsafe.Pointer, n int, size uintptr, t
 			return b[1:], nil
 		}
 
-		_, b, err = d.parseValue(b)
+		_, b, _, err = d.parseValue(b)
 		if err != nil {
 			return b, err
 		}
@@ -630,7 +622,7 @@ func (d decoder) decodeSlice(b []byte, p unsafe.Pointer, size uintptr, t reflect
 
 		b, err = decode(d, b, unsafe.Pointer(uintptr(s.data)+(uintptr(s.len)*size)))
 		if err != nil {
-			if _, r, err := d.parseValue(input); err != nil {
+			if _, r, _, err := d.parseValue(input); err != nil {
 				return r, err
 			} else {
 				b = r
@@ -691,7 +683,7 @@ func (d decoder) decodeMap(b []byte, p unsafe.Pointer, t, kt, vt reflect.Type, k
 			b = skipSpaces(b[1:])
 		}
 
-		if hasPrefix(b, "null") {
+		if hasNullPrefix(b) {
 			return b, syntaxError(b, "cannot decode object key string from 'null' value")
 		}
 
@@ -709,7 +701,7 @@ func (d decoder) decodeMap(b []byte, p unsafe.Pointer, t, kt, vt reflect.Type, k
 		b = skipSpaces(b[1:])
 
 		if b, err = decodeValue(d, b, vptr); err != nil {
-			if _, r, err := d.parseValue(input); err != nil {
+			if _, r, _, err := d.parseValue(input); err != nil {
 				return r, err
 			} else {
 				b = r
@@ -770,7 +762,7 @@ func (d decoder) decodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, e
 			b = skipSpaces(b[1:])
 		}
 
-		if hasPrefix(b, "null") {
+		if hasNullPrefix(b) {
 			return b, syntaxError(b, "cannot decode object key string from 'null' value")
 		}
 
@@ -790,7 +782,7 @@ func (d decoder) decodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, e
 
 		b, err = d.decodeInterface(b, unsafe.Pointer(&val))
 		if err != nil {
-			if _, r, err := d.parseValue(input); err != nil {
+			if _, r, _, err := d.parseValue(input); err != nil {
 				return r, err
 			} else {
 				b = r
@@ -851,7 +843,7 @@ func (d decoder) decodeMapStringRawMessage(b []byte, p unsafe.Pointer) ([]byte,
 			b = skipSpaces(b[1:])
 		}
 
-		if hasPrefix(b, "null") {
+		if hasNullPrefix(b) {
 			return b, syntaxError(b, "cannot decode object key string from 'null' value")
 		}
 
@@ -871,7 +863,7 @@ func (d decoder) decodeMapStringRawMessage(b []byte, p unsafe.Pointer) ([]byte,
 
 		b, err = d.decodeRawMessage(b, unsafe.Pointer(&val))
 		if err != nil {
-			if _, r, err := d.parseValue(input); err != nil {
+			if _, r, _, err := d.parseValue(input); err != nil {
 				return r, err
 			} else {
 				b = r
@@ -932,7 +924,7 @@ func (d decoder) decodeMapStringString(b []byte, p unsafe.Pointer) ([]byte, erro
 			b = skipSpaces(b[1:])
 		}
 
-		if hasPrefix(b, "null") {
+		if hasNullPrefix(b) {
 			return b, syntaxError(b, "cannot decode object key string from 'null' value")
 		}
 
@@ -952,7 +944,7 @@ func (d decoder) decodeMapStringString(b []byte, p unsafe.Pointer) ([]byte, erro
 
 		b, err = d.decodeString(b, unsafe.Pointer(&val))
 		if err != nil {
-			if _, r, err := d.parseValue(input); err != nil {
+			if _, r, _, err := d.parseValue(input); err != nil {
 				return r, err
 			} else {
 				b = r
@@ -1014,7 +1006,7 @@ func (d decoder) decodeMapStringStringSlice(b []byte, p unsafe.Pointer) ([]byte,
 			b = skipSpaces(b[1:])
 		}
 
-		if hasPrefix(b, "null") {
+		if hasNullPrefix(b) {
 			return b, syntaxError(b, "cannot decode object key string from 'null' value")
 		}
 
@@ -1034,7 +1026,7 @@ func (d decoder) decodeMapStringStringSlice(b []byte, p unsafe.Pointer) ([]byte,
 
 		b, err = d.decodeSlice(b, unsafe.Pointer(&buf), stringSize, sliceStringType, decoder.decodeString)
 		if err != nil {
-			if _, r, err := d.parseValue(input); err != nil {
+			if _, r, _, err := d.parseValue(input); err != nil {
 				return r, err
 			} else {
 				b = r
@@ -1098,7 +1090,7 @@ func (d decoder) decodeMapStringBool(b []byte, p unsafe.Pointer) ([]byte, error)
 			b = skipSpaces(b[1:])
 		}
 
-		if hasPrefix(b, "null") {
+		if hasNullPrefix(b) {
 			return b, syntaxError(b, "cannot decode object key string from 'null' value")
 		}
 
@@ -1118,7 +1110,7 @@ func (d decoder) decodeMapStringBool(b []byte, p unsafe.Pointer) ([]byte, error)
 
 		b, err = d.decodeBool(b, unsafe.Pointer(&val))
 		if err != nil {
-			if _, r, err := d.parseValue(input); err != nil {
+			if _, r, _, err := d.parseValue(input); err != nil {
 				return r, err
 			} else {
 				b = r
@@ -1172,7 +1164,7 @@ func (d decoder) decodeStruct(b []byte, p unsafe.Pointer, st *structType) ([]byt
 		}
 		i++
 
-		if hasPrefix(b, "null") {
+		if hasNullPrefix(b) {
 			return b, syntaxError(b, "cannot decode object key string from 'null' value")
 		}
 
@@ -1201,14 +1193,14 @@ func (d decoder) decodeStruct(b []byte, p unsafe.Pointer, st *structType) ([]byt
 			if (d.flags & DisallowUnknownFields) != 0 {
 				return b, fmt.Errorf("json: unknown field %q", k)
 			}
-			if _, b, err = d.parseValue(b); err != nil {
+			if _, b, _, err = d.parseValue(b); err != nil {
 				return b, err
 			}
 			continue
 		}
 
 		if b, err = f.codec.decode(d, b, unsafe.Pointer(uintptr(p)+f.offset)); err != nil {
-			if _, r, err := d.parseValue(input); err != nil {
+			if _, r, _, err := d.parseValue(input); err != nil {
 				return r, err
 			} else {
 				b = r
@@ -1277,37 +1269,34 @@ func (d decoder) decodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) {
 		return b, err
 	}
 
-	v, b, err := d.parseValue(b)
+	v, b, k, err := d.parseValue(b)
 	if err != nil {
 		return b, err
 	}
 
-	switch v[0] {
-	case '{':
+	switch k.Class() {
+	case Object:
 		m := make(map[string]interface{})
 		v, err = d.decodeMapStringInterface(v, unsafe.Pointer(&m))
 		val = m
 
-	case '[':
+	case Array:
 		a := make([]interface{}, 0, 10)
 		v, err = d.decodeSlice(v, unsafe.Pointer(&a), unsafe.Sizeof(a[0]), sliceInterfaceType, decoder.decodeInterface)
 		val = a
 
-	case '"':
+	case String:
 		s := ""
 		v, err = d.decodeString(v, unsafe.Pointer(&s))
 		val = s
 
-	case 'n':
-		v, err = d.decodeNull(v, nil)
-		val = nil
+	case Null:
+		v, val = nil, nil
 
-	case 't', 'f':
-		x := false
-		v, err = d.decodeBool(v, unsafe.Pointer(&x))
-		val = x
+	case Bool:
+		v, val = nil, k == True
 
-	case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+	case Num:
 		if (d.flags & UseNumber) != 0 {
 			n := Number("")
 			v, err = d.decodeNumber(v, unsafe.Pointer(&n))
@@ -1352,7 +1341,7 @@ func (d decoder) decodeMaybeEmptyInterface(b []byte, p unsafe.Pointer, t reflect
 }
 
 func (d decoder) decodeUnmarshalTypeError(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) {
-	v, b, err := d.parseValue(b)
+	v, b, _, err := d.parseValue(b)
 	if err != nil {
 		return b, err
 	}
@@ -1363,7 +1352,7 @@ func (d decoder) decodeUnmarshalTypeError(b []byte, p unsafe.Pointer, t reflect.
 }
 
 func (d decoder) decodeRawMessage(b []byte, p unsafe.Pointer) ([]byte, error) {
-	v, r, err := d.parseValue(b)
+	v, r, _, err := d.parseValue(b)
 	if err != nil {
 		return d.inputError(b, rawMessageType)
 	}
@@ -1377,7 +1366,7 @@ func (d decoder) decodeRawMessage(b []byte, p unsafe.Pointer) ([]byte, error) {
 }
 
 func (d decoder) decodeJSONUnmarshaler(b []byte, p unsafe.Pointer, t reflect.Type, pointer bool) ([]byte, error) {
-	v, b, err := d.parseValue(b)
+	v, b, _, err := d.parseValue(b)
 	if err != nil {
 		return b, err
 	}
@@ -1397,7 +1386,7 @@ func (d decoder) decodeJSONUnmarshaler(b []byte, p unsafe.Pointer, t reflect.Typ
 func (d decoder) decodeTextUnmarshaler(b []byte, p unsafe.Pointer, t reflect.Type, pointer bool) ([]byte, error) {
 	var value string
 
-	v, b, err := d.parseValue(b)
+	v, b, k, err := d.parseValue(b)
 	if err != nil {
 		return b, err
 	}
@@ -1405,11 +1394,11 @@ func (d decoder) decodeTextUnmarshaler(b []byte, p unsafe.Pointer, t reflect.Typ
 		return d.inputError(v, t)
 	}
 
-	switch v[0] {
-	case 'n':
-		_, _, err := d.parseNull(v)
+	switch k.Class() {
+	case Null:
 		return b, err
-	case '"':
+
+	case String:
 		s, _, _, err := d.parseStringUnquote(v, nil)
 		if err != nil {
 			return b, err
@@ -1423,16 +1412,22 @@ func (d decoder) decodeTextUnmarshaler(b []byte, p unsafe.Pointer, t reflect.Typ
 			u.Set(reflect.New(t))
 		}
 		return b, u.Interface().(encoding.TextUnmarshaler).UnmarshalText(s)
-	case '{':
+
+	case Bool:
+		if k == True {
+			value = "true"
+		} else {
+			value = "false"
+		}
+
+	case Num:
+		value = "number"
+
+	case Object:
 		value = "object"
-	case '[':
+
+	case Array:
 		value = "array"
-	case 't':
-		value = "true"
-	case 'f':
-		value = "false"
-	case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-		value = "number"
 	}
 
 	return b, &UnmarshalTypeError{Value: value, Type: reflect.PtrTo(t)}
@@ -1449,7 +1444,7 @@ func (d decoder) inputError(b []byte, t reflect.Type) ([]byte, error) {
 	if len(b) == 0 {
 		return nil, unexpectedEOF(b)
 	}
-	_, r, err := d.parseValue(b)
+	_, r, _, err := d.parseValue(b)
 	if err != nil {
 		return r, err
 	}
diff --git a/json/encode.go b/json/encode.go
index 720bfdd..1d54297 100644
--- a/json/encode.go
+++ b/json/encode.go
@@ -121,7 +121,7 @@ func (e encoder) encodeNumber(b []byte, p unsafe.Pointer) ([]byte, error) {
 	}
 
 	d := decoder{}
-	_, _, err := d.parseNumber(stringToBytes(string(n)))
+	_, _, _, err := d.parseNumber(stringToBytes(string(n)))
 	if err != nil {
 		return b, err
 	}
@@ -845,7 +845,7 @@ func (e encoder) encodeRawMessage(b []byte, p unsafe.Pointer) ([]byte, error) {
 	} else {
 		var err error
 		d := decoder{}
-		s, _, err = d.parseValue(v)
+		s, _, _, err = d.parseValue(v)
 		if err != nil {
 			return b, &UnsupportedValueError{Value: reflect.ValueOf(v), Str: err.Error()}
 		}
@@ -878,7 +878,7 @@ func (e encoder) encodeJSONMarshaler(b []byte, p unsafe.Pointer, t reflect.Type,
 	}
 
 	d := decoder{}
-	s, _, err := d.parseValue(j)
+	s, _, _, err := d.parseValue(j)
 	if err != nil {
 		return b, &MarshalerError{Type: t, Err: err}
 	}
diff --git a/json/json.go b/json/json.go
index 3b95e19..c3e710b 100644
--- a/json/json.go
+++ b/json/json.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
+	"math/bits"
 	"reflect"
 	"runtime"
 	"sync"
@@ -54,7 +55,7 @@ type UnsupportedValueError = json.UnsupportedValueError
 
 // AppendFlags is a type used to represent configuration options that can be
 // applied when formatting json output.
-type AppendFlags int
+type AppendFlags uint
 
 const (
 	// EscapeHTML is a formatting flag used to to escape HTML in json strings.
@@ -73,12 +74,20 @@ const (
 
 // ParseFlags is a type used to represent configuration options that can be
 // applied when parsing json input.
-type ParseFlags int
+type ParseFlags uint
 
 func (flags ParseFlags) has(f ParseFlags) bool {
 	return (flags & f) != 0
 }
 
+func (f ParseFlags) kind() Kind {
+	return Kind((f >> kindOffset) & 0xFF)
+}
+
+func (f ParseFlags) withKind(kind Kind) ParseFlags {
+	return (f & ^(ParseFlags(0xFF) << kindOffset)) | (ParseFlags(kind) << kindOffset)
+}
+
 const (
 	// DisallowUnknownFields is a parsing flag used to prevent decoding of
 	// objects to Go struct values when a field of the input does not match
@@ -130,8 +139,40 @@ const (
 	// contain a backslash. If the flag is unset, it's unknown whether the
 	// input contains a backslash.
 	noBackslash ParseFlags = 1 << 29
+
+	// Bit offset where the kind of the json value is stored.
+	//
+	// See Kind in token.go for the enum.
+	kindOffset ParseFlags = 16
+)
+
+// Kind represents the different kinds of value that exist in JSON.
+type Kind uint
+
+const (
+	Undefined Kind = 0
+
+	Null Kind = 1 // Null is not zero, so we keep zero for "undefined".
+
+	Bool  Kind = 2 // Bit two is set to 1, means it's a boolean.
+	False Kind = 2 // Bool + 0
+	True  Kind = 3 // Bool + 1
+
+	Num   Kind = 4 // Bit three is set to 1, means it's a number.
+	Uint  Kind = 5 // Num + 1
+	Int   Kind = 6 // Num + 2
+	Float Kind = 7 // Num + 3
+
+	String    Kind = 8 // Bit four is set to 1, means it's a string.
+	Unescaped Kind = 9 // String + 1
+
+	Array  Kind = 16 // Equivalent to Delim == '['
+	Object Kind = 32 // Equivalent to Delim == '{'
 )
 
+// Class returns the class of k.
+func (k Kind) Class() Kind { return Kind(1 << uint(bits.Len(uint(k))-1)) }
+
 // Append acts like Marshal but appends the json representation to b instead of
 // always reallocating a new slice.
 func Append(b []byte, x interface{}, flags AppendFlags) ([]byte, error) {
@@ -264,7 +305,7 @@ func Parse(b []byte, x interface{}, flags ParseFlags) ([]byte, error) {
 	b = skipSpaces(b)
 
 	if t == nil || p == nil || t.Kind() != reflect.Ptr {
-		_, r, err := d.parseValue(b)
+		_, r, _, err := d.parseValue(b)
 		r = skipSpaces(r)
 		if err != nil {
 			return r, err
@@ -288,7 +329,7 @@ func Parse(b []byte, x interface{}, flags ParseFlags) ([]byte, error) {
 func Valid(data []byte) bool {
 	data = skipSpaces(data)
 	d := decoder{flags: internalParseFlags(data)}
-	_, data, err := d.parseValue(data)
+	_, data, _, err := d.parseValue(data)
 	if err != nil {
 		return false
 	}
@@ -337,7 +378,7 @@ func (dec *Decoder) readValue() (v []byte, err error) {
 
 	for {
 		if len(dec.remain) != 0 {
-			v, r, err = d.parseValue(dec.remain)
+			v, r, _, err = d.parseValue(dec.remain)
 			if err == nil {
 				dec.remain, n = skipSpacesN(r)
 				dec.inputOffset += int64(len(v) + n)
diff --git a/json/parse.go b/json/parse.go
index f7f26b4..3f715d6 100644
--- a/json/parse.go
+++ b/json/parse.go
@@ -162,7 +162,7 @@ func (d decoder) parseInt(b []byte, t reflect.Type) (int64, []byte, error) {
 	if count < len(b) {
 		switch b[count] {
 		case '.', 'e', 'E': // was this actually a float?
-			v, r, err := d.parseNumber(b)
+			v, r, _, err := d.parseNumber(b)
 			if err != nil {
 				v, r = b[:count+1], b[count+1:]
 			}
@@ -214,7 +214,7 @@ func (d decoder) parseUint(b []byte, t reflect.Type) (uint64, []byte, error) {
 	if count < len(b) {
 		switch b[count] {
 		case '.', 'e', 'E': // was this actually a float?
-			v, r, err := d.parseNumber(b)
+			v, r, _, err := d.parseNumber(b)
 			if err != nil {
 				v, r = b[:count+1], b[count+1:]
 			}
@@ -280,45 +280,49 @@ parseLoop:
 	return value, b[count:], nil
 }
 
-func (d decoder) parseNull(b []byte) ([]byte, []byte, error) {
+func (d decoder) parseNull(b []byte) ([]byte, []byte, Kind, error) {
 	if hasNullPrefix(b) {
-		return b[:4], b[4:], nil
+		return b[:4], b[4:], Null, nil
 	}
 	if len(b) < 4 {
-		return nil, b[len(b):], unexpectedEOF(b)
+		return nil, b[len(b):], Undefined, unexpectedEOF(b)
 	}
-	return nil, b, syntaxError(b, "expected 'null' but found invalid token")
+	return nil, b, Undefined, syntaxError(b, "expected 'null' but found invalid token")
 }
 
-func (d decoder) parseTrue(b []byte) ([]byte, []byte, error) {
+func (d decoder) parseTrue(b []byte) ([]byte, []byte, Kind, error) {
 	if hasTruePrefix(b) {
-		return b[:4], b[4:], nil
+		return b[:4], b[4:], True, nil
 	}
 	if len(b) < 4 {
-		return nil, b[len(b):], unexpectedEOF(b)
+		return nil, b[len(b):], Undefined, unexpectedEOF(b)
 	}
-	return nil, b, syntaxError(b, "expected 'true' but found invalid token")
+	return nil, b, Undefined, syntaxError(b, "expected 'true' but found invalid token")
 }
 
-func (d decoder) parseFalse(b []byte) ([]byte, []byte, error) {
+func (d decoder) parseFalse(b []byte) ([]byte, []byte, Kind, error) {
 	if hasFalsePrefix(b) {
-		return b[:5], b[5:], nil
+		return b[:5], b[5:], False, nil
 	}
 	if len(b) < 5 {
-		return nil, b[len(b):], unexpectedEOF(b)
+		return nil, b[len(b):], Undefined, unexpectedEOF(b)
 	}
-	return nil, b, syntaxError(b, "expected 'false' but found invalid token")
+	return nil, b, Undefined, syntaxError(b, "expected 'false' but found invalid token")
 }
 
-func (d decoder) parseNumber(b []byte) (v, r []byte, err error) {
+func (d decoder) parseNumber(b []byte) (v, r []byte, kind Kind, err error) {
 	if len(b) == 0 {
 		r, err = b, unexpectedEOF(b)
 		return
 	}
 
+	// Assume it's an unsigned integer at first.
+	kind = Uint
+
 	i := 0
 	// sign
 	if b[i] == '-' {
+		kind = Int
 		i++
 	}
 
@@ -351,6 +355,7 @@ func (d decoder) parseNumber(b []byte) (v, r []byte, err error) {
 
 	// decimal part
 	if i < len(b) && b[i] == '.' {
+		kind = Float
 		i++
 		decimalStart := i
 
@@ -373,6 +378,7 @@ func (d decoder) parseNumber(b []byte) (v, r []byte, err error) {
 
 	// exponent part
 	if i < len(b) && (b[i] == 'e' || b[i] == 'E') {
+		kind = Float
 		i++
 
 		if i < len(b) {
@@ -421,21 +427,21 @@ func (d decoder) parseUnicode(b []byte) (rune, int, error) {
 	return rune(u), 4, nil
 }
 
-func (d decoder) parseStringFast(b []byte) ([]byte, []byte, bool, error) {
+func (d decoder) parseString(b []byte) ([]byte, []byte, Kind, error) {
 	if len(b) < 2 {
-		return nil, b[len(b):], false, unexpectedEOF(b)
+		return nil, b[len(b):], Undefined, unexpectedEOF(b)
 	}
 	if b[0] != '"' {
-		return nil, b, false, syntaxError(b, "expected '\"' at the beginning of a string value")
+		return nil, b, Undefined, syntaxError(b, "expected '\"' at the beginning of a string value")
 	}
 
 	n := bytes.IndexByte(b[1:], '"') + 2
 	if n <= 1 {
-		return nil, b[len(b):], false, syntaxError(b, "missing '\"' at the end of a string value")
+		return nil, b[len(b):], Undefined, syntaxError(b, "missing '\"' at the end of a string value")
 	}
 	if (d.flags.has(noBackslash) || bytes.IndexByte(b[1:n], '\\') < 0) &&
 		(d.flags.has(validAsciiPrint) || ascii.ValidPrint(b[1:n])) {
-		return b[:n], b[n:], false, nil
+		return b[:n], b[n:], Unescaped, nil
 	}
 
 	for i := 1; i < len(b); i++ {
@@ -447,41 +453,36 @@ func (d decoder) parseStringFast(b []byte) ([]byte, []byte, bool, error) {
 				case 'u':
 					_, n, err := d.parseUnicode(b[i+1:])
 					if err != nil {
-						return nil, b[i+1+n:], false, err
+						return nil, b[i+1+n:], Undefined, err
 					}
 					i += n
 				default:
-					return nil, b, false, syntaxError(b, "invalid character '%c' in string escape code", b[i])
+					return nil, b, Undefined, syntaxError(b, "invalid character '%c' in string escape code", b[i])
 				}
 			}
 
 		case '"':
-			return b[:i+1], b[i+1:], true, nil
+			return b[:i+1], b[i+1:], String, nil
 
 		default:
 			if b[i] < 0x20 {
-				return nil, b, false, syntaxError(b, "invalid character '%c' in string escape code", b[i])
+				return nil, b, Undefined, syntaxError(b, "invalid character '%c' in string escape code", b[i])
 			}
 		}
 	}
 
-	return nil, b[len(b):], false, syntaxError(b, "missing '\"' at the end of a string value")
-}
-
-func (d decoder) parseString(b []byte) ([]byte, []byte, error) {
-	s, b, _, err := d.parseStringFast(b)
-	return s, b, err
+	return nil, b[len(b):], Undefined, syntaxError(b, "missing '\"' at the end of a string value")
 }
 
 func (d decoder) parseStringUnquote(b []byte, r []byte) ([]byte, []byte, bool, error) {
-	s, b, escaped, err := d.parseStringFast(b)
+	s, b, k, err := d.parseString(b)
 	if err != nil {
 		return s, b, false, err
 	}
 
 	s = s[1 : len(s)-1] // trim the quotes
 
-	if !escaped {
+	if k == Unescaped {
 		return s, b, false, nil
 	}
 
@@ -572,13 +573,13 @@ func appendCoerceInvalidUTF8(b []byte, s []byte) []byte {
 	return b
 }
 
-func (d decoder) parseObject(b []byte) ([]byte, []byte, error) {
+func (d decoder) parseObject(b []byte) ([]byte, []byte, Kind, error) {
 	if len(b) < 2 {
-		return nil, b[len(b):], unexpectedEOF(b)
+		return nil, b[len(b):], Undefined, unexpectedEOF(b)
 	}
 
 	if b[0] != '{' {
-		return nil, b, syntaxError(b, "expected '{' at the beginning of an object value")
+		return nil, b, Undefined, syntaxError(b, "expected '{' at the beginning of an object value")
 	}
 
 	var err error
@@ -591,60 +592,60 @@ func (d decoder) parseObject(b []byte) ([]byte, []byte, error) {
 		b = skipSpaces(b)
 
 		if len(b) == 0 {
-			return nil, b, syntaxError(b, "cannot decode object from empty input")
+			return nil, b, Undefined, syntaxError(b, "cannot decode object from empty input")
 		}
 
 		if b[0] == '}' {
 			j := (n - len(b)) + 1
-			return a[:j], a[j:], nil
+			return a[:j], a[j:], Object, nil
 		}
 
 		if i != 0 {
 			if len(b) == 0 {
-				return nil, b, syntaxError(b, "unexpected EOF after object field value")
+				return nil, b, Undefined, syntaxError(b, "unexpected EOF after object field value")
 			}
 			if b[0] != ',' {
-				return nil, b, syntaxError(b, "expected ',' after object field value but found '%c'", b[0])
+				return nil, b, Undefined, syntaxError(b, "expected ',' after object field value but found '%c'", b[0])
 			}
 			b = skipSpaces(b[1:])
 			if len(b) == 0 {
-				return nil, b, unexpectedEOF(b)
+				return nil, b, Undefined, unexpectedEOF(b)
 			}
 			if b[0] == '}' {
-				return nil, b, syntaxError(b, "unexpected trailing comma after object field")
+				return nil, b, Undefined, syntaxError(b, "unexpected trailing comma after object field")
 			}
 		}
 
-		_, b, err = d.parseString(b)
+		_, b, _, err = d.parseString(b)
 		if err != nil {
-			return nil, b, err
+			return nil, b, Undefined, err
 		}
 		b = skipSpaces(b)
 
 		if len(b) == 0 {
-			return nil, b, syntaxError(b, "unexpected EOF after object field key")
+			return nil, b, Undefined, syntaxError(b, "unexpected EOF after object field key")
 		}
 		if b[0] != ':' {
-			return nil, b, syntaxError(b, "expected ':' after object field key but found '%c'", b[0])
+			return nil, b, Undefined, syntaxError(b, "expected ':' after object field key but found '%c'", b[0])
 		}
 		b = skipSpaces(b[1:])
 
-		_, b, err = d.parseValue(b)
+		_, b, _, err = d.parseValue(b)
 		if err != nil {
-			return nil, b, err
+			return nil, b, Undefined, err
 		}
 
 		i++
 	}
 }
 
-func (d decoder) parseArray(b []byte) ([]byte, []byte, error) {
+func (d decoder) parseArray(b []byte) ([]byte, []byte, Kind, error) {
 	if len(b) < 2 {
-		return nil, b[len(b):], unexpectedEOF(b)
+		return nil, b[len(b):], Undefined, unexpectedEOF(b)
 	}
 
 	if b[0] != '[' {
-		return nil, b, syntaxError(b, "expected '[' at the beginning of array value")
+		return nil, b, Undefined, syntaxError(b, "expected '[' at the beginning of array value")
 	}
 
 	var err error
@@ -657,61 +658,69 @@ func (d decoder) parseArray(b []byte) ([]byte, []byte, error) {
 		b = skipSpaces(b)
 
 		if len(b) == 0 {
-			return nil, b, syntaxError(b, "missing closing ']' after array value")
+			return nil, b, Undefined, syntaxError(b, "missing closing ']' after array value")
 		}
 
 		if b[0] == ']' {
 			j := (n - len(b)) + 1
-			return a[:j], a[j:], nil
+			return a[:j], a[j:], Array, nil
 		}
 
 		if i != 0 {
 			if len(b) == 0 {
-				return nil, b, syntaxError(b, "unexpected EOF after array element")
+				return nil, b, Undefined, syntaxError(b, "unexpected EOF after array element")
 			}
 			if b[0] != ',' {
-				return nil, b, syntaxError(b, "expected ',' after array element but found '%c'", b[0])
+				return nil, b, Undefined, syntaxError(b, "expected ',' after array element but found '%c'", b[0])
 			}
 			b = skipSpaces(b[1:])
 			if len(b) == 0 {
-				return nil, b, unexpectedEOF(b)
+				return nil, b, Undefined, unexpectedEOF(b)
 			}
 			if b[0] == ']' {
-				return nil, b, syntaxError(b, "unexpected trailing comma after object field")
+				return nil, b, Undefined, syntaxError(b, "unexpected trailing comma after object field")
 			}
 		}
 
-		_, b, err = d.parseValue(b)
+		_, b, _, err = d.parseValue(b)
 		if err != nil {
-			return nil, b, err
+			return nil, b, Undefined, err
 		}
 
 		i++
 	}
 }
 
-func (d decoder) parseValue(b []byte) ([]byte, []byte, error) {
-	if len(b) != 0 {
-		switch b[0] {
-		case '{':
-			return d.parseObject(b)
-		case '[':
-			return d.parseArray(b)
-		case '"':
-			return d.parseString(b)
-		case 'n':
-			return d.parseNull(b)
-		case 't':
-			return d.parseTrue(b)
-		case 'f':
-			return d.parseFalse(b)
-		case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			return d.parseNumber(b)
-		default:
-			return nil, b, syntaxError(b, "invalid character '%c' looking for beginning of value", b[0])
-		}
+func (d decoder) parseValue(b []byte) ([]byte, []byte, Kind, error) {
+	if len(b) == 0 {
+		return nil, b, Undefined, syntaxError(b, "unexpected end of JSON input")
 	}
-	return nil, b, syntaxError(b, "unexpected end of JSON input")
+
+	var v []byte
+	var k Kind
+	var err error
+
+	switch b[0] {
+	case '{':
+		v, b, k, err = d.parseObject(b)
+	case '[':
+		k = Array
+		v, b, k, err = d.parseArray(b)
+	case '"':
+		v, b, k, err = d.parseString(b)
+	case 'n':
+		v, b, k, err = d.parseNull(b)
+	case 't':
+		v, b, k, err = d.parseTrue(b)
+	case 'f':
+		v, b, k, err = d.parseFalse(b)
+	case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+		v, b, k, err = d.parseNumber(b)
+	default:
+		err = syntaxError(b, "invalid character '%c' looking for beginning of value", b[0])
+	}
+
+	return v, b, k, err
 }
 
 func hasNullPrefix(b []byte) bool {
diff --git a/json/token.go b/json/token.go
index 5f10c6f..93714fc 100644
--- a/json/token.go
+++ b/json/token.go
@@ -1,6 +1,10 @@
 package json
 
-import "sync"
+import (
+	"strconv"
+	"sync"
+	"unsafe"
+)
 
 // Tokenizer is an iterator-style type which can be used to progressively parse
 // through a json input.
@@ -12,31 +16,18 @@ import "sync"
 // Here is a common pattern to use a tokenizer:
 //
 //	for t := json.NewTokenizer(b); t.Next(); {
-//		switch t.Delim {
-//		case '{':
-//			...
-//		case '}':
-//			...
-//		case '[':
-//			...
-//		case ']':
-//			...
-//		case ':':
+//		switch k := t.Kind(); k.Class() {
+//		case json.Null:
 //			...
-//		case ',':
+//		case json.Bool:
 //			...
-//		}
-//
-//		switch {
-//		case t.Value.String():
-//			...
-//		case t.Value.Null():
+//		case json.Num:
 //			...
-//		case t.Value.True():
+//		case json.String:
 //			...
-//		case t.Value.False():
+//		case json.Array:
 //			...
-//		case t.Value.Number():
+//		case json.Object:
 //			...
 //		}
 //	}
@@ -78,7 +69,7 @@ type Tokenizer struct {
 	stack *stack
 
 	// Decoder used for parsing.
-	decoder decoder
+	decoder
 }
 
 // NewTokenizer constructs a new Tokenizer which reads its json input from b.
@@ -144,24 +135,31 @@ skipLoop:
 		return false
 	}
 
+	var kind Kind
 	switch t.json[0] {
 	case '"':
 		t.Delim = 0
-		t.Value, t.json, t.Err = t.decoder.parseString(t.json)
+		t.Value, t.json, kind, t.Err = t.parseString(t.json)
 	case 'n':
 		t.Delim = 0
-		t.Value, t.json, t.Err = t.decoder.parseNull(t.json)
+		t.Value, t.json, kind, t.Err = t.parseNull(t.json)
 	case 't':
 		t.Delim = 0
-		t.Value, t.json, t.Err = t.decoder.parseTrue(t.json)
+		t.Value, t.json, kind, t.Err = t.parseTrue(t.json)
 	case 'f':
 		t.Delim = 0
-		t.Value, t.json, t.Err = t.decoder.parseFalse(t.json)
+		t.Value, t.json, kind, t.Err = t.parseFalse(t.json)
 	case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
 		t.Delim = 0
-		t.Value, t.json, t.Err = t.decoder.parseNumber(t.json)
+		t.Value, t.json, kind, t.Err = t.parseNumber(t.json)
 	case '{', '}', '[', ']', ':', ',':
 		t.Delim, t.Value, t.json = Delim(t.json[0]), t.json[:1], t.json[1:]
+		switch t.Delim {
+		case '{':
+			kind = Object
+		case '[':
+			kind = Array
+		}
 	default:
 		t.Delim = 0
 		t.Value, t.json, t.Err = t.json[:1], t.json[1:], syntaxError(t.json, "expected token but found '%c'", t.json[0])
@@ -169,6 +167,7 @@ skipLoop:
 
 	t.Depth = t.depth()
 	t.Index = t.index()
+	t.flags = t.flags.withKind(kind)
 
 	if t.Delim == 0 {
 		t.IsKey = t.isKey
@@ -234,6 +233,75 @@ func (t *Tokenizer) pop(expect scope) error {
 	return nil
 }
 
+// Kind returns the kind of the value that the tokenizer is currently positioned
+// on.
+func (t *Tokenizer) Kind() Kind { return t.flags.kind() }
+
+// String returns a byte slice containing the value of the json string that the
+// tokenizer is currently pointing at.
+//
+// This method must only be called after checking the kind of the token via a
+// call to Kind.
+//
+// If the tokenizer is not positioned on a string, the behavior is undefined.
+func (t *Tokenizer) Bool() bool { return t.flags.kind() == True }
+
+// Int returns a byte slice containing the value of the json number that the
+// tokenizer is currently pointing at.
+//
+// This method must only be called after checking the kind of the token via a
+// call to Kind.
+//
+// If the tokenizer is not positioned on an integer, the behavior is undefined.
+func (t *Tokenizer) Int() int64 {
+	i, _, _ := t.parseInt(t.Value, int64Type)
+	return i
+}
+
+// Uint returns a byte slice containing the value of the json number that the
+// tokenizer is currently pointing at.
+//
+// This method must only be called after checking the kind of the token via a
+// call to Kind.
+//
+// If the tokenizer is not positioned on a positive integer, the behavior is
+// undefined.
+func (t *Tokenizer) Uint() uint64 {
+	u, _, _ := t.parseUint(t.Value, uint64Type)
+	return u
+}
+
+// Float returns a byte slice containing the value of the json number that the
+// tokenizer is currently pointing at.
+//
+// This method must only be called after checking the kind of the token via a
+// call to Kind.
+//
+// If the tokenizer is not positioned on a number, the behavior is undefined.
+func (t *Tokenizer) Float() float64 {
+	f, _ := strconv.ParseFloat(*(*string)(unsafe.Pointer(&t.Value)), 64)
+	return f
+}
+
+// String returns a byte slice containing the value of the json string that the
+// tokenizer is currently pointing at.
+//
+// This method must only be called after checking the kind of the token via a
+// call to Kind.
+//
+// When possible, the returned byte slice references the backing array of the
+// tokenizer. A new slice is only allocated if the tokenizer needed to unescape
+// the json string.
+//
+// If the tokenizer is not positioned on a string, the behavior is undefined.
+func (t *Tokenizer) String() []byte {
+	if t.flags.kind() == Unescaped && len(t.Value) > 1 {
+		return t.Value[1 : len(t.Value)-1] // unquote
+	}
+	s, _, _, _ := t.parseStringUnquote(t.Value, nil)
+	return s
+}
+
 // RawValue represents a raw json value, it is intended to carry null, true,
 // false, number, and string values only.
 type RawValue []byte
@@ -264,19 +332,14 @@ func (v RawValue) Number() bool {
 // AppendUnquote writes the unquoted version of the string value in v into b.
 func (v RawValue) AppendUnquote(b []byte) []byte {
 	d := decoder{}
-	s, r, new, err := d.parseStringUnquote(v, b)
+	s, r, _, err := d.parseStringUnquote(v, b)
 	if err != nil {
 		panic(err)
 	}
 	if len(r) != 0 {
 		panic(syntaxError(r, "unexpected trailing tokens after json value"))
 	}
-	if new {
-		b = s
-	} else {
-		b = append(b, s...)
-	}
-	return b
+	return append(b, s...)
 }
 
 // Unquote returns the unquoted version of the string value in v.


Test patch:
diff --git a/json/json_test.go b/json/json_test.go
index 51f9396..5f3f8a2 100644
--- a/json/json_test.go
+++ b/json/json_test.go
@@ -1777,3 +1777,27 @@ func BenchmarkUnmarshalField(b *testing.B) {
 
 	b.Log(v)
 }
+
+func TestKind(t *testing.T) {
+	for _, test := range []struct {
+		kind  Kind
+		class Kind
+	}{
+		{kind: 0, class: 0},
+		{kind: Null, class: Null},
+		{kind: False, class: Bool},
+		{kind: True, class: Bool},
+		{kind: Num, class: Num},
+		{kind: Uint, class: Num},
+		{kind: Int, class: Num},
+		{kind: Float, class: Num},
+		{kind: String, class: String},
+		{kind: Unescaped, class: String},
+		{kind: Array, class: Array},
+		{kind: Object, class: Object},
+	} {
+		if class := test.kind.Class(); class != test.class {
+			t.Errorf("class of kind(%d) mismatch: want=%d got=%d", test.kind, test.class, class)
+		}
+	}
+}
diff --git a/json/parse_test.go b/json/parse_test.go
index cd4083a..f835f6f 100644
--- a/json/parse_test.go
+++ b/json/parse_test.go
@@ -25,7 +25,7 @@ func TestParseString(t *testing.T) {
 	d := decoder{}
 	for _, test := range tests {
 		t.Run(test.in, func(t *testing.T) {
-			out, ext, err := d.parseString([]byte(test.in))
+			out, ext, _, err := d.parseString([]byte(test.in))
 
 			if test.err == "" {
 				if err != nil {