Repository: segmentio/encoding
Pull request #34 (2020-04-19 00:58:26)

1.14
There was a behavior change on `encoding/json` in Go 1.14: https://github.com/golang/go/pull/34804

The `unsafe_NewArray` function is still available in the runtime as well, so enable it on Go 1.14.


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 68a341d..39d86fb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2
 jobs:
   test:
     docker:
-      - image: circleci/golang:1.13
+      - image: circleci/golang:1.14
     steps:
       - checkout
       - run: go mod download
diff --git a/json/json.go b/json/json.go
index 4af5e78..6e16914 100644
--- a/json/json.go
+++ b/json/json.go
@@ -234,11 +234,12 @@ func Valid(data []byte) bool {
 
 // Decoder is documented at https://golang.org/pkg/encoding/json/#Decoder
 type Decoder struct {
-	reader io.Reader
-	buffer []byte
-	remain []byte
-	err    error
-	flags  ParseFlags
+	reader      io.Reader
+	buffer      []byte
+	remain      []byte
+	inputOffset int64
+	err         error
+	flags       ParseFlags
 }
 
 // NewDecoder is documented at https://golang.org/pkg/encoding/json/#NewDecoder
@@ -274,7 +275,8 @@ func (dec *Decoder) readValue() (v []byte, err error) {
 		if len(dec.remain) != 0 {
 			v, r, err = parseValue(dec.remain)
 			if err == nil {
-				dec.remain = skipSpaces(r)
+				dec.remain, n = skipSpacesN(r)
+				dec.inputOffset += int64(len(v) + n)
 				return
 			}
 			if len(r) != 0 {
@@ -309,7 +311,9 @@ func (dec *Decoder) readValue() (v []byte, err error) {
 		if n > 0 {
 			dec.buffer = dec.buffer[:len(dec.buffer)+n]
 		}
-		dec.remain, dec.err = skipSpaces(dec.buffer), err
+		dec.remain, n = skipSpacesN(dec.buffer)
+		dec.inputOffset += int64(n)
+		dec.err = err
 	}
 }
 
@@ -347,6 +351,13 @@ func (dec *Decoder) DontMatchCaseInsensitiveStructFields() {
 // all the copy optimizations of the decoder.
 func (dec *Decoder) ZeroCopy() { dec.flags |= ZeroCopy }
 
+// InputOffset returns the input stream byte offset of the current decoder position.
+// The offset gives the location of the end of the most recently returned token
+// and the beginning of the next token.
+func (dec *Decoder) InputOffset() int64 {
+	return dec.inputOffset
+}
+
 // Encoder is documented at https://golang.org/pkg/encoding/json/#Encoder
 type Encoder struct {
 	writer io.Writer
diff --git a/json/parse.go b/json/parse.go
index d0500d9..ac4d9c6 100644
--- a/json/parse.go
+++ b/json/parse.go
@@ -25,14 +25,19 @@ const (
 )
 
 func skipSpaces(b []byte) []byte {
+	b, _ = skipSpacesN(b)
+	return b
+}
+
+func skipSpacesN(b []byte) ([]byte, int) {
 	for i := range b {
 		switch b[i] {
 		case sp, ht, nl, cr:
 		default:
-			return b[i:]
+			return b[i:], i
 		}
 	}
-	return nil
+	return nil, 0
 }
 
 // parseInt parses a decimanl representation of an int64 from b.
diff --git a/json/reflect.go b/json/reflect.go
index d72e9e5..3f852f1 100644
--- a/json/reflect.go
+++ b/json/reflect.go
@@ -1,4 +1,4 @@
-// +build go1.14
+// +build go1.15
 
 package json
 
diff --git a/json/reflect_optimize.go b/json/reflect_optimize.go
index 1015d7e..ab8fc9e 100644
--- a/json/reflect_optimize.go
+++ b/json/reflect_optimize.go
@@ -1,4 +1,4 @@
-// +build !go1.14
+// +build !go1.15
 
 package json
 


Test patch:
diff --git a/json/golang_scanner_test.go b/json/golang_scanner_test.go
index 0be8baa..a26bace 100644
--- a/json/golang_scanner_test.go
+++ b/json/golang_scanner_test.go
@@ -47,6 +47,7 @@ var examples = []example{
 	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
 	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
 	{ex1, ex1i},
+	{"{\"\":\"<>&\u2028\u2029\"}", "{\n\t\"\": \"<>&\u2028\u2029\"\n}"}, // See golang.org/issue/34070
 }
 
 var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`
@@ -88,8 +89,8 @@ func TestCompactSeparators(t *testing.T) {
 	tests := []struct {
 		in, compact string
 	}{
-		{"{\"\u2028\": 1}", `{"\u2028":1}`},
-		{"{\"\u2029\" :2}", `{"\u2029":2}`},
+		{"{\"\u2028\": 1}", "{\"\u2028\":1}"},
+		{"{\"\u2029\" :2}", "{\"\u2029\":2}"},
 	}
 	for _, tt := range tests {
 		var buf bytes.Buffer
diff --git a/json/json_test.go b/json/json_test.go
index 22d9bc6..8b07a7d 100644
--- a/json/json_test.go
+++ b/json/json_test.go
@@ -1376,6 +1376,52 @@ func TestGithubIssue16(t *testing.T) {
 	}
 }
 
+func TestDecoderInputOffset(t *testing.T) {
+	checkOffset := func(o, expected int64) {
+		if o != expected {
+			t.Error("unexpected input offset", o, expected)
+		}
+	}
+
+	b := []byte(`{"userId": "blah"}{"userId": "blah"}
+	{"userId": "blah"}{"num": 0}`)
+	d := NewDecoder(bytes.NewReader(b))
+
+	var expected int64
+	checkOffset(d.InputOffset(), expected)
+
+	var a struct {
+		UserId string `json:"userId"`
+	}
+
+	if err := d.Decode(&a); err != nil {
+		t.Error("unexpected decode error", err)
+	}
+	expected = int64(18)
+	checkOffset(d.InputOffset(), expected)
+
+	if err := d.Decode(&a); err != nil {
+		t.Error("unexpected decode error", err)
+	}
+	expected = int64(38)
+	checkOffset(d.InputOffset(), expected)
+
+	if err := d.Decode(&a); err != nil {
+		t.Error("unexpected decode error", err)
+	}
+	expected = int64(56)
+	checkOffset(d.InputOffset(), expected)
+
+	var z struct {
+		Num int64 `json:"num"`
+	}
+	if err := d.Decode(&z); err != nil {
+		t.Error("unexpected decode error", err)
+	}
+	expected = int64(66)
+	checkOffset(d.InputOffset(), expected)
+}
+
 func TestGithubIssue18(t *testing.T) {
 	// https://github.com/segmentio/encoding/issues/18
 	b := []byte(`{