Repository: segmentio/encoding
Pull request #33 (2020-04-18 23:21:49)

Add support for json.Decoder InputOffset()
go version 1.14 adds support for getting the input offset
of a json.Decoder (https://golang.org/pkg/encoding/json/#Decoder.InputOffset)

This commit adds identical functionality to segmentio/encoding

Signed-off-by: Daniel Ferstay <dferstay@splunk.com>


Patch:
diff --git a/json/json.go b/json/json.go
index 4af5e78..6e16914 100644
--- a/json/json.go
+++ b/json/json.go
@@ -234,11 +234,12 @@ func Valid(data []byte) bool {
 
 // Decoder is documented at https://golang.org/pkg/encoding/json/#Decoder
 type Decoder struct {
-	reader io.Reader
-	buffer []byte
-	remain []byte
-	err    error
-	flags  ParseFlags
+	reader      io.Reader
+	buffer      []byte
+	remain      []byte
+	inputOffset int64
+	err         error
+	flags       ParseFlags
 }
 
 // NewDecoder is documented at https://golang.org/pkg/encoding/json/#NewDecoder
@@ -274,7 +275,8 @@ func (dec *Decoder) readValue() (v []byte, err error) {
 		if len(dec.remain) != 0 {
 			v, r, err = parseValue(dec.remain)
 			if err == nil {
-				dec.remain = skipSpaces(r)
+				dec.remain, n = skipSpacesN(r)
+				dec.inputOffset += int64(len(v) + n)
 				return
 			}
 			if len(r) != 0 {
@@ -309,7 +311,9 @@ func (dec *Decoder) readValue() (v []byte, err error) {
 		if n > 0 {
 			dec.buffer = dec.buffer[:len(dec.buffer)+n]
 		}
-		dec.remain, dec.err = skipSpaces(dec.buffer), err
+		dec.remain, n = skipSpacesN(dec.buffer)
+		dec.inputOffset += int64(n)
+		dec.err = err
 	}
 }
 
@@ -347,6 +351,13 @@ func (dec *Decoder) DontMatchCaseInsensitiveStructFields() {
 // all the copy optimizations of the decoder.
 func (dec *Decoder) ZeroCopy() { dec.flags |= ZeroCopy }
 
+// InputOffset returns the input stream byte offset of the current decoder position.
+// The offset gives the location of the end of the most recently returned token
+// and the beginning of the next token.
+func (dec *Decoder) InputOffset() int64 {
+	return dec.inputOffset
+}
+
 // Encoder is documented at https://golang.org/pkg/encoding/json/#Encoder
 type Encoder struct {
 	writer io.Writer
diff --git a/json/parse.go b/json/parse.go
index d0500d9..ac4d9c6 100644
--- a/json/parse.go
+++ b/json/parse.go
@@ -25,14 +25,19 @@ const (
 )
 
 func skipSpaces(b []byte) []byte {
+	b, _ = skipSpacesN(b)
+	return b
+}
+
+func skipSpacesN(b []byte) ([]byte, int) {
 	for i := range b {
 		switch b[i] {
 		case sp, ht, nl, cr:
 		default:
-			return b[i:]
+			return b[i:], i
 		}
 	}
-	return nil
+	return nil, 0
 }
 
 // parseInt parses a decimanl representation of an int64 from b.


Test patch:
diff --git a/json/json_test.go b/json/json_test.go
index 22d9bc6..8b07a7d 100644
--- a/json/json_test.go
+++ b/json/json_test.go
@@ -1376,6 +1376,52 @@ func TestGithubIssue16(t *testing.T) {
 	}
 }
 
+func TestDecoderInputOffset(t *testing.T) {
+	checkOffset := func(o, expected int64) {
+		if o != expected {
+			t.Error("unexpected input offset", o, expected)
+		}
+	}
+
+	b := []byte(`{"userId": "blah"}{"userId": "blah"}
+	{"userId": "blah"}{"num": 0}`)
+	d := NewDecoder(bytes.NewReader(b))
+
+	var expected int64
+	checkOffset(d.InputOffset(), expected)
+
+	var a struct {
+		UserId string `json:"userId"`
+	}
+
+	if err := d.Decode(&a); err != nil {
+		t.Error("unexpected decode error", err)
+	}
+	expected = int64(18)
+	checkOffset(d.InputOffset(), expected)
+
+	if err := d.Decode(&a); err != nil {
+		t.Error("unexpected decode error", err)
+	}
+	expected = int64(38)
+	checkOffset(d.InputOffset(), expected)
+
+	if err := d.Decode(&a); err != nil {
+		t.Error("unexpected decode error", err)
+	}
+	expected = int64(56)
+	checkOffset(d.InputOffset(), expected)
+
+	var z struct {
+		Num int64 `json:"num"`
+	}
+	if err := d.Decode(&z); err != nil {
+		t.Error("unexpected decode error", err)
+	}
+	expected = int64(66)
+	checkOffset(d.InputOffset(), expected)
+}
+
 func TestGithubIssue18(t *testing.T) {
 	// https://github.com/segmentio/encoding/issues/18
 	b := []byte(`{