nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: segmentio/encoding
Pull request #54 (2020-10-28 18:27:09)

Fix negative slice index panic in Tokenizer




Patch:
diff --git a/json/token.go b/json/token.go
index b799401..9cac4db 100644
--- a/json/token.go
+++ b/json/token.go
@@ -189,6 +189,10 @@ skipLoop:
 		case ':':
 			t.isKey = false
 		case ',':
+			if len(t.stack) == 0 {
+				t.Err = syntaxError(t.json, "found unexpected comma")
+				return false
+			}
 			if t.is(inObject) {
 				t.isKey = true
 			}


Test patch:
diff --git a/json/token_test.go b/json/token_test.go
index 69759a3..b53f765 100644
--- a/json/token_test.go
+++ b/json/token_test.go
@@ -185,6 +185,41 @@ func TestTokenizer(t *testing.T) {
 	}
 }
 
+// Regression test for syntax that caused panics in Next.
+func TestTokenizer_invalidInput(t *testing.T) {
+	tests := []struct {
+		scenario string
+		payload  []byte
+	}{
+		{
+			scenario: "bare comma",
+			payload:  []byte(","),
+		},
+		{
+			scenario: "comma after array",
+			payload:  []byte("[],"),
+		},
+		{
+			scenario: "comma after object",
+			payload:  []byte("{},"),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.scenario, func(t *testing.T) {
+			tkn := NewTokenizer(test.payload)
+
+			// This shouldn't panic
+			for tkn.Next() {
+			}
+
+			if tkn.Err == nil {
+				t.Error("expected Err to be set, got nil")
+			}
+		})
+	}
+}
+
 func BenchmarkTokenizer(b *testing.B) {
 	values := []struct {
 		scenario string