nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: amzn/ion-go
Pull request #76 (2020-06-19 17:11:42)

String with ISO-8859-1 characters

This fix is to detect strings with latine-1 (ISO-8859-1) characters.




Patch:
diff --git a/ion/bitstream.go b/ion/bitstream.go
index a0c05d2..ae536d3 100644
--- a/ion/bitstream.go
+++ b/ion/bitstream.go
@@ -9,6 +9,7 @@ import (
 	"math"
 	"math/big"
 	"time"
+	"unicode/utf8"
 )
 
 type bss uint8
@@ -647,7 +648,10 @@ func (b *bitstream) ReadString() (string, error) {
 	b.state = b.stateAfterValue()
 	b.clear()
 
-	return string(bs), nil
+	if utf8.Valid(bs) {
+		return string(bs), nil
+	}
+	return "", &UnexpectedTokenError{"String value contains non-UTF-8 runes", b.pos}
 }
 
 // ReadBytes reads a blob or clob value.


Test patch:
diff --git a/ion/integration_test.go b/ion/integration_test.go
index 00ac1ee..c729563 100644
--- a/ion/integration_test.go
+++ b/ion/integration_test.go
@@ -180,7 +180,6 @@ var malformedIonsSkipList = []string{
 	"nopPadWithAnnotations.10n",
 	"nullDotCommentInt.ion",
 	"sexpOperatorAnnotation.ion",
-	"stringWithLatinEncoding.10n",
 	"structOrderedEmpty.10n",
 	"surrogate_1.ion",
 	"surrogate_10.ion",
@@ -206,7 +205,7 @@ var equivsSkipList = []string{
 	"localSymbolTablesValuesWithAnnotations.ion",
 	"nonIVMNoOps.ion",
 	"sexps.ion",
-	"stringUtf8.ion",
+	"stringUtf8.ion", // fails on utf-16 surrogate https://github.com/amzn/ion-go/issues/75
 	"structsFieldsDiffOrder.ion",
 	"structsFieldsRepeatedNames.ion",
 	"systemSymbols.ion",
Repository: amzn/ion-go
Pull request #77 (2020-06-19 21:03:21)

Fix invalid negative int zero
Ion specs for [integer](http://amzn.github.io/ion-docs/docs/binary.html#2-and-3-int) reads:

> Zero is always stored as positive; negative zero is illegal.




Patch:
diff --git a/ion/bitstream.go b/ion/bitstream.go
index afd3451..a0c05d2 100644
--- a/ion/bitstream.go
+++ b/ion/bitstream.go
@@ -417,18 +417,21 @@ func (b *bitstream) ReadInt() (interface{}, error) {
 	}
 
 	var ret interface{}
+	isZero := false
 	switch {
 	case b.len == 0:
 		// Special case for zero.
 		ret = int64(0)
+		isZero = true
 
-	case b.len < 8, (b.len == 8 && bs[0]&0x80 == 0):
+	case b.len < 8, b.len == 8 && bs[0]&0x80 == 0:
 		// It'll fit in an int64.
 		i := int64(0)
 		for _, b := range bs {
 			i <<= 8
 			i ^= int64(b)
 		}
+		isZero = i == 0
 		if b.code == bitcodeNegInt {
 			i = -i
 		}
@@ -437,12 +440,18 @@ func (b *bitstream) ReadInt() (interface{}, error) {
 	default:
 		// Need to go big.Int.
 		i := new(big.Int).SetBytes(bs)
+		isZero = i.BitLen() == 0
 		if b.code == bitcodeNegInt {
 			i = i.Neg(i)
 		}
 		ret = i
 	}
 
+	// Zero is always stored as positive; negative zero is illegal.
+	if isZero && b.code == bitcodeNegInt {
+		return 0, &SyntaxError{"Integer zero cannot be negative", b.pos - b.len}
+	}
+
 	b.state = b.stateAfterValue()
 	b.clear()
 


Test patch:
diff --git a/ion/integration_test.go b/ion/integration_test.go
index daf975a..00ac1ee 100644
--- a/ion/integration_test.go
+++ b/ion/integration_test.go
@@ -176,8 +176,6 @@ var malformedIonsSkipList = []string{
 	"localSymbolTableWithMultipleSymbolsFields.10n",
 	"localSymbolTableWithMultipleSymbolsFields.ion",
 	"minLongWithLenTooSmall.10n",
-	"negativeIntZero.10n",
-	"negativeIntZeroLn.10n",
 	"nopPadTooShort.10n",
 	"nopPadWithAnnotations.10n",
 	"nullDotCommentInt.ion",
Repository: amzn/ion-go
Pull request #128 (2020-08-10 19:43:25)

Fix S-Expression Tests
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.




Patch:
diff --git a/ion/textreader.go b/ion/textreader.go
index 8034b8b..3e35473 100644
--- a/ion/textreader.go
+++ b/ion/textreader.go
@@ -237,7 +237,11 @@ func (t *textReader) nextBeforeTypeAnnotations() (bool, error) {
 				if err := t.verifyUnquotedSymbol(val, "annotation"); err != nil {
 					return false, err
 				}
+			} else if tok == tokenSymbolOperator {
+				return false, &SyntaxError{
+					"Annotations that include a '" + val + "' must be enclosed in quotes.", t.tok.Pos() - 1}
 			}
+
 			t.annotations = append(t.annotations, val)
 			return false, nil
 		}
diff --git a/ion/tokenizer.go b/ion/tokenizer.go
index 34aec46..5a7305b 100644
--- a/ion/tokenizer.go
+++ b/ion/tokenizer.go
@@ -229,6 +229,10 @@ func (t *tokenizer) Next() error {
 			t.unread(c)
 			return t.ok(tokenSymbolOperator, true)
 		}
+		if c2 == ' ' || isIdentifierPart(c2) {
+			t.unread(c)
+		}
+
 		return t.ok(tokenDot, false)
 
 	case c == '\'':


Test patch:
diff --git a/ion/integration_test.go b/ion/integration_test.go
index f49dfd5..8633c9c 100644
--- a/ion/integration_test.go
+++ b/ion/integration_test.go
@@ -139,7 +139,6 @@ var malformedIonsSkipList = []string{
 	"nopPadTooShort.10n",
 	"nopPadWithAnnotations.10n",
 	"nullDotCommentInt.ion",
-	"sexpOperatorAnnotation.ion",
 	"surrogate_1.ion",
 	"surrogate_10.ion",
 	"surrogate_2.ion",
@@ -161,7 +160,6 @@ var equivsSkipList = []string{
 	"localSymbolTables.ion",
 	"localSymbolTablesValuesWithAnnotations.ion",
 	"nonIVMNoOps.ion",
-	"sexps.ion",
 	"stringUtf8.ion", // fails on utf-16 surrogate https://github.com/amzn/ion-go/issues/75
 	"systemSymbols.ion",
 	"systemSymbolsAsAnnotations.ion",