Repository: amzn/ion-go (go)

Issue:
Fix CLOB Newline Normalization
https://github.com/amzn/ion-tests/commit/2a1f3f4f272256108e89f1c53bfae5b3a9ede812 introduces CLOB newline normalization in text.  This is currently skipped in the parser tests.

Pull request description:
Fixes CLOB newline processing.
Also adds escape processing in CLOB parsing to support `\xHH` escapes
which is part of #18.

#17 asks a larger question about the factoring around escape logic
in the parser, which this commit does not aim to address.

Fixes #3

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.


Patch:
diff --git a/internal/lex/lex_binary.go b/internal/lex/lex_binary.go
index e2cb141..0dcf657 100644
--- a/internal/lex/lex_binary.go
+++ b/internal/lex/lex_binary.go
@@ -110,6 +110,11 @@ Loop:
 				return x.error("unterminated short clob")
 			case !isEscapeAble(r):
 				return x.errorf("invalid character after escape: %#U", r)
+			case r == '\r':
+				// check for CR LF
+				if x.peek() == '\n' {
+					x.next()
+				}
 			case r == 'x' || r == 'X':
 				// If what is being escaped is a hex character, then we still
 				// need to make sure that escaped character is allowed.
diff --git a/ion/parse_text_simple.go b/ion/parse_text_simple.go
index a61b624..34c990e 100644
--- a/ion/parse_text_simple.go
+++ b/ion/parse_text_simple.go
@@ -17,9 +17,9 @@ package ion
 
 import (
 	"bytes"
-	"math"
-
 	"github.com/amzn/ion-go/internal/lex"
+	"math"
+	"strconv"
 )
 
 // This file contains text parsers for Null, Padding, Bool, Symbol, String, Blob, and Clob.
@@ -115,20 +115,18 @@ func doStringReplacements(str []byte) []byte {
 	return ret
 }
 
-// doClobReplacements is like doStringReplacements, but keeps \r characters as-is.
+// doClobReplacements is like doStringReplacements but is restricted to escapes that CLOBs have.
 func doClobReplacements(str []byte) []byte {
 	strLen := len(str)
 	ret := make([]byte, 0, strLen)
 	for index := 0; index < strLen; index++ {
 		switch ch := str[index]; ch {
 		case '\r':
-			// We need to treat "\r\n" as "\n", so skip extra if what comes next is "\n".
+			// We normalize "\r" and "\r\n" as "\n".
 			if index < strLen-1 && str[index+1] == '\n' {
-				ret = append(ret, '\n')
 				index++
-			} else {
-				ret = append(ret, '\r')
 			}
+			ret = append(ret, '\n')
 		case '\\':
 			if index >= strLen-1 {
 				continue
@@ -155,6 +153,12 @@ func doClobReplacements(str []byte) []byte {
 			case '\'', '"', '\\':
 				ret = append(ret, next)
 				index++
+			case 'x':
+				// Decode the hex sequence
+				index += 2
+				octet, _ := strconv.ParseUint(string(str[index:index+2]), 16, 8)
+				index += 2
+				ret = append(ret, byte(octet))
 			default:
 				// Don't have anything special to do with the next character, so
 				// just add the current character and let the next one get added


Test patch:
diff --git a/ion/parse_text_test.go b/ion/parse_text_test.go
index d8c05cc..91b3869 100644
--- a/ion/parse_text_test.go
+++ b/ion/parse_text_test.go
@@ -291,8 +291,6 @@ func TestIonTests_Text_Good(t *testing.T) {
 	filesToSkip := map[string]bool{
 		"utf16.ion": true,
 		"utf32.ion": true,
-		// TODO amzn/ion-go#3 (newline normalization in CLOB)
-		"clobNewlines.ion": true,
 	}
 
 	testFilePath := "../ion-tests/iontestdata/good"
@@ -357,8 +355,6 @@ func TestIonTests_Text_Equivalents(t *testing.T) {
 		// These files contain UTF16 and UTF32 which we do not support.
 		"stringU0001D11E.ion": true,
 		"stringUtf8.ion":      true,
-		// TODO amzn/ion-go#3 (newline normalization in CLOB)
-		"clobNewlines.ion": true,
 	}
 
 	testFilePath := "../ion-tests/iontestdata/good/equivs"