Repository: benhoyt/goawk (go)

Issue:
Add support for new \u Unicode escape sequence
As per the addition to BWK's awk and Gawk. The `\u` must be followed by 1-8 hex digits. (Look up references/sources, perhaps in the BWK awk source code and/or the Gawk manual.)

Pull request description:
Add support for new Unicode string escape \u
See similar onetrueawk and Gawk changes:
- https://github.com/onetrueawk/awk/commit/ac3084de9b64d0c838e520ea1ebdb2e1bde87b3f
- https://git.savannah.gnu.org/cgit/gawk.git/commit/?id=1bc73bfb10f0600aaccca7220b5200e220f9a71a

Also, in lexer tests, stop after first error.

Fixes #210

Patch:
diff --git a/lexer/lexer.go b/lexer/lexer.go
index e5408de..10545cc 100644
--- a/lexer/lexer.go
+++ b/lexer/lexer.go
@@ -10,6 +10,7 @@ package lexer
 import (
 	"errors"
 	"fmt"
+	"unicode/utf8"
 )
 
 // Lexer tokenizes a byte string of AWK source code. Use NewLexer to
@@ -467,7 +468,7 @@ func parseString(quote byte, ch func() byte, next func()) (string, error) {
 			c = '\v'
 			next()
 		case 'x':
-			// Hex byte of one of two hex digits
+			// Hex byte of one or two hex digits
 			next()
 			digit := hexDigit(ch())
 			if digit < 0 {
@@ -480,6 +481,29 @@ func parseString(quote byte, ch func() byte, next func()) (string, error) {
 				c = c*16 + byte(digit)
 				next()
 			}
+		case 'u':
+			// Hex Unicode character of 1-8 digits
+			next()
+			r := hexDigit(ch())
+			if r < 0 {
+				return "", errors.New("1-8 hex digits expected")
+			}
+			next()
+			for i := 0; i < 7; i++ {
+				digit := hexDigit(ch())
+				if digit < 0 {
+					break
+				}
+				next()
+				r = r*16 + digit
+			}
+			if !utf8.ValidRune(rune(r)) {
+				return "", errors.New("invalid Unicode character")
+			}
+			runeBytes := make([]byte, utf8.UTFMax)
+			n := utf8.EncodeRune(runeBytes, rune(r))
+			chars = append(chars, runeBytes[:n]...)
+			continue
 		case '0', '1', '2', '3', '4', '5', '6', '7':
 			// Octal byte of 1-3 octal digits
 			c = ch() - '0'


Test patch:
diff --git a/interp/interp_test.go b/interp/interp_test.go
index 2d8276f..6a8b04e 100644
--- a/interp/interp_test.go
+++ b/interp/interp_test.go
@@ -269,6 +269,7 @@ BEGIN {
 	{`BEGIN { print 1, 1., .1, 1e0, -1, 1e }`, "", "1 1 0.1 1 -1 1\n", "", ""},
 	{`BEGIN { print '\"' '\'' 'xy' "z" "'" '\"' }`, "", "\"'xyz'\"\n", "", "invalid char"}, // Check support for single-quoted strings
 	{`BEGIN { print "0\n1\t2\r3\a4\b5\f6\v7\x408\xf" }  # !posix`, "", "0\n1\t2\r3\a4\b5\f6\v7@8\x0f\n", "", ""},
+	{`BEGIN { print "The \u201cQUICK\u201d brown fox \u1F602" }  # !gawk`, "", "The “QUICK” brown fox 😂\n", "", ""},
 	{`{ print /foo/ }`, "food\nfoo\nxfooz\nbar\n", "1\n1\n1\n0\n", "", ""},
 	{`/[a-/`, "foo", "", "parse error at 1:1: error parsing regexp: invalid character class range: `a-)`", "terminated"},
 	{`/\Q/  # !gawk`, "", "", "parse error at 1:1: error parsing regexp: missing closing ): `(?s:\\Q)`", ""}, // Gawk produces a warning (not an error), so skip
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index c0f651f..aba4b7f 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -20,7 +20,7 @@ func TestLexer(t *testing.T) {
 		{"+# foo \n- #foo", `1:1 + "", 1:8 <newline> "", 2:1 - ""`},
 		{"+\\\n-", `1:1 + "", 2:1 - ""`},
 		{"+\\\r\n-", `1:1 + "", 2:1 - ""`},
-		{"+\\-", `1:1 + "", 1:3 <illegal> "expected \\n after \\ line continuation", 1:3 - ""`},
+		{"+\\-", `1:1 + "", 1:3 <illegal> "expected \\n after \\ line continuation"`},
 
 		// Names and keywords
 		{"x", `1:1 name "x"`},
@@ -33,14 +33,19 @@ func TestLexer(t *testing.T) {
 		{`"a\t\r\n\z\'\"\a\b\f\vb"`, `1:1 string "a\t\r\nz'\"\a\b\f\vb"`},
 		{`"x`, `1:3 <illegal> "didn't find end quote in string"`},
 		{`"foo\"`, `1:7 <illegal> "didn't find end quote in string"`},
-		{"\"x\n\"", `1:3 <illegal> "can't have newline in string", 1:3 <newline> "", 2:2 <illegal> "didn't find end quote in string"`},
+		{"\"x\n\"", `1:3 <illegal> "can't have newline in string"`},
 		{`'foo'`, `1:1 string "foo"`},
 		{`'a\t\r\n\z\'\"b'`, `1:1 string "a\t\r\nz'\"b"`},
 		{`'x`, `1:3 <illegal> "didn't find end quote in string"`},
-		{"'x\n'", `1:3 <illegal> "can't have newline in string", 1:3 <newline> "", 2:2 <illegal> "didn't find end quote in string"`},
+		{"'x\n'", `1:3 <illegal> "can't have newline in string"`},
 		{`"\x0.\x00.\x0A\x10\xff\xFF\x41"`, `1:1 string "\x00.\x00.\n\x10\xff\xffA"`},
-		{`"\xg"`, `1:4 <illegal> "1 or 2 hex digits expected", 1:4 name "g", 1:6 <illegal> "didn't find end quote in string"`},
+		{`"\xg"`, `1:4 <illegal> "1 or 2 hex digits expected"`},
 		{`"\0\78\7\77\777\0 \141 "`, `1:1 string "\x00\a8\a?\xff\x00 a "`},
+		{`"\ufg\uffg\uabcg\u201C\u1F602"`, `1:1 string "\x0fgÿg઼g“😂"`},
+		{`"\u201C"`, `1:1 string "“"`},
+		{`"\u00000041Z"`, `1:1 string "AZ"`},
+		{`"\ug"`, `1:4 <illegal> "1-8 hex digits expected"`},
+		{`"\uffffffffg"`, `1:12 <illegal> "invalid Unicode character"`},
 
 		// Number tokens
 		{"0", `1:1 number "0"`},
@@ -67,7 +72,7 @@ func TestLexer(t *testing.T) {
 		{".", `1:2 <illegal> "expected digits"`},
 
 		// Misc errors
-		{"&=", `1:2 <illegal> "unexpected char after '&'", 1:2 = ""`},
+		{"&=", `1:2 <illegal> "unexpected char after '&'"`},
 	}
 	for _, test := range tests {
 		t.Run(test.input, func(t *testing.T) {
@@ -88,6 +93,9 @@ func TestLexer(t *testing.T) {
 					}
 				}
 				strs = append(strs, fmt.Sprintf("%d:%d %s %q", pos.Line, pos.Column, tok, val))
+				if tok == ILLEGAL {
+					break
+				}
 			}
 			output := strings.Join(strs, ", ")
 			if output != test.output {