Repository: benhoyt/goawk
Pull request #134 (2022-06-05 22:40:54)

More POSIX-accurate handling of cmdline var=value escaping
Instead of using Go's strconv.Unquote, we should use the same string
parsing we do in the GoAWK lexer. Refactor the string parsing in the
lexer slightly so we can add an Unescape function and use that.


Patch:
diff --git a/goawk.go b/goawk.go
index c2705ff..8b470b4 100644
--- a/goawk.go
+++ b/goawk.go
@@ -35,7 +35,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"runtime/pprof"
-	"strconv"
 	"strings"
 	"unicode/utf8"
 
@@ -276,9 +275,9 @@ func main() {
 		}
 		name, value := v[:equals], v[equals+1:]
 		// Oddly, -v must interpret escapes (issue #129)
-		unquoted, err := strconv.Unquote(`"` + value + `"`)
+		unescaped, err := lexer.Unescape(value)
 		if err == nil {
-			value = unquoted
+			value = unescaped
 		}
 		config.Vars = append(config.Vars, name, value)
 	}
diff --git a/interp/io.go b/interp/io.go
index 4924be6..cebbd4f 100644
--- a/interp/io.go
+++ b/interp/io.go
@@ -743,9 +743,9 @@ func (p *interp) nextLine() (string, error) {
 					// Yep, set variable to value and keep going
 					name, val := matches[1], matches[2]
 					// Oddly, var=value args must interpret escapes (issue #129)
-					unquoted, err := strconv.Unquote(`"` + val + `"`)
+					unescaped, err := Unescape(val)
 					if err == nil {
-						val = unquoted
+						val = unescaped
 					}
 					err = p.setVarByName(name, val)
 					if err != nil {
diff --git a/lexer/lexer.go b/lexer/lexer.go
index afb5561..05cf33f 100644
--- a/lexer/lexer.go
+++ b/lexer/lexer.go
@@ -7,6 +7,10 @@
 // then call Scan() until the token type is EOF or ILLEGAL.
 package lexer
 
+import (
+	"errors"
+)
+
 // Lexer tokenizes a byte string of AWK source code. Use NewLexer to
 // actually create a lexer, and Scan() or ScanRegex() to get tokens.
 type Lexer struct {
@@ -181,80 +185,18 @@ func (l *Lexer) scan() (Position, Token, string) {
 		}
 	case '"', '\'':
 		// Note: POSIX awk spec doesn't allow single-quoted strings,
-		// but this helps without quoting, especially on Windows
+		// but this helps with quoting, especially on Windows
 		// where the shell quote character is " (double quote).
-		chars := make([]byte, 0, 32) // most won't require heap allocation
-		for l.ch != ch {
-			c := l.ch
-			if c == 0 {
-				return l.pos, ILLEGAL, "didn't find end quote in string"
-			}
-			if c == '\r' || c == '\n' {
-				return l.pos, ILLEGAL, "can't have newline in string"
-			}
-			if c != '\\' {
-				// Normal, non-escaped character
-				chars = append(chars, c)
-				l.next()
-				continue
-			}
-			// Escape sequence, skip over \ and process
-			l.next()
-			switch l.ch {
-			case 'n':
-				c = '\n'
-				l.next()
-			case 't':
-				c = '\t'
-				l.next()
-			case 'r':
-				c = '\r'
-				l.next()
-			case 'a':
-				c = '\a'
-				l.next()
-			case 'b':
-				c = '\b'
-				l.next()
-			case 'f':
-				c = '\f'
-				l.next()
-			case 'v':
-				c = '\v'
-				l.next()
-			case 'x':
-				// Hex byte of one of two hex digits
-				l.next()
-				digit := hexDigit(l.ch)
-				if digit < 0 {
-					return l.pos, ILLEGAL, "1 or 2 hex digits expected"
-				}
-				c = byte(digit)
-				l.next()
-				digit = hexDigit(l.ch)
-				if digit >= 0 {
-					c = c*16 + byte(digit)
-					l.next()
-				}
-			case '0', '1', '2', '3', '4', '5', '6', '7':
-				// Octal byte of 1-3 octal digits
-				c = l.ch - '0'
-				l.next()
-				for i := 0; i < 2 && l.ch >= '0' && l.ch <= '7'; i++ {
-					c = c*8 + l.ch - '0'
-					l.next()
-				}
-			default:
-				// Any other escape character is just the char
-				// itself, eg: "\z" is just "z"
-				c = l.ch
-				l.next()
-			}
-			chars = append(chars, c)
+		s, err := parseString(ch, func() byte { return l.ch }, l.next)
+		if err != nil {
+			return l.pos, ILLEGAL, err.Error()
+		}
+		if l.ch != ch {
+			return l.pos, ILLEGAL, "didn't find end quote in string"
 		}
 		l.next()
 		tok = STRING
-		val = string(chars)
+		val = s
 	case '(':
 		tok = LPAREN
 	case ')':
@@ -456,3 +398,102 @@ func (l *Lexer) choice(ch byte, one, two Token) Token {
 func (l *Lexer) PeekByte() byte {
 	return l.ch
 }
+
+// Unescape unescapes the backslash escapes in s (which shouldn't include the
+// surrounding quotes) and returns the unquoted string. It's intended for use
+// when unescaping command line var=value assignments, as required by the
+// POSIX AWK spec.
+func Unescape(s string) (string, error) {
+	i := 0
+	ch := func() byte {
+		if i >= len(s) {
+			return 0
+		}
+		return s[i]
+	}
+	next := func() {
+		i++
+	}
+	return parseString(0, ch, next)
+}
+
+// Parses a string ending with given quote character (not parsed). The ch
+// function returns the current character (or 0 at the end); the next function
+// moves forward one character.
+func parseString(quote byte, ch func() byte, next func()) (string, error) {
+	chars := make([]byte, 0, 32) // most strings won't require heap allocation
+	for {
+		c := ch()
+		if c == quote || c == 0 {
+			break
+		}
+		if c == '\r' || c == '\n' {
+			return "", errors.New("can't have newline in string")
+		}
+		if c != '\\' {
+			// Normal, non-escaped character
+			chars = append(chars, c)
+			next()
+			continue
+		}
+		// Escape sequence, skip over \ and process
+		next()
+		switch ch() {
+		case 'n':
+			c = '\n'
+			next()
+		case 't':
+			c = '\t'
+			next()
+		case 'r':
+			c = '\r'
+			next()
+		case 'a':
+			c = '\a'
+			next()
+		case 'b':
+			c = '\b'
+			next()
+		case 'f':
+			c = '\f'
+			next()
+		case 'v':
+			c = '\v'
+			next()
+		case 'x':
+			// Hex byte of one of two hex digits
+			next()
+			digit := hexDigit(ch())
+			if digit < 0 {
+				return "", errors.New("1 or 2 hex digits expected")
+			}
+			c = byte(digit)
+			next()
+			digit = hexDigit(ch())
+			if digit >= 0 {
+				c = c*16 + byte(digit)
+				next()
+			}
+		case '0', '1', '2', '3', '4', '5', '6', '7':
+			// Octal byte of 1-3 octal digits
+			c = ch() - '0'
+			next()
+			for i := 0; i < 2 && ch() >= '0' && ch() <= '7'; i++ {
+				c = c*8 + ch() - '0'
+				next()
+			}
+		default:
+			// Any other escape character is just the char
+			// itself, eg: "\z" is just "z".
+			c = ch()
+			if c == 0 {
+				// Expect backslash right at the end of the string, which is
+				// interpreted as a literal backslash (only for Unescape).
+				c = '\\'
+			}
+			next()
+		}
+		chars = append(chars, c)
+	}
+	return string(chars), nil
+}


Test patch:
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index c37097e..8e0b85c 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -32,6 +32,7 @@ func TestLexer(t *testing.T) {
 		{`"foo"`, `1:1 string "foo"`},
 		{`"a\t\r\n\z\'\"\a\b\f\vb"`, `1:1 string "a\t\r\nz'\"\a\b\f\vb"`},
 		{`"x`, `1:3 <illegal> "didn't find end quote in string"`},
+		{`"foo\"`, `1:7 <illegal> "didn't find end quote in string"`},
 		{"\"x\n\"", `1:3 <illegal> "can't have newline in string", 1:3 <newline> "", 2:2 <illegal> "didn't find end quote in string"`},
 		{`'foo'`, `1:1 string "foo"`},
 		{`'a\t\r\n\z\'\"b'`, `1:1 string "a\t\r\nz'\"b"`},
@@ -274,6 +275,40 @@ func TestAllTokens(t *testing.T) {
 	}
 }
 
+func TestUnescape(t *testing.T) {
+	tests := []struct {
+		input  string
+		output string
+		error  string
+	}{
+		{``, "", ""},
+		{`foo bar`, "foo bar", ""},
+		{`foo\tbar`, "foo\tbar", ""},
+		{"foo\nbar", "", "can't have newline in string"},
+		{`foo"`, "foo\"", ""},
+		{`O'Connor`, "O'Connor", ""},
+		{`foo\`, "foo\\", ""},
+		// Other cases tested in TestLexer string handling.
+	}
+	for _, test := range tests {
+		t.Run(test.input, func(t *testing.T) {
+			got, err := Unescape(test.input)
+			if err != nil {
+				if err.Error() != test.error {
+					t.Fatalf("expected error %q, got %q", test.error, err)
+				}
+			} else {
+				if test.error != "" {
+					t.Fatalf("expected error %q, got %q", test.error, "")
+				}
+				if got != test.output {
+					t.Fatalf("expected %q, got %q", test.output, got)
+				}
+			}
+		})
+	}
+}
+
 func benchmarkLexer(b *testing.B, repeat int, source string) {
 	fullSource := []byte(strings.Repeat(source+"\n", repeat))
 	b.ResetTimer()