Repository: benhoyt/goawk
Pull request #69 (2021-09-25 16:36:39)

make number literals behave like awk/gawk
Consider this:

```sh
for i in goawk gawk mawk nawk 'busybox awk'; do echo "$i"; $i 'BEGIN{a = 9; e = 19; print 1.2e+a}'; echo ""; done
```

The output is:

```
goawk
------------------------------------
BEGIN{a = 9; e = 19; print 1.2e+a}
                                ^
------------------------------------
parse error at 1:33: expected digits

gawk
1.228

mawk
1.228

nawk
1.228

busybox awk
1.228
```

This means that other version of awk treat things like "1e" like "1 e" (i.e. concatenation). So it is necessary to unread the 'e' if there are no digits after it.

benhoyt: I added new interp tests to test this behaviour, updated the lexer tests, and factored out `unread` into its own method on `Lexer`. Merging shortly. Thanks for your work on this!

Patch:
diff --git a/lexer/lexer.go b/lexer/lexer.go
index b4ff4d8..b79beb1 100644
--- a/lexer/lexer.go
+++ b/lexer/lexer.go
@@ -152,9 +152,13 @@ func (l *Lexer) scan() (Position, Token, string) {
 				l.next()
 				gotDigit = true
 			}
-			// Per awk/gawk, "1e" is allowed, but not "1e+"
-			if gotSign && !gotDigit {
-				return l.pos, ILLEGAL, "expected digits"
+			// Per awk/gawk, "1e" is allowed and parsed as "1 e" (with "e"
+			// considered a variable). "1e+" is parsed as "1e + ...".
+			if !gotDigit {
+				if gotSign {
+					l.unread() // unread the '+' or '-'
+				}
+				l.unread() // unread the 'e' or 'E'
 			}
 		}
 		tok = NUMBER
@@ -396,6 +400,7 @@ func (l *Lexer) next() {
 		if l.ch != 0 {
 			l.ch = 0
 			l.offset++
+			l.nextPos.Column++
 		}
 		return
 	}
@@ -410,6 +415,14 @@ func (l *Lexer) next() {
 	l.offset++
 }
 
+// Un-read the character just scanned (doesn't handle line boundaries).
+func (l *Lexer) unread() {
+	l.offset--
+	l.pos.Column--
+	l.nextPos.Column--
+	l.ch = l.src[l.offset-1]
+}
+
 func isNameStart(ch byte) bool {
 	return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')
 }


Test patch:
diff --git a/interp/interp_test.go b/interp/interp_test.go
index dcdf1ee..1d8dcfd 100644
--- a/interp/interp_test.go
+++ b/interp/interp_test.go
@@ -221,6 +221,14 @@ BEGIN {
 	{`BEGIN { printf "\0\78\7\77\777\0 \141 " }  # !awk`, "", "\x00\a8\a?\xff\x00 a ", "", ""},
 	{`BEGIN { printf "\1\78\7\77\777\1 \141 " }`, "", "\x01\a8\a?\xff\x01 a ", "", ""},
 
+	// Unusual number/exponent handling
+	{`BEGIN { e="x"; E="X"; print 1e, 1E }`, "", "1x 1X\n", "", ""},
+	{`BEGIN { e="x"; E="X"; print 1e1e, 1E1E }`, "", "10x 10X\n", "", ""},
+	{`BEGIN { a=2; print 1e+a, 1E+a, 1e+1, 1E+1 }`, "", "12 12 10 10\n", "", ""},
+	{`BEGIN { a=2; print 1e-a, 1E-a, 1e-1, 1E-1 }`, "", "1-2 1-2 0.1 0.1\n", "", ""},
+	{`BEGIN { print 1e+ }`, "", "", "parse error at 1:19: expected expression instead of }", "syntax error"},
+	{`BEGIN { print 1e- }`, "", "", "parse error at 1:19: expected expression instead of }", "syntax error"},
+
 	// Conditional ?: expression
 	{`{ print /x/?"t":"f" }`, "x\ny\nxx\nz\n", "t\nf\nt\nf\n", "", ""},
 	{`BEGIN { print 1?2?3:4:5, 1?0?3:4:5, 0?2?3:4:5 }`, "", "3 4 5\n", "", ""},
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index e983838..89b4837 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -52,15 +52,15 @@ func TestLexer(t *testing.T) {
 		{"5e+1", `1:1 number "5e+1"`},
 		{"5e-1", `1:1 number "5e-1"`},
 		{"0.", `1:1 number "0."`},
-		{"42e", `1:1 number "42e"`},
-		{"4.2e", `1:1 number "4.2e"`},
+		{"42e", `1:1 number "42", 1:3 name "e"`},
+		{"4.2e", `1:1 number "4.2", 1:4 name "e"`},
 		{"1.e3", `1:1 number "1.e3"`},
 		{"1.e3", `1:1 number "1.e3"`},
 		{"1e3foo", `1:1 number "1e3", 1:4 name "foo"`},
 		{"1e3+", `1:1 number "1e3", 1:4 + ""`},
 		{"1e3.4", `1:1 number "1e3", 1:4 number ".4"`},
-		{"1e-", `1:4 <illegal> "expected digits"`},
-		{"1e+", `1:4 <illegal> "expected digits"`},
+		{"1e-", `1:1 number "1", 1:2 name "e", 1:3 - ""`},
+		{"1e+", `1:1 number "1", 1:2 name "e", 1:3 + ""`},
 		{"42@", `1:1 number "42", 1:3 <illegal> "unexpected char"`},
 		{"0..", `1:1 number "0.", 1:4 <illegal> "expected digits"`},
 		{".", `1:2 <illegal> "expected digits"`},