Repository: benhoyt/goawk (go)

Issue:
Parsing "expr | getline > 0" fails without extra parens
When parsing a getline expression like `expr | getline > 0`, GoAWK fails to parse it where other AWKs succeed. For example:

```
$ goawk 'BEGIN {while ("ls README*" | getline > 0) print}'
<cmdline>:1:38: expected ) instead of >
BEGIN {while ("ls README*" | getline > 0) print}
                                     ^

# Gawk (and others) succeed:
$ gawk 'BEGIN {while ("ls README*" | getline > 0) print}'
README.md

# GoAWK needs extra parentheses
$ goawk 'BEGIN {while (("ls README*" | getline) > 0) print}'
README.md
```

Thanks @raygard for the bug report.

Pull request:
Fix wrong precedence of 'expr | getline' expressions
This pull request modifies `getLine()` inside parser.go so that it lets binary operators come after 'expr | getline' expressions.

I also uncommented two tests that were in TODO inside interp_test.go and added two new tests.

There were two other tests under the same TODO: (`BEGIN { print("echo foo" | getline ($0+1)); print }` and `BEGIN { print("echo foo" | getline foo()); print } function foo() { print "z" }`), but those address a separate issue.

Fixes #189 

Patch:
diff --git a/README.md b/README.md
index de74d7d..a6131b5 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 # GoAWK: an AWK interpreter with CSV support
 
 [![Documentation](https://pkg.go.dev/badge/github.com/benhoyt/goawk)](https://pkg.go.dev/github.com/benhoyt/goawk)
-[![GitHub Actions Build](https://github.com/benhoyt/goawk/workflows/Tests/badge.svg)](https://github.com/benhoyt/goawk/actions/workflows/tests.yml)
+[![GitHub Actions Build](https://github.com/benhoyt/goawk/actions/workflows/tests.yml/badge.svg)](https://github.com/benhoyt/goawk/actions/workflows/tests.yml)
 
 AWK is a fascinating text-processing language, and somehow after reading the delightfully-terse [*The AWK Programming Language*](https://ia802309.us.archive.org/25/items/pdfy-MgN0H1joIoDVoIC7/The_AWK_Programming_Language.pdf) I was inspired to write an interpreter for it in Go. So here it is, feature-complete and tested against "the one true AWK" and GNU AWK test suites.
 
diff --git a/goawk.go b/goawk.go
index f24edcf..c80b59f 100644
--- a/goawk.go
+++ b/goawk.go
@@ -47,7 +47,7 @@ import (
 )
 
 const (
-	version    = "v1.25.0"
+	version    = "v1.26.0"
 	copyright  = "GoAWK " + version + " - Copyright (c) 2022 Ben Hoyt"
 	shortUsage = "usage: goawk [-F fs] [-v var=value] [-f progfile | 'prog'] [file ...]"
 	longUsage  = `Standard AWK arguments:
@@ -366,8 +366,10 @@ argsLoop:
 
 	// Run the program!
 	interpreter, err := interp.New(prog)
+	if err != nil {
+		errorExit(err)
+	}
 	status, err := interpreter.Execute(config)
-
 	if err != nil {
 		errorExit(err)
 	}
diff --git a/internal/ast/ast.go b/internal/ast/ast.go
index 98e1307..303df70 100644
--- a/internal/ast/ast.go
+++ b/internal/ast/ast.go
@@ -292,12 +292,16 @@ func (e *NumExpr) String() string {
 	}
 }
 
-// StrExpr is a literal string like "foo".
+// StrExpr is a literal string like "foo" or a regex constant like /foo/.
 type StrExpr struct {
 	Value string
+	Regex bool
 }
 
 func (e *StrExpr) String() string {
+	if e.Regex {
+		return formatRegex(e.Value)
+	}
 	return strconv.Quote(e.Value)
 }
 
@@ -308,8 +312,7 @@ type RegExpr struct {
 }
 
 func (e *RegExpr) String() string {
-	escaped := strings.Replace(e.Regex, "/", `\/`, -1)
-	return "/" + escaped + "/"
+	return formatRegex(e.Regex)
 }
 
 // VarExpr is a variable reference (special var, global, or local).
@@ -460,6 +463,12 @@ func IsLValue(expr Expr) bool {
 	}
 }
 
+// formatRegex formats the regex string r.
+func formatRegex(r string) string {
+	escaped := strings.Replace(r, "/", `\/`, -1)
+	return "/" + escaped + "/"
+}
+
 // Stmt is the abstract syntax tree for any AWK statement.
 type Stmt interface {
 	Node
diff --git a/internal/resolver/toposort.go b/internal/resolver/toposort.go
index 745b012..311ae3e 100644
--- a/internal/resolver/toposort.go
+++ b/internal/resolver/toposort.go
@@ -57,7 +57,6 @@ func topoSort(graph map[string]map[string]struct{}) []string {
 		permMarks[n] = struct{}{}
 		delete(unmarked, n)
 		sorted = append(sorted, n)
-		return
 	}
 
 	for len(unmarked) > 0 {
diff --git a/interp/io.go b/interp/io.go
index 2652ae5..51a8b39 100644
--- a/interp/io.go
+++ b/interp/io.go
@@ -646,6 +646,7 @@ func (p *interp) ensureFields() {
 
 	switch {
 	case p.inputMode == CSVMode || p.inputMode == TSVMode:
+		// Normally fields have already been parsed by csvSplitter
 		if p.reparseCSV {
 			scanner := bufio.NewScanner(strings.NewReader(p.line))
 			scanner.Buffer(nil, maxRecordLength)
@@ -659,8 +660,6 @@ func (p *interp) ensureFields() {
 			if !scanner.Scan() {
 				p.fields = nil
 			}
-		} else {
-			// Normally fields have already been parsed by csvSplitter
 		}
 	case p.fieldSep == " ":
 		// FS space (default) means split fields on any whitespace
diff --git a/interp/iostream.go b/interp/iostream.go
index 572cdf6..34bca21 100644
--- a/interp/iostream.go
+++ b/interp/iostream.go
@@ -16,7 +16,7 @@ const (
 )
 
 var (
-	doubleCloseError = errors.New("close: stream already closed")
+	errDoubleClose = errors.New("close: stream already closed")
 )
 
 // firstError returns the first non-nil error or nil if all errors are nil.
@@ -89,7 +89,7 @@ func newOutFileStream(wc io.WriteCloser, size int) outputStream {
 
 func (s *outFileStream) Close() error {
 	if s.closed {
-		return doubleCloseError
+		return errDoubleClose
 	}
 	s.closed = true
 	flushErr := s.Writer.Flush()
@@ -130,7 +130,7 @@ func newOutCmdStream(cmd *exec.Cmd) (outputStream, error) {
 
 func (s *outCmdStream) Close() error {
 	if s.closed {
-		return doubleCloseError
+		return errDoubleClose
 	}
 	s.closed = true
 	flushErr := s.Writer.Flush()
@@ -154,7 +154,7 @@ func newOutNullStream() outputStream { return &outNullStream{io.Discard, false}
 func (s outNullStream) Flush() error { return nil }
 func (s *outNullStream) Close() error {
 	if s.closed {
-		return doubleCloseError
+		return errDoubleClose
 	}
 	s.closed = true
 	return nil
@@ -173,7 +173,7 @@ func newInFileStream(rc io.ReadCloser) inputStream {
 
 func (s *inFileStream) Close() error {
 	if s.closed {
-		return doubleCloseError
+		return errDoubleClose
 	}
 	s.closed = true
 	if err := s.ReadCloser.Close(); err != nil {
@@ -210,7 +210,7 @@ func newInCmdStream(cmd *exec.Cmd) (inputStream, error) {
 
 func (s *inCmdStream) Close() error {
 	if s.closed {
-		return doubleCloseError
+		return errDoubleClose
 	}
 	s.closed = true
 	closeErr := s.ReadCloser.Close()
diff --git a/parser/parser.go b/parser/parser.go
index 33b1e5f..d6366e2 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -124,9 +124,10 @@ type parser struct {
 	val     string   // string value of last token (or "")
 
 	// Parsing state
-	inAction  bool   // true if parsing an action (false in BEGIN or END)
-	funcName  string // function name if parsing a func, else ""
-	loopDepth int    // current loop depth (0 if not in any loops)
+	inAction           bool     // true if parsing an action (false in BEGIN or END)
+	funcName           string   // function name if parsing a func, else ""
+	loopDepth          int      // current loop depth (0 if not in any loops)
+	pendingGetlineLeft ast.Expr // saved expression to the left of |
 
 	// Variable tracking and resolving
 	multiExprs map[*ast.MultiExpr]Position // tracks comma-separated expressions
@@ -158,7 +159,7 @@ func (p *parser) program() *ast.Program {
 		p.optionalNewlines()
 		switch p.tok {
 		case EOF:
-			break
+			// End of file
 		case BEGIN:
 			p.next()
 			prog.Begin = append(prog.Begin, p.stmtsBrace())
@@ -379,6 +380,7 @@ func (p *parser) stmt() ast.Stmt {
 		p.next()
 		p.optionalNewlines()
 		body := p.loopStmts()
+		p.optionalNewlines()
 		p.expect(WHILE)
 		p.expect(LPAREN)
 		cond := p.expr()
@@ -518,21 +520,21 @@ func (p *parser) exprList(parse func() ast.Expr) []ast.Expr {
 // which skips PIPE GETLINE and GREATER expressions.
 
 // Parse a single expression.
-func (p *parser) expr() ast.Expr      { return p.getLine() }
+func (p *parser) expr() ast.Expr      { return p._assign(p.getline) }
 func (p *parser) printExpr() ast.Expr { return p._assign(p.printCond) }
 
 // Parse an "expr | getline [lvalue]" expression:
 //
 //	assign [PIPE GETLINE [lvalue]]
-func (p *parser) getLine() ast.Expr {
-	expr := p._assign(p.cond)
+func (p *parser) getline() ast.Expr {
+	// NOTE: getline is special, see https://github.com/benhoyt/goawk/pull/216
+	p.pendingGetlineLeft = nil
+	left := p.cond()
 	if p.tok == PIPE {
-		p.next()
-		p.expect(GETLINE)
-		target := p.optionalLValue()
-		return &ast.GetlineExpr{expr, target, nil}
+		p.pendingGetlineLeft = left
+		return p.cond()
 	}
-	return expr
+	return left
 }
 
 // Parse an = assignment expression:
@@ -715,6 +717,14 @@ func (p *parser) postIncr() ast.Expr {
 }
 
 func (p *parser) primary() ast.Expr {
+	if p.pendingGetlineLeft != nil {
+		p.expect(PIPE)
+		p.expect(GETLINE)
+		left := p.pendingGetlineLeft
+		p.pendingGetlineLeft = nil
+		target := p.optionalLValue()
+		return &ast.GetlineExpr{left, target, nil}
+	}
 	switch p.tok {
 	case NUMBER:
 		// AWK allows forms like "1.5e", but ParseFloat doesn't
@@ -725,7 +735,7 @@ func (p *parser) primary() ast.Expr {
 	case STRING:
 		s := p.val
 		p.next()
-		return &ast.StrExpr{s}
+		return &ast.StrExpr{Value: s}
 	case DIV, DIV_ASSIGN:
 		// If we get to DIV or DIV_ASSIGN as a primary expression,
 		// it's actually a regex.
@@ -966,7 +976,7 @@ func (p *parser) optionalLValue() ast.Expr {
 func (p *parser) regexStr(parse func() ast.Expr) ast.Expr {
 	if p.matches(DIV, DIV_ASSIGN) {
 		regex := p.nextRegex()
-		return &ast.StrExpr{regex}
+		return &ast.StrExpr{Value: regex, Regex: true}
 	}
 	return parse()
 }


Test patch:
diff --git a/goawk_test.go b/goawk_test.go
index 77679a8..43a329a 100644
--- a/goawk_test.go
+++ b/goawk_test.go
@@ -229,9 +229,6 @@ func sortedLines(data []byte) []byte {
 
 func TestGAWK(t *testing.T) {
 	skip := map[string]bool{ // TODO: fix these (at least the ones that are bugs)
-		"getline":  true, // getline syntax issues (may be okay, see grammar notes at http://pubs.opengroup.org/onlinepubs/007904975/utilities/awk.html#tag_04_06_13_14)
-		"getline3": true, // getline syntax issues (similar to above)
-
 		"gsubtst7": true, // something wrong with gsub or field split/join
 		"splitwht": true, // other awks handle split(s, a, " ") differently from split(s, a, / /)
 
@@ -483,11 +480,6 @@ func runAWKs(t *testing.T, testArgs []string, testStdin, testOutput, testError s
 
 	t.Run("awk", func(t *testing.T) {
 		t.Helper()
-		var args []string
-		if strings.Contains(awkExe, "gawk") {
-			args = append(args, "--posix")
-		}
-		args = append(args, testArgs...)
 		cmd := exec.Command(awkExe, testArgs...)
 		if runtime.GOOS != "windows" {
 			cmd.Env = []string{"LC_ALL=en_US.UTF-8"}
diff --git a/internal/parseutil/filereader_test.go b/internal/parseutil/filereader_test.go
index 24c13ad..19f5f4d 100644
--- a/internal/parseutil/filereader_test.go
+++ b/internal/parseutil/filereader_test.go
@@ -129,7 +129,7 @@ print x
 			// test result source
 			source := string(fr.Source())
 			for _, file := range tst.files {
-				if strings.Index(source, file.source) < 0 {
+				if !strings.Contains(source, file.source) {
 					t.Errorf("Source() is incorrect")
 				}
 			}
diff --git a/interp/fuzz_unexported_test.go b/interp/fuzz_unexported_test.go
index abd3a75..bf22323 100644
--- a/interp/fuzz_unexported_test.go
+++ b/interp/fuzz_unexported_test.go
@@ -68,7 +68,7 @@ func parseFloatHelper(s string) (float64, error) {
 	if s == "+nan" || s == "-nan" {
 		return math.NaN(), nil
 	}
-	if strings.Contains(s, "0x") && strings.IndexAny(s, "pP") < 0 {
+	if strings.Contains(s, "0x") && !strings.ContainsAny(s, "pP") {
 		s += "p0"
 	}
 	return strconv.ParseFloat(s, 64)
diff --git a/interp/interp_test.go b/interp/interp_test.go
index a2208b9..bb1bb3b 100644
--- a/interp/interp_test.go
+++ b/interp/interp_test.go
@@ -155,6 +155,7 @@ NR==3, NR==5 { print NR }
 	{`BEGIN { s="x"; while (s=="x") { print s; s="y" } }`, "", "x\n", "", ""},
 	{`BEGIN { s="x"; while (s!="") { print s; s="" } }`, "", "x\n", "", ""},
 	{`BEGIN { s="x"; while (s) { print s; s="" } }`, "", "x\n", "", ""},
+	{"BEGIN { do { print 1 }\nwhile (0) }", "", "1\n", "", ""},
 	// regression tests for break and continue with nested loops
 	{`
 BEGIN {
@@ -830,11 +831,10 @@ BEGIN { x[1]=3; f5(x); print x[1] }
 	{`BEGIN { print getline x+1; print x }`, "foo", "2\nfoo\n", "", ""},
 	{`BEGIN { print getline (x+1); print $0 }`, "foo", "11\nfoo\n", "", ""},
 	{`BEGIN { print getline foo(); print $0 } function foo() { print "z" }`, "foo", "z\n1\nfoo\n", "", ""},
-	// TODO: these forms don't yet work under GoAWK
-	//{`BEGIN { print("echo foo" | getline x+1); print x }`, "", "2\nfoo\n", "", ""},
-	//{`BEGIN { print("echo foo" | getline $0+1); print }`, "", "2\nfoo\n", "", ""},
-	//{`BEGIN { print("echo foo" | getline ($0+1)); print }`, "", "11\nfoo\n", "", ""},
-	//{`BEGIN { print("echo foo" | getline foo()); print } function foo() { print "z" }`, "", "z\n1\nfoo\n", "", ""},
+	{`BEGIN { print("echo foo" | getline x+1); print x }`, "", "2\nfoo\n", "", ""},
+	{`BEGIN { print("echo foo" | getline $0+1); print }`, "", "2\nfoo\n", "", ""},
+	{`BEGIN { print("echo foo" | getline ($0+1)); print }`, "", "11\nfoo\n", "", ""},
+	{`BEGIN { print("echo foo" | getline foo()); print } function foo() { print "z" }`, "", "z\n1\nfoo\n", "", ""},
 	{`BEGIN {
 	print "foo" >"out"
 	print close("out")
@@ -932,6 +932,8 @@ BEGIN { x[1]=3; f5(x); print x[1] }
 
 	// Ensure syntax errors result in errors
 	{`{ $1 = substr($1, 1, 3) print $1 }`, "", "", "parse error at 1:25: expected ; or newline between statements", "syntax error"},
+	{`BEGIN { 1 + 1 - | }`, "", "", `parse error at 1:17: expected expression instead of |`, "syntax"},
+	{`BEGIN { | }`, "", "", `parse error at 1:9: expected expression instead of |`, "syntax"},
 	{`BEGIN { f() }`, "", "", `parse error at 1:9: undefined function "f"`, "defined"},
 	{`function f() {} function f() {} BEGIN { }`, "", "", `parse error at 1:26: function "f" already defined`, "define"},
 	{`BEGIN { print (1,2),(3,4) }`, "", "", "parse error at 1:15: unexpected comma-separated expression", "syntax"},
@@ -1387,7 +1389,7 @@ BEGIN {
 			}},
 		{`BEGIN { 8 }`, "", "", `native function "r" returns more than two values`,
 			map[string]interface{}{
-				"r": func() (int, error, int) { return 0, nil, 0 },
+				"r": func() (int, int, int) { return 0, 0, 0 },
 			}},
 		{`BEGIN { print f(), f(1, 2) }`, "", "", `parse error at 1:20: "f" called with more arguments than declared`,
 			map[string]interface{}{
diff --git a/interp/iostream_test.go b/interp/iostream_test.go
index 8955f36..59c7cc4 100644
--- a/interp/iostream_test.go
+++ b/interp/iostream_test.go
@@ -64,7 +64,7 @@ func checkDoubleClose(t *testing.T, sc streamCloser) {
 		t.Fatal(err)
 	}
 	exitCode := sc.ExitCode()
-	if err := sc.Close(); !errors.Is(err, doubleCloseError) {
+	if err := sc.Close(); !errors.Is(err, errDoubleClose) {
 		t.Error("expected stream.Close() to return error on double close")
 	}
 	if sc.ExitCode() != exitCode {
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 1874fbc..468863d 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -124,6 +124,8 @@ $0 {
     a[x, y, z]
     f()
     set(a, k, v)
+    sub(/regex/, repl, s)
+    sub("regex", repl, s)
     sub(regex, repl)
     sub(regex, repl, s)
     gsub(regex, repl)