Repository: xonsh/xonsh

Issue:
Parser doesn't emit syntax errors raised from production rules
Our parser currently doesn't report `SyntaxError` exceptions raised inside of parser production rules. This is due to the fact that Yacc catches such an exception and attempts recovery by injecting `error` token in place of the current token and continues parsing the input as if the `error` token was a part of the input text. Since we don't handle `error` token at all, the parser ultimately unwinds the stack to a state where none of the rules can be matched anymore and is supposed to call `p_error`, that is a catchall handler for unhandled  errors. But there's the catch - in case of production-rule-risen syntax error the parser does NOT call the catchall handler. See the docs at https://ply.readthedocs.io/en/latest/ply.html#signalling-an-error-from-a-production. Most importantly the documentation says:
> One important aspect of manually setting an error is that the p_error() function will NOT be called in this case. If you need to issue an error message, make sure you do it in the production that raises SyntaxError.

We need to re-implement our error handling logic to be able to report such errors. There is, however, another catch - since Xonsh fist attempts to parse the input in Python mode and if that fails it resorts to parsing in shell mode, commands like `echo foo` are in fact expected to raise an error in the first (Python mode) pass and we definitely do not want to first report a syntax error and then successfully execute the `echo` command.

## xonfig

<details>

```
+------------------+----------------------+
| xonsh            | 0.9.20               |
| Git SHA          | 4be525ef             |
| Commit Date      | Aug 27 13:25:38 2020 |
| Python           | 3.8.5                |
| PLY              | 3.11                 |
| have readline    | True                 |
| prompt toolkit   | 3.0.4                |
| shell type       | prompt_toolkit       |
| pygments         | 2.6.1                |
| on posix         | True                 |
| on linux         | True                 |
| distro           | arch                 |
| on darwin        | False                |
| on windows       | False                |
| on cygwin        | False                |
| on msys2         | False                |
| is superuser     | False                |
| default encoding | utf-8                |
| xonsh encoding   | utf-8                |
| encoding errors  | surrogateescape      |
+------------------+----------------------+
```

</details>

## Expected Behavior
```
$ f"{}"
SyntaxError: f-string: empty expression not allowed
```

## Current Behavior
```
$ f"{}"
$
```

## For community
⬇️  **Please click the 👍 reaction instead of leaving a `+1` or 👍  comment**



Discussion:

Resolving patch:
diff --git a/xonsh/parsers/base.py b/xonsh/parsers/base.py
--- a/xonsh/parsers/base.py
+++ b/xonsh/parsers/base.py
@@ -461,6 +461,7 @@ def __init__(
 
         # Keeps track of the last token given to yacc (the lookahead token)
         self._last_yielded_token = None
+        self._error = None
 
     def reset(self):
         """Resets for clean parsing."""
@@ -468,6 +469,7 @@ def reset(self):
         self._last_yielded_token = None
         self._lines = None
         self.xonsh_code = None
+        self._error = None
 
     def parse(self, s, filename="<code>", mode="exec", debug_level=0):
         """Returns an abstract syntax tree of xonsh code.
@@ -493,6 +495,8 @@ def parse(self, s, filename="<code>", mode="exec", debug_level=0):
         while self.parser is None:
             time.sleep(0.01)  # block until the parser is ready
         tree = self.parser.parse(input=s, lexer=self.lexer, debug=debug_level)
+        if self._error is not None:
+            self._parse_error(self._error[0], self._error[1])
         if tree is not None:
             check_contexts(tree)
         # hack for getting modes right
@@ -609,6 +613,13 @@ def source_slice(self, start, stop):
         lines[0] = lines[0][bcol:]
         return "".join(lines)
 
+    def _set_error(self, msg, loc=None):
+        assert self._error is None
+        if loc is None:
+            loc = self.currloc(self.lineno, self.col)
+        self._error = (msg, loc)
+        raise SyntaxError()
+
     def _parse_error(self, msg, loc):
         if self.xonsh_code is None or loc is None:
             err_line_pointer = ""
@@ -945,8 +956,7 @@ def _set_args_def(self, argmts, vals, kwargs=False):
             else (argmts.args, argmts.defaults)
         )
         if vals is None and kwargs:
-            loc = self.currloc(self.lineno, self.col)
-            self._parse_error("named arguments must follow bare *", loc)
+            self._set_error("named arguments must follow bare *")
         for v in vals:
             args.append(v["arg"])
             d = v["default"]
@@ -1173,7 +1183,7 @@ def p_expr_stmt_augassign(self, p):
         store_ctx(p1)
         op = self._augassign_op[p2]
         if op is None:
-            self._parse_error(
+            self._set_error(
                 "operation {0!r} not supported".format(p2),
                 self.currloc(lineno=p.lineno, column=p.lexpos),
             )
@@ -2098,7 +2108,7 @@ def p_op_factor(self, p):
         p1 = p[1]
         op = self._term_binops[p1.value]
         if op is None:
-            self._parse_error(
+            self._set_error(
                 "operation {0!r} not supported".format(p1),
                 self.currloc(lineno=p.lineno, column=p.lexpos),
             )
@@ -2227,7 +2237,8 @@ def p_atom_lparen(self, p):
             p0 = p2[0]
             p0._lopen_lineno, p0._lopen_col = p1_tok.lineno, p1_tok.lexpos
         else:
-            self.p_error(p)
+            loc = self.currloc(lineno=p.lineno, column=p.lexpos)
+            self._set_error(f"code: {p.value}", loc)
         p[0] = p0
 
     def p_atom_lbraket(self, p):
@@ -2397,7 +2408,7 @@ def p_string_literal(self, p):
                         value_without_p, new_pref, filename=self.lexer.fname
                     ).run()
                 except SyntaxError as e:
-                    self._parse_error(
+                    self._set_error(
                         str(e), self.currloc(lineno=p1.lineno, column=p1.lexpos)
                     )
             s = ast.increment_lineno(s, p1.lineno - 1)
@@ -2425,7 +2436,7 @@ def p_string_literal(self, p):
                         p1.value, prefix, filename=self.lexer.fname
                     ).run()
                 except SyntaxError as e:
-                    self._parse_error(
+                    self._set_error(
                         str(e), self.currloc(lineno=p1.lineno, column=p1.lexpos)
                     )
             s = ast.increment_lineno(s, p1.lineno - 1)
@@ -2505,7 +2516,7 @@ def p_trailer_bang_lparen(self, p):
                     break
                 else:
                     msg = "empty macro arguments not allowed"
-                    self._parse_error(msg, self.currloc(*beg))
+                    self._set_error(msg, self.currloc(*beg))
             node = ast.Str(s=s, lineno=beg[0], col_offset=beg[1])
             elts.append(node)
         p0 = ast.Tuple(
@@ -2995,8 +3006,7 @@ def p_subproc_pipe(self, p):
         """
         p1 = p[1]
         if len(p1) > 1 and hasattr(p1[-2], "s") and p1[-2].s != "|":
-            msg = "additional redirect following non-pipe redirect"
-            self._parse_error(msg, self.currloc(lineno=self.lineno, column=self.col))
+            self._set_error("additional redirect following non-pipe redirect")
         cliargs = self._subproc_cliargs(p[3], lineno=self.lineno, col=self.col)
         p[0] = p1 + [p[2], cliargs]
 
diff --git a/xonsh/parsers/v36.py b/xonsh/parsers/v36.py
--- a/xonsh/parsers/v36.py
+++ b/xonsh/parsers/v36.py
@@ -159,7 +159,7 @@ def p_expr_stmt_annassign(self, p):
         lineno, col = lopen_loc(p1)
         if len(p[1]) > 1 or not isinstance(p1, ast.Name):
             loc = self.currloc(lineno, col)
-            self._parse_error("only single target can be annotated", loc)
+            self._set_error("only single target can be annotated", loc)
         store_ctx(p1)
         p[0] = ast.AnnAssign(
             target=p1,


Test patch: