Repository: xonsh/xonsh

Issue:
Expression `2>0` not parsed correctly
When I enter the expression `2>0` in a Xonsh shell, I would expect it to evaluate it as a boolean and print `True` - that is what Python does. Instead, Xonsh attempts to parse it as a subprocess command and gives an error:
```
xonsh.tools.XonshError: xonsh: subprocess mode: command not found: 2>
```

I'll try to work on a fix for this issue and make a PR.

## xonfig

```
+------------------+-------------------------+
| xonsh            | 0.13.3                  |
| Git SHA          | c63f75ef                |
| Commit Date      | Nov 14 13:22:10 2022    |
| Python           | 3.9.13                  |
| PLY              | 3.11                    |
| have readline    | True                    |
| prompt toolkit   | 3.0.29                  |
| shell type       | prompt_toolkit          |
| history backend  | sqlite                  |
| pygments         | 2.11.2                  |
| on posix         | True                    |
| on linux         | True                    |
| distro           | fedora                  |
| on wsl           | False                   |
| on darwin        | False                   |
| on windows       | False                   |
| on cygwin        | False                   |
| on msys2         | False                   |
| is superuser     | False                   |
| default encoding | utf-8                   |
| xonsh encoding   | utf-8                   |
| encoding errors  | surrogateescape         |
| xontrib          | []                      |
| RC file 1        | /home/yaxollum/.xonshrc |
+------------------+-------------------------+
```

## For community
⬇️  **Please click the 👍 reaction instead of leaving a `+1` or 👍  comment**



Discussion:

Resolving patch:
diff --git a/xonsh/lexer.py b/xonsh/lexer.py
--- a/xonsh/lexer.py
+++ b/xonsh/lexer.py
@@ -253,9 +253,25 @@ def handle_redirect(state, token):
     typ = token.type
     st = token.string
     key = (typ, st) if (typ, st) in token_map else typ
-    yield _new_token(token_map[key], st, token.start)
+    new_tok = _new_token(token_map[key], st, token.start)
     if state["pymode"][-1][0]:
+        if typ == IOREDIRECT:
+            # Fix Python mode code that was incorrectly recognized as an
+            # IOREDIRECT by the tokenizer (see issue #4994).
+            # The tokenizer does not know when the code should be tokenized in
+            # Python mode. By default, when it sees code like `2>`, it produces
+            # an IOREDIRECT token. This call to `get_tokens` makes the
+            # tokenizer run over the code again, knowing that it is *not* an
+            # IOREDIRECT token.
+            lineno, lexpos = token.start
+            for tok in get_tokens(
+                token.string, state["tolerant"], tokenize_ioredirects=False
+            ):
+                yield _new_token(tok.type, tok.value, (lineno, lexpos + tok.lexpos))
+        else:
+            yield new_tok
         return
+    yield new_tok
     # add a whitespace token after a redirection, if we need to
     next_tok = next(state["stream"])
     if next_tok.start == token.end:
@@ -358,7 +374,7 @@ def handle_token(state, token):
         yield _new_token("ERRORTOKEN", m, token.start)
 
 
-def get_tokens(s, tolerant):
+def get_tokens(s, tolerant, pymode=True, tokenize_ioredirects=True):
     """
     Given a string containing xonsh code, generates a stream of relevant PLY
     tokens using ``handle_token``.
@@ -366,8 +382,10 @@ def get_tokens(s, tolerant):
     state = {
         "indents": [0],
         "last": None,
-        "pymode": [(True, "", "", (0, 0))],
-        "stream": tokenize(io.BytesIO(s.encode("utf-8")).readline, tolerant),
+        "pymode": [(pymode, "", "", (0, 0))],
+        "stream": tokenize(
+            io.BytesIO(s.encode("utf-8")).readline, tolerant, tokenize_ioredirects
+        ),
         "tolerant": tolerant,
     }
     while True:
@@ -406,7 +424,7 @@ class Lexer:
 
     _tokens: tp.Optional[tuple[str, ...]] = None
 
-    def __init__(self, tolerant=False):
+    def __init__(self, tolerant=False, pymode=True):
         """
         Attributes
         ----------
@@ -419,12 +437,15 @@ def __init__(self, tolerant=False):
         tolerant : bool
             Tokenize without extra checks (e.g. paren matching).
             When True, ERRORTOKEN contains the erroneous string instead of an error msg.
+        pymode : bool
+            Start the lexer in Python mode.
 
         """
         self.fname = ""
         self.last = None
         self.beforelast = None
         self._tolerant = tolerant
+        self._pymode = pymode
         self._token_stream = iter(())
 
     @property
@@ -442,7 +463,7 @@ def reset(self):
 
     def input(self, s):
         """Calls the lexer on the string s."""
-        self._token_stream = get_tokens(s, self._tolerant)
+        self._token_stream = get_tokens(s, self._tolerant, self._pymode)
 
     def token(self):
         """Retrieves the next token."""
diff --git a/xonsh/parsers/completion_context.py b/xonsh/parsers/completion_context.py
--- a/xonsh/parsers/completion_context.py
+++ b/xonsh/parsers/completion_context.py
@@ -350,7 +350,7 @@ def __init__(
 
         self.error = None
         self.debug = debug
-        self.lexer = Lexer(tolerant=True)
+        self.lexer = Lexer(tolerant=True, pymode=False)
         self.tokens = tuple(self.used_tokens | self.artificial_tokens)
 
         yacc_kwargs = dict(
diff --git a/xonsh/tokenize.py b/xonsh/tokenize.py
--- a/xonsh/tokenize.py
+++ b/xonsh/tokenize.py
@@ -374,6 +374,7 @@ def maybe(*choices):
     StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r"\\\r?\n"),
 )
 PseudoExtras = group(r"\\\r?\n|\Z", Comment, Triple, SearchPath)
+PseudoTokenWithoutIO = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name_RE)
 PseudoToken = Whitespace + group(
     PseudoExtras, IORedirect, Number, Funny, ContStr, Name_RE
 )
@@ -859,7 +860,7 @@ def tokopen(filename):
         raise
 
 
-def _tokenize(readline, encoding, tolerant=False):
+def _tokenize(readline, encoding, tolerant=False, tokenize_ioredirects=True):
     lnum = parenlev = continued = 0
     numchars = "0123456789"
     contstr, needcont = "", 0
@@ -993,7 +994,9 @@ def _tokenize(readline, encoding, tolerant=False):
             continued = 0
 
         while pos < max:
-            pseudomatch = _compile(PseudoToken).match(line, pos)
+            pseudomatch = _compile(
+                PseudoToken if tokenize_ioredirects else PseudoTokenWithoutIO
+            ).match(line, pos)
             if pseudomatch:  # scan for tokens
                 start, end = pseudomatch.span(1)
                 spos, epos, pos = (lnum, start), (lnum, end), end
@@ -1131,7 +1134,7 @@ def _tokenize(readline, encoding, tolerant=False):
     yield TokenInfo(ENDMARKER, "", (lnum, 0), (lnum, 0), "")
 
 
-def tokenize(readline, tolerant=False):
+def tokenize(readline, tolerant=False, tokenize_ioredirects=True):
     """
     The tokenize() generator requires one argument, readline, which
     must be a callable object which provides the same interface as the
@@ -1152,12 +1155,19 @@ def tokenize(readline, tolerant=False):
 
     If ``tolerant`` is True, yield ERRORTOKEN with the erroneous string instead of
     throwing an exception when encountering an error.
+
+    If ``tokenize_ioredirects`` is True, produce IOREDIRECT tokens for special
+    io-redirection operators like ``2>``. Otherwise, treat code like ``2>`` as
+    regular Python code.
     """
     encoding, consumed = detect_encoding(readline)
     rl_gen = iter(readline, b"")
     empty = itertools.repeat(b"")
     return _tokenize(
-        itertools.chain(consumed, rl_gen, empty).__next__, encoding, tolerant
+        itertools.chain(consumed, rl_gen, empty).__next__,
+        encoding,
+        tolerant,
+        tokenize_ioredirects,
     )
 
 


Test patch:
diff --git a/tests/test_lexer.py b/tests/test_lexer.py
--- a/tests/test_lexer.py
+++ b/tests/test_lexer.py
@@ -477,3 +477,36 @@ def test_tolerant_lexer(s):
     lexer.input(s)
     error_tokens = list(tok for tok in lexer if tok.type == "ERRORTOKEN")
     assert all(tok.value in s for tok in error_tokens)  # no error messages
+
+
+@pytest.mark.parametrize(
+    "s, exp",
+    [
+        ("2>1", [("NUMBER", "2", 0), ("GT", ">", 1), ("NUMBER", "1", 2)]),
+        ("a>b", [("NAME", "a", 0), ("GT", ">", 1), ("NAME", "b", 2)]),
+        (
+            "3>2>1",
+            [
+                ("NUMBER", "3", 0),
+                ("GT", ">", 1),
+                ("NUMBER", "2", 2),
+                ("GT", ">", 3),
+                ("NUMBER", "1", 4),
+            ],
+        ),
+        (
+            "36+2>>3",
+            [
+                ("NUMBER", "36", 0),
+                ("PLUS", "+", 2),
+                ("NUMBER", "2", 3),
+                ("RSHIFT", ">>", 4),
+                ("NUMBER", "3", 6),
+            ],
+        ),
+    ],
+)
+def test_pymode_not_ioredirect(s, exp):
+    # test that Python code like `2>1` is lexed correctly
+    # as opposed to being recognized as an IOREDIRECT token (issue #4994)
+    assert check_tokens(s, exp)