Repository: xonsh/xonsh

Issue:
Update pyghooks for @$(...)
I noticed that the `@` doesn't colorize with our new operator.



Discussion:
Seems to work for me. Is it only readline?
![image](https://cloud.githubusercontent.com/assets/1038978/15452590/3942d34a-1ff5-11e6-9520-2c3657eaab63.png)

Oh sorry, your right it does work.  But further nesting messes up, eg. `$[@$(which ls)]`, see http://asciinema.org/a/70iqofez8uxk69fxeirmy6ekx I think this is related to other nesting parentheses issues with the highlighter.

So maybe this is a duplicate issue.

Sorry. I accidentally posted a reply to #981 here and then deleted it again. Sorry for the noise

Don't know if this is related (it seems to be) or whether I should open a separate issue, but it seems the parentheses highlighting/matching gets confused when there are multiple nested parens.  Here is a simple example showing this:

![](http://imgur.com/UWD4NvLl.png)

As you can see, the closing parentheses after the @(a are not colorized/balanced with their appropriate opening ones.  Not sure if this is pygments or some other piece like prompt_toolkit.

Hi @TobalJackson - thanks for reporting we are aware of this issue.  This is likely due to the pygment highlighter, which we implement in xonsh.pyghooks.  Anyone who wants to take a stab at fixing this, their help would be hugely appreciated! A PR would be very welcome....


Resolving patch:
diff --git a/xonsh/commands_cache.py b/xonsh/commands_cache.py
--- a/xonsh/commands_cache.py
+++ b/xonsh/commands_cache.py
@@ -91,7 +91,8 @@ def all_commands(self):
         only_alias = (None, True)
         for cmd in alss:
             if cmd not in allcmds:
-                allcmds[cmd] = only_alias
+                key = cmd.upper() if ON_WINDOWS else cmd
+                allcmds[key] = only_alias
         self._cmds_cache = allcmds
         return allcmds
 
diff --git a/xonsh/pyghooks.py b/xonsh/pyghooks.py
--- a/xonsh/pyghooks.py
+++ b/xonsh/pyghooks.py
@@ -12,14 +12,11 @@
 # must come before pygments imports
 from xonsh.lazyasd import load_module_in_background
 
-load_module_in_background('pkg_resources', debug='XONSH_DEBUG',
-                          replacements={'pygments.plugin': 'pkg_resources'})
-
-from pygments.lexer import inherit, bygroups, using, this
-from pygments.lexers.shell import BashLexer
+from pygments.lexer import inherit, bygroups, include
 from pygments.lexers.agile import PythonLexer
 from pygments.token import (Keyword, Name, Comment, String, Error, Number,
-                            Operator, Generic, Whitespace, Token)
+                            Operator, Generic, Whitespace, Token, Punctuation,
+                            Text)
 from pygments.style import Style
 from pygments.styles import get_style_by_name
 import pygments.util
@@ -30,55 +27,40 @@
 from xonsh.tokenize import SearchPath
 
 
-def command_token_callback(_, match, fallback):
-    """Yield Keyword token if match contains valid command,
-    otherwise fallback to fallback lexer.
-    """
-    yield match.start(1), Whitespace, match.group(1)
+load_module_in_background('pkg_resources', debug='XONSH_DEBUG',
+                          replacements={'pygments.plugin': 'pkg_resources'})
 
-    start = match.start(2)
-    cmd = match.group(2)
+
+def _command_is_valid(cmd):
     cmd_abspath = os.path.abspath(os.path.expanduser(cmd))
-    if cmd in builtins.__xonsh_commands_cache__ or \
-            (os.path.isfile(cmd_abspath) and os.access(cmd_abspath, os.X_OK)):
-        yield start, Keyword, cmd
-    else:
-        lx = fallback()
-        for i, t, v in lx.get_tokens_unprocessed(cmd):
-            yield i + start, t, v
+    return cmd in builtins.__xonsh_commands_cache__ or \
+        (os.path.isfile(cmd_abspath) and os.access(cmd_abspath, os.X_OK))
 
 
-COMMAND_TOKEN_RE = r'^(\s*)([^=\s\[\]{}()$"\'`\\<&|;]+)'
+def _command_is_autocd(cmd):
+    if not builtins.__xonsh_env__['AUTO_CD']:
+        return False
+    cmd_abspath = os.path.abspath(os.path.expanduser(cmd))
+    return os.path.isdir(cmd_abspath)
 
 
-class XonshSubprocLexer(BashLexer):
-    """Lexer for xonsh subproc mode."""
-    name = 'Xonsh subprocess lexer'
-    tokens = {'root': [(COMMAND_TOKEN_RE,
-                        functools.partial(command_token_callback,
-                                          fallback=BashLexer)),
-                       (SearchPath, String.Backtick),
-                       inherit]}
+def subproc_cmd_callback(_, match):
+    """Yield Builtin token if match contains valid command,
+    otherwise fallback to fallback lexer.
+    """
+    cmd = match.group()
+    yield match.start(), Name.Builtin if _command_is_valid(cmd) else Error, cmd
 
 
-ROOT_TOKENS = [(COMMAND_TOKEN_RE, functools.partial(command_token_callback,
-                                                    fallback=PythonLexer)),
-               (r'\?', Keyword),
-               (r'\$\w+', Name.Variable),
-               (r'\$\{', Keyword, ('pymode', )),
-               (r'[\!\$]\(', Keyword, ('subproc', )),
-               (r'[\!\$]\[', Keyword, ('subproc', )),
-               (r'@\$\(', Keyword, ('subproc', )),
-               (r'@\(', Keyword, ('pymode', )),
-               inherit, ]
+def subproc_arg_callback(_, match):
+    """Check if match contains valid path"""
+    text = match.group()
+    yield (match.start(),
+           Name.Constant if os.path.exists(os.path.expanduser(text)) else Text,
+           text)
 
-PYMODE_TOKENS = [(r'(.+)(\))', bygroups(using(this), Keyword), '#pop'),
-                 (r'(.+)(\})', bygroups(using(this), Keyword), '#pop'), ]
 
-SUBPROC_TOKENS = [
-    (r'(.+)(\))', bygroups(using(XonshSubprocLexer), Keyword), '#pop'),
-    (r'(.+)(\])', bygroups(using(XonshSubprocLexer), Keyword), '#pop'),
-]
+COMMAND_TOKEN_RE = r'[^=\s\[\]{}()$"\'`<&|;!]+(?=\s|$|\)|\]|\}|!)'
 
 
 class XonshLexer(PythonLexer):
@@ -88,14 +70,105 @@ class XonshLexer(PythonLexer):
     aliases = ['xonsh', 'xsh']
     filenames = ['*.xsh', '*xonshrc']
 
+    def __init__(self, *args, **kwargs):
+        _ = builtins.__xonsh_commands_cache__.all_commands
+        super().__init__(*args, **kwargs)
+
     tokens = {
-        'root': list(ROOT_TOKENS),
-        'pymode': PYMODE_TOKENS,
-        'subproc': SUBPROC_TOKENS,
+        'mode_switch_brackets': [
+            (r'(\$)(\{)', bygroups(Keyword, Punctuation), 'py_curly_bracket'),
+            (r'(@)(\()', bygroups(Keyword, Punctuation), 'py_bracket'),
+            (r'([\!\$])(\()', bygroups(Keyword, Punctuation),
+             ('subproc_bracket', 'subproc_start')),
+            (r'(@\$)(\()', bygroups(Keyword, Punctuation),
+             ('subproc_bracket', 'subproc_start')),
+            (r'([\!\$])(\[)', bygroups(Keyword, Punctuation),
+             ('subproc_square_bracket', 'subproc_start')),
+            (r'(g?)(`)', bygroups(String.Affix, String.Backtick),
+             'backtick_re'),
+        ],
+        'subproc_bracket': [
+            (r'\)', Punctuation, '#pop'),
+            include('subproc'),
+        ],
+        'subproc_square_bracket': [
+            (r'\]', Punctuation, '#pop'),
+            include('subproc'),
+        ],
+        'py_bracket': [
+            (r'\)', Punctuation, '#pop'),
+            include('root'),
+        ],
+        'py_curly_bracket': [
+            (r'\}', Punctuation, '#pop'),
+            include('root'),
+        ],
+        'backtick_re': [
+            (r'[\.\^\$\*\+\?\[\]\|]', String.Regex),
+            (r'({[0-9]+}|{[0-9]+,[0-9]+})\??', String.Regex),
+            (r'\\([0-9]+|[AbBdDsSwWZabfnrtuUvx\\])', String.Escape),
+            (r'`', String.Backtick, '#pop'),
+            (r'[^`\.\^\$\*\+\?\[\]\|]+', String.Backtick),
+        ],
+        'root': [
+            (r'\?', Keyword),
+            (r'(?<=\w)!', Keyword),
+            (r'\$\w+', Name.Variable),
+            (r'\(', Punctuation, 'py_bracket'),
+            (r'\{', Punctuation, 'py_curly_bracket'),
+            include('mode_switch_brackets'),
+            inherit,
+        ],
+        'subproc_start': [
+            (r'\s+', Whitespace),
+            (COMMAND_TOKEN_RE, subproc_cmd_callback, '#pop'),
+            (r'', Whitespace, '#pop'),
+        ],
+        'subproc': [
+            include('mode_switch_brackets'),
+            (r'&&|\|\|', Operator, 'subproc_start'),
+            (r'"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+            (r'(?<=\w|\s)!', Keyword, 'subproc_macro'),
+            (r'^!', Keyword, 'subproc_macro'),
+            (r';', Punctuation, 'subproc_start'),
+            (r'&|=', Punctuation),
+            (r'\|', Punctuation, 'subproc_start'),
+            (r'\s+', Text),
+            (r'[^=\s\[\]{}()$"\'`<&|;]+', subproc_arg_callback),
+            (r'<', Text),
+            (r'\$\w+', Name.Variable),
+        ],
+        'subproc_macro': [
+            (r'(\s*)([^\n]+)', bygroups(Whitespace, String)),
+            (r'', Whitespace, '#pop'),
+        ],
     }
 
+    def get_tokens_unprocessed(self, text):
+        """Check first command, then call super.get_tokens_unprocessed
+        with root or subproc state"""
+        start = 0
+        state = ('root', )
+        m = re.match(r'(\s*)({})'.format(COMMAND_TOKEN_RE), text)
+        if m is not None:
+            yield m.start(1), Whitespace, m.group(1)
+            cmd = m.group(2)
+            cmd_is_valid = _command_is_valid(cmd)
+            cmd_is_autocd = _command_is_autocd(cmd)
+
+            if cmd_is_valid or cmd_is_autocd:
+                yield (m.start(2),
+                       Name.Builtin if cmd_is_valid else Name.Constant,
+                       cmd)
+                start = m.end(2)
+                state = ('subproc', )
+
+        for i, t, v in super().get_tokens_unprocessed(text[start:], state):
+            yield i + start, t, v
+
 
-class XonshConsoleLexer(PythonLexer):
+class XonshConsoleLexer(XonshLexer):
     """Xonsh console lexer for pygments."""
 
     name = 'Xonsh console lexer'
@@ -106,17 +179,10 @@ class XonshConsoleLexer(PythonLexer):
         'root': [(r'^(>>>|\.\.\.) ', Generic.Prompt),
                  (r'\n(>>>|\.\.\.)', Generic.Prompt),
                  (r'\n(?![>.][>.][>.] )([^\n]*)', Generic.Output),
-                 (r'\n(?![>.][>.][>.] )(.*?)$', Generic.Output)] + ROOT_TOKENS,
-        'pymode': PYMODE_TOKENS,
-        'subproc': SUBPROC_TOKENS,
+                 (r'\n(?![>.][>.][>.] )(.*?)$', Generic.Output),
+                 inherit]
     }
 
-# XonshLexer & XonshSubprocLexer have to reference each other
-XonshSubprocLexer.tokens['root'] = [
-    (r'(\$\{)(.*)(\})', bygroups(Keyword, using(XonshLexer), Keyword)),
-    (r'(@\()(.+)(\))', bygroups(Keyword, using(XonshLexer), Keyword)),
-] + XonshSubprocLexer.tokens['root']
-
 
 #
 # Colors and Styles


Test patch:
diff --git a/tests/test_ptk_highlight.py b/tests/test_ptk_highlight.py
new file mode 100644
--- /dev/null
+++ b/tests/test_ptk_highlight.py
@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+"""Test XonshLexer for pygments"""
+
+import os
+import builtins
+
+import pytest
+from pygments.token import (Keyword, Name, String, Error, Number,
+                            Operator, Punctuation, Text)
+from tools import skip_if_on_windows
+
+from xonsh.platform import ON_WINDOWS
+from xonsh.built_ins import load_builtins, unload_builtins
+from xonsh.pyghooks import XonshLexer
+
+
+@pytest.yield_fixture(autouse=True)
+def load_command_cache():
+    load_builtins()
+    if ON_WINDOWS:
+        for key in ('cd', 'bash'):
+            builtins.aliases[key] = lambda *args, **kwargs: None
+    yield
+    unload_builtins()
+
+
+def check_token(code, tokens):
+    """Make sure that all tokens appears in code in order"""
+    lx = XonshLexer()
+    tks = list(lx.get_tokens(code))
+
+    for tk in tokens:
+        while tks:
+            if tk == tks[0]:
+                break
+            tks = tks[1:]
+        else:
+            msg = "Token {!r} missing: {!r}".format(tk,
+                                                    list(lx.get_tokens(code)))
+            pytest.fail(msg)
+            break
+
+
+@skip_if_on_windows
+def test_ls():
+    check_token('ls -al', [(Name.Builtin, 'ls')])
+
+
+@skip_if_on_windows
+def test_bin_ls():
+    check_token('/bin/ls -al', [(Name.Builtin, '/bin/ls')])
+
+
+def test_py_print():
+    check_token('print("hello")', [(Keyword, 'print'),
+                                   (String.Double, 'hello')])
+
+
+def test_invalid_cmd():
+    check_token('non-existance-cmd -al', [(Name, 'non')])  # parse as python
+    check_token('![non-existance-cmd -al]',
+                [(Error, 'non-existance-cmd')])  # parse as error
+    check_token('for i in range(10):', [(Keyword, 'for')])  # as py keyword
+    check_token('(1, )', [(Punctuation, '('),
+                          (Number.Integer, '1')])
+
+
+def test_multi_cmd():
+    check_token('cd && cd', [(Name.Builtin, 'cd'),
+                             (Operator, '&&'),
+                             (Name.Builtin, 'cd')])
+    check_token('cd || non-existance-cmd', [(Name.Builtin, 'cd'),
+                                            (Operator, '||'),
+                                            (Error, 'non-existance-cmd')
+                                            ])
+
+
+def test_nested():
+    check_token('echo @("hello")', [(Name.Builtin, 'echo'),
+                                    (Keyword, '@'),
+                                    (Punctuation, '('),
+                                    (String.Double, 'hello'),
+                                    (Punctuation, ')')])
+    check_token('print($(cd))', [(Keyword, 'print'),
+                                 (Punctuation, '('),
+                                 (Keyword, '$'),
+                                 (Punctuation, '('),
+                                 (Name.Builtin, 'cd'),
+                                 (Punctuation, ')'),
+                                 (Punctuation, ')')])
+    check_token(r'print(![echo "])\""])', [(Keyword, 'print'),
+                                           (Keyword, '!'),
+                                           (Punctuation, '['),
+                                           (Name.Builtin, 'echo'),
+                                           (String.Double, r'"])\""'),
+                                           (Punctuation, ']')])
+
+
+def test_path(tmpdir):
+    test_dir = str(tmpdir.mkdir('xonsh-test-highlight-path'))
+    check_token('cd {}'.format(test_dir), [(Name.Builtin, 'cd'),
+                                           (Name.Constant, test_dir)])
+    check_token('cd {}-xxx'.format(test_dir), [(Name.Builtin, 'cd'),
+                                               (Text,
+                                                '{}-xxx'.format(test_dir))
+                                               ])
+    check_token('cd X={}'.format(test_dir), [(Name.Constant, test_dir)])
+
+    with builtins.__xonsh_env__.swap(AUTO_CD=True):
+        check_token(test_dir, [(Name.Constant, test_dir)])
+
+
+def test_subproc_args():
+    check_token('cd 192.168.0.1', [(Text, '192.168.0.1')])
+
+
+def test_backtick():
+    check_token(r'echo g`.*\w+`', [(String.Affix, 'g'),
+                                   (String.Backtick, '`'),
+                                   (String.Regex, '.'),
+                                   (String.Regex, '*'),
+                                   (String.Escape, r'\w'),
+                                   ])
+
+
+def test_macro():
+    check_token(r'g!(42, *, 65)', [(Name, 'g'),
+                                   (Keyword, '!'),
+                                   (Punctuation, '('),
+                                   (Number.Integer, '42')])
+    check_token(r'echo! hello world', [(Name.Builtin, 'echo'),
+                                       (Keyword, '!'),
+                                       (String, "hello world")])
+    check_token(r'bash -c ! export var=42; echo $var',
+                [(Name.Builtin, 'bash'),
+                 (Text, '-c'),
+                 (Keyword, '!'),
+                 (String, 'export var=42; echo $var'),
+                 ])