Repository: ipython/ipython

Issue:
IPython should specify pygments entry points for its lexers
IPython implements a few pygments lexers - IPyLexer, IPythonTracebackLexer, etc., but does not install pygments entry points, so these lexers cannot be used with the pygmentize command.

Having them available to pygmentize will be helpful because pygmentize has some features that are not available from from nbconvert.



Discussion:

Resolving patch:
diff --git a/IPython/nbconvert/utils/lexers.py b/IPython/nbconvert/utils/lexers.py
--- a/IPython/nbconvert/utils/lexers.py
+++ b/IPython/nbconvert/utils/lexers.py
@@ -49,10 +49,15 @@
 line_re = re.compile('.*?\n')
 
 ipython_tokens = [
-  (r'(\%+)(\w+)\s+(\.*)(\n)', bygroups(Operator, Keyword,
+  (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
+  (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
+  (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
+  (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
+  (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
                                        using(BashLexer), Text)),
-  (r'(\%+)(\w+)\b', bygroups(Operator, Keyword)),
-  (r'^(!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+  (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
+  (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+  (r'(!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
 ]
 
 def build_ipy_lexer(python3):


Test patch:
diff --git a/IPython/nbconvert/utils/tests/test_lexers.py b/IPython/nbconvert/utils/tests/test_lexers.py
new file mode 100644
--- /dev/null
+++ b/IPython/nbconvert/utils/tests/test_lexers.py
@@ -0,0 +1,111 @@
+"""Test lexers module"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2014 The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from pygments.token import Token
+
+from IPython.nbconvert.tests.base import TestsBase
+from .. import lexers
+
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+class TestLexers(TestsBase):
+    """Collection of lexers tests"""
+    def setUp(self):
+        self.lexer = lexers.IPythonLexer()
+
+    def testIPythonLexer(self):
+        fragment = '!echo $HOME\n'
+        tokens = [
+            (Token.Operator, '!'),
+            (Token.Name.Builtin, 'echo'),
+            (Token.Text, ' '),
+            (Token.Name.Variable, '$HOME'),
+            (Token.Text, '\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+        fragment_2 = '!' + fragment
+        tokens_2 = [
+            (Token.Operator, '!!'),
+        ] + tokens[1:]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = '\t %%!\n' + fragment[1:]
+        tokens_2 = [
+            (Token.Text, '\t '),
+            (Token.Operator, '%%!'),
+            (Token.Text, '\n'),
+        ] + tokens[1:]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = 'x = ' + fragment
+        tokens_2 = [
+            (Token.Name, 'x'),
+            (Token.Text, ' '),
+            (Token.Operator, '='),
+            (Token.Text, ' '),
+        ] + tokens
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = 'x, = ' + fragment
+        tokens_2 = [
+            (Token.Name, 'x'),
+            (Token.Punctuation, ','),
+            (Token.Text, ' '),
+            (Token.Operator, '='),
+            (Token.Text, ' '),
+        ] + tokens
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = 'x, = %sx ' + fragment[1:]
+        tokens_2 = [
+            (Token.Name, 'x'),
+            (Token.Punctuation, ','),
+            (Token.Text, ' '),
+            (Token.Operator, '='),
+            (Token.Text, ' '),
+            (Token.Operator, '%'),
+            (Token.Keyword, 'sx'),
+            (Token.Text, ' '),
+        ] + tokens[1:]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = 'f = %R function () {}\n'
+        tokens_2 = [
+            (Token.Name, 'f'),
+            (Token.Text, ' '),
+            (Token.Operator, '='),
+            (Token.Text, ' '),
+            (Token.Operator, '%'),
+            (Token.Keyword, 'R'),
+            (Token.Text, ' function () {}\n'),
+        ]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = '\t%%xyz\n$foo\n'
+        tokens_2 = [
+            (Token.Text, '\t'),
+            (Token.Operator, '%%'),
+            (Token.Keyword, 'xyz'),
+            (Token.Text, '\n$foo\n'),
+        ]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
+
+        fragment_2 = '%system?\n'
+        tokens_2 = [
+            (Token.Operator, '%'),
+            (Token.Keyword, 'system'),
+            (Token.Operator, '?'),
+            (Token.Text, '\n'),
+        ]
+        self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))