Repository: sphinx-doc/sphinx

Issue:
LaTeX output: sub/superscripts and other Unicode glyphs get "TeX-escaped" even if engine is xelatex
Arguably, [TeX-escaping](https://github.com/sphinx-doc/sphinx/blob/master/sphinx/util/texescape.py) when using Unicode aware xelatex engine should be essentially limited to the few characters with special TeX meaning such as `$`, `{`, `\` and a few others.

For the initial report where the too extensive TeX escaping causes problems see #6733. 

However some Unicode code-points such as `ℯ` seem to have limited font support (particularly monospace, it is lacking both from DejaVu Sans Mono and GNU FreeMono), and should perhaps be retained at least for the near future in the escape list. Indeed such missing glyphs do not raise errors but only LaTeX warnings and they may escape attention.


Discussion:

Patch:
diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py
index 977b71f28..0a6cf8d6e 100644
--- a/sphinx/highlighting.py
+++ b/sphinx/highlighting.py
@@ -28,7 +28,7 @@
 from sphinx.locale import __
 from sphinx.pygments_styles import SphinxStyle, NoneStyle
 from sphinx.util import logging
-from sphinx.util.texescape import tex_hl_escape_map_new
+from sphinx.util.texescape import get_hlescape_func, tex_hl_escape_map_new
 
 if False:
     # For type annotation
@@ -68,9 +68,11 @@ class PygmentsBridge:
     html_formatter = HtmlFormatter
     latex_formatter = LatexFormatter
 
-    def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=None):
-        # type: (str, str, bool) -> None
+    def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=None,
+                 latex_engine=None):
+        # type: (str, str, bool, str) -> None
         self.dest = dest
+        self.latex_engine = latex_engine
 
         style = self.get_style(stylename)
         self.formatter_args = {'style': style}  # type: Dict[str, Any]
@@ -192,7 +194,8 @@ def highlight_block(self, source, lang, opts=None, force=False, location=None, *
         if self.dest == 'html':
             return hlsource
         else:
-            return hlsource.translate(tex_hl_escape_map_new)
+            escape = get_hlescape_func(self.latex_engine)
+            return escape(hlsource)
 
     def get_stylesheet(self):
         # type: () -> str
diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py
index 4e7055119..c3231d3d3 100644
--- a/sphinx/util/texescape.py
+++ b/sphinx/util/texescape.py
@@ -32,7 +32,6 @@
     ('¶', r'\P{}'),
     ('§', r'\S{}'),
     ('€', r'\texteuro{}'),
-    ('∞', r'\(\infty\)'),
     ('±', r'\(\pm\)'),
     ('→', r'\(\rightarrow\)'),
     ('‣', r'\(\rightarrow\)'),
@@ -53,6 +52,8 @@
 # A map Unicode characters to LaTeX representation
 # (for LaTeX engines which don't support unicode)
 unicode_tex_replacements = [
+    # map special Unicode characters to TeX commands
+    ('∞', r'\(\infty\)'),
     # superscript
     ('⁰', r'\(\sp{\text{0}}\)'),
     ('¹', r'\(\sp{\text{1}}\)'),
@@ -80,7 +81,8 @@
 tex_escape_map = {}  # type: Dict[int, str]
 tex_escape_map_without_unicode = {}  # type: Dict[int, str]
 tex_replace_map = {}
-tex_hl_escape_map_new = {}
+tex_hl_escape_map_new = {}  # type: Dict[int, str]
+tex_hl_escape_map_new_without_unicode = {}  # type: Dict[int, str]
 
 
 def get_escape_func(latex_engine: str) -> Callable[[str], str]:
@@ -101,6 +103,24 @@ def escape_for_unicode_latex_engine(s: str) -> str:
     return s.translate(tex_escape_map_without_unicode)
 
 
+def get_hlescape_func(latex_engine: str) -> Callable[[str], str]:
+    """Get hlescape() function for given latex_engine."""
+    if latex_engine in ('lualatex', 'xelatex'):
+        return hlescape_for_unicode_latex_engine
+    else:
+        return hlescape
+
+
+def hlescape(s: str) -> str:
+    """Escape text for LaTeX highlighter."""
+    return s.translate(tex_hl_escape_map_new)
+
+
+def hlescape_for_unicode_latex_engine(s: str) -> str:
+    """Escape text for unicode supporting LaTeX engine."""
+    return s.translate(tex_hl_escape_map_new_without_unicode)
+
+
 def escape_abbr(text: str) -> str:
     """Adjust spacing after abbreviations. Works with @ letter or other."""
     return re.sub(r'\.(?=\s|$)', r'.\@{}', text)
@@ -120,3 +140,7 @@ def init() -> None:
         if a in '[]{}\\':
             continue
         tex_hl_escape_map_new[ord(a)] = b
+        tex_hl_escape_map_new_without_unicode[ord(a)] = b
+
+    for a, b in unicode_tex_replacements:
+        tex_hl_escape_map_new[ord(a)] = b
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index 29367a79e..5facdc40c 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -653,7 +653,8 @@ def __init__(self, document, builder):
             self.elements['classoptions'] += ',' + \
                                              self.elements['extraclassoptions']
 
-        self.highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style)
+        self.highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style,
+                                                       latex_engine=self.config.latex_engine)
         self.context = []                   # type: List[Any]
         self.descstack = []                 # type: List[str]
         self.table = None                   # type: Table


Test patch:
diff --git a/tests/test_markup.py b/tests/test_markup.py
index b8c9b66d9..94d1af951 100644
--- a/tests/test_markup.py
+++ b/tests/test_markup.py
@@ -312,6 +312,24 @@ def test_inline(get_verifier, type, rst, html_expected, latex_expected):
     verifier(rst, html_expected, latex_expected)
 
 
+@pytest.mark.sphinx(confoverrides={'latex_engine': 'xelatex'})
+@pytest.mark.parametrize('type,rst,html_expected,latex_expected', [
+    (
+        # in verbatim code fragments
+        'verify',
+        '::\n\n @Γ\\∞${}',
+        None,
+        ('\\begin{sphinxVerbatim}[commandchars=\\\\\\{\\}]\n'
+         '@Γ\\PYGZbs{}∞\\PYGZdl{}\\PYGZob{}\\PYGZcb{}\n'
+         '\\end{sphinxVerbatim}'),
+    ),
+])
+def test_inline_for_unicode_latex_engine(get_verifier, type, rst,
+                                         html_expected, latex_expected):
+    verifier = get_verifier(type)
+    verifier(rst, html_expected, latex_expected)
+
+
 def test_samp_role(parse):
     # no braces
     text = ':samp:`a{b}c`'