Repository: sphinx-doc/sphinx

Issue:
LaTeX output: sub/superscripts and other Unicode glyphs get "TeX-escaped" even if engine is xelatex
Arguably, [TeX-escaping](https://github.com/sphinx-doc/sphinx/blob/master/sphinx/util/texescape.py) when using Unicode aware xelatex engine should be essentially limited to the few characters with special TeX meaning such as `$`, `{`, `\` and a few others.

For the initial report where the too extensive TeX escaping causes problems see #6733. 

However some Unicode code-points such as `ℯ` seem to have limited font support (particularly monospace, it is lacking both from DejaVu Sans Mono and GNU FreeMono), and should perhaps be retained at least for the near future in the escape list. Indeed such missing glyphs do not raise errors but only LaTeX warnings and they may escape attention.


Discussion:

Patch:
diff --git a/CHANGES b/CHANGES
index 7cc5b54d7..9d3b51cc4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -42,6 +42,8 @@ Bugs fixed
 
   .. _latex3/latex2e#173: https://github.com/latex3/latex2e/issues/173
 * #6618: LaTeX: Avoid section names at the end of a page
+* #6738: LaTeX: Do not replace unicode characters by LaTeX macros on unicode
+  supported LaTeX engines
 * #6704: linkcheck: Be defensive and handle newly defined HTTP error code
 * #6655: image URLs containing ``data:`` causes gettext builder crashed
 * #6584: i18n: Error when compiling message catalogs on Hindi
diff --git a/sphinx/util/template.py b/sphinx/util/template.py
index fd8886944..3a43db9a5 100644
--- a/sphinx/util/template.py
+++ b/sphinx/util/template.py
@@ -63,14 +63,14 @@ def render_from_file(cls, filename: str, context: Dict) -> str:
 
 
 class LaTeXRenderer(SphinxRenderer):
-    def __init__(self, template_path: str = None) -> None:
+    def __init__(self, template_path: str = None, latex_engine: str = None) -> None:
         if template_path is None:
             template_path = os.path.join(package_dir, 'templates', 'latex')
         super().__init__(template_path)
 
         # use texescape as escape filter
-        self.env.filters['e'] = texescape.escape
-        self.env.filters['escape'] = texescape.escape
+        self.env.filters['e'] = texescape.get_escape_func(latex_engine)
+        self.env.filters['escape'] = texescape.get_escape_func(latex_engine)
         self.env.filters['eabbr'] = texescape.escape_abbr
 
         # use JSP/eRuby like tagging instead because curly bracket; the default
diff --git a/sphinx/util/texescape.py b/sphinx/util/texescape.py
index 408ec1253..4e7055119 100644
--- a/sphinx/util/texescape.py
+++ b/sphinx/util/texescape.py
@@ -9,7 +9,7 @@
 """
 
 import re
-from typing import Dict
+from typing import Callable, Dict
 
 tex_replacements = [
     # map TeX special chars
@@ -46,6 +46,14 @@
     ('|', r'\textbar{}'),
     ('ℯ', r'e'),
     ('ⅈ', r'i'),
+    # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc
+    # OHM SIGN U+2126 is handled by LaTeX textcomp package
+]
+
+# A map Unicode characters to LaTeX representation
+# (for LaTeX engines which don't support unicode)
+unicode_tex_replacements = [
+    # superscript
     ('⁰', r'\(\sp{\text{0}}\)'),
     ('¹', r'\(\sp{\text{1}}\)'),
     ('²', r'\(\sp{\text{2}}\)'),
@@ -56,6 +64,7 @@
     ('⁷', r'\(\sp{\text{7}}\)'),
     ('⁸', r'\(\sp{\text{8}}\)'),
     ('⁹', r'\(\sp{\text{9}}\)'),
+    # subscript
     ('₀', r'\(\sb{\text{0}}\)'),
     ('₁', r'\(\sb{\text{1}}\)'),
     ('₂', r'\(\sb{\text{2}}\)'),
@@ -66,20 +75,32 @@
     ('₇', r'\(\sb{\text{7}}\)'),
     ('₈', r'\(\sb{\text{8}}\)'),
     ('₉', r'\(\sb{\text{9}}\)'),
-    # Greek alphabet not escaped: pdflatex handles it via textalpha and inputenc
-    # OHM SIGN U+2126 is handled by LaTeX textcomp package
 ]
 
 tex_escape_map = {}  # type: Dict[int, str]
+tex_escape_map_without_unicode = {}  # type: Dict[int, str]
 tex_replace_map = {}
 tex_hl_escape_map_new = {}
 
 
+def get_escape_func(latex_engine: str) -> Callable[[str], str]:
+    """Get escape() function for given latex_engine."""
+    if latex_engine in ('lualatex', 'xelatex'):
+        return escape_for_unicode_latex_engine
+    else:
+        return escape
+
+
 def escape(s: str) -> str:
     """Escape text for LaTeX output."""
     return s.translate(tex_escape_map)
 
 
+def escape_for_unicode_latex_engine(s: str) -> str:
+    """Escape text for unicode supporting LaTeX engine."""
+    return s.translate(tex_escape_map_without_unicode)
+
+
 def escape_abbr(text: str) -> str:
     """Adjust spacing after abbreviations. Works with @ letter or other."""
     return re.sub(r'\.(?=\s|$)', r'.\@{}', text)
@@ -87,6 +108,11 @@ def escape_abbr(text: str) -> str:
 
 def init() -> None:
     for a, b in tex_replacements:
+        tex_escape_map[ord(a)] = b
+        tex_escape_map_without_unicode[ord(a)] = b
+        tex_replace_map[ord(a)] = '_'
+
+    for a, b in unicode_tex_replacements:
         tex_escape_map[ord(a)] = b
         tex_replace_map[ord(a)] = '_'
 
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index 364067497..29367a79e 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -32,7 +32,7 @@
 from sphinx.util.docutils import SphinxTranslator
 from sphinx.util.nodes import clean_astext, get_prev_node
 from sphinx.util.template import LaTeXRenderer
-from sphinx.util.texescape import tex_escape_map, tex_replace_map
+from sphinx.util.texescape import get_escape_func, tex_replace_map
 
 try:
     from docutils.utils.roman import toRoman
@@ -500,6 +500,9 @@ def __init__(self, document, builder):
         self.compact_list = 0
         self.first_param = 0
 
+        # escape helper
+        self.escape = get_escape_func(self.config.latex_engine)
+
         # sort out some elements
         self.elements = self.builder.context.copy()
 
@@ -758,8 +761,7 @@ def generate(content, collapsed):
             for i, (letter, entries) in enumerate(content):
                 if i > 0:
                     ret.append('\\indexspace\n')
-                ret.append('\\bigletter{%s}\n' %
-                           str(letter).translate(tex_escape_map))
+                ret.append('\\bigletter{%s}\n' % self.escape(letter))
                 for entry in entries:
                     if not entry[3]:
                         continue
@@ -794,13 +796,14 @@ def generate(content, collapsed):
 
     def render(self, template_name, variables):
         # type: (str, Dict) -> str
+        renderer = LaTeXRenderer(latex_engine=self.config.latex_engine)
         for template_dir in self.builder.config.templates_path:
             template = path.join(self.builder.confdir, template_dir,
                                  template_name)
             if path.exists(template):
-                return LaTeXRenderer().render(template, variables)
+                return renderer.render(template, variables)
 
-        return LaTeXRenderer().render(template_name, variables)
+        return renderer.render(template_name, variables)
 
     def visit_document(self, node):
         # type: (nodes.Element) -> None
@@ -914,14 +917,13 @@ def visit_title(self, node):
                 if not self.elements['title']:
                     # text needs to be escaped since it is inserted into
                     # the output literally
-                    self.elements['title'] = node.astext().translate(tex_escape_map)
+                    self.elements['title'] = self.escape(node.astext())
                 self.this_is_the_title = 0
                 raise nodes.SkipNode
             else:
                 short = ''
                 if node.traverse(nodes.image):
-                    short = ('[%s]' %
-                             ' '.join(clean_astext(node).split()).translate(tex_escape_map))
+                    short = ('[%s]' % self.escape(' '.join(clean_astext(node).split())))
 
                 try:
                     self.body.append(r'\%s%s{' % (self.sectionnames[self.sectionlevel], short))
@@ -1955,8 +1957,7 @@ def visit_number_reference(self, node):
         else:
             id = node.get('refuri', '')[1:].replace('#', ':')
 
-        title = node.get('title', '%s')
-        title = str(title).translate(tex_escape_map).replace('\\%s', '%s')
+        title = self.escape(node.get('title', '%s')).replace('\\%s', '%s')
         if '\\{name\\}' in title or '\\{number\\}' in title:
             # new style format (cf. "Fig.%{number}")
             title = title.replace('\\{name\\}', '{name}').replace('\\{number\\}', '{number}')
@@ -2404,7 +2405,7 @@ def visit_docinfo(self, node):
 
     def encode(self, text):
         # type: (str) -> str
-        text = str(text).translate(tex_escape_map)
+        text = self.escape(text)
         if self.literal_whitespace:
             # Insert a blank before the newline, to avoid
             # ! LaTeX Error: There's no line here to end.
@@ -2615,33 +2616,31 @@ def generate_numfig_format(self, builder):
         ret = []  # type: List[str]
         figure = self.builder.config.numfig_format['figure'].split('%s', 1)
         if len(figure) == 1:
-            ret.append('\\def\\fnum@figure{%s}\n' %
-                       str(figure[0]).strip().translate(tex_escape_map))
+            ret.append('\\def\\fnum@figure{%s}\n' % self.escape(figure[0]).strip())
         else:
-            definition = escape_abbr(str(figure[0]).translate(tex_escape_map))
+            definition = escape_abbr(self.escape(figure[0]))
             ret.append(self.babel_renewcommand('\\figurename', definition))
             ret.append('\\makeatletter\n')
             ret.append('\\def\\fnum@figure{\\figurename\\thefigure{}%s}\n' %
-                       str(figure[1]).translate(tex_escape_map))
+                       self.escape(figure[1]))
             ret.append('\\makeatother\n')
 
         table = self.builder.config.numfig_format['table'].split('%s', 1)
         if len(table) == 1:
-            ret.append('\\def\\fnum@table{%s}\n' %
-                       str(table[0]).strip().translate(tex_escape_map))
+            ret.append('\\def\\fnum@table{%s}\n' % self.escape(table[0]).strip())
         else:
-            definition = escape_abbr(str(table[0]).translate(tex_escape_map))
+            definition = escape_abbr(self.escape(table[0]))
             ret.append(self.babel_renewcommand('\\tablename', definition))
             ret.append('\\makeatletter\n')
             ret.append('\\def\\fnum@table{\\tablename\\thetable{}%s}\n' %
-                       str(table[1]).translate(tex_escape_map))
+                       self.escape(table[1]))
             ret.append('\\makeatother\n')
 
         codeblock = self.builder.config.numfig_format['code-block'].split('%s', 1)
         if len(codeblock) == 1:
             pass  # FIXME
         else:
-            definition = str(codeblock[0]).strip().translate(tex_escape_map)
+            definition = self.escape(codeblock[0]).strip()
             ret.append(self.babel_renewcommand('\\literalblockname', definition))
             if codeblock[1]:
                 pass  # FIXME


Test patch:
diff --git a/tests/roots/test-latex-unicode/conf.py b/tests/roots/test-latex-unicode/conf.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/roots/test-latex-unicode/index.rst b/tests/roots/test-latex-unicode/index.rst
new file mode 100644
index 000000000..2abeca98f
--- /dev/null
+++ b/tests/roots/test-latex-unicode/index.rst
@@ -0,0 +1,7 @@
+test-latex-unicode
+==================
+
+* script small e: ℯ
+* double struck italic small i: ⅈ
+* superscript: ⁰, ¹
+* subscript: ₀, ₁
diff --git a/tests/test_build_latex.py b/tests/test_build_latex.py
index c48399f1b..0ebcd0b62 100644
--- a/tests/test_build_latex.py
+++ b/tests/test_build_latex.py
@@ -1439,6 +1439,30 @@ def test_index_on_title(app, status, warning):
             in result)
 
 
+@pytest.mark.sphinx('latex', testroot='latex-unicode',
+                    confoverrides={'latex_engine': 'pdflatex'})
+def test_texescape_for_non_unicode_supported_engine(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'python.tex').text()
+    print(result)
+    assert 'script small e: e' in result
+    assert 'double struck italic small i: i' in result
+    assert r'superscript: \(\sp{\text{0}}\), \(\sp{\text{1}}\)' in result
+    assert r'subscript: \(\sb{\text{0}}\), \(\sb{\text{1}}\)' in result
+
+
+@pytest.mark.sphinx('latex', testroot='latex-unicode',
+                    confoverrides={'latex_engine': 'xelatex'})
+def test_texescape_for_unicode_supported_engine(app, status, warning):
+    app.builder.build_all()
+    result = (app.outdir / 'python.tex').text()
+    print(result)
+    assert 'script small e: e' in result
+    assert 'double struck italic small i: i' in result
+    assert 'superscript: ⁰, ¹' in result
+    assert 'subscript: ₀, ₁' in result
+
+    
 @pytest.mark.sphinx('latex', testroot='basic',
                     confoverrides={'latex_elements': {'extrapackages': r'\usepackage{foo}'}})
 def test_latex_elements_extrapackages(app, status, warning):