Repository: sciunto-org/python-bibtexparser
Pull request #187 (2018-01-31 15:01:20)

Fix invalid reorders of unicode combining in latexenc
Simplifies `latex_to_unicode` and fixes modifier orders.

In particular this fixes indiscriminate reorder of unicode modifiers. When the string already contained unicode modifiers these would be moved inappropriately to the next character by `latex_to_unicode`. For example `"Jalapeño" #  u"Jalapen\u0303o"'` would be changed to `"Jalapenõ"  #  u"Jalapen\u00f5"`.


coveralls: 
[![Coverage Status](https://coveralls.io/builds/15307764/badge)](https://coveralls.io/builds/15307764)

Coverage decreased (-0.3%) to 96.742% when pulling **1e488bfda96c88925b3a67e3d30244fd033d1b14 on omangin:fix/latexenc** into **ee212b9ef17efb48c7a9eaff3a2e609a9905914f on sciunto-org:master**.


Patch:
diff --git a/bibtexparser/latexenc.py b/bibtexparser/latexenc.py
index b4ac36d..2a30f62 100644
--- a/bibtexparser/latexenc.py
+++ b/bibtexparser/latexenc.py
@@ -32,6 +32,30 @@ def string_to_latex(string):
     return ''.join(new)
 
 
+def _replace_latex(string, latex, unicod):
+    if latex in string:
+        if unicodedata.combining(unicod):
+            for m in re.finditer(re.escape(latex), string):
+                i, j = m.span()
+                # Insert after the following character,
+                if j < len(string):
+                    string = ''.join([
+                        string[:i], string[j], unicod, string[(j + 1):]])
+                else:
+                    # except if not in last position (nothing to modify)
+                    string = string[:i]
+        else:
+            # Just replace
+            string = string.replace(latex, unicod)
+    return string
+
+
+def _replace_all_latex(string, replacements):
+    for u, l in replacements:
+        string = _replace_latex(string, l, u)
+    return string
+
+
 def latex_to_unicode(string):
     """
     Convert a LaTeX string to unicode equivalent.
@@ -40,55 +64,25 @@ def latex_to_unicode(string):
     :returns: string
     """
     if '\\' in string or '{' in string:
-        for k, v in itertools.chain(unicode_to_crappy_latex1, unicode_to_latex):
-            if v in string:
-                string = string.replace(v, k)
+        string = _replace_all_latex(string, itertools.chain(
+            unicode_to_crappy_latex1, unicode_to_latex))
 
-    # If there is still very crappy items
-    if '\\' in string:
-        for k, v in unicode_to_crappy_latex2:
-            if v in string:
-                parts = string.split(str(v))
-                for key, string in enumerate(parts):
-                    if key+1 < len(parts) and len(parts[key+1]) > 0:
-                        # Change order to display accents
-                        parts[key] = parts[key] + parts[key+1][0]
-                        parts[key+1] = parts[key+1][1:]
-                string = k.join(parts)
+    # TODO Shouldn't this preserve escaped braces instead?
+    # Remove any left braces
+    string = string.replace("{", "").replace("}", "")
 
-    # Place accents at correct position
-    # LaTeX requires accents *before* the character. Unicode requires accents
-    # to be *after* the character. Hence, by a raw conversion, accents are not
-    # on the correct letter, see
-    # https://github.com/sciunto-org/python-bibtexparser/issues/121.
-    # We just swap accents positions to fix this.
-    cleaned_string = []
-    i = 0
-    while i < len(string):
-        if not unicodedata.combining(string[i]):
-            # Not a combining diacritical mark, append it
-            cleaned_string.append(string[i])
-            i += 1
-        elif i < len(string) - 1:
-            # Diacritical mark, append it but swap with next character
-            cleaned_string.append(string[i + 1])
-            cleaned_string.append(string[i])
-            i += 2
-        else:
-            # If trailing character is a combining one, just discard it
-            i += 1
+    # If there is still very crappy items
+    if '\\' in string or '{' in string:
+        string = _replace_all_latex(string, unicode_to_crappy_latex2)
 
     # Normalize unicode characters
     # Also, when converting to unicode, we should return a normalized Unicode
     # string, that is always having only compound accentuated character (letter
     # + accent) or single accentuated character (letter with accent). We choose
     # to normalize to the latter.
-    cleaned_string = unicodedata.normalize("NFC", "".join(cleaned_string))
-
-    # Remove any left braces
-    cleaned_string = cleaned_string.replace("{", "").replace("}", "")
+    string = unicodedata.normalize("NFC", u"".join(string))
 
-    return cleaned_string
+    return string
 
 
 def protect_uppercase(string):
@@ -113,6 +107,7 @@ unicode_to_latex_map = {}
 unicode_to_crappy_latex1 = []
 unicode_to_crappy_latex2 = []
 
+
 def prepare_unicode_to_latex():
     global unicode_to_latex
     global unicode_to_latex_map
@@ -2702,4 +2697,5 @@ def prepare_unicode_to_latex():
         unicode_to_crappy_latex2 = tuple((k.decode('unicode-escape'), v) for k, v in to_crappy2)
         unicode_to_latex_map = dict(unicode_to_latex)
 
+
 prepare_unicode_to_latex()


Test patch:
diff --git a/bibtexparser/tests/test_latexenc.py b/bibtexparser/tests/test_latexenc.py
index e50ff3f..b7a3f91 100644
--- a/bibtexparser/tests/test_latexenc.py
+++ b/bibtexparser/tests/test_latexenc.py
@@ -18,7 +18,9 @@
 from __future__ import unicode_literals
 import unittest
 
-from bibtexparser.latexenc import *
+from bibtexparser.latexenc import (string_to_latex, latex_to_unicode,
+                                   protect_uppercase)
+
 
 class TestLatexConverter(unittest.TestCase):
 
@@ -34,6 +36,7 @@ class TestLatexConverter(unittest.TestCase):
         expected = '{\c c}'
         self.assertEqual(result, expected)
 
+
 class TestUppercaseProtection(unittest.TestCase):
 
     def test_uppercase(self):
@@ -67,5 +70,38 @@ class TestUppercaseProtection(unittest.TestCase):
         self.assertEqual(result, expected)
 
 
+class TestUnicodeConversion(unittest.TestCase):
+
+    def test_accents(self):
+        string = "{\`a} {\\\'e} {\`e} {\\\"o}"
+        result = latex_to_unicode(string)
+        expected = 'à é è ö'
+        self.assertEqual(result, expected)
+
+    def test_ignores_trailing_modifier(self):
+        string = "a\\\'"
+        result = latex_to_unicode(string)
+        expected = 'a'
+        self.assertEqual(result, expected)
+
+    def test_special_caracter(self):
+        string = '{\c c}'
+        result = latex_to_unicode(string)
+        expected = 'ç'
+        self.assertEqual(result, expected)
+
+    def test_does_not_modify_existing_combining(self):
+        string = b'ph\xc6\xa1\xcc\x89'.decode('utf8')
+        result = latex_to_unicode(string)
+        expected = 'phở'  # normalized
+        self.assertEqual(result, expected)
+
+    def test_does_not_modify_two_existing_combining(self):
+        string = b'pho\xcc\x9b\xcc\x89'.decode('utf8')
+        result = latex_to_unicode(string)
+        expected = 'phở'  # normalized
+        self.assertEqual(result, expected)
+
+
 if __name__ == '__main__':
     unittest.main()