Repository: sciunto-org/python-bibtexparser

Issue:
convert_to_unicode
I am using the 

parser.customization = convert_to_unicode

feature, but in the utf-8 output the accents are coming out on the character before the correct one.  What would cause this? 


Discussion:

Patch:
diff --git a/bibtexparser/customization.py b/bibtexparser/customization.py
index 98cd246..c1b19b2 100644
--- a/bibtexparser/customization.py
+++ b/bibtexparser/customization.py
@@ -7,11 +7,10 @@ You can find inspiration from these functions to design yours.
 Each of them takes a record and return the modified record.
 """
 
-import itertools
 import re
 import logging
 
-from bibtexparser.latexenc import unicode_to_latex, unicode_to_crappy_latex1, unicode_to_crappy_latex2, string_to_latex, protect_uppercase
+from bibtexparser.latexenc import latex_to_unicode, string_to_latex, protect_uppercase
 
 logger = logging.getLogger(__name__)
 
@@ -495,22 +494,16 @@ def convert_to_unicode(record):
     :returns: dict -- the modified record.
     """
     for val in record:
-        if '\\' in record[val] or '{' in record[val]:
-            for k, v in itertools.chain(unicode_to_crappy_latex1, unicode_to_latex):
-                if v in record[val]:
-                    record[val] = record[val].replace(v, k)
-
-        # If there is still very crappy items
-        if '\\' in record[val]:
-            for k, v in unicode_to_crappy_latex2:
-                if v in record[val]:
-                    parts = record[val].split(str(v))
-                    for key, record[val] in enumerate(parts):
-                        if key+1 < len(parts) and len(parts[key+1]) > 0:
-                            # Change order to display accents
-                            parts[key] = parts[key] + parts[key+1][0]
-                            parts[key+1] = parts[key+1][1:]
-                    record[val] = k.join(parts)
+        if isinstance(record[val], list):
+            record[val] = [
+                latex_to_unicode(x) for x in record[val]
+            ]
+        elif isinstance(record[val], dict):
+            record[val] = {
+                k: latex_to_unicode(v) for k, v in record[val].items()
+            }
+        else:
+            record[val] = latex_to_unicode(record[val])
     return record
 
 
diff --git a/bibtexparser/latexenc.py b/bibtexparser/latexenc.py
index b919a24..e225de4 100644
--- a/bibtexparser/latexenc.py
+++ b/bibtexparser/latexenc.py
@@ -7,11 +7,14 @@
 # Etienne Posthumus (epoz)
 # Francois Boulogne <fboulogne at april dot org>
 
+import itertools
 import re
 import sys
+import unicodedata
 
-__all__ = ['string_to_latex', 'protect_uppercase', 'unicode_to_latex',
-           'unicode_to_crappy_latex1', 'unicode_to_crappy_latex2']
+__all__ = ['string_to_latex', 'latex_to_unicode', 'protect_uppercase',
+           'unicode_to_latex', 'unicode_to_crappy_latex1',
+           'unicode_to_crappy_latex2']
 
 
 def string_to_latex(string):
@@ -29,6 +32,62 @@ def string_to_latex(string):
     return ''.join(new)
 
 
+def latex_to_unicode(string):
+    """
+    Convert a LaTeX string to unicode equivalent.
+
+    :param string: string to convert
+    :returns: string
+    """
+    if '\\' in string or '{' in string:
+        for k, v in itertools.chain(unicode_to_crappy_latex1, unicode_to_latex):
+            if v in string:
+                string = string.replace(v, k)
+
+    # If there is still very crappy items
+    if '\\' in string:
+        for k, v in unicode_to_crappy_latex2:
+            if v in string:
+                parts = string.split(str(v))
+                for key, string in enumerate(parts):
+                    if key+1 < len(parts) and len(parts[key+1]) > 0:
+                        # Change order to display accents
+                        parts[key] = parts[key] + parts[key+1][0]
+                        parts[key+1] = parts[key+1][1:]
+                string = k.join(parts)
+
+    # Place accents at correct position
+    # LaTeX requires accents *before* the character. Unicode requires accents
+    # to be *after* the character. Hence, by a raw conversion, accents are not
+    # on the correct letter, see
+    # https://github.com/sciunto-org/python-bibtexparser/issues/121.
+    # We just swap accents positions to fix this.
+    cleaned_string = []
+    i = 0
+    while i < len(string):
+        if not unicodedata.combining(string[i]):
+            # Not a combining diacritical mark, append it
+            cleaned_string.append(string[i])
+            i += 1
+        elif i < len(string) - 1:
+            # Diacritical mark, append it but swap with next character
+            cleaned_string.append(string[i + 1])
+            cleaned_string.append(string[i])
+            i += 2
+        else:
+            # If trailing character is a combining one, just discard it
+            i += 1
+
+    # Normalize unicode characters
+    # Also, when converting to unicode, we should return a normalized Unicode
+    # string, that is always having only compound accentuated character (letter
+    # + accent) or single accentuated character (letter with accent). We choose
+    # to normalize to the latter.
+    cleaned_string = unicodedata.normalize("NFC", "".join(cleaned_string))
+
+    return cleaned_string
+
+
 def protect_uppercase(string):
     """
     Protect uppercase letters for bibtex


Test patch:
diff --git a/bibtexparser/tests/test_customization.py b/bibtexparser/tests/test_customization.py
index 7587d5e..c6b168b 100644
--- a/bibtexparser/tests/test_customization.py
+++ b/bibtexparser/tests/test_customization.py
@@ -86,6 +86,11 @@ class TestBibtexParserMethod(unittest.TestCase):
         result = convert_to_unicode(record)
         expected = {'toto': 'ü ü'}
         self.assertEqual(result, expected)
+        # From issue 121
+        record = {'title': '{Two Gedenk\\"uberlieferung der Angelsachsen}'}
+        result = convert_to_unicode(record)
+        expected = {'title': '{Two Gedenküberlieferung der Angelsachsen}'}
+        self.assertEqual(result, expected)
 
     ###########
     # homogenize