Repository: translate/translate

Issue:
prop2po with personality=strings fails on keys with "=" character
I have noticed a problem when converting .strings files. If key and value contain equal sign, then the resulting PO file has part of the key as a text for translation.

In .strings file I have a key 

``` properties
/* No comment provided by engineer. */
"Sample key with = sign" = "Sample key with = sign"
```

This string is converted to below message:

``` po
#: Sample+key+with
msgid "sign\" = \"Sample key with = sign"
msgstr ""
```

Please also check attached sample files.
version used: 1.9.0



Discussion:

Resolving patch:
diff --git a/translate/storage/properties.py b/translate/storage/properties.py
--- a/translate/storage/properties.py
+++ b/translate/storage/properties.py
@@ -136,66 +136,6 @@
 eol = "\n"
 
 
-def _find_delimiter(line, delimiters):
-    """Find the type and position of the delimiter in a property line.
-
-    Property files can be delimited by "=", ":" or whitespace (space for now).
-    We find the position of each delimiter, then find the one that appears
-    first.
-
-    :param line: A properties line
-    :type line: str
-    :param delimiters: valid delimiters
-    :type delimiters: list
-    :return: delimiter character and offset within *line*
-    :rtype: Tuple (delimiter char, Offset Integer)
-    """
-    delimiter_dict = {}
-    for delimiter in delimiters:
-        delimiter_dict[delimiter] = -1
-    delimiters = delimiter_dict
-    # Find the position of each delimiter type
-    for delimiter, pos in six.iteritems(delimiters):
-        prewhitespace = len(line) - len(line.lstrip())
-        pos = line.find(delimiter, prewhitespace)
-        while pos != -1:
-            if delimiters[delimiter] == -1 and line[pos-1] != u"\\":
-                delimiters[delimiter] = pos
-                break
-            pos = line.find(delimiter, pos + 1)
-    # Find the first delimiter
-    mindelimiter = None
-    minpos = -1
-    for delimiter, pos in six.iteritems(delimiters):
-        if pos == -1 or delimiter == u" ":
-            continue
-        if minpos == -1 or pos < minpos:
-            minpos = pos
-            mindelimiter = delimiter
-    if mindelimiter is None and delimiters.get(u" ", -1) != -1:
-        # Use space delimiter if we found nothing else
-        return (u" ", delimiters[" "])
-    if (mindelimiter is not None and
-        u" " in delimiters and
-        delimiters[u" "] < delimiters[mindelimiter]):
-        # If space delimiter occurs earlier than ":" or "=" then it is the
-        # delimiter only if there are non-whitespace characters between it and
-        # the other detected delimiter.
-        if len(line[delimiters[u" "]:delimiters[mindelimiter]].strip()) > 0:
-            return (u" ", delimiters[u" "])
-    return (mindelimiter, minpos)
-
-
-@deprecated("Use Dialect.find_delimiter instead")
-def find_delimeter(line):
-    """Misspelled function that is kept around in case someone relies on it.
-
-    .. deprecated:: 1.7.0
-       Use :func:`find_delimiter` instead
-    """
-    return _find_delimiter(line, DialectJava.delimiters)
-
-
 def is_line_continuation(line):
     """Determine whether *line* has a line continuation marker.
 
@@ -312,8 +252,58 @@ def encode(cls, string, encoding=None):
 
     @classmethod
     def find_delimiter(cls, line):
-        """Find the delimiter"""
-        return _find_delimiter(line, cls.delimiters)
+        """Find the type and position of the delimiter in a property line.
+
+        Property files can be delimited by "=", ":" or whitespace (space for now).
+        We find the position of each delimiter, then find the one that appears
+        first.
+
+        :param line: A properties line
+        :type line: str
+        :param delimiters: valid delimiters
+        :type delimiters: list
+        :return: delimiter character and offset within *line*
+        :rtype: Tuple (delimiter char, Offset Integer)
+        """
+        delimiter_dict = {}
+        for delimiter in cls.delimiters:
+            delimiter_dict[delimiter] = -1
+        delimiters = delimiter_dict
+        # Find the position of each delimiter type
+        for delimiter, pos in six.iteritems(delimiters):
+            start_pos = len(line) - len(line.lstrip())  # Skip initial whitespace
+            if cls.key_wrap_char != '' and line[start_pos] == cls.key_wrap_char:
+                # Skip the key if it is delimited by some char
+                start_pos += 1
+                while (line[start_pos] != cls.key_wrap_char or line[start_pos-1] == u"\\"):
+                    start_pos += 1
+            pos = line.find(delimiter, start_pos)
+            while pos != -1:
+                if delimiters[delimiter] == -1 and line[pos-1] != u"\\":
+                    delimiters[delimiter] = pos
+                    break
+                pos = line.find(delimiter, pos + 1)
+        # Find the first delimiter
+        mindelimiter = None
+        minpos = -1
+        for delimiter, pos in six.iteritems(delimiters):
+            if pos == -1 or delimiter == u" ":
+                continue
+            if minpos == -1 or pos < minpos:
+                minpos = pos
+                mindelimiter = delimiter
+        if mindelimiter is None and delimiters.get(u" ", -1) != -1:
+            # Use space delimiter if we found nothing else
+            return (u" ", delimiters[" "])
+        if (mindelimiter is not None and
+            u" " in delimiters and
+            delimiters[u" "] < delimiters[mindelimiter]):
+            # If space delimiter occurs earlier than ":" or "=" then it is the
+            # delimiter only if there are non-whitespace characters between it and
+            # the other detected delimiter.
+            if len(line[delimiters[u" "]:delimiters[mindelimiter]].strip()) > 0:
+                return (u" ", delimiters[u" "])
+        return (mindelimiter, minpos)
 
     @classmethod
     def key_strip(cls, key):


Test patch:
diff --git a/translate/storage/test_properties.py b/translate/storage/test_properties.py
--- a/translate/storage/test_properties.py
+++ b/translate/storage/test_properties.py
@@ -6,50 +6,46 @@
 from translate.storage import properties, test_monolingual
 
 
+# Note that DialectJava delimitors are [u"=", u":", u" "]
+
 def test_find_delimiter_pos_simple():
     """Simple tests to find the various delimiters"""
-    assert properties._find_delimiter(u"key=value", [u"=", u":", u" "]) == ('=', 3)
-    assert properties._find_delimiter(u"key:value", [u"=", u":", u" "]) == (':', 3)
-    assert properties._find_delimiter(u"key value", [u"=", u":", u" "]) == (' ', 3)
+    assert properties.DialectJava.find_delimiter(u"key=value") == ('=', 3)
+    assert properties.DialectJava.find_delimiter(u"key:value") == (':', 3)
+    assert properties.DialectJava.find_delimiter(u"key value") == (' ', 3)
     # NOTE this is valid in Java properties, the key is then the empty string
-    assert properties._find_delimiter(u"= value", [u"=", u":", u" "]) == ('=', 0)
+    assert properties.DialectJava.find_delimiter(u"= value") == ('=', 0)
 
 
 def test_find_delimiter_pos_multiple():
     """Find delimiters when multiple potential delimiters are involved"""
-    assert properties._find_delimiter(u"key=value:value", [u"=", u":", u" "]) == ('=', 3)
-    assert properties._find_delimiter(u"key:value=value", [u"=", u":", u" "]) == (':', 3)
-    assert properties._find_delimiter(u"key value=value", [u"=", u":", u" "]) == (' ', 3)
+    assert properties.DialectJava.find_delimiter(u"key=value:value") == ('=', 3)
+    assert properties.DialectJava.find_delimiter(u"key:value=value") == (':', 3)
+    assert properties.DialectJava.find_delimiter(u"key value=value") == (' ', 3)
 
 
 def test_find_delimiter_pos_none():
     """Find delimiters when there isn't one"""
-    assert properties._find_delimiter(u"key", [u"=", u":", u" "]) == (None, -1)
-    assert properties._find_delimiter(u"key\=\:\ ", [u"=", u":", u" "]) == (None, -1)
+    assert properties.DialectJava.find_delimiter(u"key") == (None, -1)
+    assert properties.DialectJava.find_delimiter(u"key\=\:\ ") == (None, -1)
 
 
 def test_find_delimiter_pos_whitespace():
     """Find delimiters when whitespace is involved"""
-    assert properties._find_delimiter(u"key = value", [u"=", u":", u" "]) == ('=', 4)
-    assert properties._find_delimiter(u"key : value", [u"=", u":", u" "]) == (':', 4)
-    assert properties._find_delimiter(u"key   value", [u"=", u":", u" "]) == (' ', 3)
-    assert properties._find_delimiter(u"key value = value", [u"=", u":", u" "]) == (' ', 3)
-    assert properties._find_delimiter(u"key value value", [u"=", u":", u" "]) == (' ', 3)
-    assert properties._find_delimiter(u" key = value", [u"=", u":", u" "]) == ('=', 5)
+    assert properties.DialectJava.find_delimiter(u"key = value") == ('=', 4)
+    assert properties.DialectJava.find_delimiter(u"key : value") == (':', 4)
+    assert properties.DialectJava.find_delimiter(u"key   value") == (' ', 3)
+    assert properties.DialectJava.find_delimiter(u"key value = value") == (' ', 3)
+    assert properties.DialectJava.find_delimiter(u"key value value") == (' ', 3)
+    assert properties.DialectJava.find_delimiter(u" key = value") == ('=', 5)
 
 
 def test_find_delimiter_pos_escapes():
     """Find delimiters when potential earlier delimiters are escaped"""
-    assert properties._find_delimiter(u"key\:=value", [u"=", u":", u" "]) == ('=', 5)
-    assert properties._find_delimiter(u"key\=: value", [u"=", u":", u" "]) == (':', 5)
-    assert properties._find_delimiter(u"key\   value", [u"=", u":", u" "]) == (' ', 5)
-    assert properties._find_delimiter(u"key\ key\ key\: = value", [u"=", u":", u" "]) == ('=', 16)
-
-
-def test_find_delimiter_deprecated_fn():
-    """Test that the deprecated function still actually works"""
-    assert properties.find_delimeter(u"key=value") == ('=', 3)
-    deprecated_call(properties.find_delimeter, u"key=value")
+    assert properties.DialectJava.find_delimiter(u"key\:=value") == ('=', 5)
+    assert properties.DialectJava.find_delimiter(u"key\=: value") == (':', 5)
+    assert properties.DialectJava.find_delimiter(u"key\   value") == (' ', 5)
+    assert properties.DialectJava.find_delimiter(u"key\ key\ key\: = value") == ('=', 16)
 
 
 def test_is_line_continuation():
@@ -231,7 +227,7 @@ def test_fullspec_line_continuation(self):
         assert len(prop_store.units) == 1
         unit = prop_store.units[0]
         print(unit)
-        assert properties._find_delimiter(prop_source, [u"=", u":", u" "]) == (' ', 6)
+        assert properties.DialectJava.find_delimiter(prop_source) == (' ', 6)
         assert unit.name == u"fruits"
         assert unit.source == u"apple, banana, pear, cantaloupe, watermelon, kiwi, mango"
 
@@ -326,6 +322,14 @@ def test_mac_strings_quotes(self):
         assert propunit.name == u'key with "quotes"'
         assert propunit.value == u'value with "quotes"'
 
+    def test_mac_strings_equals(self):
+        """test that equal signs inside keys/values are not mixed with delimiter"""
+        propsource = '"key with = sign" = "value with = sign";'.encode('utf-16')
+        propfile = self.propparse(propsource, personality="strings")
+        propunit = propfile.units[0]
+        assert propunit.name == u'key with = sign'
+        assert propunit.value == u'value with = sign'
+
     def test_mac_strings_serialization(self):
         """test that serializer quotes mac strings properly"""
         propsource = r'"key with \"quotes\"" = "value with \"quotes\"";'.encode('utf-16')