Repository: translate/translate

Issue:
Backwards incompatible change in multistring
The behavior of the `multistring` class with respect to string comparisons has changed in 1.14-rc1, more specifically when the `autoencode` class was removed in 4f9ee7edfc9. At least this is the case in Python 2.7.

I have experienced this in some custom Pootle code when testing out TTK from master — a multistring with non-ASCII characters was being compared against the empty byte string `''`, and it resulted in a encoding error. I can reproduce this in a test case:

``` python
from translate.misc import multistring

def test_cmp():
    t = multistring.multistring
    s1 = t("test")
    assert s1.__cmp__("test") == 0

    s2 = t(u"téßt")
    assert s2.__cmp__("test") > 0
```

(Note that for 1.13.x the multistring constructor needs an `encoding='UTF-8'` parameter in the test).

I'm not sure if this could be considered a bug, after all my understanding is apps/libraries should internally work with code points and not bytes, however this is a breaking change that might potentially affect existing consumer code so I consider it's worth bringing this up to hear people's thoughts.



Discussion:
@claudep Can you provide some additional insight?

/me is currently looking...


Resolving patch:
diff --git a/translate/misc/multistring.py b/translate/misc/multistring.py
--- a/translate/misc/multistring.py
+++ b/translate/misc/multistring.py
@@ -55,7 +55,7 @@ def cmp_compat(s1, s2):
         elif isinstance(otherstring, six.text_type):
             return cmp_compat(six.text_type(self), otherstring)
         elif isinstance(otherstring, bytes):
-            return cmp_compat(bytes(self), otherstring)
+            return cmp_compat(self.encode('utf-8'), otherstring)
         elif isinstance(otherstring, list) and otherstring:
             return cmp_compat(self, multistring(otherstring))
         else:
@@ -71,7 +71,8 @@ def __eq__(self, otherstring):
         return self.__cmp__(otherstring) == 0
 
     def __repr__(self):
-        return "multistring([" + ",".join(self.strings) + "])"
+        _repr = "multistring([" + u",".join(self.strings) + "])"
+        return _repr.encode('utf-8') if six.PY2 else _repr
 
     def replace(self, old, new, count=None):
         if count is None:


Test patch:
diff --git a/translate/misc/test_multistring.py b/translate/misc/test_multistring.py
--- a/translate/misc/test_multistring.py
+++ b/translate/misc/test_multistring.py
@@ -1,27 +1,27 @@
+# -*- coding: utf-8 -*-
 import pytest
 
 from translate.misc import multistring
 
 
 class TestMultistring:
-    type2test = multistring.multistring
 
     def test_constructor(self):
-        t = self.type2test
+        t = multistring.multistring
         s1 = t("test")
         assert type(s1) == t
         assert s1 == "test"
         assert s1.strings == ["test"]
-        s2 = t(["test", "me"])
+        s2 = t(["test", u"mé"])
         assert type(s2) == t
-        assert repr(s2) == 'multistring([test,me])'
+        assert repr(s2) == 'multistring([test,mé])'
         assert s2 == "test"
-        assert s2.strings == ["test", "me"]
+        assert s2.strings == ["test", u"mé"]
         assert s2 != s1
         pytest.raises(ValueError, t, [])
 
     def test_replace(self):
-        t = self.type2test
+        t = multistring.multistring
         s1 = t(["abcdef", "def"])
 
         result = s1.replace("e", "")
@@ -50,3 +50,12 @@ def test_replace(self):
 
         result = s2.replace("e", "g", 1)
         assert result == t([u"abcdgef", u"dgef"])
+
+    def test_comparison(self):
+        t = multistring.multistring
+        assert t("test") == "test"
+        assert t("test").__cmp__("test") == 0
+
+        assert t(u"téßt") > "test"
+        assert "test" < t(u"téßt")
+        assert t(u"téßt").__cmp__("test") > 0