Repository: web2py/web2py

Issue:
SPAN.flatten() raises UnicodeDecodeError with T-components
Code like this crashes in Py2.7 with UnicodeDecodeError for T-languages where translate produces multi-byte utf-8 encoded strings (e.g. Japanese, Korean, etc.):
```
elem = SPAN(T("Complete"))
text = elem.flatten()
```
The problem could be here:
https://github.com/web2py/web2py/blob/master/gluon/languages.py#L433
This should probably not str(self).encode() (as it seems normal that lazyT.__str__() returns an utf-8 encoded str)?

Or maybe this:
https://github.com/web2py/web2py/blob/master/gluon/_compat.py#L72
...could try to use the __str__ method first when obj is neither a str nor a unicode?



Discussion:

Resolving patch:
diff --git a/gluon/languages.py b/gluon/languages.py
--- a/gluon/languages.py
+++ b/gluon/languages.py
@@ -431,10 +431,16 @@ def xml(self):
         return str(self) if self.M else local_html_escape(str(self), quote=False)
 
     def encode(self, *a, **b):
-        return str(self).encode(*a, **b)
+        if PY2 and a[0] != 'utf8':
+            return to_unicode(str(self)).encode(*a, **b)
+        else:
+            return str(self)
 
     def decode(self, *a, **b):
-        return str(self).decode(*a, **b)
+        if PY2:
+            return str(self).decode(*a, **b)
+        else:
+            return str(self)
 
     def read(self):
         return str(self)


Test patch:
diff --git a/gluon/tests/test_languages.py b/gluon/tests/test_languages.py
--- a/gluon/tests/test_languages.py
+++ b/gluon/tests/test_languages.py
@@ -12,7 +12,9 @@
 import unittest
 
 from gluon import languages
-from gluon._compat import PY2
+from gluon._compat import PY2, to_unicode, to_bytes
+from gluon.storage import Messages
+from gluon.html import SPAN
 
 MP_WORKING = 0
 try:
@@ -108,6 +110,8 @@ def test_plain(self):
         T.force('it')
         self.assertEqual(str(T('Hello World')),
                          'Salve Mondo')
+        self.assertEqual(to_unicode(T('Hello World')),
+                         'Salve Mondo')
 
 class TestDummyApp(unittest.TestCase):
 
@@ -179,3 +183,43 @@ def test_update_all_languages(self):
         for key in ['hello', 'world', '%s %%{shop}', 'ahoy']:
             self.assertTrue(key in en_dict)
             self.assertTrue(key in pt_dict)
+
+class TestMessages(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.isdir('gluon'):
+            self.langpath = 'applications/welcome/languages'
+        else:
+            self.langpath = os.path.realpath(
+                '../../applications/welcome/languages')
+        self.http_accept_language = 'en'
+
+    def tearDown(self):
+        pass
+
+    def test_decode(self):
+        T = languages.translator(self.langpath, self.http_accept_language)
+        messages = Messages(T)
+        messages.update({'email_sent':'Email sent', 'test': "ä"})
+        self.assertEqual(to_unicode(messages.email_sent, 'utf-8'), 'Email sent')
+
+class TestHTMLTag(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.isdir('gluon'):
+            self.langpath = 'applications/welcome/languages'
+        else:
+            self.langpath = os.path.realpath(
+                '../../applications/welcome/languages')
+        self.http_accept_language = 'en'
+
+    def tearDown(self):
+        pass
+
+    def test_decode(self):
+        T = languages.translator(self.langpath, self.http_accept_language)
+        elem = SPAN(T("Complete"))
+        self.assertEqual(elem.flatten(), "Complete")
+        elem = SPAN(T("Cannot be empty", language="ru"))
+        self.assertEqual(elem.xml(), to_bytes('<span>Пустое значение недопустимо</span>'))
+        self.assertEqual(elem.flatten(), 'Пустое значение недопустимо')