Repository: google/mobly

Issue:
[Py2] Unicode is not properly handled in string conversions
In Py2, we sometimes get:
```
...
mobly/records.py", line 102, in _test_end
I 16:41:23:508 UTC     self.details = str(e)
I 16:41:23:508 UTC UnicodeEncodeError: 'ascii' codec can't encode character u'\u2022' in position 32280: ordinal not in range(128)
```
This is caused by `str` not able to handle unicode properly in py2.


Discussion:
This seems to solve the problem for all cases:
```
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
```
That's a generally frowned upon approach.
What's a better way? Care to share some examples? :)
https://stackoverflow.com/questions/3828723/why-should-we-not-use-sys-setdefaultencodingutf-8-in-a-py-script
That explains why not to use it, which I understand.
What alternatives do you suggest? Because I don't see an easy way to solve this for all places.
A better question: how should we handle unicode for py2?
Do we want to convert them into ascii which makes them lose meaning?
Or do we want to try to keep them as unicode text in our outputs?
Why not use sys.getdefaultencoding? I don't think we should be focing either way.
"sys.getdefaultencoding() is always 'ascii' on all systems in Python 2"
I'm not sure how that API solves our problem here. Can you be more explicit?
That does not seem to answer the question "how should we handle unicode for py2"
If the user sets a systemwide default encoding in py2 to something other than ascii we should be using that, and that would be determined by getdefaultencoding. If they don't touch it and it is ascii then convert to ascii. The only thing I am asserting is that Mobly should not be dictating (and changing) system wide encoding.
Does that mean the test that reproduces this issue has the default encoding changed?
Can you confirm that?
The test did not change it. It's just that the str() op is assuming ascii and trying to deal with a unicode character that came from the snippet?
If the unicode is coming from the Java side, how does calling `sys.getdefaultencoding()` on the host help us here? It'll only say the current system is 'ascii', then what do we do?
It (sys.getdefaultencoding()) will tell us that str() expects ascii and we can either check and strip unicode beforehand, or you could wrap the str() in a try.

Resolving patch:
diff --git a/mobly/records.py b/mobly/records.py
--- a/mobly/records.py
+++ b/mobly/records.py
@@ -100,7 +100,12 @@ def dump(self, content, entry_type):
         new_content['Type'] = entry_type.value
         # Use safe_dump here to avoid language-specific tags in final output.
         with open(self._path, 'a') as f:
-            yaml.safe_dump(new_content, f, explicit_start=True, indent=4)
+            yaml.safe_dump(
+                new_content,
+                f,
+                explicit_start=True,
+                allow_unicode=True,
+                indent=4)
 
 
 class TestResultEnums(object):
@@ -159,10 +164,30 @@ def __init__(self, e, position=None):
                 traceback.format_exception(e.__class__, e, exc_traceback))
         # Populate fields based on the type of the termination signal.
         if self.is_test_signal:
-            self.details = str(e.details)
+            self._set_details(e.details)
             self.extras = e.extras
         else:
-            self.details = str(e)
+            self._set_details(e)
+
+    def _set_details(self, content):
+        """Sets the `details` field.
+
+        Args:
+            content: the content to extract details from.
+        """
+        try:
+            self.details = str(content)
+        except UnicodeEncodeError:
+            if sys.version_info < (3, 0):
+                # If Py2 threw encode error, convert to unicode.
+                self.details = unicode(content)
+            else:
+                # We should never hit this in Py3, if this happens, record
+                # an encoded version of the content for users to handle.
+                logging.error(
+                    'Unable to decode "%s" in Py3, encoding in utf-8.',
+                    content)
+                self.details = content.encode('utf-8')
 
     def to_dict(self):
         result = {}
diff --git a/mobly/signals.py b/mobly/signals.py
--- a/mobly/signals.py
+++ b/mobly/signals.py
@@ -34,7 +34,7 @@ class TestSignal(Exception):
 
     def __init__(self, details, extras=None):
         super(TestSignal, self).__init__(details)
-        self.details = str(details)
+        self.details = details
         try:
             json.dumps(extras)
             self.extras = extras


Test patch:
diff --git a/tests/lib/integration_test.py b/tests/lib/integration_test.py
--- a/tests/lib/integration_test.py
+++ b/tests/lib/integration_test.py
@@ -1,13 +1,13 @@
 # Copyright 2016 Google Inc.
-# 
-# Licensed under the Apache License, Version 2.0 (the "License");
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
+# distributed under the License is distributed on an 'AS IS' BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
@@ -26,12 +26,16 @@ def setup_class(self):
         self.register_controller(mock_controller)
 
     def test_hello_world(self):
-        asserts.assert_equal(self.user_params["icecream"], 42)
-        asserts.assert_equal(self.user_params["extra_param"], "haha")
-        logging.info("This is a bare minimal test to make sure the basic MOBLY"
-                     "test flow works.")
-        asserts.explicit_pass("Hello World")
-
-
-if __name__ == "__main__":
+        asserts.assert_equal(self.user_params['icecream'], 42)
+        asserts.assert_equal(self.user_params['extra_param'], 'haha')
+        logging.info('This is a bare minimal test to make sure the basic MOBLY'
+                     ' test flow works.')
+        asserts.explicit_pass(
+            'Hello World',
+            # Use a unicode string here to make sure the full log pipeline
+            # works with unicode.
+            extras=u'\u2022')
+
+
+if __name__ == '__main__':
     test_runner.main()
diff --git a/tests/mobly/records_test.py b/tests/mobly/records_test.py
--- a/tests/mobly/records_test.py
+++ b/tests/mobly/records_test.py
@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import str
+
 import copy
+import mock
 import os
 import shutil
 import tempfile
@@ -162,6 +165,30 @@ def test_result_record_fail_with_float_extra(self):
             details=self.details,
             extras=self.float_extra)
 
+    def test_result_record_fail_with_unicode_test_signal(self):
+        record = records.TestResultRecord(self.tn)
+        record.test_begin()
+        details = u'\u2022'
+        s = signals.TestFailure(details, self.float_extra)
+        record.test_fail(s)
+        self.verify_record(
+            record=record,
+            result=records.TestResultEnums.TEST_RESULT_FAIL,
+            details=details,
+            extras=self.float_extra)
+
+    def test_result_record_fail_with_unicode_exception(self):
+        record = records.TestResultRecord(self.tn)
+        record.test_begin()
+        details = u'\u2022'
+        s = Exception(details)
+        record.test_fail(s)
+        self.verify_record(
+            record=record,
+            result=records.TestResultEnums.TEST_RESULT_FAIL,
+            details=details,
+            extras=None)
+
     def test_result_record_fail_with_json_extra(self):
         record = records.TestResultRecord(self.tn)
         record.test_begin()
@@ -344,6 +371,23 @@ def test_exception_record_deepcopy(self):
         self.assertIsNot(er, new_er)
         self.assertDictEqual(er.to_dict(), new_er.to_dict())
 
+    def test_add_controller_info(self):
+        tr = records.TestResult()
+        self.assertFalse(tr.controller_info)
+        tr.add_controller_info('MockDevice', ['magicA', 'magicB'])
+        self.assertTrue(tr.controller_info)
+        self.assertEqual(tr.controller_info['MockDevice'],
+                         ['magicA', 'magicB'])
+
+    @mock.patch('yaml.dump', side_effect=TypeError('ha'))
+    def test_add_controller_info_not_serializable(self, mock_yaml_dump):
+        tr = records.TestResult()
+        self.assertFalse(tr.controller_info)
+        tr.add_controller_info('MockDevice', ['magicA', 'magicB'])
+        self.assertTrue(tr.controller_info)
+        self.assertEqual(tr.controller_info['MockDevice'],
+                         "['magicA', 'magicB']")
+
 
 if __name__ == "__main__":
     unittest.main()