Repository: google/mobly

Issue:
Include stack traces of extra errors
Right now only the stack trace for the main error is recorded in `TeatResultRecord`.
We should store the stack traces for extra errors as well.


Discussion:

Resolving patch:
diff --git a/mobly/records.py b/mobly/records.py
--- a/mobly/records.py
+++ b/mobly/records.py
@@ -14,6 +14,7 @@
 """This module has classes for test result collection, and test result output.
 """
 
+import collections
 import itertools
 import copy
 import enum
@@ -119,23 +120,86 @@ class TestResultEnums(object):
     RECORD_EXTRA_ERRORS = 'Extra Errors'
     RECORD_DETAILS = 'Details'
     RECORD_STACKTRACE = 'Stacktrace'
+    RECORD_POSITION = 'Position'
     TEST_RESULT_PASS = 'PASS'
     TEST_RESULT_FAIL = 'FAIL'
     TEST_RESULT_SKIP = 'SKIP'
     TEST_RESULT_ERROR = 'ERROR'
 
 
+class ExceptionRecord(object):
+    """A wrapper class for representing exception objects in TestResultRecord.
+
+    Attributes:
+        exception: Exception object, the original Exception.
+        stacktrace: string, stacktrace of the Exception.
+        extras: optional serializable, this corresponds to the
+            `TestSignal.extras` field.
+        position: string, an optional label specifying the position where the
+            Exception ocurred.
+    """
+
+    def __init__(self, e, position=None):
+        self.exception = e
+        self.stacktrace = None
+        self.extras = None
+        self.position = position
+        self.is_test_signal = isinstance(e, signals.TestSignal)
+        # Record stacktrace of the exception.
+        # This check cannot be based on try...except, which messes up
+        # `exc_info`.
+        if hasattr(e, '__traceback__'):
+            exc_traceback = e.__traceback__
+        else:
+            # In py2, exception objects don't have built-in traceback, so we
+            # have to immediately retrieve stacktrace from `sys.exc_info`.
+            _, _, exc_traceback = sys.exc_info()
+        if exc_traceback:
+            self.stacktrace = ''.join(
+                traceback.format_exception(e.__class__, e, exc_traceback))
+        # Populate fields based on the type of the termination signal.
+        if self.is_test_signal:
+            self.details = str(e.details)
+            self.extras = e.extras
+        else:
+            self.details = str(e)
+
+    def to_dict(self):
+        result = {}
+        result[TestResultEnums.RECORD_DETAILS] = self.details
+        result[TestResultEnums.RECORD_POSITION] = self.position
+        result[TestResultEnums.RECORD_STACKTRACE] = self.stacktrace
+        result[TestResultEnums.RECORD_EXTRAS] = copy.deepcopy(self.extras)
+        return result
+
+
 class TestResultRecord(object):
-    """A record that holds the information of a test execution.
+    """A record that holds the information of a single test.
+
+    The record object holds all information of a test, including all the
+    exceptions occurred during the test.
+
+    A test can terminate for two reasons:
+      1. the test function executes to the end and completes naturally.
+      2. the test is terminated by an exception, which we call
+         "termination signal".
+
+    The termination signal is treated differently. Its content are extracted
+    into first-tier attributes of the record object, like `details` and
+    `stacktrace`, for easy consumption.
+
+    Note the termination signal is not always an error, it can also be explicit
+    pass signal or abort/skip signals.
 
     Attributes:
-        test_name: A string representing the name of the test method.
+        test_name: string, the name of the test.
         begin_time: Epoch timestamp of when the test started.
         end_time: Epoch timestamp of when the test ended.
-        self.uid: Unique identifier of a test.
-        self.result: Test result, PASS/FAIL/SKIP.
-        self.extras: User defined extra information of the test result.
-        self.details: A string explaining the details of the test.
+        uid: Unique identifier of a test.
+        termination_signal: ExceptionRecord, the main exception of the test.
+        extra_errors: OrderedDict, all exceptions occurred during the entire
+            test lifecycle. The order of occurrence is preserved.
+        result: TestResultEnum.TEAT_RESULT_*, PASS/FAIL/SKIP.
     """
 
     def __init__(self, t_name, t_class=None):
@@ -144,11 +208,35 @@ def __init__(self, t_name, t_class=None):
         self.begin_time = None
         self.end_time = None
         self.uid = None
+        self.termination_signal = None
+        self.extra_errors = collections.OrderedDict()
         self.result = None
-        self.extras = None
-        self.details = None
-        self.stacktrace = None
-        self.extra_errors = {}
+
+    @property
+    def details(self):
+        """String description of the cause of the test's termination.
+
+        Note a passed test can have this as well due to the explicit pass
+        signal. If the test passed implicitly, this field would be None.
+        """
+        if self.termination_signal:
+            return self.termination_signal.details
+
+    @property
+    def stacktrace(self):
+        """The stacktrace string for the exception that terminated the test.
+        """
+        if self.termination_signal:
+            return self.termination_signal.stacktrace
+
+    @property
+    def extras(self):
+        """User defined extra information of the test result.
+
+        Must be serializable.
+        """
+        if self.termination_signal:
+            return self.termination_signal.extras
 
     def test_begin(self):
         """Call this when the test begins execution.
@@ -158,7 +246,7 @@ def test_begin(self):
         self.begin_time = utils.get_current_epoch_time()
 
     def _test_end(self, result, e):
-        """Class internal function to signal the end of a test execution.
+        """Marks the end of the test logic.
 
         Args:
             result: One of the TEST_RESULT enums in TestResultEnums.
@@ -169,19 +257,25 @@ def _test_end(self, result, e):
         if self.begin_time is not None:
             self.end_time = utils.get_current_epoch_time()
         self.result = result
+        if e:
+            self.termination_signal = ExceptionRecord(e)
+
+    def update_record(self):
+        """Updates the content of a record.
+
+        Several display fields like "details" and "stacktrace" need to be
+        updated based on the content of the record object.
+
+        As the content of the record change, call this method to update all
+        the appropirate fields.
+        """
         if self.extra_errors:
-            self.result = TestResultEnums.TEST_RESULT_ERROR
-        if isinstance(e, signals.TestSignal):
-            self.details = e.details
-            _, _, exc_traceback = sys.exc_info()
-            if exc_traceback:
-                self.stacktrace = ''.join(traceback.format_tb(exc_traceback))
-            self.extras = e.extras
-        elif isinstance(e, Exception):
-            self.details = str(e)
-            _, _, exc_traceback = sys.exc_info()
-            if exc_traceback:
-                self.stacktrace = ''.join(traceback.format_tb(exc_traceback))
+            if self.result != TestResultEnums.TEST_RESULT_FAIL:
+                self.result = TestResultEnums.TEST_RESULT_ERROR
+        # If no termination signal is provided, use the first exception
+        # occurred as the termination signal.
+        if not self.termination_signal and self.extra_errors:
+            _, self.termination_signal = self.extra_errors.popitem(last=False)
 
     def test_pass(self, e=None):
         """To mark the test as passed in this record.
@@ -219,19 +313,29 @@ def test_error(self, e=None):
         """
         self._test_end(TestResultEnums.TEST_RESULT_ERROR, e)
 
-    def add_error(self, tag, e):
-        """Add extra error happened during a test mark the test result as
+    def add_error(self, position, e):
+        """Add extra error happened during a test.
+
+        If the test has passed or skipped, this will mark the test result as
         ERROR.
 
         If an error is added the test record, the record's result is equivalent
         to the case where an uncaught exception happened.
 
+        If the test record has not recorded any error, the newly added error
+        would be the main error of the test record. Otherwise the newly added
+        error is added to the record's extra errors.
+
         Args:
-            tag: A string describing where this error came from, e.g. 'on_pass'.
+            position: string, where this error occurred, e.g. 'teardown_test'.
             e: An exception object.
         """
-        self.result = TestResultEnums.TEST_RESULT_ERROR
-        self.extra_errors[tag] = str(e)
+        if self.result != TestResultEnums.TEST_RESULT_FAIL:
+            self.result = TestResultEnums.TEST_RESULT_ERROR
+        if position in self.extra_errors:
+            raise Error('An exception is already recorded with position "%s",'
+                        ' cannot reuse.' % position)
+        self.extra_errors[position] = ExceptionRecord(e, position=position)
 
     def __str__(self):
         d = self.to_dict()
@@ -259,7 +363,9 @@ def to_dict(self):
         d[TestResultEnums.RECORD_UID] = self.uid
         d[TestResultEnums.RECORD_EXTRAS] = self.extras
         d[TestResultEnums.RECORD_DETAILS] = self.details
-        d[TestResultEnums.RECORD_EXTRA_ERRORS] = self.extra_errors
+        d[TestResultEnums.RECORD_EXTRA_ERRORS] = [
+            e.to_dict() for e in self.extra_errors.values()
+        ]
         d[TestResultEnums.RECORD_STACKTRACE] = self.stacktrace
         return d
 
@@ -340,9 +446,13 @@ def add_record(self, record):
 
         A record is considered executed once it's added to the test result.
 
+        Adding the record finalizes the content of a record, so no change
+        should be made to the record afterwards.
+
         Args:
             record: A test record object to add.
         """
+        record.update_record()
         if record.result == TestResultEnums.TEST_RESULT_SKIP:
             self.skipped.append(record)
             return
@@ -375,6 +485,7 @@ def add_class_error(self, test_record):
         Args:
             test_record: A TestResultRecord object for the test class.
         """
+        test_record.update_record()
         self.error.append(test_record)
 
     def is_test_executed(self, test_name):


Test patch:
diff --git a/mobly/base_test.py b/mobly/base_test.py
--- a/mobly/base_test.py
+++ b/mobly/base_test.py
@@ -301,9 +301,9 @@ def _exec_procedure_func(self, func, tr_record):
                 executed.
         """
         try:
-          # Pass a copy of the record instead of the actual object so that it
-          # will not be modified.
-          func(copy.deepcopy(tr_record))
+            # Pass a copy of the record instead of the actual object so that it
+            # will not be modified.
+            func(copy.deepcopy(tr_record))
         except signals.TestAbortAll:
             raise
         except Exception as e:
@@ -327,6 +327,7 @@ def exec_one_test(self, test_name, test_method, args=(), **kwargs):
         tr_record = records.TestResultRecord(test_name, self.TAG)
         tr_record.test_begin()
         logging.info('%s %s', TEST_CASE_TOKEN, test_name)
+        teardown_test_failed = False
         try:
             try:
                 self._setup_test(test_name)
@@ -348,6 +349,7 @@ def exec_one_test(self, test_name, test_method, args=(), **kwargs):
                 except Exception as e:
                     logging.exception(e)
                     tr_record.add_error('teardown_test', e)
+                    teardown_test_failed = True
         except (signals.TestFailure, AssertionError) as e:
             tr_record.test_fail(e)
         except signals.TestSkip as e:
@@ -364,8 +366,10 @@ def exec_one_test(self, test_name, test_method, args=(), **kwargs):
             # Exception happened during test.
             tr_record.test_error(e)
         else:
-            tr_record.test_pass()
+            if not teardown_test_failed:
+                tr_record.test_pass()
         finally:
+            tr_record.update_record()
             if tr_record.result in (records.TestResultEnums.TEST_RESULT_ERROR,
                                     records.TestResultEnums.TEST_RESULT_FAIL):
                 self._exec_procedure_func(self._on_fail, tr_record)
diff --git a/tests/mobly/base_test_test.py b/tests/mobly/base_test_test.py
--- a/tests/mobly/base_test_test.py
+++ b/tests/mobly/base_test_test.py
@@ -250,7 +250,7 @@ def test_something(self):
     def test_teardown_test_assert_fail(self):
         class MockBaseTest(base_test.BaseTestClass):
             def teardown_test(self):
-                asserts.assert_true(False, MSG_EXPECTED_EXCEPTION)
+                asserts.fail(MSG_EXPECTED_EXCEPTION)
 
             def test_something(self):
                 pass
@@ -259,7 +259,7 @@ def test_something(self):
         bt_cls.run()
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
-        self.assertIsNone(actual_record.details)
+        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
@@ -277,10 +277,9 @@ def test_something(self):
         bt_cls.run()
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
-        self.assertIsNone(actual_record.details)
+        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
-        expected_extra_error = {"teardown_test": MSG_EXPECTED_EXCEPTION}
-        self.assertEqual(actual_record.extra_errors, expected_extra_error)
+        self.assertFalse(actual_record.extra_errors)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
@@ -372,10 +371,10 @@ def test_something(self):
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
-        my_mock.assert_called_once_with("on_fail")
+        my_mock.assert_called_once_with('on_fail')
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
-        self.assertIsNone(actual_record.details)
+        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
@@ -463,8 +462,8 @@ def test_something(self):
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
-        self.assertEqual(actual_record.extra_errors,
-                         {'teardown_test': 'This is an expected exception.ha'})
+        self.assertEqual(actual_record.extra_errors['teardown_test'].details,
+                         'This is an expected exception.ha')
         self.assertIsNone(actual_record.extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
@@ -504,12 +503,12 @@ def test_something(self):
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
-        actual_record = bt_cls.results.error[0]
+        actual_record = bt_cls.results.failed[0]
         self.assertIn('_on_fail', actual_record.extra_errors)
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
-        expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
+        expected_summary = ("Error 0, Executed 1, Failed 1, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
 
@@ -526,8 +525,8 @@ def test_something(self):
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
         actual_record = bt_cls.results.error[0]
-        expected_extra_error = {'_on_pass': expected_msg}
-        self.assertEqual(actual_record.extra_errors, expected_extra_error)
+        self.assertEqual(actual_record.extra_errors['_on_pass'].details,
+                         expected_msg)
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertIsNone(actual_record.extras)
@@ -549,12 +548,58 @@ def test_something(self):
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, "Test Body Exception.")
         self.assertIsNone(actual_record.extras)
-        self.assertEqual(actual_record.extra_errors["teardown_test"],
-                         "Details=This is an expected exception., Extras=None")
+        self.assertEqual(actual_record.extra_errors['teardown_test'].details,
+                         MSG_EXPECTED_EXCEPTION)
+        self.assertIsNone(actual_record.extra_errors['teardown_test'].extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
 
+    def test_exception_objects_in_record(self):
+        """Checks that the exception objects are correctly tallied.
+        """
+        expected_termination_signal = Exception('Test Body Exception.')
+        expected_extra_error = Exception('teardown_test Exception.')
+
+        class MockBaseTest(base_test.BaseTestClass):
+            def teardown_test(self):
+                raise expected_extra_error
+
+            def test_something(self):
+                raise expected_termination_signal
+
+        bt_cls = MockBaseTest(self.mock_test_cls_configs)
+        bt_cls.run()
+        actual_record = bt_cls.results.error[0]
+        self.assertIs(actual_record.termination_signal.exception,
+                      expected_termination_signal)
+        self.assertIsNotNone(actual_record.termination_signal.stacktrace)
+        self.assertEqual(len(actual_record.extra_errors), 1)
+        extra_error = actual_record.extra_errors['teardown_test']
+        self.assertIs(extra_error.exception, expected_extra_error)
+        self.assertIsNotNone(extra_error.stacktrace)
+        self.assertIsNone(actual_record.extras)
+
+    def test_promote_extra_errors_to_termination_signal(self):
+        """If no termination singal is specified, use the first extra error as
+        the termination signal.
+        """
+        expected_extra_error = Exception('teardown_test Exception.')
+
+        class MockBaseTest(base_test.BaseTestClass):
+            def teardown_test(self):
+                raise expected_extra_error
+
+            def test_something(self):
+                pass
+
+        bt_cls = MockBaseTest(self.mock_test_cls_configs)
+        bt_cls.run()
+        actual_record = bt_cls.results.error[0]
+        self.assertFalse(actual_record.extra_errors)
+        self.assertEqual(actual_record.details, 'teardown_test Exception.')
+        self.assertIsNotNone(actual_record.stacktrace)
+
     def test_explicit_pass_but_teardown_test_raises_an_exception(self):
         """Test record result should be marked as ERROR as opposed to PASS.
         """
@@ -564,16 +609,17 @@ def teardown_test(self):
                 asserts.assert_true(False, MSG_EXPECTED_EXCEPTION)
 
             def test_something(self):
-                asserts.explicit_pass("Test Passed!")
+                asserts.explicit_pass('Test Passed!')
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
         actual_record = bt_cls.results.error[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
-        self.assertEqual(actual_record.details, "Test Passed!")
+        self.assertEqual(actual_record.details, 'Test Passed!')
         self.assertIsNone(actual_record.extras)
-        self.assertEqual(actual_record.extra_errors["teardown_test"],
-                         "Details=This is an expected exception., Extras=None")
+        self.assertEqual(actual_record.extra_errors['teardown_test'].details,
+                         MSG_EXPECTED_EXCEPTION)
+        self.assertIsNone(actual_record.extra_errors['teardown_test'].extras)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
@@ -606,8 +652,8 @@ def test_something(self):
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertEqual(actual_record.extras, MOCK_EXTRA)
-        self.assertEqual(actual_record.extra_errors,
-                         {'_on_pass': MSG_EXPECTED_EXCEPTION})
+        self.assertEqual(actual_record.extra_errors['_on_pass'].details,
+                         MSG_EXPECTED_EXCEPTION)
         expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
@@ -622,14 +668,13 @@ def test_something(self):
 
         bt_cls = MockBaseTest(self.mock_test_cls_configs)
         bt_cls.run()
-        actual_record = bt_cls.results.error[0]
-        self.assertEqual(bt_cls.results.failed, [])
+        actual_record = bt_cls.results.failed[0]
         self.assertEqual(actual_record.test_name, self.mock_test_name)
         self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
         self.assertEqual(actual_record.extras, MOCK_EXTRA)
-        self.assertEqual(actual_record.extra_errors,
-                         {'_on_fail': MSG_EXPECTED_EXCEPTION})
-        expected_summary = ("Error 1, Executed 1, Failed 0, Passed 0, "
+        self.assertEqual(actual_record.extra_errors['_on_fail'].details,
+                         MSG_EXPECTED_EXCEPTION)
+        expected_summary = ("Error 0, Executed 1, Failed 1, Passed 0, "
                             "Requested 1, Skipped 0")
         self.assertEqual(bt_cls.results.summary_str(), expected_summary)
 
diff --git a/tests/mobly/records_test.py b/tests/mobly/records_test.py
--- a/tests/mobly/records_test.py
+++ b/tests/mobly/records_test.py
@@ -28,6 +28,7 @@
 class RecordsTest(unittest.TestCase):
     """This test class tests the implementation of classes in mobly.records.
     """
+
     def setUp(self):
         self.tn = "test_name"
         self.details = "Some details about the test execution."
@@ -38,7 +39,8 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.tmp_path)
 
-    def verify_record(self, record, result, details, extras):
+    def verify_record(self, record, result, details, extras, stacktrace=None):
+        record.update_record()
         # Verify each field.
         self.assertEqual(record.test_name, self.tn)
         self.assertEqual(record.result, result)
@@ -58,9 +60,15 @@ def verify_record(self, record, result, details, extras):
         d[records.TestResultEnums.RECORD_END_TIME] = record.end_time
         d[records.TestResultEnums.RECORD_UID] = None
         d[records.TestResultEnums.RECORD_CLASS] = None
-        d[records.TestResultEnums.RECORD_EXTRA_ERRORS] = {}
-        d[records.TestResultEnums.RECORD_STACKTRACE] = None
+        d[records.TestResultEnums.RECORD_EXTRA_ERRORS] = []
+        d[records.TestResultEnums.RECORD_STACKTRACE] = stacktrace
         actual_d = record.to_dict()
+        # Verify stacktrace partially match as stacktraces often have file path
+        # in them.
+        if stacktrace:
+            stacktrace_key = records.TestResultEnums.RECORD_STACKTRACE
+            self.assertTrue(
+                d.pop(stacktrace_key) in actual_d.pop(stacktrace_key))
         self.assertDictEqual(actual_d, d)
         # Verify that these code paths do not cause crashes and yield non-empty
         # results.
@@ -123,16 +131,14 @@ def test_result_record_fail_stacktrace(self):
         # Verify stacktrace separately if we expect a non-None value.
         # Because stacktrace includes file names and line numbers, we can't do
         # a simple equality check.
-        self.assertTrue('Something failed.' in record.stacktrace)
-        self.assertTrue(
-            'in test_result_record_fail_stacktrace\n    raise Exception' in
-            record.stacktrace)
-        record.stacktrace = None
         self.verify_record(
             record=record,
             result=records.TestResultEnums.TEST_RESULT_FAIL,
             details='Something failed.',
-            extras=None)
+            extras=None,
+            stacktrace='in test_result_record_fail_stacktrace\n    '
+            'raise Exception(\'Something failed.\')\nException: '
+            'Something failed.\n')
 
     def test_result_record_fail_with_float_extra(self):
         record = records.TestResultRecord(self.tn)