Repository: google/mobly

Issue:
New summary output.
The current summary log has limited debug information and is in json.

We need to consider what other useful information we can include in the summary log.
Like information on the invocation status, and device monitoring info.

Also consider using a different format so it's possible to stream this file instead of only writing it out in the end.
This will make the summary log more crash proof and takes less memory.


Discussion:
E.g. AndroidDevice should probably dump adb version used in test
Duplicate of #81 
Will address the issue raised in #81 in this as well.

Resolving patch:
diff --git a/mobly/config_parser.py b/mobly/config_parser.py
--- a/mobly/config_parser.py
+++ b/mobly/config_parser.py
@@ -37,8 +37,8 @@ def _validate_test_config(test_config):
     """
     required_key = keys.Config.key_testbed.value
     if required_key not in test_config:
-        raise MoblyConfigError('Required key %s missing in test config.' %
-                               required_key)
+        raise MoblyConfigError(
+            'Required key %s missing in test config.' % required_key)
 
 
 def _validate_testbed_name(name):
@@ -109,8 +109,8 @@ def load_test_config_file(test_config_path, tb_filters=None):
         if len(tbs) != len(tb_filters):
             raise MoblyConfigError(
                 'Expect to find %d test bed configs, found %d. Check if'
-                ' you have the correct test bed names.' %
-                (len(tb_filters), len(tbs)))
+                ' you have the correct test bed names.' % (len(tb_filters),
+                                                           len(tbs)))
         configs[keys.Config.key_testbed.value] = tbs
     mobly_params = configs.get(keys.Config.key_mobly_params.value, {})
     # Decide log path.
@@ -166,6 +166,8 @@ class TestRunConfig(object):
         user_params: dict, all the parameters to be consumed by the test logic.
         register_controller: func, used by test classes to register controller
                              modules.
+        summary_writer: records.TestSummaryWriter, used to write elements to
+                        the test result summary file.
     """
 
     def __init__(self):
@@ -174,6 +176,7 @@ def __init__(self):
         self.controller_configs = None
         self.user_params = None
         self.register_controller = None
+        self.summary_writer = None
 
     def copy(self):
         """Returns a deep copy of the current config.
diff --git a/mobly/logger.py b/mobly/logger.py
--- a/mobly/logger.py
+++ b/mobly/logger.py
@@ -20,6 +20,7 @@
 import re
 import sys
 
+from mobly import records
 from mobly import utils
 
 log_line_format = '%(asctime)s.%(msecs).03d %(levelname)s %(message)s'
@@ -162,11 +163,24 @@ def _setup_test_logger(log_path, prefix=None, filename=None):
     if filename is None:
         filename = get_log_file_timestamp()
         utils.create_dir(log_path)
+    # TODO(#270): Deprecate `test_run_details.txt` when we remove old output
+    # format support.
     fh = logging.FileHandler(os.path.join(log_path, 'test_run_details.txt'))
     fh.setFormatter(f_formatter)
     fh.setLevel(logging.DEBUG)
-    log.addHandler(ch)
     log.addHandler(fh)
+    # Write logger output to files
+    fh_info = logging.FileHandler(
+        os.path.join(log_path, records.OUTPUT_FILE_INFO_LOG))
+    fh_info.setFormatter(f_formatter)
+    fh_info.setLevel(logging.INFO)
+    fh_debug = logging.FileHandler(
+        os.path.join(log_path, records.OUTPUT_FILE_DEBUG_LOG))
+    fh_debug.setFormatter(f_formatter)
+    fh_debug.setLevel(logging.DEBUG)
+    log.addHandler(ch)
+    log.addHandler(fh_info)
+    log.addHandler(fh_debug)
     log.log_path = log_path
     logging.log_path = log_path
 
@@ -224,4 +238,3 @@ def normalize_log_line_timestamp(log_line_timestamp):
     norm_tp = log_line_timestamp.replace(' ', '_')
     norm_tp = norm_tp.replace(':', '-')
     return norm_tp
-
diff --git a/mobly/records.py b/mobly/records.py
--- a/mobly/records.py
+++ b/mobly/records.py
@@ -11,19 +11,88 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""This module is where all the record definitions and record containers live.
+"""This module has classes for test result collection, and test result output.
 """
 
 import itertools
+import copy
+import enum
 import json
 import logging
 import pprint
 import sys
 import traceback
+import yaml
 
 from mobly import signals
 from mobly import utils
 
+# File names for the default files output by 
+OUTPUT_FILE_INFO_LOG = 'test_log.INFO'
+OUTPUT_FILE_DEBUG_LOG = 'test_log.DEBUG'
+OUTPUT_FILE_SUMMARY = 'test_summary.yaml'
+
+
+class TestSummaryEntryType(enum.Enum):
+    """Constants used to identify the type of entries in test summary file.
+
+    Test summary file contains multiple yaml documents. In order to parse this
+    file efficiently, the write adds the type of each entry when it writes the
+    entry to the file.
+
+    The idea is similar to how `TestResult.json_str` categorizes different
+    sections of a `TestResult` object in the serialized format.
+    """
+    RECORD = 'Record'
+    SUMMARY = 'Summary'
+    CONTROLLER_INFO = 'ControllerInfo'
+
+
+class Error(Exception):
+    """Raised for errors in records."""
+
+
+class TestSummaryWriter(object):
+    """Writer for the test result summary file of a test run.
+
+    For each test run, a writer is created to stream test results to the
+    summary file on disk.
+
+    The serialization and writing of the `TestResult` object is intentionally
+    kept out of `TestResult` class and put in this class. Because `TestResult`
+    can be operated on by suites, like `+` operation, and it is difficult to
+    guarantee the consistency between `TestResult` in memory and the files on
+    disk. Also, this separation makes it easier to provide a more generic way
+    for users to consume the test summary, like via a database instead of a
+    file.
+    """
+
+    def __init__(self, path):
+        self._path = path
+
+    def dump(self, content, entry_type):
+        """Dumps a dictionary as a yaml document to the summary file.
+
+        Each call to this method dumps a separate yaml document to the same
+        summary file associated with a test run.
+
+        The content of the dumped dictionary has an extra field `TYPE` that
+        specifies the type of each yaml document, which is the flag for parsers
+        to identify each document.
+
+        Args:
+            content: dictionary, the content to serialize and write.
+            entry_type: a member of enum TestSummaryEntryType.
+
+        Raises:
+            recoreds.Error is raised if an invalid entry type is passed in.
+        """
+        new_content = copy.deepcopy(content)
+        new_content['Type'] = entry_type.value
+        content_str = yaml.dump(new_content, explicit_start=True, indent=4)
+        with open(self._path, 'a') as f:
+            f.write(content_str)
+
 
 class TestResultEnums(object):
     """Enums used for TestResultRecord class.
@@ -188,6 +257,8 @@ def to_dict(self):
     def json_str(self):
         """Converts this test record to a string in json format.
 
+        TODO(#270): Deprecate with old output format.
+
         Format of the json string is:
             {
                 'Test Name': <test name>,
@@ -251,7 +322,7 @@ def __add__(self, r):
                 # '+' operator for TestResult is only valid when multiple
                 # TestResult objs were created in the same test run, which means
                 # the controller info would be the same across all of them.
-                # TODO(angli): have a better way to validate this situation.
+                # TODO(xpconanfan): have a better way to validate this situation.
                 setattr(sum_result, name, l_value)
         return sum_result
 
@@ -276,9 +347,9 @@ def add_record(self, record):
 
     def add_controller_info(self, name, info):
         try:
-            json.dumps(info)
+            yaml.dump(info)
         except TypeError:
-            logging.warning('Controller info for %s is not JSON serializable!'
+            logging.warning('Controller info for %s is not YAML serializable!'
                             ' Coercing it to string.' % name)
             self.controller_info[name] = str(info)
             return
@@ -323,6 +394,8 @@ def is_all_pass(self):
     def json_str(self):
         """Converts this test result to a string in json format.
 
+        TODO(#270): Deprecate with old output format.
+
         Format of the json string is:
             {
                 'Results': [


Test patch:
diff --git a/mobly/base_test.py b/mobly/base_test.py
--- a/mobly/base_test.py
+++ b/mobly/base_test.py
@@ -87,6 +87,7 @@ def __init__(self, configs):
         self.user_params = configs.user_params
         self.register_controller = configs.register_controller
         self.results = records.TestResult()
+        self.summary_writer = configs.summary_writer
         self.current_test_name = None
         self._generated_test_table = collections.OrderedDict()
 
@@ -383,6 +384,8 @@ def exec_one_test(self, test_name, test_method, args=(), **kwargs):
             elif tr_record.result == records.TestResultEnums.TEST_RESULT_SKIP:
                 self._exec_procedure_func(self._on_skip, tr_record)
             self.results.add_record(tr_record)
+            self.summary_writer.dump(tr_record.to_dict(),
+                                     records.TestSummaryEntryType.RECORD)
 
     def _assert_function_name_in_stack(self, expected_func_name):
         """Asserts that the current stack contains the given function name."""
@@ -504,6 +507,8 @@ def _skip_remaining_tests(self, exception):
                 test_record = records.TestResultRecord(test_name, self.TAG)
                 test_record.test_skip(exception)
                 self.results.add_record(test_record)
+                self.summary_writer.dump(test_record.to_dict(),
+                                         records.TestSummaryEntryType.RECORD)
 
     def run(self, test_names=None):
         """Runs tests within a test class.
@@ -534,6 +539,8 @@ def run(self, test_names=None):
             class_record.test_begin()
             class_record.test_error(e)
             self.results.add_class_error(class_record)
+            self.summary_writer.dump(class_record.to_dict(),
+                                     records.TestSummaryEntryType.RECORD)
             return self.results
         logging.info('==========> %s <==========', self.TAG)
         # Devise the actual test methods to run in the test class.
@@ -565,6 +572,8 @@ def run(self, test_names=None):
             class_record.test_error(e)
             self._exec_procedure_func(self._on_fail, class_record)
             self.results.add_class_error(class_record)
+            self.summary_writer.dump(class_record.to_dict(),
+                                     records.TestSummaryEntryType.RECORD)
             self._skip_remaining_tests(e)
             self._safe_exec_func(self.teardown_class)
             return self.results
diff --git a/mobly/test_runner.py b/mobly/test_runner.py
--- a/mobly/test_runner.py
+++ b/mobly/test_runner.py
@@ -158,7 +158,7 @@ def destroy(objects):
 
         def get_info(objects):
             [Optional] Gets info from the controller objects used in a test
-            run. The info will be included in test_result_summary.json under
+            run. The info will be included in test_summary.yaml under
             the key 'ControllerInfo'. Such information could include unique
             ID, version, or anything that could be useful for describing the
             test bed and debugging.
@@ -262,9 +262,7 @@ class to execute.
                 (self._test_bed_name, config.test_bed_name))
         self._test_run_infos.append(
             TestRunner._TestRunInfo(
-                config=config,
-                test_class=test_class,
-                tests=tests))
+                config=config, test_class=test_class, tests=tests))
 
     def _run_test_class(self, config, test_class, tests=None):
         """Instantiates and executes a test class.
@@ -300,6 +298,8 @@ def run(self):
             raise Error('No tests to execute.')
         start_time = logger.get_log_file_timestamp()
         log_path = os.path.join(self._log_dir, self._test_bed_name, start_time)
+        summary_writer = records.TestSummaryWriter(
+            os.path.join(log_path, records.OUTPUT_FILE_SUMMARY))
         logger.setup_test_logger(log_path, self._test_bed_name)
         try:
             for test_run_info in self._test_run_infos:
@@ -308,7 +308,7 @@ def run(self):
                 test_config.log_path = log_path
                 test_config.register_controller = functools.partial(
                     self._register_controller, test_config)
-
+                test_config.summary_writer = summary_writer
                 try:
                     self._run_test_class(test_config, test_run_info.test_class,
                                          test_run_info.tests)
@@ -319,6 +319,11 @@ def run(self):
                 finally:
                     self._unregister_controllers()
         finally:
+            # Write controller info and summary to summary file.
+            summary_writer.dump(self.results.controller_info,
+                                records.TestSummaryEntryType.CONTROLLER_INFO)
+            summary_writer.dump(self.results.summary_dict(),
+                                records.TestSummaryEntryType.SUMMARY)
             # Stop and show summary.
             msg = '\nSummary for test run %s@%s: %s\n' % (
                 self._test_bed_name, start_time, self.results.summary_str())
@@ -438,7 +443,7 @@ def _unregister_controllers(self):
     def _write_results_json_str(self, log_path):
         """Writes out a json file with the test result info for easy parsing.
 
-        TODO(angli): This should be replaced by standard log record mechanism.
+        TODO(#270): Deprecate with old output format.
         """
         path = os.path.join(log_path, 'test_run_summary.json')
         with open(path, 'w') as f:
diff --git a/tests/mobly/base_test_test.py b/tests/mobly/base_test_test.py
--- a/tests/mobly/base_test_test.py
+++ b/tests/mobly/base_test_test.py
@@ -40,6 +40,7 @@ class SomeError(Exception):
 class BaseTestTest(unittest.TestCase):
     def setUp(self):
         self.mock_test_cls_configs = config_parser.TestRunConfig()
+        self.mock_test_cls_configs.summary_writer = mock.Mock()
         self.mock_test_cls_configs.log_path = '/tmp'
         self.mock_test_cls_configs.user_params = {"some_param": "hahaha"}
         self.mock_test_cls_configs.reporter = mock.MagicMock()
diff --git a/tests/mobly/output_test.py b/tests/mobly/output_test.py
new file mode 100755
--- /dev/null
+++ b/tests/mobly/output_test.py
@@ -0,0 +1,94 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+import unittest
+import yaml
+
+from mobly import config_parser
+from mobly import records
+from mobly import test_runner
+
+from tests.lib import mock_controller
+from tests.lib import integration_test
+
+
+class OutputTest(unittest.TestCase):
+    """This test class has unit tests for the implementation of Mobly's output
+    files.
+    """
+
+    def setUp(self):
+        self.tmp_dir = tempfile.mkdtemp()
+        self.base_mock_test_config = config_parser.TestRunConfig()
+        self.base_mock_test_config.test_bed_name = 'SampleTestBed'
+        self.base_mock_test_config.controller_configs = {}
+        self.base_mock_test_config.user_params = {
+            'icecream': 42,
+            'extra_param': 'haha'
+        }
+        self.base_mock_test_config.log_path = self.tmp_dir
+        self.log_dir = self.base_mock_test_config.log_path
+        self.test_bed_name = self.base_mock_test_config.test_bed_name
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def test_output(self):
+        """Verifies the expected output files from a test run.
+
+        * Files are correctly created.
+        * Basic sanity checks of each output file.
+        """
+        mock_test_config = self.base_mock_test_config.copy()
+        mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
+        my_config = [{
+            'serial': 'xxxx',
+            'magic': 'Magic1'
+        }, {
+            'serial': 'xxxx',
+            'magic': 'Magic2'
+        }]
+        mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
+        tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
+        tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
+        tr.run()
+        output_dir = os.path.join(self.log_dir, self.test_bed_name, 'latest')
+        summary_file_path = os.path.join(output_dir,
+                                         records.OUTPUT_FILE_SUMMARY)
+        debug_log_path = os.path.join(output_dir,
+                                      records.OUTPUT_FILE_DEBUG_LOG)
+        info_log_path = os.path.join(output_dir, records.OUTPUT_FILE_INFO_LOG)
+        self.assertTrue(os.path.isfile(summary_file_path))
+        self.assertTrue(os.path.isfile(debug_log_path))
+        self.assertTrue(os.path.isfile(info_log_path))
+        summary_entries = []
+        with open(summary_file_path) as f:
+            for entry in yaml.load_all(f):
+                self.assertTrue(entry['Type'])
+                summary_entries.append(entry)
+        with open(debug_log_path, 'r') as f:
+            content = f.read()
+            self.assertIn('DEBUG', content)
+            self.assertIn('INFO', content)
+        with open(info_log_path, 'r') as f:
+            content = f.read()
+            self.assertIn('INFO', content)
+            self.assertNotIn('DEBUG', content)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/mobly/records_test.py b/tests/mobly/records_test.py
--- a/tests/mobly/records_test.py
+++ b/tests/mobly/records_test.py
@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import shutil
+import tempfile
+import yaml
+
 from future.tests.base import unittest
 
 from mobly import records
@@ -28,6 +33,10 @@ def setUp(self):
         self.details = "Some details about the test execution."
         self.float_extra = 12345.56789
         self.json_extra = {"ha": "whatever"}
+        self.tmp_path = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_path)
 
     def verify_record(self, record, result, details, extras):
         # Verify each field.
@@ -295,6 +304,19 @@ def test_is_test_executed(self):
         self.assertTrue(tr.is_test_executed(record1.test_name))
         self.assertFalse(tr.is_test_executed(self.tn + 'ha'))
 
+    def test_summary_write_dump(self):
+        s = signals.TestFailure(self.details, self.float_extra)
+        record1 = records.TestResultRecord(self.tn)
+        record1.test_begin()
+        record1.test_fail(s)
+        dump_path = os.path.join(self.tmp_path, 'ha.yaml')
+        writer = records.TestSummaryWriter(dump_path)
+        writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD)
+        with open(dump_path, 'r') as f:
+            content = yaml.load(f)
+            self.assertEqual(content['Type'],
+                             records.TestSummaryEntryType.RECORD.value)
+
 
 if __name__ == "__main__":
     unittest.main()