Repository: fitbenchmarking/fitbenchmarking

Issue:
Create an option to toggle the 3rd party output grabber
As observed in #500 the output grabber is not working on windows and as part of #511 to enable windows test a solution was to have the output grabber to pass through. It would also be useful to have this as a option within FitBenchmarking.


Discussion:

Patch:
diff --git a/examples/all_softwares.ini b/examples/all_softwares.ini
index b054ae82..dc589f6d 100644
--- a/examples/all_softwares.ini
+++ b/examples/all_softwares.ini
@@ -196,7 +196,7 @@ software: dfo
 
 # file_name specifies the file path to write the logs to
 # default is fitbenchmarking.log
-#file_name: fitbenchmarking.log 
+#file_name: fitbenchmarking.log
 
 # append specifies whether to log in append mode or not.
 # If append mode is active, the log file will be extended with each subsequent
@@ -210,3 +210,9 @@ software: dfo
 #                                           ERROR, and CRITICAL
 # default is INFO
 #level: INFO
+
+# external_output toggles whether the output grabber collects the third party
+#                 output. We note that for the windows operating system the
+#                 default will be off.
+# default True (yes/no can also be used)
+#external_output: yes
diff --git a/examples/options_template.ini b/examples/options_template.ini
index 671d2ba6..dced07d0 100644
--- a/examples/options_template.ini
+++ b/examples/options_template.ini
@@ -196,7 +196,7 @@
 
 # file_name specifies the file path to write the logs to
 # default is fitbenchmarking.log
-#file_name: fitbenchmarking.log 
+#file_name: fitbenchmarking.log
 
 # append specifies whether to log in append mode or not.
 # If append mode is active, the log file will be extended with each subsequent
@@ -210,3 +210,9 @@
 #                                           ERROR, and CRITICAL
 # default is INFO
 #level: INFO
+
+# external_output toggles whether the output grabber collects the third party
+#                 output. We note that for the windows operating system the
+#                 default will be off.
+# default True (yes/no can also be used)
+#external_output: yes
diff --git a/fitbenchmarking/core/fitbenchmark_one_problem.py b/fitbenchmarking/core/fitbenchmark_one_problem.py
index a65f5292..0dfbde15 100644
--- a/fitbenchmarking/core/fitbenchmark_one_problem.py
+++ b/fitbenchmarking/core/fitbenchmark_one_problem.py
@@ -30,7 +30,7 @@ def fitbm_one_prob(problem, options):
     :return: list of all results
     :rtype: list of fibenchmarking.utils.fitbm_result.FittingResult
     """
-    grabbed_output = output_grabber.OutputGrabber()
+    grabbed_output = output_grabber.OutputGrabber(options)
     results = []
 
     software = options.software
@@ -85,7 +85,7 @@ def benchmark(controller, minimizers, options):
     :return: list of all results
     :rtype: list of fibenchmarking.utils.fitbm_result.FittingResult
     """
-    grabbed_output = output_grabber.OutputGrabber()
+    grabbed_output = output_grabber.OutputGrabber(options)
     problem = controller.problem
 
     results_problem = []
diff --git a/fitbenchmarking/core/fitting_benchmarking.py b/fitbenchmarking/core/fitting_benchmarking.py
index 17b970f3..22c8aa41 100644
--- a/fitbenchmarking/core/fitting_benchmarking.py
+++ b/fitbenchmarking/core/fitting_benchmarking.py
@@ -34,7 +34,7 @@ def fitbenchmark_group(group_name, options, data_dir):
              the problem group and the location of the results
     :rtype: tuple(list, str)
     """
-    grabbed_output = output_grabber.OutputGrabber()
+    grabbed_output = output_grabber.OutputGrabber(options)
 
     # Extract problem definitions
     problem_group = misc.get_problem_files(data_dir)
diff --git a/fitbenchmarking/utils/default_options.ini b/fitbenchmarking/utils/default_options.ini
index 36555638..4e97f1ad 100644
--- a/fitbenchmarking/utils/default_options.ini
+++ b/fitbenchmarking/utils/default_options.ini
@@ -196,7 +196,7 @@ results_dir: results
 
 # file_name specifies the file path to write the logs to
 # default is fitbenchmarking.log
-file_name: fitbenchmarking.log 
+file_name: fitbenchmarking.log
 
 # append specifies whether to log in append mode or not.
 # If append mode is active, the log file will be extended with each subsequent
@@ -210,3 +210,9 @@ append: no
 #                                           ERROR, and CRITICAL
 # default is INFO
 level: INFO
+
+# external_output toggles whether the output grabber collects the third party
+#                 output. We note that for the windows operating system the
+#                 default will be off.
+# default True (yes/no can also be used)
+external_output: yes
diff --git a/fitbenchmarking/utils/options.py b/fitbenchmarking/utils/options.py
index 14721ec1..f752852e 100644
--- a/fitbenchmarking/utils/options.py
+++ b/fitbenchmarking/utils/options.py
@@ -75,6 +75,11 @@ class Options(object):
         self.log_file = logging.getstr('file_name')
         self.log_level = logging.getstr('level')
 
+        try:
+            self.external_output = logging.getboolean('external_output')
+        except ValueError:
+            error_message.append(template.format('external_output', "boolean"))
+
         # sys.exit() will be addressed in future FitBenchmarking
         # error handling issue
         if error_message != []:
@@ -116,7 +121,8 @@ class Options(object):
                               'table_type': list_to_string(self.table_type)}
         config['LOGGING'] = {'file_name': self.log_file,
                              'level': self.log_level,
-                             'append': self.log_append}
+                             'append': self.log_append,
+                             'external_output': self.external_output}
 
         with open(file_name, 'w') as f:
             config.write(f)
diff --git a/fitbenchmarking/utils/output_grabber.py b/fitbenchmarking/utils/output_grabber.py
index baf8d967..e1e444b7 100644
--- a/fitbenchmarking/utils/output_grabber.py
+++ b/fitbenchmarking/utils/output_grabber.py
@@ -1,28 +1,37 @@
 import os
 import sys
+import platform
 
 from fitbenchmarking.utils.log import get_logger
 
 LOGGER = get_logger()
 
+
 class OutputGrabber(object):
     """
     Class used to grab standard output or another stream.
     """
     escape_char = "\b"
 
-    def __init__(self):
+    def __init__(self, options):
 
         self.origstream = sys.stdout
         self.origstreamfd = self.origstream.fileno()
         self.capturedtext = ""
+        # From issue 500 the output grabber does not currently on windows, thus
+        # we set the __enter__ and __exit__ functions to pass through for this
+        # case
+        self.system = platform.system() != "Windows"
+        self.external_output = options.external_output
 
     def __enter__(self):
-        self.start()
+        if self.system and self.external_output:
+            self.start()
         return self
 
     def __exit__(self, type, value, traceback):
-        self.stop()
+        if self.system and self.external_output:
+            self.stop()
 
     def start(self):
         """


Test patch:
diff --git a/fitbenchmarking/utils/tests/test_options.py b/fitbenchmarking/utils/tests/test_options.py
index da5e66a3..acc98f5f 100644
--- a/fitbenchmarking/utils/tests/test_options.py
+++ b/fitbenchmarking/utils/tests/test_options.py
@@ -42,6 +42,7 @@ class OptionsTests(unittest.TestCase):
             file_name: THE_LOG.log
             append: yes
             level: debug
+            external_output: no
             """
         incorrect_config_str = """
             [FITTING]
@@ -51,6 +52,7 @@ class OptionsTests(unittest.TestCase):
             make_plots: incorrect_falue
             [LOGGING]
             append: sure
+            external_output: maybe
             """
         opts = {'MINIMIZERS': {'scipy': ['nonesense',
                                          'another_fake_minimizer'],
@@ -67,7 +69,8 @@ class OptionsTests(unittest.TestCase):
                              'results_dir': 'new_results'},
                 'LOGGING': {'file_name': 'THE_LOG.log',
                             'append': True,
-                            'level': 'debug'}}
+                            'level': 'debug',
+                            'external_output': False}}
 
         opts_file = 'test_options_tests_{}.txt'.format(
             datetime.datetime.now())
@@ -163,6 +166,18 @@ class OptionsTests(unittest.TestCase):
         logging_opts = self.options['LOGGING']
         self.assertEqual(logging_opts['append'], options.log_append)
 
+    def test_log_external_output_false(self):
+        with self.assertRaises(exceptions.OptionsError) as cm:
+            Options(file_name=self.options_file_incorrect)
+        excep = cm.exception
+        self.assertIn('external_output', excep._obj_message)
+
+    def test_log_external_output_true(self):
+        options = Options(file_name=self.options_file)
+        logging_opts = self.options['LOGGING']
+        self.assertEqual(logging_opts['external_output'],
+                         options.external_output)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/fitbenchmarking/utils/tests/test_output_grabber.py b/fitbenchmarking/utils/tests/test_output_grabber.py
index 45bb404f..a9700f40 100644
--- a/fitbenchmarking/utils/tests/test_output_grabber.py
+++ b/fitbenchmarking/utils/tests/test_output_grabber.py
@@ -2,13 +2,16 @@ from __future__ import (absolute_import, division, print_function)
 import unittest
 
 from fitbenchmarking.utils.output_grabber import OutputGrabber
-
+from fitbenchmarking.utils.options import Options
 
 class OutputGrabberTests(unittest.TestCase):
 
+    def setUp(self):
+        self.options = Options()
+
     def test_correct_output(self):
         output_string = 'This is the correct output string\nSecond line'
-        output = OutputGrabber()
+        output = OutputGrabber(self.options)
         with output:
             print(output_string)
         # print adds an extra \n
@@ -16,12 +19,11 @@ class OutputGrabberTests(unittest.TestCase):
 
     def test_incorrect_output(self):
         output_string = 'This is the correct output string\nSecond line'
-        incorrect_output_sting = 'This is the incorrect string'
-        output = OutputGrabber()
+        incorrect_output_sting = 'This is the incorrect string\n'
+        output = OutputGrabber(self.options)
         with output:
             print(output_string)
-        # print adds an extra \n
-        assert output.capturedtext != incorrect_output_sting + "\n"
+        assert output.capturedtext != incorrect_output_sting
 
 
 if __name__ == "__main__":