Repository: fitbenchmarking/fitbenchmarking

Issue:
Store all runtimes and select average in results processing
Currently we take the mean of the runtimes after each fit.

We should store all runtimes at this point and pass it through to the results processing.
Results processing can then take the mean for now with the idea of having a user facing option to select the proper average in a future feature.

Discussion:

Patch:
diff --git a/fitbenchmarking/core/fitting_benchmarking.py b/fitbenchmarking/core/fitting_benchmarking.py
index e79062db..6c600ee9 100644
--- a/fitbenchmarking/core/fitting_benchmarking.py
+++ b/fitbenchmarking/core/fitting_benchmarking.py
@@ -517,11 +517,11 @@ def loop_over_hessians(controller, options, grabbed_output, checkpointer):
                             hess_name)
 
             # Perform the fit a number of times specified by num_runs
-            accuracy, runtime, emissions = perform_fit(
+            accuracy, runtimes, emissions = perform_fit(
                 controller, options, grabbed_output)
             result_args = {'controller': controller,
                            'accuracy': accuracy,
-                           'runtime': runtime,
+                           'runtimes': runtimes,
                            'emissions': emissions}
             if problem.multifit:
                 # for multifit problems, multiple accuracy values are stored
@@ -556,8 +556,8 @@ def perform_fit(controller, options, grabbed_output):
     :type options: fitbenchmarking.utils.options.Options
     :param grabbed_output: Object that removes third part output from console
     :type grabbed_output: fitbenchmarking.utils.output_grabber.OutputGrabber
-    :return: The chi squared, runtime, and emissions of the fit.
-    :rtype: tuple(float, float, float)
+    :return: The chi squared, runtimes and emissions of the fit.
+    :rtype: tuple(float, list[float], float)
     """
     num_runs = options.num_runs
 
@@ -573,18 +573,17 @@ def perform_fit(controller, options, grabbed_output):
             controller.prepare()
             with emissions_tracker:
                 # Calls timeit repeat with repeat = num_runs and number = 1
-                runtime_list = timeit.Timer(
+                runtimes = timeit.Timer(
                     stmt=controller.execute
                 ).repeat(num_runs, 1)
             if track_emissions:
                 # stop emissions tracking after all runs have completed
                 emissions = emissions_tracker.final_emissions / num_runs
 
-            runtime = sum(runtime_list) / num_runs
             controller.cleanup()
             controller.check_attributes()
-        min_time = np.min(runtime_list)
-        ratio = np.max(runtime_list) / min_time
+        min_time = np.min(runtimes)
+        ratio = np.max(runtimes) / min_time
         tol = 4
         if ratio > tol:
             warnings.warn(
@@ -606,7 +605,7 @@ def perform_fit(controller, options, grabbed_output):
 
         accuracy_check = any(np.isnan(n) for n in accuracy) \
             if controller.problem.multifit else np.isnan(accuracy)
-        if np.isnan(runtime) or accuracy_check:
+        if np.isnan(runtimes).any() or accuracy_check:
             raise ControllerAttributeError(
                 "Either the computed runtime or accuracy values were a NaN.")
     except ValidationException as ex:
@@ -633,11 +632,11 @@ def perform_fit(controller, options, grabbed_output):
     controller.timer.reset()
 
     if controller.flag in [3, 6, 7]:
-        # If there was an exception, set the runtime and
+        # If there was an exception, set the runtimes and
         # cost function value to be infinite
         emissions = np.inf
-        runtime = np.inf
         multi_fit = controller.problem.multifit
+        runtimes = [np.inf] * num_runs
         controller.final_params = \
             None if not multi_fit \
             else [None] * len(controller.data_x)
@@ -649,4 +648,4 @@ def perform_fit(controller, options, grabbed_output):
         # been respected by the minimizer and set error
         # flag if not
         controller.check_bounds_respected()
-    return accuracy, runtime, emissions
+    return accuracy, runtimes, emissions
diff --git a/fitbenchmarking/core/results_output.py b/fitbenchmarking/core/results_output.py
index 1eb1e083..f5afe706 100644
--- a/fitbenchmarking/core/results_output.py
+++ b/fitbenchmarking/core/results_output.py
@@ -261,7 +261,7 @@ def _process_best_results(results: 'List[FittingResult]') -> 'FittingResult':
     for result in results[1:]:
         if best.accuracy > result.accuracy:
             best = result
-        if fastest.runtime > result.runtime:
+        if fastest.mean_runtime > result.mean_runtime:
             fastest = result
         if lowest.emissions > result.emissions:
             lowest = result
@@ -270,7 +270,7 @@ def _process_best_results(results: 'List[FittingResult]') -> 'FittingResult':
 
     for result in results:
         result.min_accuracy = best.accuracy
-        result.min_runtime = fastest.runtime
+        result.min_runtime = fastest.mean_runtime
         result.min_emissions = lowest.emissions
 
     return best
diff --git a/fitbenchmarking/results_processing/compare_table.py b/fitbenchmarking/results_processing/compare_table.py
index 8efa0736..b9e0a970 100644
--- a/fitbenchmarking/results_processing/compare_table.py
+++ b/fitbenchmarking/results_processing/compare_table.py
@@ -71,7 +71,7 @@ class CompareTable(Table):
         acc_abs = result.accuracy
 
         runtime_rel = result.norm_runtime
-        runtime_abs = result.runtime
+        runtime_abs = result.mean_runtime
 
         return [[acc_rel, runtime_rel],
                 [acc_abs, runtime_abs]]
diff --git a/fitbenchmarking/results_processing/fitting_report.py b/fitbenchmarking/results_processing/fitting_report.py
index 32c92352..35597ea9 100644
--- a/fitbenchmarking/results_processing/fitting_report.py
+++ b/fitbenchmarking/results_processing/fitting_report.py
@@ -87,7 +87,7 @@ def create_prob_group(result, support_pages_dir, options):
             initial_guess=result.ini_function_params,
             minimizer=result.modified_minimizer_name(),
             accuracy=f"{result.accuracy:.4g}",
-            runtime=f"{result.runtime:.4g}",
+            runtime=f"{result.mean_runtime:.4g}",
             emissions=f"{result.emissions:.4g}",
             is_best_fit=result.is_best_fit,
             initial_plot_available=init_success,
diff --git a/fitbenchmarking/results_processing/runtime_table.py b/fitbenchmarking/results_processing/runtime_table.py
index df62bb68..d11bf699 100644
--- a/fitbenchmarking/results_processing/runtime_table.py
+++ b/fitbenchmarking/results_processing/runtime_table.py
@@ -61,5 +61,5 @@ class RuntimeTable(Table):
         :rtype: tuple(float, float)
         """
         rel_value = result.norm_runtime
-        abs_value = result.runtime
+        abs_value = result.mean_runtime
         return rel_value, abs_value
diff --git a/fitbenchmarking/utils/checkpoint.py b/fitbenchmarking/utils/checkpoint.py
index 10a27020..4cf7ba61 100644
--- a/fitbenchmarking/utils/checkpoint.py
+++ b/fitbenchmarking/utils/checkpoint.py
@@ -86,7 +86,8 @@ class Checkpoint:
             'fin_params': _compress(result.params),
             'fin_params_str': result.fin_function_params,
             'accuracy': result.accuracy,
-            'runtime': result.runtime,
+            'mean_runtime': result.mean_runtime,
+            'runtimes': result.runtimes,
             'emissions': result.emissions,
             'flag': result.error_flag,
             'software': result.software,
@@ -264,7 +265,8 @@ class Checkpoint:
                 new_result.params = _decompress(r['fin_params'])
                 new_result.fin_function_params = r['fin_params_str']
                 new_result.accuracy = r['accuracy']
-                new_result.runtime = r['runtime']
+                new_result.mean_runtime = r['mean_runtime']
+                new_result.runtimes = r['runtimes']
                 new_result.emissions = r['emissions']
                 new_result.error_flag = r['flag']
                 new_result.software = r['software']
diff --git a/fitbenchmarking/utils/fitbm_result.py b/fitbenchmarking/utils/fitbm_result.py
index add361e0..c760716e 100644
--- a/fitbenchmarking/utils/fitbm_result.py
+++ b/fitbenchmarking/utils/fitbm_result.py
@@ -26,7 +26,7 @@ class FittingResult:
     def __init__(self,
                  controller: 'Controller',
                  accuracy: 'float | list[float]' = np.inf,
-                 runtime: 'float' = np.inf,
+                 runtimes: 'float | list[float]' = np.inf,
                  emissions: 'float' = np.inf,
                  dataset: 'Optional[int]' = None) -> None:
         """
@@ -36,8 +36,8 @@ class FittingResult:
         :type controller: controller.base_controller.Controller
         :param accuracy: The score for the fitting, defaults to np.inf
         :type accuracy: float | list[float], optional
-        :param runtime: The average runtime of the fit, defaults to np.inf
-        :type runtime: float | list[float], optional
+        :param runtimes: All runtimes of the fit, defaults to np.inf
+        :type runtimes: float | list[float], optional
         :param emissions: The average emissions for the fit, defaults to np.inf
         :type emissions: float | list[float], optional
         :param dataset: The index of the dataset (Only used for MultiFit),
@@ -74,7 +74,8 @@ class FittingResult:
             self.params = controller.final_params[dataset]
             self.accuracy = accuracy[dataset]
 
-        self.runtime = runtime
+        self.mean_runtime = np.average(runtimes)
+        self.runtimes = runtimes
         self.emissions = emissions
 
         # Details of options used for this run
@@ -158,7 +159,8 @@ class FittingResult:
                 "Jacobian": self.jacobian_tag,
                 "Hessian": self.hessian_tag,
                 "Accuracy": self.accuracy,
-                "Runtime": self.runtime,
+                "Mean Runtime": self.mean_runtime,
+                "Runtimes": self.runtimes,
                 "Emissions": self.emissions}
 
         return get_printable_table("FittingResult", info)
@@ -247,7 +249,7 @@ class FittingResult:
             if self.min_runtime in [np.nan, np.inf]:
                 self._norm_runtime = np.inf
             else:
-                self._norm_runtime = self.runtime / self.min_runtime
+                self._norm_runtime = self.mean_runtime / self.min_runtime
         return self._norm_runtime
 
     @norm_runtime.setter


Test patch:
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
index e98bbc7e..e13a244b 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
@@ -114,7 +114,7 @@ class BenchmarkTests(unittest.TestCase):
                 fitbm_result.FittingResult(
                     controller=controller,
                     accuracy=1,
-                    runtime=1
+                    runtimes=[1.0]
                 )
                 for _ in range(self.scipy_len)
             ]
@@ -148,7 +148,7 @@ class BenchmarkTests(unittest.TestCase):
                 fitbm_result.FittingResult(
                     controller=controller,
                     accuracy=1,
-                    runtime=1)
+                    runtimes=[1.0])
                 for _ in range(self.scipy_len)
             ]
             results.extend(list_results)
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
index e47ffa99..23f9bc15 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
@@ -82,7 +82,7 @@ class LoopOverBenchmarkProblemsTests(unittest.TestCase):
             fitbm_result.FittingResult(
                 controller=controllers[i],
                 accuracy=1,
-                runtime=1)
+                runtimes=[1])
             for i in range(scipy_len)
         ]
         self.individual_problem_results = [
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_hessians.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_hessians.py
index 581b77e6..4333af50 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_hessians.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_hessians.py
@@ -123,7 +123,8 @@ class LoopOverHessiansTests(unittest.TestCase):
         """
         Clean up after the test
         """
-        rmtree(self.options.results_dir)
+        if os.path.isdir(self.options.results_dir):
+            rmtree(self.options.results_dir)
 
     def test_single_hessian(self):
         """
@@ -209,7 +210,9 @@ class LoopOverHessiansTests(unittest.TestCase):
         grabbed_output = output_grabber.OutputGrabber(options)
         controller.final_params = [[0.1, 0.1], [0.1, 0.1]]
         controller.parameter_set = 0
-        perform_fit.return_value = ([0.1, 0.2], [0.1, 0.01], [10e-3, 10e-4])
+        perform_fit.return_value = ([0.1, 0.2],
+                                    [0.1, 0.01],
+                                    [10e-3, 10e-4])
         results = loop_over_hessians(controller=controller,
                                      options=options,
                                      grabbed_output=grabbed_output,
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_minimizers.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_minimizers.py
index 86438cc1..b27ca1e0 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_minimizers.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_minimizers.py
@@ -102,7 +102,7 @@ class LoopOverMinimizersTests(unittest.TestCase):
         self.result = fitbm_result.FittingResult(
             controller=self.controller,
             accuracy=1,
-            runtime=1)
+            runtimes=[1])
         self.cp = Checkpoint(self.options)
 
     def mock_func_call(self, *args, **kwargs):
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_software.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_software.py
index d2de4bef..e8904e63 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_software.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_software.py
@@ -78,7 +78,7 @@ class LoopOverSoftwareTests(unittest.TestCase):
         controller.parameter_set = 0
         self.result_args = {'controller': controller,
                             'accuracy': 1,
-                            'runtime': 1}
+                            'runtimes': [1]}
         self.cp = Checkpoint(self.options)
 
     def mock_func_call(self, *args, **kwargs):
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_starting_values.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_starting_values.py
index 970487c0..b8cc3ae4 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_starting_values.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_starting_values.py
@@ -74,7 +74,7 @@ class LoopOverStartingValuesTests(unittest.TestCase):
         controller.parameter_set = 0
         self.result_args = {'controller': controller,
                             'accuracy': 1,
-                            'runtime': 1}
+                            'runtimes': [1]}
         self.cp = Checkpoint(self.options)
 
     def mock_func_call(self, *args, **kwargs):
diff --git a/fitbenchmarking/core/tests/test_results_output.py b/fitbenchmarking/core/tests/test_results_output.py
index dab4239f..636b3324 100644
--- a/fitbenchmarking/core/tests/test_results_output.py
+++ b/fitbenchmarking/core/tests/test_results_output.py
@@ -421,11 +421,11 @@ class ProcessBestResultsTests(unittest.TestCase):
             results, self.options = load_mock_results(
                 {'results_dir': self.results_dir})
             self.results = results[:5]
-            for r, accuracy, runtime in zip(self.results,
-                                            [2, 1, 5, 3, 4],
-                                            [5, 4, 1, 2, 3]):
+            for r, accuracy, mean_runtime in zip(self.results,
+                                                 [2, 1, 5, 3, 4],
+                                                 [5, 4, 1, 2, 3]):
                 r.accuracy = accuracy
-                r.runtime = runtime
+                r.mean_runtime = mean_runtime
             self.best = _process_best_results(self.results)
 
     def test_returns_best_result(self):
@@ -464,11 +464,11 @@ class ProcessBestResultsTests(unittest.TestCase):
         Test that min_runtime is set correctly.
         """
         fastest = self.results[2]
-        self.assertEqual(self.results[0].min_runtime, fastest.runtime)
-        self.assertEqual(self.results[1].min_runtime, fastest.runtime)
-        self.assertEqual(self.results[2].min_runtime, fastest.runtime)
-        self.assertEqual(self.results[3].min_runtime, fastest.runtime)
-        self.assertEqual(self.results[4].min_runtime, fastest.runtime)
+        self.assertEqual(self.results[0].min_runtime, fastest.mean_runtime)
+        self.assertEqual(self.results[1].min_runtime, fastest.mean_runtime)
+        self.assertEqual(self.results[2].min_runtime, fastest.mean_runtime)
+        self.assertEqual(self.results[3].min_runtime, fastest.mean_runtime)
+        self.assertEqual(self.results[4].min_runtime, fastest.mean_runtime)
 
 
 if __name__ == "__main__":
diff --git a/fitbenchmarking/test_files/checkpoint.json b/fitbenchmarking/test_files/checkpoint.json
index 849e77ae..35f372e2 100644
--- a/fitbenchmarking/test_files/checkpoint.json
+++ b/fitbenchmarking/test_files/checkpoint.json
@@ -40,7 +40,12 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 0.2,
-        "runtime": 15.0,
+        "mean_runtime": 15.0,
+        "runtimes": [
+                14.0,
+                15.0,
+                16.0
+        ],
         "emissions": 0.001,
         "flag": 0,
         "software": "s1",
@@ -62,7 +67,12 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 0.3,
-        "runtime": 14.0,
+        "mean_runtime": 14.0,
+        "runtimes": [
+                13.0,
+                14.0,
+                15.0
+        ],
         "emissions": 0.0001,
         "flag": 0,
         "software": "s1",
@@ -84,7 +94,12 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 0.4,
-        "runtime": 13.0,
+        "mean_runtime": 13.0,
+        "runtimes": [
+                12.0,
+                13.0,
+                14.0
+        ],
         "emissions": 0.001,
         "flag": 0,
         "software": "s0",
@@ -106,7 +121,10 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": Infinity,
-        "runtime": Infinity,
+        "mean_runtime": Infinity,
+        "runtimes": [
+                Infinity
+        ],
         "emissions": Infinity,
         "flag": 4,
         "software": "s0",
@@ -128,7 +146,12 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 0.6,
-        "runtime": 11.0,
+        "mean_runtime": 11.0,
+        "runtimes": [
+                10.0,
+                11.0,
+                12.0
+        ],
         "emissions": 1e-05,
         "flag": 0,
         "software": "s1",
@@ -150,7 +173,12 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 0.7,
-        "runtime": 10.0,
+        "mean_runtime": 10.0,
+        "runtimes": [
+                9.0,
+                10.0,
+                11.0
+        ],
         "emissions": 1e-05,
         "flag": 0,
         "software": "s1",
@@ -172,7 +200,10 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 0.8,
-        "runtime": 1.0,
+        "mean_runtime": 1.0,
+        "runtimes": [
+                1.0
+        ],
         "emissions": 1e-07,
         "flag": 0,
         "software": "s0",
@@ -194,7 +225,10 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 1.0,
-        "runtime": 1.0,
+        "mean_runtime": 1.0,
+        "runtimes": [
+                1.0
+        ],
         "emissions": 1e-07,
         "flag": 0,
         "software": "s1",
@@ -216,7 +250,10 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 1.0,
-        "runtime": 1.0,
+        "mean_runtime": 1.0,
+        "runtimes": [
+                1.0
+        ],
         "emissions": 1e-07,
         "flag": 0,
         "software": "s1",
@@ -238,7 +275,11 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 2.0,
-        "runtime": 2.0,
+        "mean_runtime": 2.0,
+        "runtimes": [
+                1.0,
+                3.0
+        ],
         "emissions": 1e-06,
         "flag": 0,
         "software": "s0",
@@ -260,7 +301,10 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": Infinity,
-        "runtime": Infinity,
+        "mean_runtime": Infinity,
+        "runtimes": [
+                Infinity
+        ],
         "emissions": Infinity,
         "flag": 0,
         "software": "s0",
@@ -282,7 +326,12 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 3.0,
-        "runtime": 3.0,
+        "mean_runtime": 3.0,
+        "runtimes": [
+                6.0,
+                1.5,
+                1.5
+        ],
         "emissions": 1e-05,
         "flag": 0,
         "software": "s1",
@@ -304,7 +353,12 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 3.0,
-        "runtime": 3.0,
+        "mean_runtime": 3.0,
+        "runtimes": [
+                2.0,
+                3.0,
+                1.0
+        ],
         "emissions": 1e-05,
         "flag": 0,
         "software": "s1",
@@ -326,7 +380,12 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": 0.2,
-        "runtime": 15.0,
+        "mean_runtime": 15.0,
+        "runtimes": [
+                10.0,
+                15.0,
+                20.0
+        ],
         "emissions": 0.001,
         "flag": 0,
         "software": "s0",
@@ -348,7 +407,10 @@
         "fin_params": "J-;,Rz!!!\")PUK$dn,NFg!!!!h5O\\XQzAM*",
         "fin_params_str": "a=1.0, b=1.0",
         "accuracy": Infinity,
-        "runtime": Infinity,
+        "mean_runtime": Infinity,
+        "runtimes": [
+                Infinity
+        ],
         "emissions": Infinity,
         "flag": 0,
         "software": "s0",
diff --git a/fitbenchmarking/utils/tests/test_checkpoint.py b/fitbenchmarking/utils/tests/test_checkpoint.py
index 9066fa58..30deab0c 100644
--- a/fitbenchmarking/utils/tests/test_checkpoint.py
+++ b/fitbenchmarking/utils/tests/test_checkpoint.py
@@ -68,12 +68,13 @@ def generate_results():
         controller.final_params = [0.1+i, 0.2+i]
         controller.flag = 0
         results[key].append(FittingResult(controller=controller,
-                                          accuracy=0.02+i, runtime=0.15+i))
+                                          accuracy=0.02+i,
+                                          runtimes=[0.15+i]))
         controller.minimizer = 'CG'
         controller.final_params = [0.3+i, 0.4+i]
         controller.flag = 2
         results[key].append(FittingResult(controller=controller,
-                                          accuracy=0.3+i, runtime=0.01+i))
+                                          accuracy=0.3+i, runtimes=[0.01+i]))
         cost_func.jacobian = jac[1]
         controller = ScipyController(cost_func)
         controller.parameter_set = 0
@@ -82,7 +83,7 @@ def generate_results():
         controller.final_params = [0.5+i, 0.6+i]
         controller.flag = 0
         results[key].append(FittingResult(controller=controller,
-                                          accuracy=0.4, runtime=13))
+                                          accuracy=0.4, runtimes=[13]))
         cost_func = WeightedNLLSCostFunc(problems[1])
         jac[1].method = '3-point'
         cost_func.jacobian = jac[0]
@@ -93,12 +94,12 @@ def generate_results():
         controller.final_params = [0.7+i, 0.8+i]
         controller.flag = 0
         results[key].append(FittingResult(controller=controller,
-                                          accuracy=1.2, runtime=0.15))
+                                          accuracy=1.2, runtimes=[0.15]))
         controller.minimizer = 'CG'
         controller.final_params = [0.9+i, 0.01+i]
         controller.flag = 0
         results[key].append(FittingResult(controller=controller,
-                                          accuracy=1.3, runtime=0.14))
+                                          accuracy=1.3, runtimes=[0.14]))
         cost_func.jacobian = jac[1]
         controller = ScipyController(cost_func)
         controller.parameter_set = 0
@@ -107,7 +108,7 @@ def generate_results():
         controller.final_params = [0.11+i, 0.21+i]
         controller.flag = 0
         results[key].append(FittingResult(controller=controller,
-                                          accuracy=1.4, runtime=0.13))
+                                          accuracy=1.4, runtimes=[0.13]))
     return results
 
 
diff --git a/fitbenchmarking/utils/tests/test_fitbm_result.py b/fitbenchmarking/utils/tests/test_fitbm_result.py
index 0350761f..db93fe52 100644
--- a/fitbenchmarking/utils/tests/test_fitbm_result.py
+++ b/fitbenchmarking/utils/tests/test_fitbm_result.py
@@ -59,12 +59,13 @@ class FitbmResultTests(unittest.TestCase):
         self.controller = controller
 
         self.accuracy = 10
-        self.runtime = 0.01
+        self.mean_runtime = 0.01
+        self.runtimes = [0.005, 0.015, 0.01]
         self.emissions = 0.001
         self.result = FittingResult(
             controller=controller,
             accuracy=self.accuracy,
-            runtime=self.runtime,
+            runtimes=self.runtimes,
             emissions=self.emissions)
 
         self.min_accuracy = 0.1
@@ -77,27 +78,29 @@ class FitbmResultTests(unittest.TestCase):
         Test that the fitting result can be printed as a readable string.
         """
         expected = textwrap.dedent('''\
-            +===============================+
-            | FittingResult                 |
-            +===============================+
-            | Cost Function | NLLSCostFunc  |
-            +-------------------------------+
-            | Problem       | cubic         |
-            +-------------------------------+
-            | Software      | scipy         |
-            +-------------------------------+
-            | Minimizer     | Newton-CG     |
-            +-------------------------------+
-            | Jacobian      | scipy 2-point |
-            +-------------------------------+
-            | Hessian       | analytic      |
-            +-------------------------------+
-            | Accuracy      | 10            |
-            +-------------------------------+
-            | Runtime       | 0.01          |
-            +-------------------------------+
-            | Emissions     | 0.001         |
-            +-------------------------------+''')
+            +======================================+
+            | FittingResult                        |
+            +======================================+
+            | Cost Function | NLLSCostFunc         |
+            +--------------------------------------+
+            | Problem       | cubic                |
+            +--------------------------------------+
+            | Software      | scipy                |
+            +--------------------------------------+
+            | Minimizer     | Newton-CG            |
+            +--------------------------------------+
+            | Jacobian      | scipy 2-point        |
+            +--------------------------------------+
+            | Hessian       | analytic             |
+            +--------------------------------------+
+            | Accuracy      | 10                   |
+            +--------------------------------------+
+            | Mean Runtime  | 0.01                 |
+            +--------------------------------------+
+            | Runtimes      | [0.005, 0.015, 0.01] |
+            +--------------------------------------+
+            | Emissions     | 0.001                |
+            +--------------------------------------+''')
 
         for i, (r, e) in enumerate(zip(str(self.result).splitlines(),
                                        expected.splitlines())):
@@ -113,7 +116,7 @@ class FitbmResultTests(unittest.TestCase):
         problem = self.controller.problem
 
         chi_sq = [10, 5, 1]
-        runtime = 0.01
+        runtimes = [0.005, 0.015, 0.01]
         controller.final_params = [np.array([1, 3, 4, 4]),
                                    np.array([2, 3, 57, 8]),
                                    np.array([4, 2, 5, 1])]
@@ -134,7 +137,7 @@ class FitbmResultTests(unittest.TestCase):
         result = FittingResult(
             controller=controller,
             accuracy=chi_sq,
-            runtime=runtime,
+            runtimes=runtimes,
             dataset=1)
 
         self.assertTrue(
@@ -151,6 +154,31 @@ class FitbmResultTests(unittest.TestCase):
             np.isclose(controller.final_params[1], result.params).all())
         self.assertEqual(chi_sq[1], result.accuracy)
 
+    def test_mean_runtime_calculation(self):
+        """
+        Tests the mean calculation with in FittingResults
+        """
+        controller = self.controller
+        result = FittingResult(
+            controller=controller,
+            runtimes=[0.005, 0.015, 0.01])
+        self.assertEqual(0.01, result.mean_runtime)
+
+        result = FittingResult(
+            controller=controller,
+            runtimes=[1.00, 2.00, 3.00])
+        self.assertEqual(2.00, result.mean_runtime)
+
+        result = FittingResult(
+            controller=controller,
+            runtimes=[np.inf, 2.00, 3.00])
+        self.assertEqual(np.inf, result.mean_runtime)
+
+        result = FittingResult(
+            controller=controller,
+            runtimes=[np.inf, np.inf, np.inf])
+        self.assertEqual(np.inf, result.mean_runtime)
+
     def test_norm_acc_finite_min(self):
         """
         Test that sanitised names are correct when min_chi_sq is finite.
@@ -171,7 +199,7 @@ class FitbmResultTests(unittest.TestCase):
         """
         Test that sanitised names are correct when min_runtime is finite.
         """
-        expected = self.runtime / self.min_runtime
+        expected = self.mean_runtime / self.min_runtime
         self.assertEqual(self.result.norm_runtime, expected)
 
     def test_norm_runtime_infinite_min(self):