Repository: fitbenchmarking/fitbenchmarking

Issue:
Allow users to comparing minimizers from scipy, mantid, other in one table
Allow user to run a fit benchmarking with a mixture of different sources of minimizers (e.g. mantid and scipy) and where the relative scores of these against selected benchmarking problems is shown in a table. 

At present FitBenchmarking only create tables which shows comparisons in scores between minimizers available from one software, and separate tables are produced for each software 

Discussion:

Patch:
diff --git a/example_scripts/example_runScripts.py b/example_scripts/example_runScripts.py
index 3e178346..5d74ae5c 100644
--- a/example_scripts/example_runScripts.py
+++ b/example_scripts/example_runScripts.py
@@ -44,34 +44,26 @@ from fitting_benchmarking import do_fitting_benchmark as fitBenchmarking
 from results_output import save_results_tables as printTables
 
 # SPECIFY THE SOFTWARE/PACKAGE CONTAINING THE MINIMIZERS YOU WANT TO BENCHMARK
-software = 'scipy'
+software = ['mantid', 'scipy']
 software_options = {'software': software}
 
 # User defined minimizers
-# minimizers = {"mantid": ["BFGS",
-#                          "Conjugate gradient (Fletcher-Reeves imp.)",
-#                          "Conjugate gradient (Polak-Ribiere imp.)",
-#                          "Damped GaussNewton",
-#                          "Levenberg-Marquardt",
-#                          "Levenberg-MarquardtMD",
-#                          "Simplex",
-#                          "SteepestDescent",
-#                          "Trust Region"],
+# custom_minimizers = {"mantid": ["BFGS", "Simplex"],
 #               "scipy": ["lm", "trf", "dogbox"]}
-minimizers = None
+custom_minimizers = None
 
 
 # SPECIFY THE MINIMIZERS YOU WANT TO BENCHMARK, AND AS A MINIMUM FOR THE SOFTWARE YOU SPECIFIED ABOVE
 if len(sys.argv) > 1:
-    # Read custom minimizer options from file
-    software_options['minimizer_options'] = current_path + sys.argv[1]
-elif minimizers:
-    # Custom minimizer options:
-    software_options['minimizer_options'] = minimizers
+  # Read custom minimizer options from file
+  software_options['minimizer_options'] = current_path + sys.argv[1]
+elif custom_minimizers:
+  # Custom minimizer options:
+  software_options['minimizer_options'] = custom_minimizers
 else:
-    # Using default minimizers from
-    # fitbenchmarking/fitbenchmarking/minimizers_list_default.json
-    software_options['minimizer_options'] = None
+  # Using default minimizers from
+  # fitbenchmarking/fitbenchmarking/minimizers_list_default.json
+  software_options['minimizer_options'] = None
 
 # Benchmark problem directories
 benchmark_probs_dir = os.path.join(fitbenchmarking_folder,
@@ -103,22 +95,22 @@ color_scale = [(1.1, 'ranking-top-1'),
 # problem_sets = ["Neutron_data", "NIST/average_difficulty"]
 problem_sets = ["NIST/average_difficulty"]
 for sub_dir in problem_sets:
-    # generate group label/name used for problem set
-    label = sub_dir.replace('/', '_')
+  # generate group label/name used for problem set
+  label = sub_dir.replace('/', '_')
 
-    # Problem data directory
-    data_dir = os.path.join(benchmark_probs_dir, sub_dir)
+  # Problem data directory
+  data_dir = os.path.join(benchmark_probs_dir, sub_dir)
 
-    print('\nRunning the benchmarking on the {} problem set\n'.format(label))
-    results_per_group, results_dir = fitBenchmarking(group_name=label, software_options=software_options,
-                                                     data_dir=data_dir,
-                                                     use_errors=use_errors, results_dir=results_dir)
+  print('\nRunning the benchmarking on the {} problem set\n'.format(label))
+  results_per_group, results_dir = fitBenchmarking(group_name=label, software_options=software_options,
+                                                   data_dir=data_dir,
+                                                   use_errors=use_errors, results_dir=results_dir)
 
-    print('\nProducing output for the {} problem set\n'.format(label))
-    for idx, group_results in enumerate(results_per_group):
-        # Display the runtime and accuracy results in a table
-        printTables(software_options, group_results,
-                    group_name=label, use_errors=use_errors,
-                    color_scale=color_scale, results_dir=results_dir)
+  print('\nProducing output for the {} problem set\n'.format(label))
+  for idx, group_results in enumerate(results_per_group):
+    # Display the runtime and accuracy results in a table
+    printTables(software_options, group_results,
+                group_name=label, use_errors=use_errors,
+                color_scale=color_scale, results_dir=results_dir)
 
-    print('\nCompleted benchmarking for {} problem set\n'.format(sub_dir))
+  print('\nCompleted benchmarking for {} problem set\n'.format(sub_dir))
diff --git a/example_scripts/example_runScripts_expert.py b/example_scripts/example_runScripts_expert.py
index 8653b058..4155b6c7 100644
--- a/example_scripts/example_runScripts_expert.py
+++ b/example_scripts/example_runScripts_expert.py
@@ -42,7 +42,7 @@ fitbenchmarking_folder = os.path.abspath(os.path.join(current_path, os.pardir))
 scripts_folder = os.path.join(fitbenchmarking_folder, 'fitbenchmarking')
 sys.path.insert(0, scripts_folder)
 
-from fitting_benchmarking import do_fitbm_group
+from fitting_benchmarking import do_benchmarking
 from utils import misc
 from utils import create_dirs
 from results_output import save_tables, generate_tables, \
@@ -50,15 +50,15 @@ from results_output import save_tables, generate_tables, \
 from resproc import visual_pages
 
 # SPECIFY THE SOFTWARE/PACKAGE CONTAINING THE MINIMIZERS YOU WANT TO BENCHMARK
-software = 'mantid'
+# software = ['mantid', 'scipy', 'mantid']
+software = ['mantid', 'scipy']
 software_options = {'software': software}
 
 # User defined minimizers
-custom_minimizers = {"mantid": ["BFGS",
-                                "Conjugate gradient (Fletcher-Reeves imp.)",
-                                "Conjugate gradient (Polak-Ribiere imp.)"],
-                     "scipy": ["lm"]}
-# custom_minimizers = None
+# custom_minimizers = {"mantid": ["BFGS",
+#                                 "Damped GaussNewton"],
+#                      "scipy": ["lm", "trf"]}
+custom_minimizers = None
 
 # SPECIFY THE MINIMIZERS YOU WANT TO BENCHMARK, AND AS A MINIMUM FOR THE SOFTWARE YOU SPECIFIED ABOVE
 if len(sys.argv) > 1:
@@ -120,30 +120,28 @@ for sub_dir in problem_sets:
     results_dir = create_dirs.results(results_dir)
     group_results_dir = create_dirs.group_results(results_dir, group_name)
 
-    # All parameters inputed by the user are stored in an object
+    # All parameters inputted by the user are stored in an object
     user_input = misc.save_user_input(software, minimizers, group_name,
                                       group_results_dir, use_errors)
 
     # Loops through group of problems and benchmark them
-    prob_results = None
-    prob_results = \
-        [do_fitbm_group(user_input, block) for block in problem_groups[group_name]]
+    prob_results = do_benchmarking(user_input, problem_groups, group_name)
 
     print('\nProducing output for the {} problem set\n'.format(group_name))
     for idx, group_results in enumerate(prob_results):
-
         # Creates the results directory where the tables are located
         tables_dir = create_dirs.restables_dir(results_dir, group_name)
 
+        if isinstance(software, list):
+            minimizers = sum(minimizers, [])
+
         # Creates the problem names with links to the visual display pages
         # in rst
-        linked_problems = \
-            visual_pages.create_linked_probs(group_results,
-                                             group_name, results_dir)
+        linked_problems = visual_pages.create_linked_probs(group_results,
+                                                           group_name, results_dir)
 
         # Generates accuracy and runtime normalised tables and summary tables
-        norm_acc_rankings, norm_runtimes, sum_cells_acc, sum_cells_runtime = \
-            generate_tables(group_results, minimizers)
+        norm_acc_rankings, norm_runtimes, sum_cells_acc, sum_cells_runtime = generate_tables(group_results, minimizers)
 
         # Creates an accuracy table
         acc_tbl = create_acc_tbl(minimizers, linked_problems, norm_acc_rankings, use_errors, color_scale)
diff --git a/fitbenchmarking/fitting_benchmarking.py b/fitbenchmarking/fitting_benchmarking.py
index 6b257091..1de80940 100644
--- a/fitbenchmarking/fitting_benchmarking.py
+++ b/fitbenchmarking/fitting_benchmarking.py
@@ -61,13 +61,43 @@ def do_fitting_benchmark(group_name, software_options, data_dir,
   user_input = misc.save_user_input(software, minimizers, group_name,
                                     group_results_dir, use_errors)
 
-  prob_results = None
-  prob_results = \
-      [do_fitbm_group(user_input, block) for block in problem_groups[group_name]]
+  prob_results = do_benchmarking(user_input, problem_groups, group_name)
 
   return prob_results, results_dir
 
 
+def do_benchmarking(user_input, problem_groups, group_name):
+  """
+  Loops through software and benchmarks each problem within the problem
+  group.
+
+  @param user_input :: all the information specified by the user
+  @param problem_block :: array of paths to problem files in the group
+  @param problem_name :: name of group to be benchmarked
+
+  @returns :: array of result objects, per problem
+  """
+
+  if not isinstance(user_input, list):
+    prob_results = []
+    for block in problem_groups[group_name]:
+      prob_results.append(do_fitbm_group(user_input, block))
+  else:
+    list_prob_results = []
+    for user in user_input:
+      for block in problem_groups[group_name]:
+        list_prob_results.append(do_fitbm_group(user, block))
+    prob_results = []
+    tuple_prob_results = zip(*list_prob_results)
+    for tup in tuple_prob_results:
+      min_results = []
+      for i in tup:
+        min_results += i
+      prob_results.append(min_results)
+
+  return [prob_results]
+
+
 def do_fitbm_group(user_input, problem_block):
   """
   Fit benchmark a specific group of problems.
diff --git a/fitbenchmarking/resproc/numpy_restables.py b/fitbenchmarking/resproc/numpy_restables.py
index 5893dbc6..c0212808 100644
--- a/fitbenchmarking/resproc/numpy_restables.py
+++ b/fitbenchmarking/resproc/numpy_restables.py
@@ -49,10 +49,10 @@ def create_accuracy_runtime_tbls(results_per_test, minimizers):
     for test_idx in range(0, len(results_per_test)):
         for minimiz_idx in range(0, len(minimizers)):
             accuracy_tbl[test_idx, minimiz_idx] = \
-            results_per_test[test_idx][minimiz_idx].chi_sq
+                results_per_test[test_idx][minimiz_idx].chi_sq
 
             time_tbl[test_idx, minimiz_idx] = \
-            results_per_test[test_idx][minimiz_idx].runtime
+                results_per_test[test_idx][minimiz_idx].runtime
 
     return accuracy_tbl, time_tbl
 
diff --git a/fitbenchmarking/results_output.py b/fitbenchmarking/results_output.py
index 80a37058..07943202 100644
--- a/fitbenchmarking/results_output.py
+++ b/fitbenchmarking/results_output.py
@@ -58,6 +58,9 @@ def save_results_tables(software_options, results_per_test, group_name,
     """
 
     minimizers, software = utils.misc.get_minimizers(software_options)
+    if isinstance(software, list):
+        minimizers = sum(minimizers, [])
+
     tables_dir = create_dirs.restables_dir(results_dir, group_name)
     linked_problems = \
         visual_pages.create_linked_probs(results_per_test, group_name, results_dir)
diff --git a/fitbenchmarking/utils/fitbm_result.py b/fitbenchmarking/utils/fitbm_result.py
index 89c5123a..1fca0ded 100644
--- a/fitbenchmarking/utils/fitbm_result.py
+++ b/fitbenchmarking/utils/fitbm_result.py
@@ -43,3 +43,6 @@ class FittingResult(object):
         self.minimizer = None
         self.ini_function_def = None
         self.fin_function_def = None
+
+    def __repr__(self):
+        return 'Fitting problem class: minimizer = {0}'.format(self.minimizer)
diff --git a/fitbenchmarking/utils/misc.py b/fitbenchmarking/utils/misc.py
index f9d100a4..52a61a30 100644
--- a/fitbenchmarking/utils/misc.py
+++ b/fitbenchmarking/utils/misc.py
@@ -39,28 +39,36 @@ def get_minimizers(software_options):
     @returns :: an array of strings containing minimizer names and software used
     """
     try:
+
         software = software_options['software']
         minimizer_options = software_options['minimizer_options']
-        if not minimizer_options:
-            current_path = os.path.dirname(os.path.realpath(__file__))
-            fitbm_path = os.path.abspath(os.path.join(current_path, os.pardir))
-            minimizer_file = os.path.join(fitbm_path,
-                                          "minimizers_list_default.json")
-            minimizers_list = json.load(open(minimizer_file))
-        elif isinstance(minimizer_options, str):
-            minimizers_list = json.load(open(minimizer_options))
-        elif isinstance(minimizer_options, dict):
-            minimizers_list = minimizer_options
-        else:
-            raise ValueError('minimizer_options required to be None, string '
-                             'or dictionary, type(minimizer_options) '
-                             '= {}'.format(type(minimizer_options)))
-        minimizers = minimizers_list[software]
+        if not isinstance(software, list):
+            software = [software]
+        minimizers = []
+        for x in software:
+            if not minimizer_options:
+                current_path = os.path.dirname(os.path.realpath(__file__))
+                fitbm_path = os.path.abspath(os.path.join(current_path, os.pardir))
+                minimizer_file = os.path.join(fitbm_path,
+                                              "minimizers_list_default.json")
+                minimizers_list = json.load(open(minimizer_file))
+            elif isinstance(minimizer_options, str):
+                minimizers_list = json.load(open(minimizer_options))
+            elif isinstance(minimizer_options, dict):
+                minimizers_list = minimizer_options
+            else:
+                raise ValueError('minimizer_options required to be None, string '
+                                 'or dictionary, type(minimizer_options) '
+                                 '= {}'.format(type(minimizer_options)))
+            minimizers.append(minimizers_list[x])
     except:
         raise ValueError('software_options required to be a dictionary with '
                          'keys software and minimizer_options')
+    if len(software) > 1:
+        return minimizers, software
 
-    return minimizers, software
+    else:
+        return minimizers[0], software[0]
 
 
 def setup_fitting_problems(data_dir, group_name):
@@ -83,12 +91,15 @@ def save_user_input(software, minimizers, group_name, results_dir, use_errors):
     @params :: please check the user_input.py file in the utils dir.
     @returns :: an object containing all the information specified by the user.
     """
-
-    uinput = user_input.UserInput()
-    uinput.software = software
-    uinput.minimizers = minimizers
-    uinput.group_name = group_name
-    uinput.group_results_dir = results_dir
-    uinput.use_errors = use_errors
-
+    if isinstance(software, str):
+        uinput = user_input.UserInput(software, minimizers, group_name,
+                                      results_dir, use_errors)
+    elif isinstance(software, list):
+        uinput = []
+        for i in range(len(software)):
+            uinput.append(user_input.UserInput(software[i], minimizers[i],
+                                               group_name,
+                                               results_dir, use_errors))
+    else:
+        raise TypeError('Software input required to be a string or list')
     return uinput
diff --git a/fitbenchmarking/utils/user_input.py b/fitbenchmarking/utils/user_input.py
index 965b300b..c2526ba6 100644
--- a/fitbenchmarking/utils/user_input.py
+++ b/fitbenchmarking/utils/user_input.py
@@ -25,15 +25,15 @@ class UserInput(object):
     Structure to hold all the user inputted data.
     """
 
-    def __init__(self):
+    def __init__(self, software, minimizers, group_name, results_dir, use_errors):
         # The software that is benchmarked e.g. scipy, mantid etc.
-        self.software = None
+        self.software = software
         # The minimizers inside that certain software that are
         # being benchmarked
-        self.minimizers = None
+        self.minimizers = minimizers
         # The name of the problem group to be analysed e.g. neutron
-        self.group_name = None
+        self.group_name = group_name
         # Director path in which to put the results for each problem group
-        self.group_results_dir = None
+        self.group_results_dir = results_dir
         # Whether or not to consider error bars in the fitting process
-        self.use_errors = None
+        self.use_errors = use_errors


Test patch:
diff --git a/fitbenchmarking/utils/tests/test_misc.py b/fitbenchmarking/utils/tests/test_misc.py
index da7daa62..f2357df8 100644
--- a/fitbenchmarking/utils/tests/test_misc.py
+++ b/fitbenchmarking/utils/tests/test_misc.py
@@ -19,162 +19,178 @@ from misc import setup_fitting_problems
 
 class CreateDirsTests(unittest.TestCase):
 
-    def base_path(self):
-        """
-        Helper function that returns the path to
-        /fitbenchmarking/benchmark_problems
-        """
+  def base_path(self):
+    """
+    Helper function that returns the path to
+    /fitbenchmarking/benchmark_problems
+    """
 
-        current_dir = os.path.dirname(os.path.realpath(__file__))
-        parent_dir = os.path.dirname(os.path.normpath(test_dir))
-        main_dir = os.path.dirname(os.path.normpath(parent_dir))
-        root_dir = os.path.dirname(os.path.normpath(main_dir))
-        bench_prob_dir = os.path.join(root_dir, 'benchmark_problems')
+    current_dir = os.path.dirname(os.path.realpath(__file__))
+    parent_dir = os.path.dirname(os.path.normpath(test_dir))
+    main_dir = os.path.dirname(os.path.normpath(parent_dir))
+    root_dir = os.path.dirname(os.path.normpath(main_dir))
+    bench_prob_dir = os.path.join(root_dir, 'benchmark_problems')
 
-        return bench_prob_dir
+    return bench_prob_dir
 
-    def all_neutron_problems(self):
-        """
-        Helper function that returns the names of all neutron problems.
-        """
+  def all_neutron_problems(self):
+    """
+    Helper function that returns the names of all neutron problems.
+    """
 
-        neutron_problems = [['ENGINX193749_calibration_peak19.txt',
-                             'ENGINX193749_calibration_peak20.txt',
-                             'ENGINX193749_calibration_peak23.txt',
-                             'ENGINX193749_calibration_peak5.txt',
-                             'ENGINX193749_calibration_peak6.txt',
-                             'ENGINX236516_vanadium_bank1_10brk.txt',
-                             'ENGINX236516_vanadium_bank1_20brk.txt',
-                             'EVS14188-90_Gaussian_peaks_1.txt',
-                             'EVS14188-90_Gaussian_peaks_2.txt',
-                             'GEMpeak1.txt',
-                             'WISH17701_peak1.txt', 'WISH17701_peak2.txt',
-                             'WISH17701_peak3.txt', 'WISH17701_peak4.txt',
-                             'WISH17701_peak5.txt', 'WISH17701_peak6.txt',
-                             'WISH17701_peak7.txt', 'WISH17701_peak8.txt',
-                             'WISH17701_peak9.txt']]
+    neutron_problems = [['ENGINX193749_calibration_peak19.txt',
+                         'ENGINX193749_calibration_peak20.txt',
+                         'ENGINX193749_calibration_peak23.txt',
+                         'ENGINX193749_calibration_peak5.txt',
+                         'ENGINX193749_calibration_peak6.txt',
+                         'ENGINX236516_vanadium_bank1_10brk.txt',
+                         'ENGINX236516_vanadium_bank1_20brk.txt',
+                         'EVS14188-90_Gaussian_peaks_1.txt',
+                         'EVS14188-90_Gaussian_peaks_2.txt',
+                         'GEMpeak1.txt',
+                         'WISH17701_peak1.txt', 'WISH17701_peak2.txt',
+                         'WISH17701_peak3.txt', 'WISH17701_peak4.txt',
+                         'WISH17701_peak5.txt', 'WISH17701_peak6.txt',
+                         'WISH17701_peak7.txt', 'WISH17701_peak8.txt',
+                         'WISH17701_peak9.txt']]
 
-        return neutron_problems
+    return neutron_problems
 
-    def all_nist_problems(self):
-        """
-        Helper function that returns the names of Nist low diff problems.
-        """
+  def all_nist_problems(self):
+    """
+    Helper function that returns the names of Nist low diff problems.
+    """
 
-        nist_ld_problems = [['Misra1a.dat', 'Chwirut2.dat', 'Chwirut1.dat',
-                             'Lanczos3.dat', 'Gauss1.dat', 'Gauss2.dat',
-                             'DanWood.dat', 'Misra1b.dat']]
+    nist_ld_problems = [['Misra1a.dat', 'Chwirut2.dat', 'Chwirut1.dat',
+                         'Lanczos3.dat', 'Gauss1.dat', 'Gauss2.dat',
+                         'DanWood.dat', 'Misra1b.dat']]
 
-        return nist_ld_problems
+    return nist_ld_problems
 
-    def get_minimizers_file(self):
+  def get_minimizers_file(self):
 
-        current_path = os.path.dirname(os.path.realpath(__file__))
-        utils_path = os.path.abspath(os.path.join(current_path, os.pardir))
-        fitbm_path = os.path.abspath(os.path.join(utils_path, os.pardir))
-        minimizer_json = os.path.join(fitbm_path,
-                                      "minimizers_list_default.json")
-        return minimizer_json
+    current_path = os.path.dirname(os.path.realpath(__file__))
+    utils_path = os.path.abspath(os.path.join(current_path, os.pardir))
+    fitbm_path = os.path.abspath(os.path.join(utils_path, os.pardir))
+    minimizer_json = os.path.join(fitbm_path,
+                                  "minimizers_list_default.json")
+    return str(minimizer_json)
 
-    def test_getMinimizers_load_correct_minimizers_mantid_default(self):
+  def test_getMinimizers_load_correct_minimizers_mantid_default(self):
 
-        software_options = {'software': 'mantid', 'minimizer_options': None}
+    software_options = {'software': 'mantid', 'minimizer_options': None}
 
-        minimizers, _ = get_minimizers(software_options)
-        minmizers_expected = \
-            ["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)",
-             "Conjugate gradient (Polak-Ribiere imp.)", "Damped GaussNewton",
-             "Levenberg-Marquardt", "Levenberg-MarquardtMD", "Simplex",
-             "SteepestDescent", "Trust Region"]
+    minimizers, _ = get_minimizers(software_options)
+    minmizers_expected = \
+        ["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)",
+         "Conjugate gradient (Polak-Ribiere imp.)", "Damped GaussNewton",
+         "Levenberg-Marquardt", "Levenberg-MarquardtMD", "Simplex",
+         "SteepestDescent", "Trust Region"]
 
-        self.assertListEqual(minmizers_expected, minimizers)
+    self.assertListEqual(minmizers_expected, minimizers)
 
-    def test_getMinimizers_load_correct_minimizers_scipy_default(self):
+  def test_getMinimizers_load_correct_minimizers_scipy_default(self):
 
-        software_options = {'software': 'scipy', 'minimizer_options': None}
+    software_options = {'software': 'scipy', 'minimizer_options': None}
 
-        minimizers, _ = get_minimizers(software_options)
-        minmizers_expected = ["lm", "trf", "dogbox"]
+    minimizers, _ = get_minimizers(software_options)
+    minmizers_expected = ["lm", "trf", "dogbox"]
 
-        self.assertListEqual(minmizers_expected, minimizers)
+    self.assertListEqual(minmizers_expected, minimizers)
 
-    def test_getMinimizers_load_correct_minimizers_scipy_min_list(self):
+  def test_getMinimizers_load_correct_minimizers_scipy_min_list(self):
 
-        software_options = {'software': 'scipy',
-                            'minimizer_options': {"scipy": ["lm", "trf"]}}
+    software_options = {'software': 'scipy',
+                        'minimizer_options': {"scipy": ["lm", "trf"]}}
 
-        minimizers, _ = get_minimizers(software_options)
-        minmizers_expected = ["lm", "trf"]
+    minimizers, _ = get_minimizers(software_options)
+    minmizers_expected = ["lm", "trf"]
 
-        self.assertListEqual(minmizers_expected, minimizers)
+    self.assertListEqual(minmizers_expected, minimizers)
 
-    def test_getMinimizers_load_correct_minimizers_mantid_min_list(self):
+  def test_getMinimizers_load_correct_minimizers_mantid_min_list(self):
 
-        software_options = {'software': 'mantid',
-                            'minimizer_options': {"mantid": ["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)", "Conjugate gradient (Polak-Ribiere imp.)"]}}
+    software_options = {'software': 'mantid',
+                        'minimizer_options': {"mantid": ["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)", "Conjugate gradient (Polak-Ribiere imp.)"]}}
 
-        minimizers, _ = get_minimizers(software_options)
-        minmizers_expected = ["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)", "Conjugate gradient (Polak-Ribiere imp.)"]
+    minimizers, _ = get_minimizers(software_options)
+    minmizers_expected = ["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)", "Conjugate gradient (Polak-Ribiere imp.)"]
 
-        self.assertListEqual(minmizers_expected, minimizers)
+    self.assertListEqual(minmizers_expected, minimizers)
 
-    def test_getMinimizers_load_correct_minimizers_scipy_read_file(self):
-        software_options = {'software': 'scipy',
-                            'minimizer_options': self.get_minimizers_file()}
+  def test_getMinimizers_load_correct_minimizers_scipy_read_file(self):
+    software_options = {'software': 'scipy',
+                        'minimizer_options': self.get_minimizers_file()}
 
-        minimizers, _ = get_minimizers(software_options)
-        minmizers_expected = ["lm", "trf", "dogbox"]
+    minimizers, _ = get_minimizers(software_options)
+    minmizers_expected = ["lm", "trf", "dogbox"]
 
-        self.assertListEqual(minmizers_expected, minimizers)
+    self.assertListEqual(minmizers_expected, minimizers)
 
-    def test_getMinimizers_load_correct_minimizers_mantid_read_file(self):
+  def test_getMinimizers_load_correct_minimizers_mantid_read_file(self):
 
-        software_options = {'software': 'mantid',
-                            'minimizer_options': self.get_minimizers_file()}
+    software_options = {'software': 'mantid',
+                        'minimizer_options': self.get_minimizers_file()}
 
-        minimizers, _ = get_minimizers(software_options)
-        minmizers_expected = ["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)",
-                              "Conjugate gradient (Polak-Ribiere imp.)",
-                              "Damped GaussNewton",
-                              "Levenberg-Marquardt", "Levenberg-MarquardtMD",
-                              "Simplex", "SteepestDescent",
-                              "Trust Region"]
+    minimizers, _ = get_minimizers(software_options)
+    minmizers_expected = ["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)",
+                          "Conjugate gradient (Polak-Ribiere imp.)",
+                          "Damped GaussNewton",
+                          "Levenberg-Marquardt", "Levenberg-MarquardtMD",
+                          "Simplex", "SteepestDescent",
+                          "Trust Region"]
 
-        self.assertListEqual(minmizers_expected, minimizers)
+    self.assertListEqual(minmizers_expected, minimizers)
 
-    def test_getMinimizers_value_error_incorrect_input_software_options(self):
+  def test_getMinimizers_load_correct_compare_minimizers_mantid_read_file(self):
 
-        software_options = 2
+    software_options = {'software': ['mantid', 'scipy'],
+                        'minimizer_options': self.get_minimizers_file()}
 
-        self.assertRaises(ValueError, get_minimizers, software_options)
+    minimizers, _ = get_minimizers(software_options)
+    minmizers_expected = [["BFGS", "Conjugate gradient (Fletcher-Reeves imp.)",
+                           "Conjugate gradient (Polak-Ribiere imp.)",
+                           "Damped GaussNewton",
+                           "Levenberg-Marquardt", "Levenberg-MarquardtMD",
+                           "Simplex", "SteepestDescent",
+                           "Trust Region"],
+                          ["lm", "trf", "dogbox"]]
 
-    def test_getMinimizers_value_error_incorrect_input_minimizer_options(self):
+    self.assertListEqual(minmizers_expected, minimizers)
 
-        software_options = {'software': 'mantid',
-                            'minimizer_options': 1}
+  def test_getMinimizers_value_error_incorrect_input_software_options(self):
 
-        self.assertRaises(ValueError, get_minimizers, software_options)
+    software_options = 2
 
-    def test_setupFittingProblems_get_correct_nist_probs(self):
+    self.assertRaises(ValueError, get_minimizers, software_options)
 
-        data_dir = os.path.join(self.base_path(), 'NIST', 'low_difficulty')
-        nist_problems = self.all_nist_problems()
+  def test_getMinimizers_value_error_incorrect_input_minimizer_options(self):
 
-        problem_groups = setup_fitting_problems(data_dir, 'NIST_low_diff')
-        problem_groups_expected = {'NIST_low_diff': nist_problems}
+    software_options = {'software': 'mantid',
+                        'minimizer_options': 1}
 
-        self.assertTrue(problem_groups_expected, problem_groups)
+    self.assertRaises(ValueError, get_minimizers, software_options)
 
-    def test_setupFittingProblems_get_correct_neutron_probs(self):
+  def test_setupFittingProblems_get_correct_nist_probs(self):
 
-        data_dir = os.path.join(self.base_path(), 'Neutron_data')
-        neutron_problems = self.all_neutron_problems()
+    data_dir = os.path.join(self.base_path(), 'NIST', 'low_difficulty')
+    nist_problems = self.all_nist_problems()
 
-        problem_groups = setup_fitting_problems(data_dir, 'Neutron')
-        problem_groups_expected = {'nist': neutron_problems}
+    problem_groups = setup_fitting_problems(data_dir, 'NIST_low_diff')
+    problem_groups_expected = {'NIST_low_diff': nist_problems}
 
-        self.assertTrue(problem_groups_expected, problem_groups)
+    self.assertTrue(problem_groups_expected, problem_groups)
+
+  def test_setupFittingProblems_get_correct_neutron_probs(self):
+
+    data_dir = os.path.join(self.base_path(), 'Neutron_data')
+    neutron_problems = self.all_neutron_problems()
+
+    problem_groups = setup_fitting_problems(data_dir, 'Neutron')
+    problem_groups_expected = {'nist': neutron_problems}
+
+    self.assertTrue(problem_groups_expected, problem_groups)
 
 
 if __name__ == "__main__":
-    unittest.main()
+  unittest.main()