Repository: fitbenchmarking/fitbenchmarking

Issue:
Documentation of code out of sync in top level files 
Code documentation of functions, function parameters and code comment has gone out of sync for fitting_benchmarking.py and fitbenchmark_one_problem.py and some of the functions it calls.

Fix:
* documentation for all function parameters
* check these are not out of date (and therefore currently wrong)
* add code explaining the code in do_benchmarking (in this function should problem_groups be problem_group and block be file?)

These two level python are particular important to be well documented since they form entry points to the other parts of FitBenchmarking.

Observed while testing #199

Discussion:

Patch:
diff --git a/example_scripts/example_runScripts.py b/example_scripts/example_runScripts.py
index 0da68d4b..3aea5da5 100644
--- a/example_scripts/example_runScripts.py
+++ b/example_scripts/example_runScripts.py
@@ -9,8 +9,8 @@ import os
 import sys
 import glob
 
-from fitbenchmarking.fitting_benchmarking import do_fitting_benchmark as fitBenchmarking
-from fitbenchmarking.results_output import save_results_tables as printTables
+from fitbenchmarking.fitting_benchmarking import fitbenchmark_group
+from fitbenchmarking.results_output import save_results_tables
 
 
 def main(args):
@@ -56,10 +56,10 @@ def main(args):
     # e.g. lower that 1.1 -> light yellow, higher than 3 -> dark red
     # Change these values to suit your needs
     color_scale = [(1.1, 'ranking-top-1'),
-                (1.33, 'ranking-top-2'),
-                (1.75, 'ranking-med-3'),
-                (3, 'ranking-low-4'),
-                (float('nan'), 'ranking-low-5')]
+                   (1.33, 'ranking-top-2'),
+                   (1.75, 'ranking-med-3'),
+                   (3, 'ranking-low-4'),
+                   (float('nan'), 'ranking-low-5')]
 
     # ADD WHICH PROBLEM SETS TO TEST AGAINST HERE
     # Do this, in this example file, by selecting sub-folders in benchmark_probs_dir
@@ -82,16 +82,21 @@ def main(args):
             continue
 
         print('\nRunning the benchmarking on the {} problem set\n'.format(label))
-        results_per_group, results_dir = fitBenchmarking(group_name=label, software_options=software_options,
-                                                         data_dir=data_dir,
-                                                         use_errors=use_errors, results_dir=results_dir)
+        results, results_dir = \
+            fitbenchmark_group(group_name=label,
+                               software_options=software_options,
+                               data_dir=data_dir,
+                               use_errors=use_errors,
+                               results_dir=results_dir)
 
         print('\nProducing output for the {} problem set\n'.format(label))
-        for _, group_results in enumerate(results_per_group):
-            # Display the runtime and accuracy results in a table
-            printTables(software_options, group_results,
-                        group_name=label, use_errors=use_errors,
-                        color_scale=color_scale, results_dir=results_dir)
+        # Display the runtime and accuracy results in a table
+        save_results_tables(software_options=software_options,
+                            results_per_test=results,
+                            group_name=label,
+                            use_errors=use_errors,
+                            color_scale=color_scale,
+                            results_dir=results_dir)
 
         print('\nCompleted benchmarking for {} problem set\n'.format(sub_dir))
 
diff --git a/example_scripts/example_runScripts_SasView.py b/example_scripts/example_runScripts_SasView.py
index cb53861a..ae6c888b 100644
--- a/example_scripts/example_runScripts_SasView.py
+++ b/example_scripts/example_runScripts_SasView.py
@@ -34,8 +34,8 @@ except:
           '******************************************')
     sys.exit()
 
-from fitbenchmarking.fitting_benchmarking import do_fitting_benchmark as fitBenchmarking
-from fitbenchmarking.results_output import save_results_tables as printTables
+from fitbenchmarking.fitting_benchmarking import fitbenchmark_group
+from fitbenchmarking.results_output import save_results_tables
 
 # SPECIFY THE SOFTWARE/PACKAGE CONTAINING THE MINIMIZERS YOU WANT TO BENCHMARK
 software = ['sasview']
@@ -100,15 +100,20 @@ for sub_dir in problem_sets:
     data_dir = os.path.join(benchmark_probs_dir, sub_dir)
 
     print('\nRunning the benchmarking on the {} problem set\n'.format(label))
-    results_per_group, results_dir = fitBenchmarking(group_name=label, software_options=software_options,
-                                                     data_dir=data_dir,
-                                                     use_errors=use_errors, results_dir=results_dir)
+    results, results_dir = fitbenchmark_group(group_name=label,
+                                              software_options=software_options,
+                                              data_dir=data_dir,
+                                              use_errors=use_errors,
+                                              results_dir=results_dir)
 
     print('\nProducing output for the {} problem set\n'.format(label))
-    for idx, group_results in enumerate(results_per_group):
-        # Display the runtime and accuracy results in a table
-        printTables(software_options, group_results,
-                    group_name=label, use_errors=use_errors,
-                    color_scale=color_scale, results_dir=results_dir)
+
+    # Display the runtime and accuracy results in a table
+    save_results_tables(software_options=software_options,
+                        results_per_test=results,
+                        group_name=label,
+                        use_errors=use_errors,
+                        color_scale=color_scale,
+                        results_dir=results_dir)
 
     print('\nCompleted benchmarking for {} problem set\n'.format(sub_dir))
diff --git a/example_scripts/example_runScripts_expert.py b/example_scripts/example_runScripts_expert.py
index faeaaef7..8e0799ee 100644
--- a/example_scripts/example_runScripts_expert.py
+++ b/example_scripts/example_runScripts_expert.py
@@ -10,7 +10,7 @@ import os
 import sys
 import glob
 
-from fitbenchmarking.fitting_benchmarking import do_benchmarking
+from fitbenchmarking.fitting_benchmarking import _benchmark
 from fitbenchmarking.utils import misc
 from fitbenchmarking.utils import create_dirs
 from fitbenchmarking.results_output import save_tables, generate_tables, \
@@ -90,10 +90,11 @@ def main(argv):
         # Processes software_options dictionary into Fitbenchmarking format
         minimizers, software = misc.get_minimizers(software_options)
 
-        # Sets up the problem groups specified by the user by providing
+        # Sets up the problem group specified by the user by providing
         # a respective data directory.
-        problem_groups = misc.setup_fitting_problems(data_dir, group_name)
+        problem_group = misc.setup_fitting_problems(data_dir)
 
+        # Create output dirs
         results_dir = create_dirs.results(results_dir)
         group_results_dir = create_dirs.group_results(results_dir, group_name)
 
@@ -102,35 +103,35 @@ def main(argv):
                                           group_results_dir, use_errors)
 
         # Loops through group of problems and benchmark them
-        prob_results = do_benchmarking(user_input, problem_groups, group_name)
+        prob_results = _benchmark(user_input, problem_group)
 
         print('\nProducing output for the {} problem set\n'.format(group_name))
-        for idx, group_results in enumerate(prob_results):
-            # Creates the results directory where the tables are located
-            tables_dir = create_dirs.restables_dir(results_dir, group_name)
 
-            if isinstance(software, list):
-                minimizers = sum(minimizers, [])
+        # Creates the results directory where the tables are located
+        tables_dir = create_dirs.restables_dir(results_dir, group_name)
 
-            # Creates the problem names with links to the visual display pages
-            # in rst
-            linked_problems = visual_pages.create_linked_probs(group_results,
-                                                               group_name, results_dir)
+        if isinstance(software, list):
+            minimizers = sum(minimizers, [])
 
-            # Generates accuracy and runtime normalised tables and summary tables
-            norm_acc_rankings, norm_runtimes, sum_cells_acc, sum_cells_runtime = generate_tables(group_results, minimizers)
+        # Creates the problem names with links to the visual display pages
+        # in rst
+        linked_problems = visual_pages.create_linked_probs(prob_results,
+                                                           group_name, results_dir)
 
-            # Creates an accuracy table
-            acc_tbl = create_acc_tbl(minimizers, linked_problems, norm_acc_rankings, use_errors, color_scale)
+        # Generates accuracy and runtime tables and summary tables
+        acc_rankings, runtimes, sum_cells_acc, sum_cells_runtime = generate_tables(prob_results, minimizers)
 
-            # Creates an runtime table
-            runtime_tbl = create_runtime_tbl(minimizers, linked_problems, norm_runtimes, use_errors, color_scale)
+        # Creates an accuracy table
+        acc_tbl = create_acc_tbl(minimizers, linked_problems, acc_rankings, use_errors, color_scale)
 
-            # Saves accuracy minimizer results
-            save_tables(tables_dir, acc_tbl, use_errors, group_name, 'acc')
+        # Creates an runtime table
+        runtime_tbl = create_runtime_tbl(minimizers, linked_problems, runtimes, use_errors, color_scale)
 
-            # Saves runtime minimizer results
-            save_tables(tables_dir, runtime_tbl, use_errors, group_name, 'runtime')
+        # Saves accuracy minimizer results
+        save_tables(tables_dir, acc_tbl, use_errors, group_name, 'acc')
+
+        # Saves runtime minimizer results
+        save_tables(tables_dir, runtime_tbl, use_errors, group_name, 'runtime')
 
         print('\nCompleted benchmarking for {} problem set\n'.format(sub_dir))
 
diff --git a/example_scripts/example_runScripts_mantid.py b/example_scripts/example_runScripts_mantid.py
index 1f4b9e3d..e9493d8e 100644
--- a/example_scripts/example_runScripts_mantid.py
+++ b/example_scripts/example_runScripts_mantid.py
@@ -14,7 +14,7 @@ import glob
 
 try:
     import mantid.simpleapi as msapi
-except:
+except ImportError:
     print('******************************************\n'
           'Mantid is not yet installed on your computer\n'
           'To install, go to the directory where setup.py is located and simply type:\n'
@@ -22,8 +22,8 @@ except:
           '******************************************')
     sys.exit()
 
-from fitbenchmarking.fitting_benchmarking import do_fitting_benchmark as fitBenchmarking
-from fitbenchmarking.results_output import save_results_tables as printTables
+from fitbenchmarking.fitting_benchmarking import fitbenchmark_group
+from fitbenchmarking.results_output import save_results_tables
 
 
 def main(argv):
@@ -33,7 +33,7 @@ def main(argv):
 
     # User defined minimizers
     custom_minimizers = {"mantid": ["Levenberg-Marquardt"],
-                        "scipy": ["lm", "trf", "dogbox"]}
+                         "scipy": ["lm", "trf", "dogbox"]}
     # custom_minimizers = None
 
     # SPECIFY THE MINIMIZERS YOU WANT TO BENCHMARK, AND AS A MINIMUM FOR THE SOFTWARE YOU SPECIFIED ABOVE
@@ -53,7 +53,7 @@ def main(argv):
     # Benchmark problem directories
     fitbenchmarking_folder = os.path.abspath(os.path.join(current_path, os.pardir))
     benchmark_probs_dir = os.path.join(fitbenchmarking_folder,
-                                    'benchmark_problems')
+                                       'benchmark_problems')
 
     """
     Modify results_dir to specify where the results of the fit should be saved
@@ -69,10 +69,10 @@ def main(argv):
     # e.g. lower that 1.1 -> light yellow, higher than 3 -> dark red
     # Change these values to suit your needs
     color_scale = [(1.1, 'ranking-top-1'),
-                (1.33, 'ranking-top-2'),
-                (1.75, 'ranking-med-3'),
-                (3, 'ranking-low-4'),
-                (float('nan'), 'ranking-low-5')]
+                   (1.33, 'ranking-top-2'),
+                   (1.75, 'ranking-med-3'),
+                   (3, 'ranking-low-4'),
+                   (float('nan'), 'ranking-low-5')]
 
     # ADD WHICH PROBLEM SETS TO TEST AGAINST HERE
     # Do this, in this example file, by selecting sub-folders in benchmark_probs_dir
@@ -93,16 +93,21 @@ def main(argv):
             continue
 
         print('\nRunning the benchmarking on the {} problem set\n'.format(label))
-        results_per_group, results_dir = fitBenchmarking(group_name=label, software_options=software_options,
-                                                        data_dir=data_dir,
-                                                        use_errors=use_errors, results_dir=results_dir)
+        results, results_dir = fitbenchmark_group(group_name=label,
+                                                  software_options=software_options,
+                                                  data_dir=data_dir,
+                                                  use_errors=use_errors,
+                                                  results_dir=results_dir)
 
         print('\nProducing output for the {} problem set\n'.format(label))
-        for idx, group_results in enumerate(results_per_group):
-            # Display the runtime and accuracy results in a table
-            printTables(software_options, group_results,
-                        group_name=label, use_errors=use_errors,
-                        color_scale=color_scale, results_dir=results_dir)
+
+        # Display the runtime and accuracy results in a table
+        save_results_tables(software_options=software_options,
+                            results_per_test=results,
+                            group_name=label,
+                            use_errors=use_errors,
+                            color_scale=color_scale,
+                            results_dir=results_dir)
 
         print('\nCompleted benchmarking for {} problem set\n'.format(sub_dir))
 
diff --git a/fitbenchmarking/fitting_benchmarking.py b/fitbenchmarking/fitting_benchmarking.py
index 249129e7..09bbe3e4 100644
--- a/fitbenchmarking/fitting_benchmarking.py
+++ b/fitbenchmarking/fitting_benchmarking.py
@@ -15,89 +15,70 @@ from fitbenchmarking.utils import create_dirs, misc
 from fitbenchmarking.fitbenchmark_one_problem import fitbm_one_prob
 
 
-def do_fitting_benchmark(group_name, software_options, data_dir,
-                         use_errors=True, results_dir=None):
-  """
-  This function does the fitting benchmarking for a
-  specified group of problems.
-
-  @param group_name :: is the name (label) for a group. E.g. the name for the group of problems in
-                       "NIST/low_difficulty" may be picked to be NIST_low_difficulty
-  @param software_options :: dictionary containing software used in fitting the problem, list of minimizers and
-                             location of json file contain minimizers
-  @param data_dir :: full path of a directory that holds a group of problem definition files
-  @param use_errors :: whether to use errors on the data or not
-  @param results_dir :: directory in which to put the results. None
-                      means results directory is created for you
-
-  @returns :: array of fitting results for the problem group and
-              the path to the results directory
-  """
-
-  logger.info("Loading minimizers from {0}".format(software_options['software']))
-  minimizers, software = misc.get_minimizers(software_options)
-  # create dir with paths to all problem definitions in data_dir
-  problem_group = misc.setup_fitting_problems(data_dir, group_name)
-
-  results_dir = create_dirs.results(results_dir)
-  group_results_dir = create_dirs.group_results(results_dir, group_name)
-
-  user_input = misc.save_user_input(software, minimizers, group_name,
-                                    group_results_dir, use_errors)
-
-  prob_results = do_benchmarking(user_input, problem_group, group_name)
-
-  return prob_results, results_dir
-
-
-def do_benchmarking(user_input, problem_groups, group_name):
-  """
-  Loops through software and benchmarks each problem within the problem
-  group.
-
-  @param user_input :: all the information specified by the user
-  @param problem_groups :: dictionary containing the paths to problem files in the group
-  @param group_name :: is the name (label) for a group. E.g. the name for the group of problems in
-                       "NIST/low_difficulty" may be picked to be NIST_low_difficulty
-
-  @returns :: array of result objects, per problem
-  """
-
-  if not isinstance(user_input, list):
-    prob_results = []
-    for block in problem_groups[group_name]:
-      prob_results.append(do_fitbm_group(user_input, block))
-  else:
-    list_prob_results = []
-    for user in user_input:
-      for block in problem_groups[group_name]:
-        list_prob_results.append(do_fitbm_group(user, block))
-    prob_results = []
-    tuple_prob_results = zip(*list_prob_results)
-    for tup in tuple_prob_results:
-      min_results = []
-      for i in tup:
-        min_results += i
-      prob_results.append(min_results)
-    prob_results = [prob_results]
-
-  return prob_results
-
-
-def do_fitbm_group(user_input, problem_block):
-  """
-  Fit benchmark a specific group of problems.
-
-  @param user_input :: all the information specified by the user
-  @param problem_block :: array of paths to problem files in the group
-
-  @returns :: array of result objects, per problem
-  """
-
-  results_per_problem = []
-  for prob_file in problem_block:
-    problem = parse.parse_problem_file(prob_file)
-    results_prob = fitbm_one_prob(user_input, problem)
-    results_per_problem.extend(results_prob)
-
-  return results_per_problem
+def fitbenchmark_group(group_name, software_options, data_dir,
+                       use_errors=True, results_dir=None):
+    """
+    Gather the user input and list of paths. Call benchmarking on these.
+
+    @param group_name :: is the name (label) for a group. E.g. the name for the group of problems in
+                         "NIST/low_difficulty" may be picked to be NIST_low_difficulty
+    @param software_options :: dictionary containing software used in fitting the problem, list of minimizers and
+                               location of json file contain minimizers
+    @param data_dir :: full path of a directory that holds a group of problem definition files
+    @param use_errors :: whether to use errors on the data or not
+    @param results_dir :: directory in which to put the results. None
+                          means results directory is created for you
+
+    @returns :: array of fitting results for the problem group and
+                the path to the results directory
+    """
+
+    logger.info("Loading minimizers from {0}".format(
+        software_options['software']))
+    minimizers, software = misc.get_minimizers(software_options)
+
+    # create list of paths to all problem definitions in data_dir
+    problem_group = misc.get_problem_files(data_dir)
+
+    results_dir = create_dirs.results(results_dir)
+    group_results_dir = create_dirs.group_results(results_dir, group_name)
+
+    user_input = misc.save_user_input(software, minimizers, group_name,
+                                      group_results_dir, use_errors)
+
+    prob_results = _benchmark(user_input, problem_group)
+
+    return prob_results, results_dir
+
+
+def _benchmark(user_input, problem_group):
+    """
+    Loops through software and benchmarks each problem within the problem
+    group.
+
+    @param user_input :: all the information specified by the user
+    @param problem_group :: list of paths to problem files in the group
+                            e.g. ['NIST/low_difficulty/file1.dat',
+                                  'NIST/low_difficulty/file2.dat',
+                                  ...]
+
+    @returns :: array of result objects, per problem per user_input
+    """
+
+    parsed_problems = [parse.parse_problem_file(p) for p in problem_group]
+
+    if not isinstance(user_input, list):
+        list_prob_results = [fitbm_one_prob(user_input, p) for p in parsed_problems]
+
+    else:
+        list_prob_results = [fitbm_one_prob(u, p)
+                             for u in user_input
+                             for p in parsed_problems]
+
+    # Flatten
+    list_prob_results = [res
+                         for tmp_list in list_prob_results
+                         for res in tmp_list]
+
+    return list_prob_results
+
diff --git a/fitbenchmarking/results_output.py b/fitbenchmarking/results_output.py
index 1e28ffd6..8dadd7fc 100644
--- a/fitbenchmarking/results_output.py
+++ b/fitbenchmarking/results_output.py
@@ -36,9 +36,9 @@ def save_results_tables(software_options, results_per_test, group_name,
     """
 
     minimizers, software = misc.get_minimizers(software_options)
-    comparison_type = software_options.get('comparison_type', None)
+    comparison_mode = software_options.get('comparison_mode', None)
 
-    if comparison_type is None:
+    if comparison_mode is None:
         if 'options_file' in software_options:
             options_file = software_options['options_file']
             comparison_mode = options.get_option(options_file=options_file,
@@ -85,6 +85,62 @@ def save_results_tables(software_options, results_per_test, group_name,
     logging.shutdown()
 
 
+def create_acc_tbl(minimizers, linked_problems, accuracy_tbl, use_errors,
+                   color_scale, comparison_mode='abs'):
+    """
+    API function to create an accuracy table using the given parameters.
+    For usage, see the expert example script.
+
+    @param minimizers :: array of minimizers used in fitting
+    @param linked_problems :: array of the problems that were fitted
+    @param accuracy_tbl :: numpy array of the accuracy results
+    @param using_errors :: boolean whether to use errors or not
+    @param color_scale :: color scale for coloring the cells
+    @param comparison_mode :: str to select between 'abs', 'rel', 'both' for
+                              the style of comparison returned
+
+    @returns :: rst table of the results
+    """
+    # Save accuracy table for this group of fit problems
+    tbl_acc_indiv = rst_table.create(minimizers, linked_problems,
+                                     accuracy_tbl,
+                                     comparison_type='accuracy',
+                                     comparison_dim='',
+                                     using_errors=use_errors,
+                                     color_scale=color_scale,
+                                     comparison_mode=comparison_mode)
+
+    return tbl_acc_indiv
+
+
+def create_runtime_tbl(minimizers, linked_problems, runtime_tbl, use_errors,
+                       color_scale, comparison_mode='abs'):
+    """
+    API function to create a runtime table using the given parameters.
+    For usage, see the expert example script.
+
+    @param minimizers :: array of minimizers used in fitting
+    @param linked_problems :: array of the problems that were fitted
+    @param runtime_tbl :: numpy array of the runtime results
+    @param using_errors :: boolean whether to use errors or not
+    @param color_scale :: color scale for coloring the cells
+    @param comparison_mode :: str to select between 'abs', 'rel', 'both' for
+                              the style of comparison returned
+
+    @returns :: rst table of the results
+    """
+    # Save runtime table for this group of fit problems
+    tbl_runtime_indiv = rst_table.create(minimizers, linked_problems,
+                                         runtime_tbl,
+                                         comparison_type='runtime',
+                                         comparison_dim='',
+                                         using_errors=use_errors,
+                                         color_scale=color_scale,
+                                         comparison_mode=comparison_mode)
+
+    return tbl_runtime_indiv
+
+
 def generate_tables(results_per_test, minimizers):
     """
     Generates accuracy and runtime tables, with both normalised and absolute results, and summary tables.
diff --git a/fitbenchmarking/utils/create_dirs.py b/fitbenchmarking/utils/create_dirs.py
index c973b504..8b982a90 100644
--- a/fitbenchmarking/utils/create_dirs.py
+++ b/fitbenchmarking/utils/create_dirs.py
@@ -9,8 +9,7 @@ import shutil
 
 def results(results_dir):
     """
-    Creates the results folder where the script that is run in the
-    console and eventually calls this is located.
+    Creates the results folder in the working directory.
 
     @param results_dir :: path to the results directory, results dir
                           name or None
diff --git a/fitbenchmarking/utils/misc.py b/fitbenchmarking/utils/misc.py
index 06746d74..f29a7f54 100644
--- a/fitbenchmarking/utils/misc.py
+++ b/fitbenchmarking/utils/misc.py
@@ -52,22 +52,6 @@ def get_minimizers(software_options):
         return minimizers, software
 
 
-def setup_fitting_problems(data_dir, group_name):
-    """
-    Sets up a problem group specified by the user by providing
-    a respective data directory.
-
-    @param group_name :: is the name (label) for a group. E.g. the name for the group of problems in
-                       "NIST/low_difficulty" may be picked to be NIST_low_difficulty
-    @param data_dir :: full path of a directory that holds a group of problem definition files
-    @returns :: the problem group dictionary
-    """
-    problem_group = {}
-    problem_group[group_name] = get_problem_files(data_dir)
-
-    return problem_group
-
-
 def save_user_input(software, minimizers, group_name, results_dir, use_errors):
     """
     All parameters inputed by the user are stored in an object.
@@ -92,20 +76,20 @@ def get_problem_files(data_dir):
     """
     Gets all the problem definition files from the specified problem set directory.
 
-    @param dirs :: array of directories that contain the problems
+    @param data_dir :: directory containing the problems
 
-    @returns :: array containing paths to all the problems
+    @returns :: array containing of paths to the problems
+                e.g. In NIST we would have
+                [low_difficulty/file1.txt, ..., ...]
     """
-    probs_all = []
-    dirs = filter(os.path.isdir, os.listdir(data_dir))
-    if dirs == [] or dirs == ['data_files']:
-        dirs.insert(0, data_dir)
-    for directory in dirs:
-        test_data = glob.glob(directory + '/*.*')
-        problems = [os.path.join(directory, data) for data in test_data]
-        problems.sort()
-        for problem in problems:
-            logger.info(problem)
-        probs_all.append(problems)
-
-    return probs_all
+
+    test_data = glob.glob(data_dir + '/*.*')
+    if test_data == []:
+        raise ValueError('"{}" not recognised as a dataset.'.format(data_dir) +
+                         'Check that it contains problem files and try again.') 
+    problems = [os.path.join(data_dir, data) for data in test_data]
+    problems.sort()
+    for problem in problems:
+        logger.info(problem)
+
+    return problems


Test patch:
diff --git a/fitbenchmarking/utils/tests/test_misc.py b/fitbenchmarking/utils/tests/test_misc.py
index af52b66f..a656f534 100644
--- a/fitbenchmarking/utils/tests/test_misc.py
+++ b/fitbenchmarking/utils/tests/test_misc.py
@@ -6,7 +6,6 @@ import shutil
 import json
 
 from fitbenchmarking.utils.misc import get_minimizers
-from fitbenchmarking.utils.misc import setup_fitting_problems
 from fitbenchmarking.utils.misc import get_problem_files
 
 
@@ -167,23 +166,13 @@ class CreateDirsTests(unittest.TestCase):
 
     self.assertRaises(ValueError, get_minimizers, software_options)
 
-  def test_setupFittingProblems_get_correct_nist_probs(self):
+  def test_getProblemFiles_get_correct_nist_probs(self):
 
     data_dir = os.path.join(self.base_path(), 'NIST', 'low_difficulty')
     nist_problems = self.all_nist_problems()
 
-    problem_groups = setup_fitting_problems(data_dir, 'NIST_low_diff')
-    problem_groups_expected = {'NIST_low_diff': nist_problems}
-
-    self.assertTrue(problem_groups_expected, problem_groups)
-
-  def test_setupFittingProblems_get_correct_neutron_probs(self):
-
-    data_dir = os.path.join(self.base_path(), 'Neutron')
-    neutron_problems = self.all_neutron_problems()
-
-    problem_groups = setup_fitting_problems(data_dir, 'Neutron')
-    problem_groups_expected = {'nist': neutron_problems}
+    problem_groups = get_problem_files(data_dir)
+    problem_groups_expected = nist_problems
 
     self.assertTrue(problem_groups_expected, problem_groups)
 
@@ -193,14 +182,15 @@ class CreateDirsTests(unittest.TestCase):
     neutron_problems = self.all_neutron_problems()
 
     paths_to_neutron_problems = \
-        get_problem_files(base_path_neutron)[0]
+        get_problem_files(base_path_neutron)
     # Please see the above for loop comments for
     # a description of this one
     paths_to_neutron_problems_expected = []
     for neutron_level_group in neutron_problems:
         paths_to_level_group = \
             [os.path.join(base_path_neutron, neutron_prob_name)
-            for neutron_prob_name in neutron_level_group]
+             for neutron_prob_name in neutron_level_group]
+
         paths_to_neutron_problems_expected.append(paths_to_level_group)
 
     self.assertListEqual(paths_to_neutron_problems_expected[0],