Repository: fitbenchmarking/fitbenchmarking

Issue:
Use a better first example 
**Description of the documentation**
The suggested example is to use the `simple_tests` problem set.

This includes Mantid data, and so will fail if Mantid isn't installed.  Change the default to be, e.g., a set of NIST problems, so that it can be run independently of third party software.



Discussion:
wathen: > Ok here is my review:
> 
>     1. Run fine but take a while to do so, maybe use a smaller/easier test set? Also some strange warnings (hard to tell where these are coming from):
> 
> 
> ```
>  <string>:1: RuntimeWarning: invalid value encountered in power 
>  'NoneType' object has no attribute 'T'
> ```
> 
>     1. Run fine with the expected warning for mantid problems:
> 
> 
> ```
> Missing dependencies for fit.
> Details: Requirements are missing for mantid parser: No module named 'mantid'
> ```
> 
>     1. Raises the expected error along with an informative message (but with weird spaces between words):
> 
> 
> ```
> Error while running FitBenchmarking. Exiting. See below for more information.
> FitBenchmarking ran with no results
> Details: The user chosen options and/or problem setup resulted in  all minimizers and/or parsers raising an exception.  This is likely due to the way `algorithm_type` was set  in the options or the selected problem set requires  additional software to be installed. Please review your  options setup and/or problem set then re-run FitBenmarking.
> ```
1) I changed the default to NIST low diff data set however this still produces the warning you have seen. Maybe it would be better to run with a smaller number of minimizers as a default. I will create an issue for this.

3) Fixed spacing issues.  


Patch:
diff --git a/docs/source/users/running.rst b/docs/source/users/running.rst
index 20b9cb21..387cef6f 100644
--- a/docs/source/users/running.rst
+++ b/docs/source/users/running.rst
@@ -10,7 +10,7 @@ directory along with a template for passing in options.
 To run with any of these problem sets, you should pass the path to the
 `fitbenchmarking` command.
 
-  e.g ``fitbenchmarking examples/benchmarking_problems/simple_tests``
+  e.g ``fitbenchmarking examples/benchmark_problems/NIST/low_difficulty``
 
 An options file can also be passed with the ``-o`` argument.
 The template in `examples` is heavily documented and serves as a good
diff --git a/fitbenchmarking/core/fitting_benchmarking.py b/fitbenchmarking/core/fitting_benchmarking.py
index c27a7a64..2b5f69a4 100644
--- a/fitbenchmarking/core/fitting_benchmarking.py
+++ b/fitbenchmarking/core/fitting_benchmarking.py
@@ -9,13 +9,14 @@ from __future__ import absolute_import, division, print_function
 from collections import defaultdict
 import timeit
 import warnings
+import os
 
 import numpy as np
 
 from fitbenchmarking.controllers.controller_factory import ControllerFactory
 from fitbenchmarking.parsing.parser_factory import parse_problem_file
 from fitbenchmarking.utils.exceptions import NoResultsError, \
-    UnknownMinimizerError, UnsupportedMinimizerError
+    UnknownMinimizerError, UnsupportedMinimizerError, MissingSoftwareError
 from fitbenchmarking.jacobian.jacobian_factory import create_jacobian
 from fitbenchmarking.utils import fitbm_result, misc, output_grabber
 
@@ -61,15 +62,6 @@ def benchmark(options, data_dir):
 
     options.minimizers = minimizer_dict
 
-    # If the results are an empty list then this means that all minimizers
-    # raise an exception and the tables will produce errors if they run.
-    if results == []:
-        message = "The user chosen options setup resulted in all minimizers " \
-                  "raising an exception. This is likely due to the way " \
-                  "`algorithm_type` was set in the options. Please review " \
-                  "your options setup and re-run FitBenmarking."
-        raise NoResultsError(message)
-
     # Used to group elements in list by name
     results_dict = defaultdict(list)
     for problem_result in results:
@@ -92,32 +84,49 @@ def loop_over_benchmark_problems(problem_group, options):
              list of failed problems and dictionary of unselected minimizers
     :rtype: tuple(list, list, dict)
     """
-
     grabbed_output = output_grabber.OutputGrabber(options)
     results = []
     failed_problems = []
     for i, p in enumerate(problem_group):
-        with grabbed_output:
-            parsed_problem = parse_problem_file(p, options)
-        parsed_problem.correct_data()
-
-        name = parsed_problem.name
-
+        problem_passed = True
         info_str = " Running data from: {} {}/{}".format(
-            name, i + 1, len(problem_group))
+            os.path.basename(p), i + 1, len(problem_group))
         LOGGER.info('\n' + '#' * (len(info_str) + 1))
         LOGGER.info(info_str)
         LOGGER.info('#' * (len(info_str) + 1))
+        try:
+            with grabbed_output:
+                parsed_problem = parse_problem_file(p, options)
+                parsed_problem.correct_data()
+        except Exception as e:
+            info_str = " Could not run data from: {} {}/{}".format(
+                p, i + 1, len(problem_group))
+            LOGGER.warn(e)
+            problem_passed = False
+
+        if problem_passed:
+            ##############################
+            # Loops over starting values #
+            ##############################
+            problem_results, problem_fails, \
+                unselected_minimzers, minimizer_dict = \
+                loop_over_starting_values(
+                    parsed_problem, options, grabbed_output)
+            results.extend(problem_results)
+            failed_problems.extend(problem_fails)
 
-        ##############################
-        # Loops over starting values #
-        ##############################
-        problem_results, problem_fails, \
-            unselected_minimzers, minimizer_dict = \
-            loop_over_starting_values(parsed_problem, options, grabbed_output)
+    # If the results are an empty list then this means that all minimizers
+    # raise an exception and the tables will produce errors if they run.
+    if results == []:
+        message = "The user chosen options and/or problem setup resulted in " \
+                  "all minimizers and/or parsers raising an exception. " \
+                  "This is likely due to the way `algorithm_type` was set " \
+                  "in the options or the selected problem set requires " \
+                  "additional software to be installed. Please review your " \
+                  "options setup and/or problem set then re-run " \
+                  "FitBenmarking."
+        raise NoResultsError(message)
 
-        results.extend(problem_results)
-        failed_problems.extend(problem_fails)
     return results, failed_problems, unselected_minimzers, minimizer_dict
 
 
diff --git a/fitbenchmarking/parsing/nist_parser.py b/fitbenchmarking/parsing/nist_parser.py
index d29573e7..29bc41c7 100644
--- a/fitbenchmarking/parsing/nist_parser.py
+++ b/fitbenchmarking/parsing/nist_parser.py
@@ -57,11 +57,15 @@ class NISTParser(Parser):
             nist_func_definition(function=fitting_problem.equation,
                                  param_names=starting_values[0].keys())
         fitting_problem.format = "nist"
+        try:
+            jacobian = self._parse_jacobian(name)
+            fitting_problem.jacobian = \
+                nist_jacobian_definition(jacobian=jacobian,
+                                         param_names=starting_values[0].keys())
+        except NoJacobianError:
+            LOGGER.warn("WARNING: Could not find analytic Jacobian "
+                        "information for {} problem".format(name))
 
-        jacobian = self._parse_jacobian(name)
-        fitting_problem.jacobian = \
-            nist_jacobian_definition(jacobian=jacobian,
-                                     param_names=starting_values[0].keys())
         return fitting_problem
 
     def _parse_jacobian(self, name):


Test patch:
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
index 0a0b37b6..43743a80 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
@@ -160,23 +160,6 @@ class BenchmarkTests(unittest.TestCase):
         self.shared_tests(expected_names, expected_unselected_minimzers,
                           expected_minimzers)
 
-    @mock.patch('{}.loop_over_benchmark_problems'.format(FITTING_DIR))
-    def test_check_no_results_produced(self, loop_over_benchmark_problems):
-        """
-        Checks benchmarking raises an error when no results are produced
-        """
-
-        results = []
-        problem_fails = []
-        expected_unselected_minimzers = {"scipy": []}
-        expected_minimzers = copy.copy(self.all_minimzers)
-        loop_over_benchmark_problems.return_value = \
-            (results, problem_fails, expected_unselected_minimzers,
-             expected_minimzers)
-        with self.assertRaises(NoResultsError):
-            _, _, _ = \
-                benchmark(self.options, self.default_parsers_dir)
-
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
index 1c2d2f5b..e56d758b 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
@@ -9,7 +9,7 @@ import unittest
 import mock
 
 from fitbenchmarking import mock_problems
-from fitbenchmarking.utils import fitbm_result
+from fitbenchmarking.utils import fitbm_result, exceptions
 from fitbenchmarking.core.fitting_benchmarking import \
     loop_over_benchmark_problems
 from fitbenchmarking.parsing.parser_factory import parse_problem_file
@@ -134,7 +134,6 @@ class LoopOverBenchmarkProblemsTests(unittest.TestCase):
         self.problem_fails = ['Random_failed_problem_1',
                               'Random_failed_problem_2']
         loop_over_starting_values.side_effect = self.mock_func_call
-        self.problem_group = []
         self.problem_group = [os.path.join(self.default_parsers_dir,
                                            "cubic.dat")]
 
@@ -142,6 +141,20 @@ class LoopOverBenchmarkProblemsTests(unittest.TestCase):
         expected_list_length = len(self.list_results)
         self.shared_tests(expected_list_length, expected_problem_fails)
 
+    @mock.patch('{}.loop_over_starting_values'.format(FITTING_DIR))
+    def test_check_no_results_produced(self, loop_over_starting_values):
+        """
+        Checks that multiple failed problems are reported correctly
+        """
+        loop_over_starting_values.side_effect = self.mock_func_call
+        problem_group = [os.path.join(self.default_parsers_dir,
+                                      "META.txt")]
+
+        with self.assertRaises(exceptions.NoResultsError):
+            _, _, _ = \
+                loop_over_benchmark_problems(problem_group,
+                                             self.options)
+
 
 if __name__ == "__main__":
     unittest.main()