Repository: pints-team/pints

Issue:
Add `max_evaluations` stopping criteria
To OptimisationController

When comparing optimisers, or trying to write some hybrid/combined method, the concept of an "evaluation budget" is quite common, and can be easily implemented if we set a `max_evaluations` stopping criterion 


Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e79ec596..7b1b6208 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
 ## Unreleased
 
 ### Added
+- [#1462](https://github.com/pints-team/pints/pull/1461) The `OptimisationController` now has a stopping criterion `max_evaluations`.
 - [#1460](https://github.com/pints-team/pints/pull/1460) Added the `Adam` local optimiser.
 - [#1459](https://github.com/pints-team/pints/pull/1459) Added the `iRprop-` local optimiser.
 - [#1456](https://github.com/pints-team/pints/pull/1456) Added an optional `translation` to `ScalingTransform` and added a `UnitCubeTransformation` class.
diff --git a/pints/_optimisers/__init__.py b/pints/_optimisers/__init__.py
index 2c99a600..7d93b029 100644
--- a/pints/_optimisers/__init__.py
+++ b/pints/_optimisers/__init__.py
@@ -455,6 +455,9 @@ class OptimisationController(object):
         self._unchanged_threshold = 1          # smallest significant f change
         self.set_max_unchanged_iterations()
 
+        # Maximum evaluations
+        self._max_evaluations = None
+
         # Threshold value
         self._threshold = None
 
@@ -470,6 +473,13 @@ class OptimisationController(object):
         """
         return self._evaluations
 
+    def max_evaluations(self):
+        """
+        Returns the maximum number of evaluations if this stopping criteria is
+        set, or ``None`` if it is not. See :meth:`set_max_evaluations`.
+        """
+        return self._max_evaluations
+
     def f_guessed_tracking(self):
         """
         Returns ``True`` if the controller is set to track the optimiser
@@ -539,6 +549,7 @@ class OptimisationController(object):
         has_stopping_criterion = False
         has_stopping_criterion |= (self._max_iterations is not None)
         has_stopping_criterion |= (self._unchanged_max_iterations is not None)
+        has_stopping_criterion |= (self._max_evaluations is not None)
         has_stopping_criterion |= (self._threshold is not None)
         if not has_stopping_criterion:
             raise ValueError('At least one stopping criterion must be set.')
@@ -617,7 +628,8 @@ class OptimisationController(object):
 
             # Add fields to log
             max_iter_guess = max(self._max_iterations or 0, 10000)
-            max_eval_guess = max_iter_guess * pop_size
+            max_eval_guess = max(
+                self._max_evaluations or 0, max_iter_guess * pop_size)
             logger.add_counter('Iter.', max_value=max_iter_guess)
             logger.add_counter('Eval.', max_value=max_eval_guess)
             logger.add_float('Best')
@@ -691,6 +703,14 @@ class OptimisationController(object):
                     halt_message = ('No significant change for ' +
                                     str(unchanged_iterations) + ' iterations.')
 
+                # Maximum number of evaluations
+                if (self._max_evaluations is not None and
+                        evaluations >= self._max_evaluations):
+                    running = False
+                    halt_message = (
+                        'Maximum number of evaluations ('
+                        + str(self._max_evaluations) + ') reached.')
+
                 # Threshold value
                 halt = (self._threshold is not None
                         and f_new < self._threshold)
@@ -827,6 +847,21 @@ class OptimisationController(object):
         """
         self._log_to_screen = True if enabled else False
 
+    def set_max_evaluations(self, evaluations=None):
+        """
+        Adds a stopping criterion, allowing the routine to halt after the
+        given number of ``evaluations``.
+
+        This criterion is disabled by default. To enable, pass in any positive
+        integer. To disable again, use ``set_max_evaluations(None)``.
+        """
+        if evaluations is not None:
+            evaluations = int(evaluations)
+            if evaluations < 0:
+                raise ValueError(
+                    'Maximum number of evaluations cannot be negative.')
+        self._max_evaluations = evaluations
+
     def set_max_iterations(self, iterations=10000):
         """
         Adds a stopping criterion, allowing the routine to halt after the


Test patch:
diff --git a/pints/tests/test_opt_optimisation_controller.py b/pints/tests/test_opt_controller.py
similarity index 95%
rename from pints/tests/test_opt_optimisation_controller.py
rename to pints/tests/test_opt_controller.py
index 8d8d3151..d9b06cc5 100755
--- a/pints/tests/test_opt_optimisation_controller.py
+++ b/pints/tests/test_opt_controller.py
@@ -193,6 +193,23 @@ class TestOptimisationController(unittest.TestCase):
         self.assertEqual(x.shape, (2, ))
         self.assertTrue(b.check(x))
 
+    def test_stopping_max_evaluations(self):
+        # Runs an optimisation with the max_fevals stopping criterion.
+
+        r = pints.toy.TwistedGaussianLogPDF(2, 0.01)
+        x = np.array([0, 1.01])
+        b = pints.RectangularBoundaries([-0.01, 0.95], [0.01, 1.05])
+        s = 0.01
+        opt = pints.OptimisationController(r, x, s, b, method=method)
+        opt.set_log_to_screen(True)
+        opt.set_max_unchanged_iterations(None)
+        opt.set_max_evaluations(10)
+        self.assertEqual(opt.max_evaluations(), 10)
+        self.assertRaises(ValueError, opt.set_max_evaluations, -1)
+        with StreamCapture() as c:
+            opt.run()
+            self.assertIn('Halting: Maximum number of evaluations', c.text())
+
     def test_stopping_max_iterations(self):
         # Runs an optimisation with the max_iter stopping criterion.