Repository: benchopt/benchopt (python)

Issue:
API rename stopping_strategy to sampling_strategy
It would make the way benchopt samples convergence curves easier to understand, whereas now there can be a confusion between stopping_strategy (how is the curved sampled) and stopping_criterion (where does the curve stop)

Pull request description:
API deprecate solver.stopping_strategy in favor of solver.sampling_strategy
closes #577 

Patch:
diff --git a/benchopt/base.py b/benchopt/base.py
index 21ffb76..07ad8f4 100644
--- a/benchopt/base.py
+++ b/benchopt/base.py
@@ -37,7 +37,7 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
       disk. This utility is necessary to reduce the impact of loading the
       result from the disk in the benchmark.
 
-    Note that two ``stopping_strategy`` can be used to construct the benchmark
+    Note that two ``sampling_strategy`` can be used to construct the benchmark
     curve:
 
     - ``'iteration'``: call the run method with max_iter number increasing
@@ -57,16 +57,25 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
 
     @property
     def _solver_strategy(self):
-        """ Change stop_strategy to stopping_strategy """
+        """Change stop_strategy and stopping_strategy to sampling_strategy."""
+        # XXX remove in 1.5
         if hasattr(self, 'stop_strategy'):
             warnings.warn(
-                "'stop_strategy' attribute is deprecated, "
-                "use 'stopping_strategy' instead",
+                "'stop_strategy' attribute is deprecated and will be "
+                "removed in benchopt 1.5, use 'sampling_strategy' instead.",
                 FutureWarning
             )
             return self.stop_strategy
-        elif hasattr(self, 'stopping_strategy'):
+        # XXX remove in 1.5
+        if hasattr(self, 'stopping_strategy'):
+            warnings.warn(
+                "'stopping_strategy' attribute is deprecated and will be "
+                "removed in benchopt 1.5, use 'sampling_strategy' instead.",
+                FutureWarning
+            )
             return self.stopping_strategy
+        elif hasattr(self, 'sampling_strategy'):
+            return self.sampling_strategy
         else:
             return self.stopping_criterion.strategy
 
@@ -141,7 +150,7 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
         This function should not return the parameters which will be
         retrieved by a subsequent call to get_result.
 
-        If `stopping_strategy` is set to `"callback"`, then `run` should call
+        If `sampling_strategy` is set to `"callback"`, then `run` should call
         the callback at each iteration. The callback will compute the time,
         the objective function and store relevant quantities for BenchOpt.
         Else, the `stop_val` parameter should be specified.
@@ -206,7 +215,7 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
         Parameters
         ----------
         stop_val : int or float, (default: 1)
-            If ``stopping_strategy`` is 'iteration', this should be an integer
+            If ``sampling_strategy`` is 'iteration', this should be an integer
             corresponding to the number of iterations the solver is run for.
             If it is 'callback', it is an integer corresponding to the number
             of times the callback is called.
diff --git a/benchopt/plotting/generate_html.py b/benchopt/plotting/generate_html.py
index 3122705..b056898 100644
--- a/benchopt/plotting/generate_html.py
+++ b/benchopt/plotting/generate_html.py
@@ -230,24 +230,26 @@ def shape_solvers_for_html(df, objective_column):
 
         color, marker = get_solver_style(solver)
         # to preserve support of previous benchopt version
-        # where 'stopping_strategy' wasn't saved in solver meta
-        try:
-            stopping_strategy = df_filtered['stopping_strategy'].unique()
-        except KeyError:
-            stopping_strategy = ["Time"]
-
-        if len(stopping_strategy) != 1:
-            found_stopping_strategies = ', '.join(
-                f"`{item}`" for item in stopping_strategy
+        # where 'sampling_strategy' wasn't saved in solver meta
+        if "sampling_strategy" in df_filtered:
+            sampling_strategy = df_filtered['sampling_strategy'].unique()
+        elif "stopping_strategy" in df_filtered:
+            sampling_strategy = df_filtered['stopping_strategy'].unique()
+        else:
+            sampling_strategy = ["Time"]
+
+        if len(sampling_strategy) != 1:
+            found_sampling_strategies = ', '.join(
+                f"`{item}`" for item in sampling_strategy
             )
 
             raise Exception(
-                "Solver can be run using only one stopping strategy. "
-                f"Expected one stopping strategy "
-                f"but found {found_stopping_strategies}"
+                "Solver can be run using only one sampling strategy. "
+                f"Expected one sampling strategy "
+                f"but found {found_sampling_strategies}"
             )
 
-        stopping_strategy = stopping_strategy[0]
+        sampling_strategy = sampling_strategy[0]
 
         solver_data[solver] = {
             'scatter': {
@@ -262,7 +264,7 @@ def shape_solvers_for_html(df, objective_column):
             },
             'color': color,
             'marker': marker,
-            'stopping_strategy': stopping_strategy
+            'sampling_strategy': sampling_strategy,
         }
 
     return solver_data
diff --git a/benchopt/runner.py b/benchopt/runner.py
index a5deb8d..6860e89 100644
--- a/benchopt/runner.py
+++ b/benchopt/runner.py
@@ -32,7 +32,7 @@ def run_one_resolution(objective, solver, meta, stop_val):
     stop_val : int | float
         Corresponds to stopping criterion, such as
         tol or max_iter for the solver. It depends
-        on the stopping_strategy for the solver.
+        on the sampling_strategy for the solver.
 
     Returns
     -------
@@ -95,7 +95,7 @@ def run_one_to_cvg(benchmark, objective, solver, meta, stopping_criterion,
 
         if solver._solver_strategy == "callback":
 
-            # If stopping strategy is 'callback', only call once to get the
+            # If sampling_strategy is 'callback', only call once to get the
             # results up to convergence.
             callback = _Callback(
                 objective, meta, stopping_criterion
@@ -182,11 +182,11 @@ def run_one_solver(benchmark, dataset, objective, solver, n_repetitions,
     states = []
     run_statistics = []
 
-    # get stopping strategy
+    # get sampling strategy
     # for plotting purpose consider 'callback' as 'iteration'
-    stopping_strategy = solver._solver_strategy
-    if stopping_strategy == 'callback':
-        stopping_strategy = 'iteration'
+    sampling_strategy = solver._solver_strategy
+    if sampling_strategy == 'callback':
+        sampling_strategy = 'iteration'
 
     # get objective description
     # use `obj_` instead of `objective_` to avoid conflicts with
@@ -202,7 +202,7 @@ def run_one_solver(benchmark, dataset, objective, solver, n_repetitions,
             solver_name=str(solver),
             data_name=str(dataset),
             idx_rep=rep,
-            stopping_strategy=stopping_strategy.capitalize(),
+            sampling_strategy=sampling_strategy.capitalize(),
             obj_description=obj_description,
             solver_description=inspect.cleandoc(solver.__doc__ or ""),
         )
diff --git a/benchopt/stopping_criterion.py b/benchopt/stopping_criterion.py
index d1a9e9d..bd94852 100644
--- a/benchopt/stopping_criterion.py
+++ b/benchopt/stopping_criterion.py
@@ -1,8 +1,8 @@
 import time
 import math
 
-# Possible stop strategies
-STOPPING_STRATEGIES = ['iteration', 'tolerance', 'callback']
+# Possible curve sampling strategies
+SAMPLING_STRATEGIES = ['iteration', 'tolerance', 'callback']
 
 EPS = 1e-10
 PATIENCE = 3
@@ -64,8 +64,8 @@ class StoppingCriterion():
     def __init__(self, strategy=None, key_to_monitor='objective_value',
                  **kwargs):
 
-        assert strategy in STOPPING_STRATEGIES, (
-            f"strategy should be in {STOPPING_STRATEGIES}. Got '{strategy}'."
+        assert strategy in SAMPLING_STRATEGIES, (
+            f"strategy should be in {SAMPLING_STRATEGIES}. Got '{strategy}'."
         )
 
         self.kwargs = kwargs
@@ -88,7 +88,7 @@ class StoppingCriterion():
             Object to format string to display the progress of the solver.
         solver : BaseSolver
             The solver for which this stopping criterion is called. Used to get
-            overridden ``stopping_strategy`` and ``get_next``.
+            overridden ``sampling_strategy`` and ``get_next``.
 
         Returns
         -------
@@ -111,8 +111,8 @@ class StoppingCriterion():
         strategy = self.strategy
         if solver is not None:
             strategy = solver._solver_strategy
-        assert strategy in STOPPING_STRATEGIES, (
-            f"stopping_strategy should be in {STOPPING_STRATEGIES}. "
+        assert strategy in SAMPLING_STRATEGIES, (
+            f"sampling_strategy should be in {SAMPLING_STRATEGIES}. "
             f"Got '{strategy}'."
         )
 
@@ -445,7 +445,7 @@ class SingleRunCriterion(StoppingCriterion):
     stop_val : int or float, (default: 1)
         Value of ``stop_val`` with which the objective function will be called.
         This value will be passed as ``n_iter`` or ``tol`` parameter for the
-        ``run`` method of solver with ``stopping_strategy`` respectively equals
+        ``run`` method of solver with ``sampling_strategy`` respectively equals
         to ``'iteration'`` or ``'tolerance'``, or the number of callback calls
         minus one for the ``'callback'`` strategy.
     """
diff --git a/benchopt/utils/temp_benchmark.py b/benchopt/utils/temp_benchmark.py
index c6690c4..5c1fa04 100644
--- a/benchopt/utils/temp_benchmark.py
+++ b/benchopt/utils/temp_benchmark.py
@@ -1,4 +1,5 @@
 import os
+import inspect
 import tempfile
 import contextlib
 
@@ -13,13 +14,13 @@ dummy_objective = (DUMMY_BENCHMARK_PATH / 'objective.py').read_text()
 dummy_solvers = [
     (DUMMY_BENCHMARK_PATH / "solvers" / p).read_text()
     for p in os.listdir(DUMMY_BENCHMARK_PATH / "solvers")
-    if p.endswith(".py")
+    if p.endswith(".py") and not p.startswith("template_")
 ]
 
 dummy_datasets = [
     (DUMMY_BENCHMARK_PATH / "datasets" / p).read_text()
     for p in os.listdir(DUMMY_BENCHMARK_PATH / "datasets")
-    if p.endswith(".py")
+    if p.endswith(".py") and not p.startswith("template_")
 ]
 
 
@@ -61,17 +62,17 @@ def temp_benchmark(
         (temp_path / "solvers").mkdir()
         (temp_path / "datasets").mkdir()
         with open(temp_path / "objective.py", "w") as f:
-            f.write(objective)
-        for idx, dataset in enumerate(datasets):
+            f.write(inspect.cleandoc(objective))
+        for idx, solver in enumerate(solvers):
             with open(temp_path / "solvers" / f"solver_{idx}.py", "w") as f:
-                f.write(dataset)
+                f.write(inspect.cleandoc(solver))
 
-        for idx, solver in enumerate(solvers):
+        for idx, dataset in enumerate(datasets):
             with open(temp_path / "datasets" / f"dataset_{idx}.py", "w") as f:
-                f.write(solver)
+                f.write(inspect.cleandoc(dataset))
 
         if config is not None:
             with open(temp_path / "config.yml", "w") as f:
-                f.write(solver)
+                f.write(config)
 
         yield Benchmark(temp_path)
diff --git a/doc/advanced.rst b/doc/advanced.rst
index 46b858e..8ca3b85 100644
--- a/doc/advanced.rst
+++ b/doc/advanced.rst
@@ -170,9 +170,9 @@ For some solvers, such as solver relying on just-in-time compilation with
 ``numba`` or ``jax``, the first iteration might be longer due to "warmup"
 effects. To avoid having such effect in the benchmark results, it is usually
 advised to call the solver once before running the benchmark, in the
-``Solver.set_objective`` method. For solvers with ``stopping_strategy`` in
+``Solver.set_objective`` method. For solvers with ``sampling_strategy`` in
 ``{'tolerance',  'iteration'}``, simply calling the ``Solver.run`` with a
-simple enough value is usually enough. For solvers with ``stopping_strategy``
+simple enough value is usually enough. For solvers with ``sampling_strategy``
 set to ``'callback'``, it is possible to call ``Solver.run_once``, which will
 call the ``run`` method with a simple callback that does not compute the
 objective value and stops after ``n_iter`` calls to callback (default to 1).
@@ -187,8 +187,8 @@ objective value and stops after ``n_iter`` calls to callback (default to 1).
             ...
             # Cache pre-compilation and other one-time setups that should
             # not be included in the benchmark timing.
-            self.run(1)  # For stopping_strategy == 'iteration' | 'tolerance'
-            self.run_once()  # For stopping_strategy == 'callback'
+            self.run(1)  # For sampling_strategy == 'iteration' | 'tolerance'
+            self.run_once()  # For sampling_strategy == 'callback'
 
 
 .. |update_params| replace:: ``update_parameters``
diff --git a/doc/how.rst b/doc/how.rst
index ecafa81..6f1dc90 100644
--- a/doc/how.rst
+++ b/doc/how.rst
@@ -132,7 +132,7 @@ A solver must define three methods:
      or a tolerance parameter ``tol``. Alternatively, a ``callback`` function that will be
      called at each iteration can be passed. The callback should return ``False`` once the
      computation should stop.
-     The parameter ``stop_value`` is controlled by the ``stopping_strategy``,
+     The parameter ``stop_value`` is controlled by the ``sampling_strategy``,
      see below for details.
 
    - ``get_result()``: This method returns a variable that can be passed
@@ -141,8 +141,8 @@ A solver must define three methods:
 
 **Stop strategy:**
 
-A solver should also define a ``stopping_strategy`` as class attribute.
-This ``stopping_strategy`` can be:
+A solver should also define a ``sampling_strategy`` as class attribute.
+This ``sampling_strategy`` can be:
 
     - ``'iteration'``: in this case the ``run`` method of the solver
       is parametrized by the number of iterations computed. The parameter
@@ -178,7 +178,7 @@ with no other dependencies. Here is an example:
 .. literalinclude:: ../benchopt/tests/test_benchmarks/dummy_benchmark/solvers/python_pgd.py
 
 For solvers that allow access to each iterate of the solution, using ``"callback"``
-as a ``stopping_strategy`` implies a slight modification for ``run``. A ``callback``
+as a ``sampling_strategy`` implies a slight modification for ``run``. A ``callback``
 should be called at each iteration with parameter the current value of the iterate.
 Here is an example in the same situation as above:
 
diff --git a/doc/performance_curves.rst b/doc/performance_curves.rst
index 8b131f5..a4ea1a6 100644
--- a/doc/performance_curves.rst
+++ b/doc/performance_curves.rst
@@ -8,17 +8,17 @@ With "*performance curves*", we refer to the evolution of the performance as a
 function of the computational budget.
 With ``benchopt``, there are different ways to vary the computational budget
 to sample the performance curve.
-They are chosen by the ``stopping_strategy`` attribute of each solver.
+They are chosen by the ``sampling_strategy`` attribute of each solver.
 
 1. Using iterations or tolerance
 --------------------------------
 
 
-The first way is to use ``Solver.stopping_strategy = "iteration"`` or ``Solver.stopping_strategy = "tolerance"``.
+The first way is to use ``Solver.sampling_strategy = "iteration"`` or ``Solver.sampling_strategy = "tolerance"``.
 This is used for black box solvers, where one can only get the result of the solver for a given number of iterations or for a given numerical tolerance.
-This stopping strategy creates curves by calling ``Solver.run(stop_val)`` several times with different values for the ``stop_val`` parameter:
+This sampling strategy creates curves by calling ``Solver.run(stop_val)`` several times with different values for the ``stop_val`` parameter:
 
-- if the solver's ``stopping_strategy`` is ``"iteration"``, ``stop_val`` is the number of iterations passed to ``run``.
+- if the solver's ``sampling_strategy`` is ``"iteration"``, ``stop_val`` is the number of iterations passed to ``run``.
   It increases geometrically by at least 1, starting from 1 with a factor :math:`\rho=1.5`.
   Note that the first call uses a number of iterations of 0.
   The value from one call to the other follows:
@@ -27,7 +27,7 @@ This stopping strategy creates curves by calling ``Solver.run(stop_val)`` severa
 
     \text{stop_val} = \max(\text{stop_val} + 1, \text{int}(\rho * \text{stop_val}))
 
-- if the solver's ``stopping_strategy`` is ``"tolerance"``, the ``stop_val`` parameter corresponds to the numerical tolerance.
+- if the solver's ``sampling_strategy`` is ``"tolerance"``, the ``stop_val`` parameter corresponds to the numerical tolerance.
   It decreases geometrically by a factor :math:`\rho=1.5` between each call to ``run``, starting from 1 at the second call.
   Note that the first call uses a tolerance of 1e38.
   The value from one call to the other follows:
@@ -47,9 +47,9 @@ For more advanced configurations, the evolution of ``stop_val`` can be controlle
 
 Restarting the solver from scratch, though inevitable to handle black box solvers, can be costly.
 
-When a solver exposes the intermediate values of the iterates, it is possible to create the curve in a single solver run, by using ``stopping_strategy = "callback"``.
+When a solver exposes the intermediate values of the iterates, it is possible to create the curve in a single solver run, by using ``sampling_strategy = "callback"``.
 In that case, the argument passed to ``Solver.run`` will be a callable object, ``callback``.
-Like with ``stopping_strategy == "iteration"``, the objective is computed after a number of callback's calls that grows geometrically. If the objective was computed after :math:`n` calls, the objective and timing will be computed again when reaching :math:`\max(n+1, \rho * n)` calls to the callback.
+Like with ``sampling_strategy == "iteration"``, the objective is computed after a number of callback's calls that grows geometrically. If the objective was computed after :math:`n` calls, the objective and timing will be computed again when reaching :math:`\max(n+1, \rho * n)` calls to the callback.
 The callback makes sure we do not account for objective computation time and also check for convergence every time the objective is computed (as described in the next section).
 It returns ``False`` when the solver should be stopped. A classical usage pattern is:
 
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index bc3c2f8..c5dc17c 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -5,6 +5,17 @@ What's new
 
 .. currentmodule:: benchopt
 
+.. _changes_1_5:
+
+Version 1.5 (dev)
+-----------------
+
+API
+~~~
+
+- ``stopping_strategy`` attribute is replaced by ``sampling_strategy`` to clarify
+  the concept, by `Mathurin Massias`_ (:gh:`585`).
+
 .. _changes_1_4:
 
 Version 1.4 - 03/07/2023


Test patch:
diff --git a/benchopt/tests/test_benchmarks.py b/benchopt/tests/test_benchmarks.py
index 883564d..8eb1182 100644
--- a/benchopt/tests/test_benchmarks.py
+++ b/benchopt/tests/test_benchmarks.py
@@ -1,9 +1,12 @@
 import pytest
 import numpy as np
 
+import benchopt
+from benchopt.cli.main import run
 from benchopt.runner import _Callback
-from benchopt.stopping_criterion import STOPPING_STRATEGIES
+from benchopt.stopping_criterion import SAMPLING_STRATEGIES
 from benchopt.utils import product_param
+from benchopt.utils.temp_benchmark import temp_benchmark
 
 
 def test_benchmark_objective(benchmark, dataset_simu):
@@ -89,8 +92,8 @@ def test_solver_class(benchmark, solver_class):
 
     # Check that the solver_class uses a valid stopping_strategy
     if hasattr(solver_class, 'stopping_strategy'):
-        msg = f"stopping_strategy should be in {STOPPING_STRATEGIES}."
-        assert solver_class.stopping_strategy in STOPPING_STRATEGIES, msg
+        msg = f"stopping_strategy should be in {SAMPLING_STRATEGIES}."
+        assert solver_class.stopping_strategy in SAMPLING_STRATEGIES, msg
 
     # Check that the solver_class uses a valid callable to override get_next.
     if hasattr(solver_class, 'get_next'):
@@ -215,3 +218,39 @@ def _test_solver_one_objective(solver, objective):
 
             diff = val_eps - val_star
             assert diff >= 0
+
+
+def test_deprecated_stopping_strategy():
+    # XXX remove in 1.5
+    assert benchopt.__version__ < '1.5'
+
+    solver1 = """from benchopt import BaseSolver
+    import numpy as np
+
+    class Solver(BaseSolver):
+        name = 'solver1'
+        stopping_strategy = 'iteration'
+
+        def run(self, n_iter): pass
+
+        def set_objective(self, X, y, lmbd):
+            self.n_features = X.shape[1]
+
+        def get_result(self, **data):
+            return np.zeros(self.n_features)
+    """
+
+    solver2 = solver1.replace("stopping_strategy", "sampling_strategy")
+    solver2 = solver2.replace("solver1", "solver2")
+
+    with temp_benchmark(solvers=[solver1, solver2]) as benchmark:
+        with pytest.warns(
+                FutureWarning,
+                match="'stopping_strategy' attribute is deprecated"):
+            run([str(benchmark.benchmark_dir),
+                 *'-s solver1 -d test-dataset -n 1 -r 1 --no-plot'.split()],
+                standalone_mode=False)
+
+        run([str(benchmark.benchmark_dir),
+             *'-s solver2 -d test-dataset -n 1 -r 1 --no-plot'.split()],
+            standalone_mode=False)
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/julia_pgd.py b/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/julia_pgd.py
index 3ef713d..4d5bdfb 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/julia_pgd.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/julia_pgd.py
@@ -17,7 +17,7 @@ class Solver(JuliaSolver):
 
     # Config of the solver
     name = 'Julia-PGD'
-    stopping_strategy = 'iteration'
+    sampling_strategy = 'iteration'
 
     def set_objective(self, X, y, lmbd):
         self.X, self.y, self.lmbd = X, y, lmbd
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/python_pgd_callback.py b/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/python_pgd_callback.py
index 5d72498..7143f65 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/python_pgd_callback.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/python_pgd_callback.py
@@ -11,7 +11,7 @@ class Solver(BaseSolver):
 
     # any parameter defined here is accessible as a class attribute
     parameters = {'use_acceleration': [False, True]}
-    stopping_strategy = "callback"
+    sampling_strategy = "callback"
 
     def set_objective(self, X, y, lmbd):
         self.X, self.y, self.lmbd = X, y, lmbd
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/r_pgd.py b/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/r_pgd.py
index 8383bcb..8a1f369 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/r_pgd.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/solvers/r_pgd.py
@@ -26,7 +26,7 @@ class Solver(BaseSolver):
 
     install_cmd = 'conda'
     requirements = ['r-base', 'rpy2']
-    stopping_strategy = 'iteration'
+    sampling_strategy = 'iteration'
     support_sparse = False
 
     def set_objective(self, X, y, lmbd):
diff --git a/benchopt/tests/test_benchmarks/requirement_benchmark/solvers/baseline.py b/benchopt/tests/test_benchmarks/requirement_benchmark/solvers/baseline.py
index e68b1a7..d973661 100644
--- a/benchopt/tests/test_benchmarks/requirement_benchmark/solvers/baseline.py
+++ b/benchopt/tests/test_benchmarks/requirement_benchmark/solvers/baseline.py
@@ -3,7 +3,7 @@ from benchopt import BaseSolver
 
 class Solver(BaseSolver):
     name = 'Dummy solver'
-    stopping_strategy = "callback"
+    sampling_strategy = "callback"
 
     def set_objective(self, X):
         self.X = X
diff --git a/benchopt/tests/test_stopping_criterion.py b/benchopt/tests/test_stopping_criterion.py
index 339a133..fbeda3f 100644
--- a/benchopt/tests/test_stopping_criterion.py
+++ b/benchopt/tests/test_stopping_criterion.py
@@ -1,12 +1,12 @@
 import pytest
 import numpy as np
 
-from benchopt.stopping_criterion import STOPPING_STRATEGIES
+from benchopt.stopping_criterion import SAMPLING_STRATEGIES
 from benchopt.stopping_criterion import SufficientDescentCriterion
 from benchopt.stopping_criterion import SufficientProgressCriterion
 
 
-@pytest.mark.parametrize('strategy', STOPPING_STRATEGIES)
+@pytest.mark.parametrize('strategy', SAMPLING_STRATEGIES)
 @pytest.mark.parametrize('criterion_class', [
     SufficientDescentCriterion, SufficientProgressCriterion
 ])
@@ -27,7 +27,7 @@ def test_max_iter(criterion_class, strategy):
     assert status == 'max_runs', "Should stop on max_runs"
 
 
-@pytest.mark.parametrize('strategy', STOPPING_STRATEGIES)
+@pytest.mark.parametrize('strategy', SAMPLING_STRATEGIES)
 @pytest.mark.parametrize('criterion_class', [
     SufficientDescentCriterion, SufficientProgressCriterion
 ])
@@ -43,7 +43,7 @@ def test_timeout(criterion_class, strategy):
     assert status == 'timeout', "Should stop on timeout"
 
 
-@pytest.mark.parametrize('strategy', STOPPING_STRATEGIES)
+@pytest.mark.parametrize('strategy', SAMPLING_STRATEGIES)
 @pytest.mark.parametrize('criterion_class', [
     SufficientDescentCriterion, SufficientProgressCriterion
 ])
@@ -71,7 +71,7 @@ def test_diverged(criterion_class, strategy):
     assert status == 'diverged', "Should stop on diverged"
 
 
-@pytest.mark.parametrize('strategy', STOPPING_STRATEGIES)
+@pytest.mark.parametrize('strategy', SAMPLING_STRATEGIES)
 @pytest.mark.parametrize('criterion_class', [
     SufficientDescentCriterion, SufficientProgressCriterion
 ])