Repository: benchopt/benchopt
Pull request #735 (2024-06-04 10:37:14)

ENH uncache errors run

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/benchopt/benchopt/pull/735?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 80.94%. Comparing base [(`9afa0a4`)](https://app.codecov.io/gh/benchopt/benchopt/commit/9afa0a445803be7bce1d3a4ed7f6fd9d352e7203?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) to head [(`5e6ed26`)](https://app.codecov.io/gh/benchopt/benchopt/commit/5e6ed26657c5e163313d5c131f3a4cab95851303?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt).

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #735      +/-   ##
==========================================
+ Coverage   80.90%   80.94%   +0.04%     
==========================================
  Files          45       45              
  Lines        3058     3065       +7     
==========================================
+ Hits         2474     2481       +7     
  Misses        584      584              
```



</details>

Patch:
diff --git a/README.rst b/README.rst
index b93b5a6..10f6574 100644
--- a/README.rst
+++ b/README.rst
@@ -44,7 +44,7 @@ It is also possible to use the **latest development version**. To do so, run ins
 
 .. code-block:: bash
 
-    pip install --pre benchopt -U -i https://test.pypi.org/simple
+    pip install git+https://github.com/benchopt/benchopt.git
 
 
 Getting started
diff --git a/benchopt/benchmark.py b/benchopt/benchmark.py
index cd67017..c1a9f12 100644
--- a/benchopt/benchmark.py
+++ b/benchopt/benchmark.py
@@ -344,8 +344,8 @@ class Benchmark:
         that are already in cache.
         """
 
-        # Create a cached function the computations in the benchmark folder
-        # and handle cases where we force the run.
+        # Create a cached version of `func` and handle cases where we force
+        # the run.
         func_cached = self.mem.cache(func, ignore=ignore)
         if force:
             assert not collect, "Cannot collect and force computation."
diff --git a/benchopt/runner.py b/benchopt/runner.py
index 08608d4..01b5432 100644
--- a/benchopt/runner.py
+++ b/benchopt/runner.py
@@ -150,7 +150,8 @@ def run_one_to_cvg(benchmark, objective, solver, meta, stopping_criterion,
             if to_save is not None:
                 with open(meta["final_results"], 'wb') as f:
                     pickle.dump(to_save, f)
-
+    if ctx.status in ['diverged', 'error', 'interrupted']:
+        raise RuntimeError(ctx.status)
     return curve, ctx.status
 
 
@@ -256,12 +257,17 @@ def run_one_solver(benchmark, dataset, objective, solver, n_repetitions,
             timeout=timeout / n_repetitions,
             output=output,
         )
-        curve, status = run_one_to_cvg_cached(
-            benchmark=benchmark, objective=objective,
-            solver=solver, meta=meta,
-            stopping_criterion=stopping_criterion,
-            force=force, output=output, pdb=pdb
+        args_run_one_to_cvg = dict(
+            benchmark=benchmark, objective=objective, solver=solver, meta=meta,
+            stopping_criterion=stopping_criterion, force=force, output=output,
+            pdb=pdb
         )
+        try:
+            curve, status = run_one_to_cvg_cached(
+                **args_run_one_to_cvg
+            )
+        except RuntimeError as e:
+            status = e.args[0]
         if status in ['diverged', 'error', 'interrupted', 'not ready']:
             run_statistics = []
             break
diff --git a/doc/names.inc b/doc/names.inc
index f0c3399..09ff255 100644
--- a/doc/names.inc
+++ b/doc/names.inc
@@ -27,3 +27,7 @@
 .. _Christopher Marouani: https://github.com/chris-mrn
 
 .. _Pierre-Antoine Comby: https://github.com/paquiteau
+
+.. _Virginie Loison: https://www.linkedin.com/in/virginie-loison/
+
+.. _Julie Alberge: https://www.linkedin.com/in/julie-alberge
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index aac322c..bda81ab 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -36,6 +36,9 @@ CLI
 FIX
 ~~~
 
+- Disable caching of diverged/errored runs. By `Julie Alberge`_ and 
+  `Virginie Loison`_ (:gh:`735`)
+
 - Fix pickling of dynamic modules to allow for nested parallelism in
   distributed runs. By `Thomas Moreau`_ (:gh:`713`)
 


Test patch:
diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
index 52b3b33..edf1ef3 100644
--- a/benchopt/tests/test_runner.py
+++ b/benchopt/tests/test_runner.py
@@ -1,3 +1,4 @@
+import time
 import numpy as np
 
 import pytest
@@ -9,6 +10,9 @@ from benchopt.benchmark import _check_patterns
 from benchopt.benchmark import _extract_options
 from benchopt.benchmark import _extract_parameters
 from benchopt.benchmark import _list_parametrized_classes
+from benchopt.utils.temp_benchmark import temp_benchmark
+from benchopt.tests.utils import CaptureRunOutput
+from benchopt.cli.main import run
 
 
 class MockOutput:
@@ -252,3 +256,63 @@ def test_extract_parameters():
         assert _extract_parameters(f"{token}") == [token]
         assert _extract_parameters(f"'{token}'") == [token]
         assert _extract_parameters(f"\"{token}\"") == [token]
+
+
+def test_error_caching(no_debug_test):
+
+    objective = """from benchopt import BaseObjective
+
+        class Objective(BaseObjective):
+            name = "test_obj"
+            min_benchopt_version = "0.0.0"
+
+            def set_data(self, X, y): pass
+            def get_one_result(self): pass
+            def evaluate_result(self, beta): return dict(value=1)
+            def get_objective(self): return dict(X=0, y=0)
+    """
+
+    solver1 = """from benchopt import BaseSolver
+
+    class Solver(BaseSolver):
+        name = "failing-solver"
+        sampling_strategy = 'iteration'
+        def set_objective(self, X, y): pass
+        def run(self, n_iter):
+            raise ValueError('Failing solver.')
+        def get_result(self): return dict(beta=1)
+    """
+
+    solver2 = """from benchopt import BaseSolver
+
+    class Solver(BaseSolver):
+        name = "normal-solver"
+        sampling_strategy = 'iteration'
+        def set_objective(self, X, y): pass
+        def run(self, n_iter): pass
+        def get_result(self): return dict(beta=1)
+    """
+
+    dataset = """from benchopt import BaseDataset
+
+    class Dataset(BaseDataset):
+        name = "dataset"
+        def get_data(self):
+            return dict(X=0, y=1)
+    """
+
+    with temp_benchmark(objective=objective,
+                        solvers=[solver1, solver2],
+                        datasets=[dataset]) as benchmark:
+        with CaptureRunOutput() as out:
+            for it in range(2):
+                run([str(benchmark.benchmark_dir),
+                    *' -d dataset --no-plot -r 1 -n 1'.split()],
+                    standalone_mode=False)
+                # benchmark is too quick to run, without sleep output files
+                # have the same name and the unlinking fails:
+                if it == 0:
+                    time.sleep(1.1)
+
+    # error message should be displayed twice
+    out.check_output("ValueError: Failing solver.", repetition=2)