Repository: benchopt/benchopt
Pull request #800 (2025-05-06 21:51:41)

ENH add --no-cache option to disable caching
### Checks before merging PR
- [x] added documentation for any new feature
- [x] added unit test
- [x] edited the [what's new](../../whatsnew.rst) (if applicable)


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/benchopt/benchopt/pull/800?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) Report
Attention: Patch coverage is `62.50000%` with `3 lines` in your changes missing coverage. Please review.
> Project coverage is 82.28%. Comparing base [(`3997bba`)](https://app.codecov.io/gh/benchopt/benchopt/commit/3997bba5a807b90cab95a1ece7a12a5b75da2b99?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) to head [(`feac16a`)](https://app.codecov.io/gh/benchopt/benchopt/commit/feac16a8ba71da7a286c1f63c7108cc4888a7a12?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt).
> Report is 1 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #800      +/-   ##
==========================================
- Coverage   82.33%   82.28%   -0.05%     
==========================================
  Files          47       47              
  Lines        3187     3195       +8     
==========================================
+ Hits         2624     2629       +5     
- Misses        563      566       +3     
```

</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>

Patch:
diff --git a/.github/workflows/expose_link.yml b/.github/workflows/expose_link.yml
index 470bd0a..b353b5d 100644
--- a/.github/workflows/expose_link.yml
+++ b/.github/workflows/expose_link.yml
@@ -1,10 +1,5 @@
 name: Expose result artifact
 on: [status]
-# Cancel in-progress workflows when pushing
-# a new commit on the same branch
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
 jobs:
   circleci_result_artifact_redirector_job:
     runs-on: ubuntu-latest
diff --git a/benchopt/benchmark.py b/benchopt/benchmark.py
index 6b7766a..60b569a 100644
--- a/benchopt/benchmark.py
+++ b/benchopt/benchmark.py
@@ -70,9 +70,12 @@ class Benchmark:
         Caching mechanism for the benchmark.
     """
     def __init__(
-        self, benchmark_dir, allow_meta_from_json=False,
+            self, benchmark_dir,
+            no_cache=False,
+            allow_meta_from_json=False,
     ):
         self.benchmark_dir = Path(benchmark_dir)
+        self.no_cache = no_cache
 
         global _RUNNING_BENCHMARK
         _RUNNING_BENCHMARK = self
@@ -329,6 +332,8 @@ class Benchmark:
 
     def get_cache_location(self):
         "Get the location for the cache of the benchmark."
+        if self.no_cache:
+            return None
         benchopt_cache_dir = get_setting("cache")
         if benchopt_cache_dir is None:
             return self.benchmark_dir / CACHE_DIR
@@ -345,6 +350,9 @@ class Benchmark:
         if it exists, or None if it does not. This is useful to gather results
         that are already in cache.
         """
+        if self.no_cache:
+            assert not collect, "Cannot collect when using `--no-cache`."
+            return func
 
         # Create a cached version of `func` and handle cases where we force
         # the run.
diff --git a/benchopt/cli/main.py b/benchopt/cli/main.py
index 17055ad..6d0f82d 100644
--- a/benchopt/cli/main.py
+++ b/benchopt/cli/main.py
@@ -62,6 +62,7 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
         "pdb",
         "profile",
         "env_name",
+        "no_cache",
         "output",
     ]
     return [cli_kwargs[name] for name in return_names]
@@ -170,6 +171,10 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               help="Run the benchmark in the conda environment "
               "named <env_name>. To install the required solvers and "
               "datasets, see the command `benchopt install`.")
+@click.option('--no-cache',
+              is_flag=True,
+              help='If set, disable the cache on disk for the run. Note that '
+              'this makes the run less tolerant to errors, use with caution.')
 @click.option("--output", default="None", type=str,
               help="Filename for the result output. "
               "If given, the results will "
@@ -191,7 +196,7 @@ def run(config_file=None, **kwargs):
         benchmark, solver_names, forced_solvers, dataset_names,
         objective_filters, max_runs, n_repetitions, timeout, no_timeout,
         n_jobs, slurm, collect, plot, display, html, pdb, do_profile,
-        env_name, output_name
+        env_name, no_cache, output_name
     ) = _get_run_args(kwargs, config)
 
     # If --no-timeout is set and --timeout is not, skip these blocks
@@ -212,7 +217,7 @@ def run(config_file=None, **kwargs):
                 timeout = pd.to_timedelta(timeout).total_seconds()
 
     # Create the Benchmark object
-    benchmark = Benchmark(benchmark)
+    benchmark = Benchmark(benchmark, no_cache=no_cache)
 
     if benchmark.min_version is not None:
         from packaging.version import parse
diff --git a/benchopt/runner.py b/benchopt/runner.py
index 62a1058..49c9dc9 100644
--- a/benchopt/runner.py
+++ b/benchopt/runner.py
@@ -433,7 +433,8 @@ def run_benchmark(benchmark_path, solver_names=None, forced_solvers=(),
                   dataset_names=None, objective_filters=None, max_runs=10,
                   n_repetitions=1, timeout=None, n_jobs=1, slurm=None,
                   plot_result=True, display=True, html=True,  collect=False,
-                  show_progress=True, pdb=False, output_name="None"):
+                  show_progress=True, pdb=False, no_cache=False,
+                  output_name="None"):
     """Run full benchmark.
 
     Parameters
@@ -480,6 +481,10 @@ def run_benchmark(benchmark_path, solver_names=None, forced_solvers=(),
         If show_progress is set to True, display the progress of the benchmark.
     pdb : bool
         If pdb is set to True, open a debugger on error.
+    no_cache : bool
+        If set to True, this deactivates the caching mechanism integrated in
+        benchopt. Note that this makes the run less tolerant to errors, use it
+        with caution.
     output_name : str
         Filename for the parquet output. If given, the results will
         be stored at <BENCHMARK>/outputs/<filename>.parquet.
@@ -492,7 +497,7 @@ def run_benchmark(benchmark_path, solver_names=None, forced_solvers=(),
         by the objective is not the same for all parameters, the missing data
         is set to `NaN`.
     """
-    benchmark = Benchmark(benchmark_path)
+    benchmark = Benchmark(benchmark_path, no_cache=no_cache)
     solvers = benchmark.check_solver_patterns(
         solver_names + list(forced_solvers)
     )
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index 3656604..7f07089 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -19,11 +19,14 @@ Major change
 CLI
 ---
 
+- Add ``--no-cache`` option to ``benchopt run``, to disable caching.
+  By `Thomas Moreau`_ (:gh:`800`)
+
 - Add ``--gpu`` flag to ``benchopt install``, to handle different requirements
   for GPU and CPU. By `Mathurin Massias`_ (:gh:`793`)
 
-- Make it possible to run ``benchopt`` as ``python -m benchopt``, to ease running
-  in various environment and debugging. By `Rémi Flamary`_ (:gh:`685`)
+- Make it possible to run ``benchopt`` as ``python -m benchopt``, to ease
+  running in various environment and debugging. By `Rémi Flamary`_ (:gh:`685`)
 
 API
 ---


Test patch:
diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
index 269a7f7..7d7bc35 100644
--- a/benchopt/tests/test_runner.py
+++ b/benchopt/tests/test_runner.py
@@ -1,5 +1,3 @@
-import time
-
 import pytest
 import numpy as np
 import pandas as pd
@@ -286,66 +284,6 @@ def test_extract_parameters():
         assert _extract_parameters(f"\"{token}\"") == [token]
 
 
-def test_error_caching(no_debug_log):
-
-    objective = """from benchopt import BaseObjective
-
-        class Objective(BaseObjective):
-            name = "test_obj"
-            min_benchopt_version = "0.0.0"
-
-            def set_data(self, X, y): pass
-            def get_one_result(self): pass
-            def evaluate_result(self, beta): return dict(value=1)
-            def get_objective(self): return dict(X=0, y=0)
-    """
-
-    solver1 = """from benchopt import BaseSolver
-
-    class Solver(BaseSolver):
-        name = "failing-solver"
-        sampling_strategy = 'iteration'
-        def set_objective(self, X, y): pass
-        def run(self, n_iter):
-            raise ValueError('Failing solver.')
-        def get_result(self): return dict(beta=1)
-    """
-
-    solver2 = """from benchopt import BaseSolver
-
-    class Solver(BaseSolver):
-        name = "normal-solver"
-        sampling_strategy = 'iteration'
-        def set_objective(self, X, y): pass
-        def run(self, n_iter): pass
-        def get_result(self): return dict(beta=1)
-    """
-
-    dataset = """from benchopt import BaseDataset
-
-    class Dataset(BaseDataset):
-        name = "dataset"
-        def get_data(self):
-            return dict(X=0, y=1)
-    """
-
-    with temp_benchmark(objective=objective,
-                        solvers=[solver1, solver2],
-                        datasets=[dataset]) as benchmark:
-        with CaptureRunOutput() as out:
-            for it in range(2):
-                run([str(benchmark.benchmark_dir),
-                    *' -d dataset --no-plot -r 1 -n 1'.split()],
-                    standalone_mode=False)
-                # benchmark is too quick to run, without sleep output files
-                # have the same name and the unlinking fails:
-                if it == 0:
-                    time.sleep(1.1)
-
-    # error message should be displayed twice
-    out.check_output("ValueError: Failing solver.", repetition=2)
-
-
 # Under windows, the function needs to be pickleable
 # for parallel jobs to work with joblib
 @pytest.mark.parametrize('n_jobs', [1, 2])
@@ -453,3 +391,92 @@ def test_prefix_with_same_parameters():
         assert "s" in df['p_solver_type'].unique()
         assert "s" not in df['p_dataset_type'].unique()
         assert "d" in df['p_dataset_type'].unique()
+
+
+class TestCache:
+    """Test the cache of the benchmark."""
+
+    objective = """from benchopt import BaseObjective
+
+        class Objective(BaseObjective):
+            name = "test_obj"
+            min_benchopt_version = "0.0.0"
+
+            def set_data(self, X, y): pass
+            def get_one_result(self): pass
+            def evaluate_result(self, beta): return dict(value=1)
+            def get_objective(self): return dict(X=0, y=0)
+    """
+
+    solver = """from benchopt import BaseSolver
+
+    class Solver(BaseSolver):
+        name = "test-solver"
+        sampling_strategy = 'run_once'
+        def set_objective(self, X, y): pass
+        def run(self, _): print("#RUN_SOLVER")
+        def get_result(self): return dict(beta=1)
+    """
+
+    dataset = """from benchopt import BaseDataset
+
+    class Dataset(BaseDataset):
+        name = "test-dataset"
+        def get_data(self): return dict(X=0, y=1)
+    """
+
+    @pytest.mark.parametrize('n_reps', [1, 4])
+    def test_cache(self, no_debug_log, n_reps):
+        with temp_benchmark(
+                objective=self.objective, solvers=self.solver,
+                datasets=self.dataset
+        ) as benchmark:
+            with CaptureRunOutput() as out:
+                for it in range(3):
+                    run([str(benchmark.benchmark_dir),
+                        *f'--no-plot -r {n_reps}'.split()],
+                        standalone_mode=False)
+
+        # Check that the run are only call once per repetition, but not cached
+        # when using multiple repetitions
+        out.check_output("#RUN_SOLVER", repetition=n_reps)
+
+    @pytest.mark.parametrize('n_reps', [1, 4])
+    def test_no_cache(self, no_debug_log, n_reps):
+        with temp_benchmark(
+                objective=self.objective, solvers=self.solver,
+                datasets=self.dataset
+        ) as benchmark:
+            with CaptureRunOutput() as out:
+                for it in range(3):
+                    run([str(benchmark.benchmark_dir),
+                        *f'--no-plot -r {n_reps} --no-cache'.split()],
+                        standalone_mode=False)
+
+        # Check that the run is not cached when using --no-cache
+        out.check_output("#RUN_SOLVER", repetition=n_reps * 3)
+
+    def test_no_error_caching(self, no_debug_log):
+
+        solver_fail = """from benchopt import BaseSolver
+
+        class Solver(BaseSolver):
+            name = "failing-solver"
+            sampling_strategy = 'iteration'
+            def set_objective(self, X, y): pass
+            def run(self, n_iter):
+                raise ValueError('Failing solver.')
+            def get_result(self): return dict(beta=1)
+        """
+
+        with temp_benchmark(objective=self.objective,
+                            solvers=[self.solver, solver_fail],
+                            datasets=self.dataset) as benchmark:
+            with CaptureRunOutput() as out:
+                for it in range(3):
+                    run([str(benchmark.benchmark_dir),
+                        *' -d test-dataset --no-plot -r 1 -n 1'.split()],
+                        standalone_mode=False)
+
+        # error message should be displayed twice
+        out.check_output("ValueError: Failing solver.", repetition=3)
diff --git a/benchopt/tests/utils/capture_run_output.py b/benchopt/tests/utils/capture_run_output.py
index 7c98794..3ba46dc 100644
--- a/benchopt/tests/utils/capture_run_output.py
+++ b/benchopt/tests/utils/capture_run_output.py
@@ -75,12 +75,17 @@ class BenchoptRunOutputProcessor:
         if len(self.result_files) >= 1 and delete_result_files:
             for result_file in self.result_files:
                 result_path = Path(result_file)
-                result_path.unlink()  # remove result file
+                self.safe_unlink(result_path)  # remove result file
                 result_dir = result_path.parents[0]
                 stem = result_path.stem
                 for html_file in result_dir.glob(f'*{stem}*.html'):
                     # remove html files associated with this results
-                    html_file.unlink()
+                    self.safe_unlink(html_file)
+
+    def safe_unlink(self, file):
+        # Avoid error when the file is no present due to conficting names.
+        if file.exists():
+            file.unlink()
 
     def check_output(self, pattern, repetition=None):