Repository: benchopt/benchopt
Pull request #803 (2025-06-02 11:35:14)

DOC add SLOPE benchmark to readme
This PR adds the slope benchmark to the readme

### Checks before merging PR
- [ ] added documentation for any new feature
- [ ] added unit test
- [ ] edited the [what's new](../../whatsnew.rst) (if applicable)


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/benchopt/benchopt/pull/803?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 82.34%. Comparing base [(`a4da218`)](https://app.codecov.io/gh/benchopt/benchopt/commit/a4da2180e1a7cfc8d5d958b9c5bcb1973cabdf9e?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) to head [(`e768d4f`)](https://app.codecov.io/gh/benchopt/benchopt/commit/e768d4f8217c1ed455e61d7a79bdd59f1126aa9a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt).

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##             main     #803   +/-   ##
=======================================
  Coverage   82.34%   82.34%           
=======================================
  Files          47       47           
  Lines        3195     3195           
=======================================
  Hits         2631     2631           
  Misses        564      564           
```

</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>

Patch:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 86daea9..e55305c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -38,7 +38,7 @@ jobs:
         --outdir dist/
         .
     - name: Wait for all tests to succeed
-      uses: fountainhead/action-wait-for-check@v1.0.0
+      uses: fountainhead/action-wait-for-check@v1.2.0
       id: wait-for-tests
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/README.rst b/README.rst
index ae55025..a2b1973 100644
--- a/README.rst
+++ b/README.rst
@@ -194,6 +194,9 @@ Available benchmarks
    * - `Bilevel Optimization <https://github.com/benchopt/benchmark_bilevel>`_
      - `Results <https://benchopt.github.io/results/benchmark_bilevel.html>`__
      - |Build Status Bilevel|
+   * - `Sorted L-One Penalized Estimation (SLOPE) <https://github.com/benchopt/benchmark_slope>`_
+     - `Results <https://benchopt.github.io/results/benchmark_slope.html>`__
+     - |Build Status SLOPE|
 
 
 
@@ -247,3 +250,5 @@ Available benchmarks
    :target: https://github.com/benchopt/benchmark_resnet_classif/actions
 .. |Build Status Bilevel| image:: https://github.com/benchopt/benchmark_bilevel/actions/workflows/main.yml/badge.svg
    :target: https://github.com/benchopt/benchmark_bilevel/actions
+.. |Build Status SLOPE| image:: https://github.com/benchopt/benchmark_slope/actions/workflows/ci.yml/badge.svg
+   :target: https://github.com/benchopt/benchmark_slope/actions
diff --git a/benchopt/base.py b/benchopt/base.py
index dd4a3c2..8ed635c 100644
--- a/benchopt/base.py
+++ b/benchopt/base.py
@@ -65,7 +65,7 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
             or 'iteration'
         )
 
-    def _set_objective(self, objective, output=None):
+    def _set_objective(self, objective):
         """Store the objective for hashing/pickling and check its compatibility
 
         Parameters
@@ -82,7 +82,6 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
             If skip is False, the reason should be None.
         """
         self._objective = objective
-        self._output = output
 
         objective_dict = objective.get_objective()
         assert objective_dict is not None, (
@@ -92,13 +91,10 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
 
         # Check if the objective is compatible with the solver
         skip, reason = self.skip(**objective_dict)
-        if skip:
-            if self._output:
-                self._output.skip(reason)
-            return True
+        if not skip:
+            self.set_objective(**objective_dict)
 
-        self.set_objective(**objective_dict)
-        return False
+        return skip, reason
 
     @abstractmethod
     def set_objective(self, **objective_dict):
@@ -206,9 +202,6 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
             the solver on an easy to solve problem.
         """
 
-        if hasattr(self, '_output') and self._output is not None:
-            self._output.progress('caching warmup times.')
-
         if self._solver_strategy == "callback":
             stopping_criterion = (
                 SingleRunCriterion(stop_val=stop_val)
@@ -244,7 +237,7 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
         self._warmup_done = True
 
     @staticmethod
-    def _reconstruct(module_filename, parameters, objective, output,
+    def _reconstruct(module_filename, parameters, objective,
                      pickled_module_hash=None, benchmark_dir=None):
         set_benchmark_module(benchmark_dir)
         Solver = _reconstruct_class(
@@ -252,15 +245,14 @@ class BaseSolver(ParametrizedNameMixin, DependenciesMixin, ABC):
         )
         obj = Solver.get_instance(**parameters)
         if objective is not None:
-            obj._set_objective(objective, output=output)
+            obj._set_objective(objective)
         return obj
 
     def __reduce__(self):
         module_hash = get_file_hash(self._module_filename)
         objective = getattr(self, '_objective', None)
-        output = getattr(self, '_output', None)
         return self._reconstruct, (
-            self._module_filename, self._parameters, objective, output,
+            self._module_filename, self._parameters, objective,
             module_hash, str(self._import_ctx._benchmark_dir)
         )
 
diff --git a/benchopt/runner.py b/benchopt/runner.py
index 49c9dc9..7138122 100644
--- a/benchopt/runner.py
+++ b/benchopt/runner.py
@@ -211,7 +211,7 @@ def run_one_solver(benchmark, dataset, objective, solver, n_repetitions,
             res = _run_one_to_cvg_cached(**kwargs)
             return res if res is not None else ([], 'not run yet')
 
-    # Set objective an skip if necessary.
+    # Set objective and skip if necessary.
     skip, reason = objective.set_dataset(dataset)
     if skip:
         output.skip(reason, objective=True)
@@ -241,8 +241,9 @@ def run_one_solver(benchmark, dataset, objective, solver, n_repetitions,
             n_repetitions = 1
 
     for rep in range(n_repetitions):
-        skip = solver._set_objective(objective, output=output)
+        skip, reason = solver._set_objective(objective)
         if skip:
+            output.skip(reason)
             return []
 
         output.set(rep=rep)
@@ -282,8 +283,8 @@ def run_one_solver(benchmark, dataset, objective, solver, n_repetitions,
         if status in ['diverged', 'error', 'interrupted', 'not run yet']:
             run_statistics = []
             break
-        run_statistics.extend(curve)
         states.append(status)
+        run_statistics.extend(curve)
 
     else:
         if 'max_runs' in states:
diff --git a/benchopt/utils/conda_env_cmd.py b/benchopt/utils/conda_env_cmd.py
index 8d0e646..8674de8 100644
--- a/benchopt/utils/conda_env_cmd.py
+++ b/benchopt/utils/conda_env_cmd.py
@@ -40,6 +40,8 @@ EMPTY_ENV = """
 channels:
   - conda-forge
   - nodefaults
+dependencies:
+  - python=3.10
 """
 
 
diff --git a/doc/user_guide/advanced.rst b/doc/user_guide/advanced.rst
index 893402d..c364e88 100644
--- a/doc/user_guide/advanced.rst
+++ b/doc/user_guide/advanced.rst
@@ -57,6 +57,11 @@ If ``slurm_time`` is not set in the config file, benchopt uses by default
 the value of ``--timeout`` multiplied by ``1.5`` for each job.
 Note that the logs of each benchmark run can be found in ``./benchopt_run/``.
 
+``slurm_array_parallelism`` can be used to limit the number of simultaneous
+jobs that are run. This is useful when the cluster has no limit on the number of
+simultaneous jobs that can be run. By default, this value is not set, leaving it to
+the cluster to adjust this.
+
 As we rely on ``joblib.Memory`` for caching the results, the cache should work
 exactly as if you were running the computation sequentially, as long as you have
 a shared file-system between the nodes used for the computations.
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index 7f07089..f7807f1 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -64,6 +64,9 @@ FIX
 - Fix ignored ``--minimal`` option in ``benchopt install``.
   By `Lionel Kusch`_ (:gh:`786`)
 
+- Fix cache miss when order of the solver changes.
+  By `Thomas Moreau`_ (:gh:`806`)
+
 .. _changes_1_6:
 
 Version 1.6 - 15/07/2024


Test patch:
diff --git a/benchopt/tests/dummy_benchmark/objective.py b/benchopt/tests/dummy_benchmark/objective.py
index 81e9ce3..4588842 100644
--- a/benchopt/tests/dummy_benchmark/objective.py
+++ b/benchopt/tests/dummy_benchmark/objective.py
@@ -39,9 +39,13 @@ class Objective(BaseObjective):
         return dict(beta=np.zeros(self.X.shape[1]))
 
     def evaluate_result(self, beta):
-        diff = self.y - self.X.dot(beta)
-        mse = .5 * diff.dot(diff)
-        regularization = abs(beta).sum()
+        if isinstance(beta, int):
+            mse = 1
+            regularization = 1
+        else:
+            diff = self.y - self.X.dot(beta)
+            mse = .5 * diff.dot(diff)
+            regularization = abs(beta).sum()
         objective_value = mse + self.lmbd * regularization
 
         # To test for multiple type of return value, makes this depend on the
diff --git a/benchopt/tests/test_benchmarks.py b/benchopt/tests/test_benchmarks.py
index 592e091..dbb4dc1 100644
--- a/benchopt/tests/test_benchmarks.py
+++ b/benchopt/tests/test_benchmarks.py
@@ -164,7 +164,7 @@ def test_solver(check_test, benchmark, solver_class):
 
         objective.set_dataset(dataset)
         solver = solver_class.get_instance()
-        skip = solver._set_objective(objective)
+        skip, reason = solver._set_objective(objective)
         if skip:
             continue
         solver_ran_once = True
diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
index 7d7bc35..1326b43 100644
--- a/benchopt/tests/test_runner.py
+++ b/benchopt/tests/test_runner.py
@@ -480,3 +480,26 @@ class TestCache:
 
         # error message should be displayed twice
         out.check_output("ValueError: Failing solver.", repetition=3)
+
+    @pytest.mark.parametrize('n_reps', [1, 4])
+    def test_cache_order(self, no_debug_log, n_reps):
+        with temp_benchmark(
+                objective=self.objective, datasets=self.dataset,
+                solvers=[
+                    self.solver,
+                    self.solver.replace("test-solver", "test-solver2")
+                ]
+        ) as benchmark:
+            with CaptureRunOutput() as out:
+                run([str(benchmark.benchmark_dir),
+                     *"-s test-solver -s test-solver2".split(),
+                     *f'--no-plot -r {n_reps}'.split()],
+                    standalone_mode=False)
+                run([str(benchmark.benchmark_dir),
+                     *"-s test-solver2 -s test-solver".split(),
+                    *f'--no-plot -r {n_reps}'.split()],
+                    standalone_mode=False)
+
+        # Check that the run are only call once per repetition, but not cached
+        # when using multiple repetitions
+        out.check_output("#RUN_SOLVER", repetition=n_reps * 2)