Repository: benchopt/benchopt (python)

Issue:
ENH Better error message for `get_result` return format
Hello,

I ran into the following error
```
TypeError: benchopt_benchmarks.bench.objective.Objective.evaluate_result() argument after ** must be a mapping, not numpy.ndarray
```
because I was using an older benchmark as a template that returned a unique numpy array.

The fix was to return a dictionary which is easy but the error message was  abit cryptic IMHO (not everyone has access to @tomMoral !).

I suggest to change the error message to something that explicitly say a dictionary is required.

Pull request:
Cln improve get result error
Fixes #696 


Patch:
diff --git a/README.rst b/README.rst
index 47b4943..70d2e76 100644
--- a/README.rst
+++ b/README.rst
@@ -2,13 +2,13 @@
    :width: 350
    :align: center
 
-*—Making your optimization benchmarks simple and open—*
+*—Making your ML and optimization benchmarks simple and open—*
 
 ----
 
 |Test Status| |codecov| |Documentation| |Python 3.6+| |install-per-months| |discord| |SWH|
 
-``Benchopt`` is a benchmarking suite for optimization algorithms.
+``Benchopt`` is a benchmarking suite tailored for machine learning workflows.
 It is built for simplicity, transparency, and reproducibility.
 It is implemented in Python but can run algorithms written in **many programming languages**.
 
@@ -113,7 +113,7 @@ or (why not) discuss new features to be added and future development directions
 Citing Benchopt
 ---------------
 
-``Benchopt`` is a continuous effort to make reproducible and transparent optimization benchmarks.
+``Benchopt`` is a continuous effort to make reproducible and transparent ML and optimization benchmarks.
 Join us in this endeavor! If you use ``benchopt`` in a scientific publication, please cite
 
 .. code-block:: bibtex
diff --git a/benchopt/base.py b/benchopt/base.py
index 9a4f188..6cf2f70 100644
--- a/benchopt/base.py
+++ b/benchopt/base.py
@@ -420,12 +420,20 @@ class BaseObjective(ParametrizedNameMixin, DependenciesMixin, ABC):
         """
         pass
 
-    def __call__(self, solver_result):
+    def __call__(self, result):
         """Used to call the evaluation of the objective.
 
         This allows standardizing the output to a dictionary.
         """
-        objective_dict = self.evaluate_result(**solver_result)
+        if not isinstance(result, dict):
+            raise TypeError(
+                "The result returned by `Solver.get_result` should be a dict "
+                "whose keys are the arguments of `Objective.evaluate_result`. "
+                f"Got {result}."
+
+            )
+
+        objective_dict = self.evaluate_result(**result)
 
         if not isinstance(objective_dict, dict):
             objective_dict = {'value': objective_dict}
diff --git a/benchopt/callback.py b/benchopt/callback.py
index 5a5a25c..1e4a5c4 100644
--- a/benchopt/callback.py
+++ b/benchopt/callback.py
@@ -86,7 +86,6 @@ class _Callback:
         Return True if the solver should be stopped.
         """
         result = self.solver.get_result()
-
         objective_dict = self.objective(result)
         self.curve.append(dict(
             **self.meta, stop_val=self.it,
diff --git a/benchopt/runner.py b/benchopt/runner.py
index 38ea254..b4f3cdd 100644
--- a/benchopt/runner.py
+++ b/benchopt/runner.py
@@ -49,8 +49,8 @@ def run_one_resolution(objective, solver, meta, stop_val):
     t_start = time.perf_counter()
     solver.run(stop_val)
     delta_t = time.perf_counter() - t_start
-    solver_result = solver.get_result()
-    objective_dict = objective(solver_result)
+    result = solver.get_result()
+    objective_dict = objective(result)
 
     # Add system info in results
     info = get_sys_info()
diff --git a/doc/index.rst b/doc/index.rst
index c3c4ab6..ab5af7f 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -2,12 +2,12 @@
 ``Benchopt``
 =============
 
-*—Making your optimization benchmarks simple and open—*
+*—Making your ML and optimization benchmarks simple and open—*
 
 
 |Python 3.6+| |PyPI version| |License|
 
-Benchopt is a benchmarking suite for optimization algorithms.
+Benchopt is a benchmarking suite tailored for machine learning workflows.
 It is built for simplicity, transparency, and reproducibility.
 It is implemented in Python but can run algorithms written in many programming languages.
 
@@ -76,7 +76,7 @@ or (why not) discuss new features to be added and future development directions
 Citing Benchopt
 ---------------
 
-Benchopt is a continuous effort to make reproducible and transparent optimization benchmarks.
+Benchopt is a continuous effort to make reproducible and transparent ML and optimization benchmarks.
 Join this endeavor! If you use benchopt in a scientific publication, please cite
 
 .. code-block:: bibtex


Test patch:
diff --git a/benchopt/tests/test_benchmark_features.py b/benchopt/tests/test_benchmark_features.py
index 0d07c90..a9b125d 100644
--- a/benchopt/tests/test_benchmark_features.py
+++ b/benchopt/tests/test_benchmark_features.py
@@ -2,7 +2,6 @@ import pytest
 import tempfile
 
 from benchopt.cli.main import run
-from benchopt.cli.main import test as _cmd_test
 from benchopt.utils.temp_benchmark import temp_benchmark
 from benchopt.utils.dynamic_modules import _load_class_from_module
 
@@ -31,21 +30,6 @@ def test_template_dataset():
     DUMMY_BENCHMARK.get_datasets()
 
 
-def test_template_solver():
-    # Make sure that importing template_dataset raises an error.
-    with pytest.raises(ImportError):
-        template_dataset = (
-            DUMMY_BENCHMARK_PATH / 'solvers' / 'template_solver.py'
-        )
-        _load_class_from_module(
-            template_dataset, 'Solver', DUMMY_BENCHMARK_PATH
-        )
-
-    # Make sure that this error is not raised when listing all solvers from
-    # the benchmark.
-    DUMMY_BENCHMARK.get_solvers()
-
-
 def test_benchmark_submodule():
     with pytest.raises(ValueError, match="raises an error"):
         run([
@@ -327,78 +311,3 @@ def test_run_once_callback(n_iter):
             ], standalone_mode=False)
 
         out.check_output(rf"RUNONCE\({n_iter}\)", repetition=1)
-
-
-def test_warm_up():
-
-    solver1 = """from benchopt import BaseSolver
-    import numpy as np
-
-    class Solver(BaseSolver):
-        name = 'solver1'
-        sampling_strategy = 'iteration'
-
-        def set_objective(self, X, y, lmbd):
-            self.n_features = X.shape[1]
-
-        def warm_up(self):
-            print("WARMUP")
-            self.run_once(1)
-
-        def run(self, n_iter): pass
-
-        def get_result(self, **data):
-            return {'beta': np.zeros(self.n_features)}
-    """
-
-    with temp_benchmark(solvers=[solver1]) as benchmark:
-        with CaptureRunOutput() as out:
-            run([
-                str(benchmark.benchmark_dir),
-                *'-s solver1 -d test-dataset -n 0 -r 5 --no-plot'.split(),
-                *'-o dummy*[reg=0.5]'.split()
-            ], standalone_mode=False)
-
-        # Make sure warmup is called exactly once
-        out.check_output("WARMUP", repetition=1)
-
-
-def test_pre_run_hook():
-
-    solver1 = """from benchopt import BaseSolver
-    import numpy as np
-
-    class Solver(BaseSolver):
-        name = 'solver1'
-        sampling_strategy = 'iteration'
-
-        def set_objective(self, X, y, lmbd):
-            self.n_features = X.shape[1]
-
-        def pre_run_hook(self, n_iter):
-            self._pre_run_hook_n_iter = n_iter
-
-        def run(self, n_iter):
-            assert self._pre_run_hook_n_iter == n_iter
-
-        def get_result(self, **data):
-            return {'beta': np.zeros(self.n_features)}
-    """
-
-    with temp_benchmark(solvers=[solver1]) as benchmark:
-        with CaptureRunOutput() as out:
-            run([
-                str(benchmark.benchmark_dir),
-                *'-s solver1 -d test-dataset -n 0 -r 5 --no-plot '
-                '-o dummy*[reg=0.5]'.split()
-            ], standalone_mode=False)
-
-        with CaptureRunOutput() as out:
-            with pytest.raises(SystemExit, match="False"):
-                _cmd_test([
-                    str(benchmark.benchmark_dir), '-k', 'solver1',
-                    '--skip-install', '-v'
-                ], standalone_mode=False)
-
-        # Make sure warmup is called exactly once
-        out.check_output("3 passed, 1 skipped, 7 deselected", repetition=1)
diff --git a/benchopt/tests/test_solvers.py b/benchopt/tests/test_solvers.py
new file mode 100644
index 0000000..2ecf886
--- /dev/null
+++ b/benchopt/tests/test_solvers.py
@@ -0,0 +1,128 @@
+import pytest
+
+from benchopt.cli.main import run
+from benchopt.cli.main import test as _cmd_test
+from benchopt.utils.temp_benchmark import temp_benchmark
+from benchopt.stopping_criterion import SAMPLING_STRATEGIES
+from benchopt.utils.dynamic_modules import _load_class_from_module
+
+from benchopt.tests import DUMMY_BENCHMARK
+from benchopt.tests import DUMMY_BENCHMARK_PATH
+from benchopt.tests.utils import CaptureRunOutput
+
+
+def test_template_solver():
+    # Make sure that importing template_dataset raises an error.
+    with pytest.raises(ImportError):
+        template_dataset = (
+            DUMMY_BENCHMARK_PATH / 'solvers' / 'template_solver.py'
+        )
+        _load_class_from_module(
+            template_dataset, 'Solver', DUMMY_BENCHMARK_PATH
+        )
+
+    # Make sure that this error is not raised when listing all solvers from
+    # the benchmark.
+    DUMMY_BENCHMARK.get_solvers()
+
+
+def test_warm_up():
+
+    solver1 = """from benchopt import BaseSolver
+    import numpy as np
+
+    class Solver(BaseSolver):
+        name = 'solver1'
+        sampling_strategy = 'iteration'
+
+        def set_objective(self, X, y, lmbd):
+            self.n_features = X.shape[1]
+
+        def warm_up(self):
+            print("WARMUP")
+            self.run_once(1)
+
+        def run(self, n_iter): pass
+
+        def get_result(self):
+            return {'beta': np.zeros(self.n_features)}
+    """
+
+    with temp_benchmark(solvers=[solver1]) as benchmark:
+        with CaptureRunOutput() as out:
+            run([
+                str(benchmark.benchmark_dir),
+                *'-s solver1 -d test-dataset -n 0 -r 5 --no-plot'.split(),
+                *'-o dummy*[reg=0.5]'.split()
+            ], standalone_mode=False)
+
+        # Make sure warmup is called exactly once
+        out.check_output("WARMUP", repetition=1)
+
+
+def test_pre_run_hook():
+
+    solver1 = """from benchopt import BaseSolver
+    import numpy as np
+
+    class Solver(BaseSolver):
+        name = 'solver1'
+        sampling_strategy = 'iteration'
+
+        def set_objective(self, X, y, lmbd):
+            self.n_features = X.shape[1]
+
+        def pre_run_hook(self, n_iter):
+            self._pre_run_hook_n_iter = n_iter
+
+        def run(self, n_iter):
+            assert self._pre_run_hook_n_iter == n_iter
+
+        def get_result(self):
+            return {'beta': np.zeros(self.n_features)}
+    """
+
+    with temp_benchmark(solvers=[solver1]) as benchmark:
+        with CaptureRunOutput() as out:
+            run([
+                str(benchmark.benchmark_dir),
+                *'-s solver1 -d test-dataset -n 0 -r 5 --no-plot '
+                '-o dummy*[reg=0.5]'.split()
+            ], standalone_mode=False)
+
+        with CaptureRunOutput() as out:
+            with pytest.raises(SystemExit, match="False"):
+                _cmd_test([
+                    str(benchmark.benchmark_dir), '-k', 'solver1',
+                    '--skip-install', '-v'
+                ], standalone_mode=False)
+
+        # Make sure warmup is called exactly once
+        out.check_output("3 passed, 1 skipped, 7 deselected", repetition=1)
+
+
+@pytest.mark.parametrize('strategy', SAMPLING_STRATEGIES)
+def test_invalid_get_result(strategy):
+
+    solver1 = f"""from benchopt import BaseSolver
+    import numpy as np
+
+    class Solver(BaseSolver):
+        name = 'solver1'
+        strategy = '{strategy}'
+
+        def set_objective(self, X, y, lmbd): pass
+        def run(self, n_iter): pass
+
+        def get_result(self):
+            return 0
+    """
+
+    with temp_benchmark(solvers=[solver1]) as benchmark:
+        with pytest.raises(TypeError, match='get_result` should be a dict '):
+            with CaptureRunOutput():
+                run([
+                    str(benchmark.benchmark_dir),
+                    *'-s solver1 -d test-dataset -n 0 -r 5 --no-plot'.split(),
+                    *'-o dummy*[reg=0.5]'.split()
+                ], standalone_mode=False)