Repository: benchopt/benchopt
Pull request #703 (2024-04-23 16:06:23)

Add parameters as columns in result file
### Checks before merging PR
- [x] added documentation for any new feature
- [x] added unit test
- [x] edited the [what's new](../../whatsnew.rst) (if applicable)


codecov-commenter: ## [Codecov](https://app.codecov.io/gh/benchopt/benchopt/pull/703?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) Report
> Merging [#703](https://app.codecov.io/gh/benchopt/benchopt/pull/703?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) (e88520b) into [main](https://app.codecov.io/gh/benchopt/benchopt/commit/e3a3b0dac8cf591a7318bab38a43cdeec3528713?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) (e3a3b0d) will **decrease** coverage by `0.04%`.
> The diff coverage is `46.15%`.

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #703      +/-   ##
==========================================
- Coverage   54.30%   54.27%   -0.04%     
==========================================
  Files          45       45              
  Lines        2994     3007      +13     
  Branches      560      565       +5     
==========================================
+ Hits         1626     1632       +6     
- Misses       1243     1249       +6     
- Partials      125      126       +1     
```



</details>

Patch:
diff --git a/benchopt/cli/helpers.py b/benchopt/cli/helpers.py
index 3e92326..9f4cd3e 100644
--- a/benchopt/cli/helpers.py
+++ b/benchopt/cli/helpers.py
@@ -305,8 +305,8 @@ def info(benchmark, solver_names, dataset_names, env_name='False',
     print(f"Info regarding the benchmark '{benchmark.name}'")
 
     # validate solvers and datasets
-    benchmark.validate_dataset_patterns(dataset_names)
-    benchmark.validate_solver_patterns(solver_names)
+    benchmark.check_dataset_patterns(dataset_names)
+    benchmark.check_solver_patterns(solver_names)
 
     # get solvers and datasets in the benchmark
     all_solvers = benchmark.get_solvers()
diff --git a/benchopt/runner.py b/benchopt/runner.py
index 1d54a45..f4bc783 100644
--- a/benchopt/runner.py
+++ b/benchopt/runner.py
@@ -245,15 +245,18 @@ def run_one_solver(benchmark, dataset, objective, solver, n_repetitions,
         output.set(rep=rep)
 
         # Get meta
-        meta = dict(
-            objective_name=str(objective),
-            solver_name=str(solver),
-            data_name=str(dataset),
-            idx_rep=rep,
-            sampling_strategy=sampling_strategy.capitalize(),
-            obj_description=obj_description,
-            solver_description=inspect.cleandoc(solver.__doc__ or ""),
-        )
+        meta = {
+            'objective_name': str(objective),
+            'obj_description': obj_description,
+            'solver_name': str(solver),
+            'solver_description': inspect.cleandoc(solver.__doc__ or ""),
+            'data_name': str(dataset),
+            'idx_rep': rep,
+            'sampling_strategy': sampling_strategy.capitalize(),
+            **{f"p_obj_{k}": v for k, v in objective._parameters.items()},
+            **{f"p_solver_{k}": v for k, v in solver._parameters.items()},
+            **{f"p_dataset_{k}": v for k, v in dataset._parameters.items()},
+        }
 
         stopping_criterion = solver._stopping_criterion.get_runner_instance(
             solver=solver,
diff --git a/benchopt/utils/conda_env_cmd.py b/benchopt/utils/conda_env_cmd.py
index 368a0d3..7ecb05d 100644
--- a/benchopt/utils/conda_env_cmd.py
+++ b/benchopt/utils/conda_env_cmd.py
@@ -14,7 +14,6 @@ from ..config import get_setting
 
 SHELL = get_setting('shell')
 CONDA_CMD = get_setting('conda_cmd')
-PIP_CMD = f"{sys.executable} -m pip"
 
 # On windows, calling conda without call exit the cmd script:
 # https://github.com/conda/conda/issues/12418
@@ -177,7 +176,7 @@ def delete_conda_env(env_name):
                capture_stdout=True)
 
 
-def get_cmd_from_requirements(packages):
+def get_env_file_from_requirements(packages):
     """Process the packages from requirements and create the install cmd.
 
     This detects the packages that need to be installed with pip and also
@@ -191,35 +190,29 @@ def get_cmd_from_requirements(packages):
             "The use of ':' to specify the channel of a dependency is "
             "deprecated. Please use '::' instead.", DeprecationWarning
         )
+        packages = [pkg.replace(":", "::", 1) for pkg in packages]
 
-    cmd = []
     conda_packages = [pkg for pkg in packages if not pkg.startswith('pip::')]
     if conda_packages:
-
-        conda_packages = [
-            pkg.replace(":", "::") if ":" in pkg and "::" not in pkg else pkg
-            for pkg in conda_packages
-        ]
-        channels = ' '.join(set(
-            f"-c {pkg.rsplit('::', 1)[0]}"
+        channels = '\n  - '.join(sorted(set(
+            pkg.rsplit('::', 1)[0]
             for pkg in conda_packages if '::' in pkg
-        ))
-        conda_packages = ' '.join(
+        )))
+        channels = f"channels:\n  - {channels}\n" if channels else ""
+        conda_packages = '\n  - '.join(sorted(set(
             pkg.rsplit('::', 1)[-1] for pkg in conda_packages
-        )
-        cmd.append(
-            f"{CONDA_CMD} install --update-all -y {channels} {conda_packages}"
-        )
+        )))
+        env = f"{channels}dependencies:\n  - {conda_packages}"
+    else:
+        env = "dependencies:"
 
-    # TODO: simplify in benchopt 1.7
-    pip_packages = [
-        pkg.replace("pip::", "").replace("pip:", "")
-        for pkg in packages if pkg.startswith('pip:')
-    ]
+    pip_packages = '\n    - '.join(sorted(set(
+        pkg.replace("pip::", "") for pkg in packages if pkg.startswith('pip::')
+    )))
     if pip_packages:
-        packages = ' '.join(pip_packages)
-        cmd.append(f"{PIP_CMD} install {packages}")
-    return cmd
+        env += f"\n  - pip\n  - pip:\n    - {pip_packages}"
+
+    return env
 
 
 def install_in_conda_env(*packages, env_name=None, force=False, quiet=False):
@@ -227,18 +220,25 @@ def install_in_conda_env(*packages, env_name=None, force=False, quiet=False):
     if len(packages) == 0:
         return
 
-    cmd = get_cmd_from_requirements(packages)
-    if force:
-        cmd = [c + ' --force-reinstall' for c in cmd]
-    cmd = '\n'.join(cmd)
-
-    error_msg = ("Failed to conda install packages "
-                 f"{packages if len(packages) > 1 else packages[0]}\n"
-                 "Error:{output}")
-    _run_shell_in_conda_env(
-        cmd, env_name=env_name, raise_on_error=error_msg,
-        capture_stdout=quiet
-    )
+    env = get_env_file_from_requirements(packages)
+    if DEBUG:
+        print(f"\ninstalling env packages:\n{'-' * 40}{env}{'-' * 40}")
+
+    with NamedTemporaryFile(mode='w+', prefix='env_', suffix='.yml') as f:
+        f.write(env)
+        f.flush()
+        cmd = (
+            f"{CONDA_CMD} env update -n {env_name} -f {f.name}"
+            f"{' -q' if quiet else ''}"
+        )
+
+        error_msg = (
+            f"Failed to conda install packages {packages}\nError:{{output}}"
+        )
+        _run_shell_in_conda_env(
+            cmd, env_name=env_name, raise_on_error=error_msg,
+            capture_stdout=quiet
+        )
 
 
 def shell_install_in_conda_env(script, env_name=None, quiet=False):
diff --git a/doc/benchmark_workflow/manage_benchmark_results.rst b/doc/benchmark_workflow/manage_benchmark_results.rst
index 277897a..470f255 100644
--- a/doc/benchmark_workflow/manage_benchmark_results.rst
+++ b/doc/benchmark_workflow/manage_benchmark_results.rst
@@ -3,18 +3,47 @@
 Manage benchmark results
 ========================
 
+.. _benchmark_results:
+
+Description of the benchmark results DataFrame
+-----------------------------------------------
+
+
+Once the benchmark is run, the results are stored as a ``pd.DataFrame`` in a
+``.parquet`` file, located in a directory ``./outputs`` in the benchmark
+directory, with a ``.parquet`` file.
+By default, the name of the file include the date and time of the run,
+as ``benchopt_run_<date>_<time>.parquet`` but a custom name can be given using
+the :option:`--output` option of ``benchopt run``.
+The DataFrame contains the following columns:
+
+- ``objective_name|solver_name|data_name``: the names of the different benchopt
+  components used for this line.
+- ``obj_description|solver_description``: A more verbose description of the
+  objective and solver, displayed in the HTML page.
+- ``p_obj_p``: the value of the objective's parameter ``p``.
+- ``p_solver_p``: the value of the solver's parameter ``p``.
+- ``p_dateset_p``: the value of the dataset's parameter ``p``.
+- ``time``: the time taken to run the solver until this point of the performance curve.
+- ``stop_val``: the number of iterations or the tolerance reached by the solver.
+- ``idx_rep``: If multiple repetitions are run for each solver with ``--n-rep``,
+  this column contains the repetition number.
+- ``sampling_strategy``: The sampling strategy used to generate the performance
+  curve of this solver. This allow to adapt the plot in the HTML depending on
+  each solver.
+- ``objective_k``: The value associated to the key ``k`` in the
+  ``Objective.evaluate_result`` dictionary.
+
+The remaining columns are informations about the system used to run the
+benchmark, with keys ``{'env_OMP_NUM_THREADS', 'platform', 'platform-architecture', 'platform-release', 'platform-version', 'system-cpus', 'system-processor', 'system-ram (GB)', 'version-cuda', 'version-numpy', 'version-scipy', 'benchmark-git-tag'}``.
+
+
 .. _collect_results:
 
 Collect benchmark results
 -------------------------
 
-Once the benchmark is run, the results are stored in a directory
-``./results`` in the benchmark directory, with a ``.parquet`` file.
-By default, the name of the file include the date and time of the run,
-but a custom name can be given using the :option:`--output` option of
-``benchopt run``.
-
-This result file is produced only once the full benchmark has been run.
+The result file is produced only once the full benchmark has been run.
 When the benchmark is run in parallel, the results that have already been
 computed can be collected using the :option:`--collect` option with
 ``benchopt run``. Adding this option with the same command line will
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index 041264a..1ac744e 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -26,6 +26,11 @@ API
   with ``::`` instead of ``:``. This allow specifying URL channels.
   By `Thomas Moreau`_ (:gh:`758`)
 
+- Add ``Objective``, ``Solver`` and ``Dataset`` parameters as columns in the
+  result DataFrame. The parameters' names are respectively prefixed with
+  ``p_obj_|p_solver_|p_dataset_`` to avoid collapse between the different
+  components. By `Melvine Nargeot`_  and `Thomas Moreau`_ (:gh:`703`).
+
 FIX
 ---
 
@@ -38,6 +43,7 @@ FIX
 - Fix the ``skip`` API for objectives that was leading to a display error.
   By `Thomas Moreau`_ (:gh:`763`)
 
+- Fix the ``info`` command By `Pierre-Antoine Comby`_ (:gh:`&67`)
 
 .. _changes_1_6:
 


Test patch:
diff --git a/benchopt/tests/test_deprecation.py b/benchopt/tests/test_deprecation.py
new file mode 100644
index 0000000..9e68874
--- /dev/null
+++ b/benchopt/tests/test_deprecation.py
@@ -0,0 +1,28 @@
+import pytest
+
+from benchopt.utils.conda_env_cmd import get_env_file_from_requirements
+
+
+# TODO: remove this test in benchopt 1.7
+def test_deprecated_channel_spec():
+
+    with pytest.warns(DeprecationWarning):
+        env = get_env_file_from_requirements(["chan:pkg"])
+    assert env == "channels:\n  - chan\ndependencies:\n  - pkg"
+
+    with pytest.warns(DeprecationWarning):
+        env = get_env_file_from_requirements(["pip:pkg"])
+    assert env == "dependencies:\n  - pip\n  - pip:\n    - pkg"
+
+    with pytest.warns(DeprecationWarning):
+        env = get_env_file_from_requirements(["pip:git+https://test.org"])
+    assert env == (
+        "dependencies:\n  - pip\n  - pip:\n    - git+https://test.org"
+    )
+
+    with pytest.warns(DeprecationWarning):
+        env = get_env_file_from_requirements(["pkg1", "chan:pkg2", "pip:pkg3"])
+    assert env == (
+        "channels:\n  - chan\n"
+        "dependencies:\n  - pkg1\n  - pkg2\n  - pip\n  - pip:\n    - pkg3"
+    )
diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
index 04e1b3d..269a7f7 100644
--- a/benchopt/tests/test_runner.py
+++ b/benchopt/tests/test_runner.py
@@ -2,6 +2,7 @@ import time
 
 import pytest
 import numpy as np
+import pandas as pd
 
 from benchopt.tests import TEST_DATASET
 from benchopt.tests import TEST_OBJECTIVE
@@ -368,3 +369,87 @@ def test_benchopt_run_script(n_jobs, no_debug_log):
 
     # Make sure the results were saved in a result file
     assert len(out.result_files) == 1, out.output
+
+
+def test_prefix_with_same_parameters():
+    from benchopt import run_benchmark
+
+    solver1 = """from benchopt import BaseSolver
+
+        class Solver(BaseSolver):
+            name = "solver1"
+            sampling_strategy = 'iteration'
+            parameters = dict(seed=[3, 27])
+            def set_objective(self, X, y): pass
+            def run(self, n_iter): pass
+            def get_result(self): return dict(beta=1)
+    """
+
+    # Different name and extra parameter
+    solver2 = (
+        solver1.replace("solver1", "solver2")
+        .replace('seed=[3, 27]', 'seed=[2, 28], type=["s"]')
+    )
+
+    dataset1 = """from benchopt import BaseDataset
+
+        class Dataset(BaseDataset):
+            name = "dataset1"
+            parameters = dict(seed=[3, 27])
+            def get_data(self):
+                return dict(X=0, y=1)
+    """
+
+    # Different name and extra parameter
+    dataset2 = (
+        dataset1.replace("dataset1", "dataset2")
+        .replace('seed=[3, 27]', 'seed=[2, 28], type=["d"]')
+    )
+
+    objective = """from benchopt import BaseObjective
+
+        class Objective(BaseObjective):
+            name = "test_obj"
+            min_benchopt_version = "0.0.0"
+
+            parameters = dict(test_p=[4])
+            def set_data(self, X, y): pass
+            def get_one_result(self): pass
+            def evaluate_result(self, beta): return dict(value=1)
+            def get_objective(self): return dict(X=0, y=0)
+    """
+
+    with temp_benchmark(solvers=[solver1, solver2],
+                        datasets=[dataset1, dataset2],
+                        objective=objective
+                        ) as benchmark:
+        run_benchmark(
+            str(benchmark.benchmark_dir),
+            solver_names=["solver1", "solver2"],
+            dataset_names=["dataset1", "dataset2"],
+            max_runs=1, n_repetitions=1, n_jobs=1, plot_result=False
+        )
+
+        df = pd.read_parquet(benchmark.get_result_file())
+
+        assert "p_solver_seed" in df.columns
+        assert "p_solver_type" in df.columns
+        assert "p_dataset_seed" in df.columns
+        assert "p_dataset_type" in df.columns
+        assert "p_obj_test_p" in df.columns
+
+        assert df.query("p_solver_seed.isna()").shape[0] == 0
+        no_type = df.query("p_solver_type.isna()")['solver_name'].unique()
+        assert all('solver1' in s for s in no_type)
+
+        assert df.query("p_dataset_seed.isna()").shape[0] == 0
+        no_type = df.query("p_dataset_type.isna()")['data_name'].unique()
+        assert all('dataset1' in s for s in no_type)
+
+        assert df.query("p_obj_test_p.isna()").shape[0] == 0
+
+        # No mixing
+        assert "d" not in df['p_solver_type'].unique()
+        assert "s" in df['p_solver_type'].unique()
+        assert "s" not in df['p_dataset_type'].unique()
+        assert "d" in df['p_dataset_type'].unique()
diff --git a/benchopt/utils/tests/test_conda_env_cmd.py b/benchopt/utils/tests/test_conda_env_cmd.py
index ba740d9..20da224 100644
--- a/benchopt/utils/tests/test_conda_env_cmd.py
+++ b/benchopt/utils/tests/test_conda_env_cmd.py
@@ -1,6 +1,4 @@
-import pytest
-
-from benchopt.utils.conda_env_cmd import get_cmd_from_requirements
+from benchopt.utils.conda_env_cmd import get_env_file_from_requirements
 from benchopt.config import get_setting
 
 
@@ -9,54 +7,32 @@ CONDA_CMD = get_setting("conda_cmd")
 
 def test_requirements_conda():
 
-    packages = ["dep1", "dep2", "chan1::dep3"]
-    cmd = get_cmd_from_requirements(packages)
-
-    assert len(cmd) == 1
-    cmd = cmd[0]
-
-    assert CONDA_CMD in cmd, f"Should use {CONDA_CMD} to install deps."
-    assert " dep1" in cmd, f"missing dep1 in cmd: {cmd}"
-    assert " dep2" in cmd, f"missing dep2 in cmd: {cmd}"
-    assert " dep3" in cmd, f"missing dep3 in cmd: {cmd}"
-    assert " -c chan1 " in cmd, f"missing channel in cmd: {cmd}"
-
-
-def test_deprecated_channel():
+    packages = ["dep1", "dep2"]
+    env = get_env_file_from_requirements(packages)
+    assert env == "dependencies:\n  - dep1\n  - dep2"
 
-    packages = ["chan:dep"]
-    with pytest.warns(DeprecationWarning):
-        cmd = get_cmd_from_requirements(packages)
 
-    assert len(cmd) == 1
-    cmd = cmd[0]
-
-    assert CONDA_CMD in cmd, f"Should use {CONDA_CMD} to install deps."
-    assert " dep" in cmd, f"missing dep in cmd: {cmd}"
-    assert " -c chan " in cmd, f"missing channel in cmd: {cmd}"
+def test_requirements_conda_with_channels():
+    packages = ["dep1", "dep2", "chan1::dep3", "chan2::dep4", "chan1::dep2"]
+    env = get_env_file_from_requirements(packages)
+    assert env == (
+        "channels:\n  - chan1\n  - chan2\n"
+        "dependencies:\n  - dep1\n  - dep2\n  - dep3\n  - dep4"
+    )
 
 
 def test_requirements_pip():
 
     packages = ["pip::dep1", "pip::dep2"]
-    cmd = get_cmd_from_requirements(packages)
-
-    assert len(cmd) == 1
-    cmd = cmd[0]
-
-    assert 'pip' in cmd, "Should use pip to install deps."
-    assert " dep1" in cmd, f"missing dep1 in cmd: {cmd}"
-    assert " dep2" in cmd, f"missing dep2 in cmd: {cmd}"
+    env = get_env_file_from_requirements(packages)
+    assert env == "dependencies:\n  - pip\n  - pip:\n    - dep1\n    - dep2"
 
 
 def test_requirements_mixed():
 
-    packages = ["dep1", "pip::dep2"]
-    cmd = get_cmd_from_requirements(packages)
-
-    assert len(cmd) == 2
-
-    assert CONDA_CMD in cmd[0], f"Should use {CONDA_CMD} to install dep1."
-    assert " dep1" in cmd[0], f"missing dep1 in cmd: {cmd[0]}"
-    assert 'pip ' in cmd[1], "Should use pip to install dep2."
-    assert " dep2" in cmd[1], f"missing dep2 in cmd: {cmd[1]}"
+    packages = ["dep1", "chan1::dep2", "pip::dep2", "pip::dep2"]
+    env = get_env_file_from_requirements(packages)
+    assert env == (
+        "channels:\n  - chan1\ndependencies:\n  - dep1\n  - dep2\n"
+        "  - pip\n  - pip:\n    - dep2"
+    )