Repository: benchopt/benchopt
Pull request #701 (2024-04-17 13:26:03)

Add "paths" key in available config keys
Allow the key `paths` in benchmark config.

tomMoral: This would close #627
codecov-commenter: ## [Codecov](https://app.codecov.io/gh/benchopt/benchopt/pull/701?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 54.30%. Comparing base [(`e3a3b0d`)](https://app.codecov.io/gh/benchopt/benchopt/commit/e3a3b0dac8cf591a7318bab38a43cdeec3528713?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) to head [(`0f5d789`)](https://app.codecov.io/gh/benchopt/benchopt/pull/701?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt).

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##             main     #701   +/-   ##
=======================================
  Coverage   54.30%   54.30%           
=======================================
  Files          45       45           
  Lines        2994     2994           
  Branches      560      560           
=======================================
  Hits         1626     1626           
  Misses       1243     1243           
  Partials      125      125           
```



</details>

Patch:
diff --git a/benchopt/base.py b/benchopt/base.py
index 3b3923f..25a979b 100644
--- a/benchopt/base.py
+++ b/benchopt/base.py
@@ -360,6 +360,16 @@ class BaseObjective(ParametrizedNameMixin, DependenciesMixin, ABC):
       evaluated. This should be a dictionary where the keys correspond to the
       keyword arguments of `evaluate_result`.
 
+    Optionally, the `Solver` can implement the following methods to change its
+    behavior:
+
+    - `save_final_results(**result)`: Return the data to be saved from the
+       results of the solver. It will be saved as a `.pkl` file in the
+       `output/results` folder, and link to the benchmark results.
+
+    - `get_next(stop_val)`: Return the next iteration where the result will be
+      evaluated.
+
     This class is also used to specify information about the benchmark.
     In particular, it should have the following class attributes:
 
@@ -420,6 +430,20 @@ class BaseObjective(ParametrizedNameMixin, DependenciesMixin, ABC):
         """
         pass
 
+    def save_final_results(self, **solver_result):
+        """Save the final results of the solver.
+
+        Parameters
+        ----------
+        solver_result : dict
+            All values needed to compute the objective metrics. This dictionary
+            is retrieved by calling ``solver_result = Solver.get_result()``.
+        Returns
+        -------
+        dict of values to save
+        """
+        pass
+
     def __call__(self, result):
         """Used to call the evaluation of the objective.
 
diff --git a/benchopt/benchmark.py b/benchopt/benchmark.py
index f5e51a4..cd67017 100644
--- a/benchopt/benchmark.py
+++ b/benchopt/benchmark.py
@@ -24,6 +24,10 @@ from .utils.shell_cmd import _run_shell_in_conda_env
 from .config import RAISE_INSTALL_ERROR
 
 
+# Global variable to access the benchmark currently running globally
+_RUNNING_BENCHMARK = None
+
+# Constant to name cache directory and folder of slurm outputs
 CACHE_DIR = '__cache__'
 SLURM_JOB_NAME = 'benchopt_run'
 
@@ -40,6 +44,11 @@ MISSING_DEPS_MSG = (
 SUBSTITUTIONS = {"*": ".*"}
 
 
+def get_running_benchmark():
+    """Return the benchmark currently running."""
+    return _RUNNING_BENCHMARK
+
+
 class Benchmark:
     """Benchmark exposes all constituents of the benchmark folder.
 
@@ -65,6 +74,8 @@ class Benchmark:
     ):
         self.benchmark_dir = Path(benchmark_dir)
 
+        global _RUNNING_BENCHMARK
+        _RUNNING_BENCHMARK = self
         set_benchmark_module(self.benchmark_dir)
 
         # Load the benchmark metadat defined in `objective.py` or
diff --git a/benchopt/config.py b/benchopt/config.py
index 1330d5c..3e45092 100644
--- a/benchopt/config.py
+++ b/benchopt/config.py
@@ -7,7 +7,6 @@ from pathlib import Path
 from collections.abc import Iterable
 from benchopt.constants import PLOT_KINDS
 
-
 BOOLEAN_STATES = configparser.ConfigParser.BOOLEAN_STATES
 CONFIG_FILE_NAME = 'benchopt.yml'
 
@@ -43,7 +42,9 @@ DEFAULT_GLOBAL_CONFIG = {
 
 DEFAULT_BENCHMARK_CONFIG = {
     "plots": list(PLOT_KINDS),
-    "plot_configs": {}
+    "plot_configs": {},
+    "data_home": "",
+    "data_paths": {}
 }
 
 """
@@ -59,7 +60,7 @@ DEFAULT_BENCHMARK_CONFIG = {
     - relative_suboptimality_curve
     - bar_chart
 
-* ``plot_configs``, *list*: list of saved views that can be easily display for
+* ``plot_configs``, *dict*: list of saved views that can be easily display for
   the plot. Each view corresponds to a name, with specified values to select
   either:
 
@@ -86,6 +87,36 @@ DEFAULT_BENCHMARK_CONFIG = {
   the ``Save as view`` button in the plot controls and downloading eiher the
   new HTML file to save them or the config file in th erepo of the benchmark,
   so that these saved views are embeded in the next plot results automatically.
+
+* ``data_home``, *str*: Allows users to define a home path where the function
+  ``get_data_path()`` search data files defined in ``data_paths``.
+
+* ``data_paths``, *dict*: Allows users to store some data files in custom
+  locations. If you are writing your own benchmark, you can use this
+  options to allow users to use custom location for data files.
+  Otherwise, please refer to the benchmark's documentation you
+  use to know if you can use this option and which paths you must define.
+
+  For example, if the benchmark asks you to define a data path,
+  you can proceed as follows in the benchmark's ``config.yaml`` file:
+
+  .. code-block:: yaml
+    data_home: path/to/data/home
+
+    data_paths:
+        my_data_file: path/to/my/file.npz
+
+  In your benchmark's datasets, you can use the ``get_data_path()``
+  to retrieve the paths:
+
+  .. code-block:: python
+    from benchopt.config import get_data_path
+
+    path = get_data_path('my_data_file')
+
+    # The "path" variable now contains "path/to/data/home/path/to/my/file.npz"
+
+  If no paths are provided, the default path used is ``your_benchmark/data``
 """
 
 
@@ -261,6 +292,38 @@ def get_setting(name, config_file=None, benchmark_name=None,
     return value
 
 
+def get_data_path(key: str = None):
+    """
+    Parameters
+    ----------
+    key (str): The key to look for in the configuration file.
+
+    Returns
+    -------
+    If the key exists in config, it returns the path associated to the key.
+    If the key does not exist, it returns the path ended by the key name.
+    """
+    from .benchmark import get_running_benchmark
+    benchmark = get_running_benchmark()
+
+    data_home = benchmark.get_setting("data_home")
+
+    if data_home == "":
+        data_home = benchmark.benchmark_dir
+
+    path = Path(data_home)
+
+    if key is not None:
+        data_paths = benchmark.get_setting("data_paths")
+
+        if key in data_paths and data_paths[key] is not None:
+            path = path / Path(data_paths[key])
+        else:
+            path = path / key
+
+    return path
+
+
 def parse_value(value, default_value):
     if isinstance(default_value, bool):
         # convert string 0/1/true/false/yes/no/on/off to boolean
diff --git a/benchopt/runner.py b/benchopt/runner.py
index 773a2a8..08608d4 100644
--- a/benchopt/runner.py
+++ b/benchopt/runner.py
@@ -1,9 +1,10 @@
 import time
 import inspect
+import pickle
 
 from datetime import datetime
 
-from joblib import Parallel, delayed
+from joblib import Parallel, delayed, hash
 
 from .callback import _Callback
 from .utils.sys_info import get_sys_info
@@ -93,6 +94,20 @@ def run_one_to_cvg(benchmark, objective, solver, meta, stopping_criterion,
     solver._warm_up()
 
     curve = []
+
+    # Augment the metadata with final_results if necessary.
+    base_method = getattr(
+        super(type(objective), objective),
+        'save_final_results', None
+    )
+
+    has_save_final_results = objective.save_final_results is not base_method
+    if has_save_final_results:
+        final_results = benchmark.get_output_folder() / 'final_results'
+        final_results /= f"{hash(meta)}.pkl"
+        final_results.parent.mkdir(exist_ok=True, parents=True)
+        meta["final_results"] = str(final_results)
+
     with exception_handler(output, pdb=pdb) as ctx:
 
         if solver._solver_strategy == "callback":
@@ -129,6 +144,12 @@ def run_one_to_cvg(benchmark, objective, solver, meta, stopping_criterion,
                 stop, ctx.status, stop_val = stopping_criterion.should_stop(
                     stop_val, curve
                 )
+        # Only run if save_final_results is defined in the objective.
+        if has_save_final_results:
+            to_save = objective.save_final_results(**solver.get_result())
+            if to_save is not None:
+                with open(meta["final_results"], 'wb') as f:
+                    pickle.dump(to_save, f)
 
     return curve, ctx.status
 
diff --git a/doc/names.inc b/doc/names.inc
index 493254c..f0c3399 100644
--- a/doc/names.inc
+++ b/doc/names.inc
@@ -25,3 +25,5 @@
 .. _Pierre Ablin: https://github.com/pierreablin
 
 .. _Christopher Marouani: https://github.com/chris-mrn
+
+.. _Pierre-Antoine Comby: https://github.com/paquiteau
diff --git a/doc/user_guide/advanced.rst b/doc/user_guide/advanced.rst
index e13a32a..3c0c46b 100644
--- a/doc/user_guide/advanced.rst
+++ b/doc/user_guide/advanced.rst
@@ -216,3 +216,10 @@ It assume that the python script is located at the same level as the benchmark f
 
 .. |SlurmExecutor| replace:: ``submitit.SlurmExecutor``
 .. _SlurmExecutor: https://github.com/facebookincubator/submitit/blob/main/submitit/slurm/slurm.py#L214
+
+.. _save_final_results:
+
+Saving Final Results of a Solver
+--------------------------------
+
+Using the `save_final_results(**results)` method of the objective function to retrieve the results to save. They are saved in `outputs/final_results/` directory and reference is added in the benchmark `.parquet` file.
diff --git a/doc/user_guide/index.rst b/doc/user_guide/index.rst
index 445266b..2e6658f 100644
--- a/doc/user_guide/index.rst
+++ b/doc/user_guide/index.rst
@@ -13,3 +13,4 @@ User guide
     advanced
     ml_benchmark
     performance_curves
+    tweak_datasets
diff --git a/doc/user_guide/tweak_datasets.rst b/doc/user_guide/tweak_datasets.rst
new file mode 100644
index 0000000..d654823
--- /dev/null
+++ b/doc/user_guide/tweak_datasets.rst
@@ -0,0 +1,54 @@
+.. _tweak_datasets:
+
+Tweak a dataset
+===============
+
+Paths to data files:
+--------------------
+
+For benchmark users
+~~~~~~~~~~~~~~~~~~~
+
+If the benchmark maker allows you to customize the path to the data files,
+you can modify the benchmark's configuration file with the keys `data_home` and `data_paths`.
+The `data_home` key allows you to define a home path where the benchmark will search data files defined in `data_paths`.
+The default value for `data_home` is the benchmark directory.
+The `data_paths` define paths for each file needed by the benchmark. For instance :
+
+.. code-block:: yaml
+
+    data_home: /path/to/data_home/folder
+    data_paths:
+        the_key: /path/to/the/file.ext
+
+With this config, the benchmark will retrieve the file located in `/path/to/data_home/folder/path/to/the/file.ext`
+
+To know which keys are needed, please refer to the benchmark documentation.
+
+For benchmark makers
+~~~~~~~~~~~~~~~~~~~~
+
+You can use **path configuration** to allow benchmark's users to customize the path to the data files.
+Benchopt provides a function `config.get_data_path(key)` that can be used to retrieve user's custom paths that have been filled in config.
+
+.. code-block:: python
+
+    from benchopt import config
+
+    [...]
+
+    class Dataset(BaseDataset):
+
+    [...]
+
+        def get_data(self):
+            path = config.get_data_path(key="the_key_name")
+
+The benchmark's user can now update his config file with the following content :
+
+.. code-block:: yaml
+
+    data_paths:
+        the_key_name: path/to/file.ext
+
+The variable `path` will contain `{benchmark_dir}/path/to/file.ext`.
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index 44e24d1..aac322c 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -13,6 +13,9 @@ Version 1.6 - In development
 API
 ~~~
 
+- Add a `save_final_results` method to Objective. If implemented it is run after the last solver iteration, to get desired outputs to be saved to file system.
+  By `Pierre-Antoine Comby`_ (:gh:`722`)
+
 - Add native way to do cross-validation in a benchmark with
   ``Objective.cv`` attribute that change split for each repetition.
   By `Christopher Marouani`_ and `Thomas Moreau`_ (:gh:`623`).


Test patch:
diff --git a/benchopt/tests/dummy_benchmark/config.yml b/benchopt/tests/dummy_benchmark/config.yml
deleted file mode 100644
index e097349..0000000
--- a/benchopt/tests/dummy_benchmark/config.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-plots:
-- objective_curve
-- suboptimality_curve
-- relative_suboptimality_curve
-- bar_chart
-
-plot_configs:
-  Init:
-    objective_column: objective_value
-    kind: suboptimality_curve
-    ylim: [5e-11, 100]
-    scale: loglog
-  View 2:
-    objective_column: objective_mse
-    kind: objective_curve
-    xlim: [1, 120]
-    ylim: [5e1, 3.8e2]
-    scale: semilog-x
-    xaxis_type: Iteration
\ No newline at end of file
diff --git a/benchopt/tests/test_benchmark_features.py b/benchopt/tests/test_benchmark_features.py
index a9b125d..5697427 100644
--- a/benchopt/tests/test_benchmark_features.py
+++ b/benchopt/tests/test_benchmark_features.py
@@ -113,6 +113,42 @@ def test_objective_no_cv(no_debug_test):
                 standalone_mode=False)
 
 
+def test_objective_save_final_results(no_debug_test):
+    save_final = """
+    from benchopt import BaseObjective
+
+    class Objective(BaseObjective):
+        name = "cross_val"
+
+        min_benchopt_version = "0.0.0"
+
+        def set_data(self, X, y): self.X, self.y = X, y
+        def get_one_result(self): return 0
+        def evaluate_result(self, beta): return dict(value=1)
+
+        def save_final_results(self, beta):
+            return "test_value"
+
+        def get_objective(self):
+            return dict(X=self.X, y=self.y, lmbd=1)
+
+    """
+
+    import pandas as pd
+    import pickle
+
+    with temp_benchmark(objective=save_final) as benchmark:
+        with CaptureRunOutput(delete_result_files=False) as out:
+            run([
+                str(benchmark.benchmark_dir),
+                *('-s python-pgd -d test-dataset -n 1 -r 1 --no-plot').split()
+            ],  standalone_mode=False)
+        data = pd.read_parquet(out.result_files[0])
+        with open(data.loc[0, "final_results"], "rb") as final_result_file:
+            final_results = pickle.load(final_result_file)
+    assert final_results == "test_value"
+
+
 def test_objective_cv_splitter(no_debug_test):
 
     objective = """from benchopt import BaseObjective, safe_import_context
@@ -311,3 +347,53 @@ def test_run_once_callback(n_iter):
             ], standalone_mode=False)
 
         out.check_output(rf"RUNONCE\({n_iter}\)", repetition=1)
+
+
+@pytest.mark.parametrize("test_case", ["without_data_home", "with_data_home"])
+def test_paths_config_key(test_case):
+    if test_case == "without_data_home":
+        config = """
+            data_paths:
+                data: /path/to/data
+        """
+    elif test_case == "with_data_home":
+        config = """
+            data_home: /path/to/home_data
+            data_paths:
+                data: path/to/data
+        """
+    else:
+        raise Exception("Invalid test case value")
+
+    custom_dataset = """
+        from benchopt import BaseDataset
+        from benchopt import config
+        import numpy as np
+
+        class Dataset(BaseDataset):
+            name = "custom_dataset"
+            def get_data(self):
+                path = config.get_data_path(key="data")
+                print(f"PATH${path}")
+
+                return dict(X=np.random.rand(3, 2), y=np.ones((3,)))
+    """
+
+    with temp_benchmark(datasets=[custom_dataset], config=config) as benchmark:
+        with CaptureRunOutput() as out:
+            run([
+                str(benchmark.benchmark_dir),
+                *'-s solver-test -d custom_dataset'
+                 ' -n 0 -r 1 --no-plot '
+                '-o dummy*[reg=0.5]'.split()
+            ], standalone_mode=False)
+
+        if test_case == "without_data_home":
+            out.check_output(r"path/to/data", repetition=1)
+        elif test_case == "with_data_home":
+            out.check_output(
+                r"/path/to/home_data/path/to/data",
+                repetition=1
+            )
+        else:
+            raise Exception("Invalid test case value")
diff --git a/benchopt/utils/tests/test_parquet.py b/benchopt/utils/tests/test_parquet.py
index 6fbf0d6..490c4b2 100644
--- a/benchopt/utils/tests/test_parquet.py
+++ b/benchopt/utils/tests/test_parquet.py
@@ -14,7 +14,6 @@ from benchopt.utils.temp_benchmark import temp_benchmark
 
 from benchopt.config import DEFAULT_BENCHMARK_CONFIG
 
-from benchopt.tests import DUMMY_BENCHMARK
 from benchopt.tests import SELECT_ONE_PGD
 from benchopt.tests import SELECT_ONE_SIMULATED
 from benchopt.tests import SELECT_ONE_OBJECTIVE
@@ -55,9 +54,29 @@ def test_parquet_metadata():
 
 def test_metadata_saving():
 
-    dummy_config = yaml.safe_load(
-        DUMMY_BENCHMARK.get_config_file().read_text()
-    )
+    dummy_config = {
+        'plot_configs': {
+            'Init': {
+                'kind': 'suboptimality_curve',
+                'objective_column': 'objective_value',
+                'scale': 'loglog',
+                'ylim': ['5e-11', 100]
+            },
+            'View 2': {
+                'kind': 'objective_curve',
+                'objective_column': 'objective_mse',
+                'scale': 'semilog-x',
+                'xaxis_type': 'Iteration',
+                'xlim': [1, 120],
+                'ylim': ['5e1', '3.8e2']
+            }
+        }, 'plots': [
+            'objective_curve',
+           'suboptimality_curve',
+           'relative_suboptimality_curve',
+           'bar_chart'
+        ]
+    }
 
     with temp_benchmark() as benchmark:
         # Check that the computation caching is working properly.
@@ -71,7 +90,11 @@ def test_metadata_saving():
             run(run_cmd, 'benchopt', standalone_mode=False)
 
         config = get_metadata(Path(out.result_files[0]))
-        assert config == DEFAULT_BENCHMARK_CONFIG
+        expected_config = {
+            k: v for k, v in DEFAULT_BENCHMARK_CONFIG.items()
+            if k in ['plots', 'plot_configs']
+        }
+        assert config == expected_config
 
         config_file = benchmark.get_config_file()
         with config_file.open('w') as f: