Repository: benchopt/benchopt
Pull request #718 (2024-05-24 15:42:45)

ENH download data
In some cases, we want to make sure the data is properly loaded when installing the benchmark, by calling `get_data` on certain datasets.
This PR adds a `--download` flag to `benchopt install` to allow calling `get_data` on any installed datasets.

This is useful in particular on big cluster where internet is not available on the computing nodes, or to avoid multiple process all trying to download the data at the same time.

### Checks before merging PR
- [x] added documentation for any new feature
- [x] added unit test
- [x] edited the [what's new](../../whatsnew.rst) (if applicable)


codecov[bot]: ## [Codecov](https://app.codecov.io/gh/benchopt/benchopt/pull/718?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) Report
Attention: Patch coverage is `96.42857%` with `2 lines` in your changes are missing coverage. Please review.
> Project coverage is 80.74%. Comparing base [(`70763cd`)](https://app.codecov.io/gh/benchopt/benchopt/commit/70763cdb6834ae339caaa027b31d1aa6a5f20789?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt) to head [(`7761afb`)](https://app.codecov.io/gh/benchopt/benchopt/commit/7761afb072ed347e1982429d98cfd45e55a698e9?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=benchopt).

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main     #718      +/-   ##
==========================================
- Coverage   80.78%   80.74%   -0.04%     
==========================================
  Files          45       45              
  Lines        3024     3044      +20     
==========================================
+ Hits         2443     2458      +15     
- Misses        581      586       +5     
```



</details>

Patch:
diff --git a/benchopt/benchmark.py b/benchopt/benchmark.py
index 691f154..f5e51a4 100644
--- a/benchopt/benchmark.py
+++ b/benchopt/benchmark.py
@@ -1,6 +1,7 @@
 import re
 import click
 import warnings
+import itertools
 from pathlib import Path
 
 from .config import get_setting
@@ -17,6 +18,7 @@ from .utils.terminal_output import YELLOW
 
 from .utils.conda_env_cmd import install_in_conda_env
 from .utils.conda_env_cmd import shell_install_in_conda_env
+from .utils.shell_cmd import _run_shell_in_conda_env
 
 # Get config values
 from .config import RAISE_INSTALL_ERROR
@@ -135,10 +137,11 @@ class Benchmark:
         "List all available solver names for the benchmark."
         return [s.name for s in self.get_solvers()]
 
-    def check_solver_patterns(self, solver_patterns):
+    def check_solver_patterns(self, solver_patterns, class_only=False):
         "Check that the patterns are valid and return selected configurations."
         return _check_patterns(
-            self.get_solvers(), solver_patterns, name_type='solver'
+            self.get_solvers(), solver_patterns, name_type='solver',
+            class_only=class_only
         )
 
     def get_datasets(self):
@@ -149,10 +152,11 @@ class Benchmark:
         "List all available dataset names for the benchmark."
         return [d.name for d in self.get_datasets()]
 
-    def check_dataset_patterns(self, dataset_patterns):
+    def check_dataset_patterns(self, dataset_patterns, class_only=False):
         "Check that the patterns are valid and return selected configurations."
         return _check_patterns(
-            self.get_datasets(), dataset_patterns, name_type='dataset'
+            self.get_datasets(), dataset_patterns, name_type='dataset',
+            class_only=class_only
         )
 
     def _list_benchmark_classes(self, base_class):
@@ -393,14 +397,14 @@ class Benchmark:
 
     def install_all_requirements(self, include_solvers, include_datasets,
                                  minimal=False, env_name=None,
-                                 force=False, quiet=False):
+                                 force=False, quiet=False, download=False):
         """Install all classes that are required for the run.
 
         Parameters
         ----------
-        include_solvers : list of str
+        include_solvers : list of BaseSolver
             patterns to select solvers to install.
-        include_datasets : list of str
+        include_datasets : list of BaseDataset
             patterns to select datasets to install.
         minimal : bool (default: False)
             only install requirements for the objective function.
@@ -411,28 +415,12 @@ class Benchmark:
             If set to True, forces reinstallation when using conda.
         quiet : bool (default: False)
             If True, silences the output of install commands.
+        download : bool (default: False)
+            If True, make sure the data are downloaded on the computer.
         """
         # Collect all classes matching one of the patterns
         print("Collecting packages:")
 
-        install_solvers = not minimal
-        install_datasets = not minimal
-
-        # If -d is used but not -s, then does not install any solver
-        if len(include_solvers) == 0 and len(include_datasets) > 0:
-            install_solvers = False
-
-        # If -s is used but not -d, then does not install any dataset
-        if len(include_datasets) == 0 and len(include_solvers) > 0:
-            install_datasets = False
-
-        # If -d or -s are followed by 'all' then all
-        # solvers or datasets are included
-        if 'all' in include_solvers:
-            include_solvers = []
-        if 'all' in include_datasets:
-            include_datasets = []
-
         check_installs, missings = [], []
         objective = self.get_benchmark_objective()
         conda_reqs, shell_install_scripts, post_install_hooks, missing_deps = (
@@ -446,31 +434,23 @@ class Benchmark:
 
         if len(shell_install_scripts) > 0 or len(conda_reqs) > 0:
             check_installs += [objective]
-        for list_classes, include_patterns, to_install in [
-                (self.get_solvers(), include_solvers, install_solvers),
-                (self.get_datasets(), include_datasets, install_datasets)
-        ]:
-            include_patterns = _check_name_lists(include_patterns)
-            for klass in list_classes:
-                for klass_parameters in product_param(klass.parameters):
-                    name = klass._get_parametrized_name(**klass_parameters)
-                    if is_matched(name, include_patterns) and to_install:
-                        reqs, scripts, hooks, missing = (
-                            klass.collect(env_name=env_name, force=force)
-                        )
-                        # If a class is not importable but has no requirements,
-                        # it might be because the requirements are specified
-                        # as global ones in the Objective. Otherwise, raise a
-                        # comprehensible error.
-                        if missing is not None:
-                            missings.append(missing)
-
-                        conda_reqs += reqs
-                        shell_install_scripts += scripts
-                        post_install_hooks += hooks
-                        if len(scripts) > 0 or len(reqs) > 0:
-                            check_installs += [klass]
-                        break
+        to_install = itertools.chain(include_datasets, include_solvers)
+        for klass in to_install:
+            reqs, scripts, hooks, missing = (
+                klass.collect(env_name=env_name, force=force)
+            )
+            # If a class is not importable but has no requirements,
+            # it might be because the requirements are specified
+            # as global ones in the Objective. Otherwise, raise a
+            # comprehensible error.
+            if missing is not None:
+                missings.append(missing)
+
+            conda_reqs += reqs
+            shell_install_scripts += scripts
+            post_install_hooks += hooks
+            if len(scripts) > 0 or len(reqs) > 0:
+                check_installs += [klass]
         print('... done')
 
         # Install the collected requirements
@@ -480,7 +460,10 @@ class Benchmark:
         if len(list_install) == 0:
             self.check_missing(missings)
             print("All required solvers are already installed.")
+            if download:
+                self.download_all_data(include_datasets, env_name, quiet)
             return
+
         print(f"Installing required packages for:\n{list_install}\n...",
               end='', flush=True)
         install_in_conda_env(
@@ -519,6 +502,18 @@ class Benchmark:
             )
         print(' done')
 
+        if download:
+            self.download_all_data(include_datasets, env_name, quiet)
+
+    def download_all_data(self, datasets, env_name, quiet):
+        if len(datasets) == 0:
+            return
+        cmd = f"benchopt check-data {self.benchmark_dir} -d "
+        cmd += "-d ".join(d.name for d in datasets)
+        _run_shell_in_conda_env(
+            cmd, env_name=env_name, raise_on_error=True, capture_stdout=False
+        )
+
     def check_missing(self, missings):
         # Check that classes not importable, with no requirements, only depends
         # on global requirements specified in Objective.requirements.
@@ -737,7 +732,8 @@ def _extract_parameters(string):
         raise ValueError(f"Invalid name '{original}', evaluated as {string}.")
 
 
-def _check_patterns(all_classes, patterns, name_type='dataset'):
+def _check_patterns(all_classes, patterns, name_type='dataset',
+                    class_only=False):
     """Check the patterns and return a list of selected classes and params.
 
     Raise an error if a pattern does not match any dataset name,
@@ -756,6 +752,8 @@ def _check_patterns(all_classes, patterns, name_type='dataset'):
     name_type: str
         Used to raise sensible error depending on the type of classes which
         are selected.
+    class_only: bool
+        Only return the classes that are matched, without a list of parameters.
 
     Returns
     -------
@@ -766,7 +764,10 @@ def _check_patterns(all_classes, patterns, name_type='dataset'):
     # If no patterns is provided or all is provided, return all the classes.
     if (patterns is None or len(patterns) == 0
             or any(p == 'all' for p, *_ in patterns)):
-        return [(cls, cls.parameters) for cls in all_classes]
+        all_valid_patterns = [(cls, cls.parameters) for cls in all_classes]
+        if not class_only:
+            return all_valid_patterns
+        return set(cls for cls, _ in all_valid_patterns)
 
     # Patterns can be either str or dict. Convert everything to 3-tuple with
     # (cls_name, args, kwargs). cls_name and kwargs correspond to class and
@@ -837,7 +838,10 @@ def _check_patterns(all_classes, patterns, name_type='dataset'):
         params.update(kwargs)
         all_valid_patterns.append((cls, params))
 
-    return all_valid_patterns
+    if not class_only:
+        return all_valid_patterns
+
+    return set(cls for cls, _ in all_valid_patterns)
 
 
 def _list_parametrized_classes(*classes, check_installed=True):
diff --git a/benchopt/cli/helpers.py b/benchopt/cli/helpers.py
index 3c8287e..c0dc4c9 100644
--- a/benchopt/cli/helpers.py
+++ b/benchopt/cli/helpers.py
@@ -1,10 +1,11 @@
+import json
 import click
 import pprint
 import tarfile
+import warnings
+import traceback
 from pathlib import Path
 from collections.abc import Iterable
-import warnings
-import json
 
 from benchopt.config import set_setting
 from benchopt.config import get_setting
@@ -21,7 +22,7 @@ from benchopt.config import GLOBAL_CONFIG_FILE_MODE
 from benchopt.utils.dynamic_modules import _load_class_from_module
 from benchopt.utils.shell_cmd import _run_shell_in_conda_env
 from benchopt.utils.terminal_output import colorify
-from benchopt.utils.terminal_output import RED, GREEN, TICK, CROSS
+from benchopt.utils.terminal_output import BLUE, RED, GREEN, TICK, CROSS
 
 
 ARCHIVE_ELEMENTS = [
@@ -465,3 +466,33 @@ def check_install(benchmark, module_filename, base_class_name):
         module_filename, base_class_name, benchmark.benchmark_dir
     )
     klass.is_installed(raise_on_not_installed=True)
+
+
+@helpers.command(
+    help="Ensure that the data of a list of Datasets are available\n\n"
+    "The names of the datasets to check/download are specified as a list.",
+    hidden=True
+)
+@click.argument('benchmark', default=Path.cwd(), type=click.Path(exists=True),
+                shell_complete=complete_benchmarks)
+@click.option('--dataset', '-d', 'dataset_names',
+              metavar="<dataset_name>", multiple=True, type=str,
+              help="Check data from the dataset <dataset_name>.",
+              shell_complete=complete_datasets)
+def check_data(benchmark, dataset_names):
+    warnings.simplefilter("ignore")
+
+    # benchmark
+    benchmark = Benchmark(benchmark)
+    datasets = benchmark.check_dataset_patterns(dataset_names, class_only=True)
+
+    # Get class to check
+    print("Loading data:")
+    for cls in datasets:
+        print(f"- {colorify(cls.name, BLUE)} ...", end='', flush=True)
+        try:
+            cls.get_instance().get_data()
+            print(colorify(f" {TICK}", GREEN))
+        except Exception:
+            print(colorify(f" {CROSS}", RED))
+            traceback.print_exc()
diff --git a/benchopt/cli/main.py b/benchopt/cli/main.py
index 45a51bc..979d032 100644
--- a/benchopt/cli/main.py
+++ b/benchopt/cli/main.py
@@ -369,6 +369,9 @@ def run(config_file=None, **kwargs):
               shell_complete=complete_config_files,
               help="YAML configuration file containing benchmark options, "
               "whose solvers and datasets will be installed.")
+@click.option('--download', is_flag=True,
+              help="If this flag is set, call `Dataset.get_data` for all "
+              "datasets, to make sure the data are present on the system.")
 @click.option('--env', '-e', 'env_name',
               flag_value='True', type=str, default='False',
               help="Install all requirements in a dedicated "
@@ -395,7 +398,7 @@ def run(config_file=None, **kwargs):
 def install(
         benchmark, minimal, solver_names, dataset_names, config_file=None,
         force=False, recreate=False, env_name='False', confirm=False,
-        quiet=False):
+        quiet=False, download=False):
 
     if config_file is not None:
         with open(config_file, "r") as f:
@@ -407,10 +410,8 @@ def install(
             forced_solvers = config.get("force-solver", tuple())
             solver_names = list(set(solver_names).union(set(forced_solvers)))
 
-    # Check that the dataset/solver patterns match actual dataset
+    # Instanciate the benchmark
     benchmark = Benchmark(benchmark)
-    benchmark.check_dataset_patterns(dataset_names)
-    benchmark.check_solver_patterns(solver_names)
 
     # Get a list of all conda envs
     default_conda_env, conda_envs = list_conda_envs()
@@ -460,11 +461,26 @@ def install(
         # create environment if necessary
         create_conda_env(env_name, recreate=recreate, quiet=quiet)
 
+    # List solver and datasets classes to install
+    if len(dataset_names) == 0 and len(solver_names) > 0:
+        datasets = []
+    else:
+        datasets = benchmark.check_dataset_patterns(
+            dataset_names, class_only=True
+        )
+    if len(solver_names) == 0 and len(dataset_names) > 0:
+        solvers = []
+    else:
+        solvers = benchmark.check_solver_patterns(
+            solver_names, class_only=True
+        )
+
     # install requirements
     print("# Install", flush=True)
     benchmark.install_all_requirements(
-        include_solvers=solver_names, include_datasets=dataset_names,
+        include_solvers=solvers, include_datasets=datasets,
         minimal=minimal, env_name=env_name, force=force, quiet=quiet,
+        download=download
     )
 
 
diff --git a/benchopt/utils/dependencies_mixin.py b/benchopt/utils/dependencies_mixin.py
index bbd857b..04e3794 100644
--- a/benchopt/utils/dependencies_mixin.py
+++ b/benchopt/utils/dependencies_mixin.py
@@ -182,7 +182,7 @@ class DependenciesMixin:
             post_install_hooks = [cls._post_install_hook]
         else:
             env_suffix = f" in '{env_name}'" if env_name else ''
-            colored_cls_name = colorify(f'{cls.name}', YELLOW)
+            colored_cls_name = colorify(cls.name, YELLOW)
             print(
                 f"- {colored_cls_name} already available{env_suffix}\n"
                 f"  No ImportError raised from {cls._module_filename}."
diff --git a/doc/user_guide/advanced.rst b/doc/user_guide/advanced.rst
index 8939a32..e13a32a 100644
--- a/doc/user_guide/advanced.rst
+++ b/doc/user_guide/advanced.rst
@@ -124,6 +124,7 @@ First, all code that need to be imported should be placed under
             └── submodule1.py  # some more helpers
 
 Then, these modules and packages can be imported as a regular package, i.e.,
+
 .. code-block::
 
     from benchopt import safe_import_context
@@ -214,4 +215,4 @@ It assume that the python script is located at the same level as the benchmark f
 .. _update_params: https://github.com/facebookincubator/submitit/blob/main/submitit/slurm/slurm.py#L386
 
 .. |SlurmExecutor| replace:: ``submitit.SlurmExecutor``
-.. _SlurmExecutor: https://github.com/facebookincubator/submitit/blob/main/submitit/slurm/slurm.py#L214
\ No newline at end of file
+.. _SlurmExecutor: https://github.com/facebookincubator/submitit/blob/main/submitit/slurm/slurm.py#L214
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index 4867902..44e24d1 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -27,6 +27,9 @@ CLI
 - Add ``--collect`` option to allow gathering results which are already
   in cache in a single parquet file. By `Thomas Moreau`_ (:gh:`710`)
 
+- Add ``--download`` option in ``benchopt install`` to allow downloading
+  the data when installing the benchmark. By `Thomas Moreau`_ (:gh:`718`)
+
 FIX
 ~~~
 


Test patch:
diff --git a/benchopt/cli/tests/test_cli.py b/benchopt/cli/tests/test_cli.py
index 4924cdb..dfd573c 100644
--- a/benchopt/cli/tests/test_cli.py
+++ b/benchopt/cli/tests/test_cli.py
@@ -29,7 +29,6 @@ from benchopt.cli.main import run
 from benchopt.cli.main import install
 from benchopt.cli.helpers import clean
 from benchopt.cli.helpers import archive
-from benchopt.cli.helpers import check_install
 from benchopt.cli.process_results import plot
 from benchopt.cli.process_results import generate_results
 
@@ -457,14 +456,14 @@ class TestInstallCmd:
     def test_invalid_dataset(self):
         with pytest.raises(click.BadParameter, match=r"invalid_dataset"):
             install([str(DUMMY_BENCHMARK_PATH), '-d', 'invalid_dataset',
-                     '-s', 'pgd'], 'benchopt', standalone_mode=False)
+                     '-y'], 'benchopt', standalone_mode=False)
 
     def test_invalid_solver(self):
         with pytest.raises(click.BadParameter, match=r"invalid_solver"):
-            install([str(DUMMY_BENCHMARK_PATH), '-s', 'invalid_solver'],
-                    'benchopt', standalone_mode=False)
+            install([str(DUMMY_BENCHMARK_PATH), '-s', 'invalid_solver',
+                     '-y'], 'benchopt', standalone_mode=False)
 
-    def test_benchopt_install(self):
+    def test_valid_call(self):
         with CaptureRunOutput() as out:
             install(
                 [str(DUMMY_BENCHMARK_PATH), '-d', SELECT_ONE_SIMULATED, '-s',
@@ -474,6 +473,25 @@ class TestInstallCmd:
         out.check_output(f"Installing '{DUMMY_BENCHMARK.name}' requirements")
         out.check_output("already available\n", repetition=3)
 
+    def test_download_data(self):
+
+        # solver with missing dependency specified
+        dataset = """from benchopt import BaseDataset
+
+            class Dataset(BaseDataset):
+                name = 'test_dataset'
+                def get_data(self): print("LOAD DATA")
+        """
+        with temp_benchmark(datasets=[dataset]) as benchmark:
+            with CaptureRunOutput() as out:
+                install([
+                    *f'{benchmark.benchmark_dir} -d test_dataset '
+                    '-y --download'.split()
+                ], 'benchopt', standalone_mode=False)
+
+        out.check_output("LOAD DATA", repetition=1)
+        out.check_output("Loading data:", repetition=1)
+
     def test_existing_empty_env(self, empty_env_name):
         msg = (
             f"`benchopt` is not installed in existing env '{empty_env_name}'"
@@ -877,33 +895,3 @@ class TestArchiveCmd:
     def test_shell_complete(self):
         # Completion for benchmark name
         _test_shell_completion(archive, [], BENCHMARK_COMPLETION_CASES)
-
-
-class TestCheckInstallCmd:
-    def test_solver_installed(self):
-        pgd_solver = DUMMY_BENCHMARK_PATH / 'solvers' / 'python_pgd.py'
-        with pytest.raises(SystemExit, match=r'0'):
-            check_install([
-                str(DUMMY_BENCHMARK_PATH), str(pgd_solver.resolve()), 'Solver'
-            ], 'benchopt')
-
-    def test_solver_does_not_exists(self):
-        pgd_solver = DUMMY_BENCHMARK_PATH / 'solvers' / 'invalid.py'
-        with pytest.raises(FileNotFoundError, match=r'invalid.py'):
-            check_install([
-                str(DUMMY_BENCHMARK_PATH), str(pgd_solver.resolve()), 'Solver'
-            ], 'benchopt')
-
-    def test_dataset_installed(self):
-        pgd_solver = DUMMY_BENCHMARK_PATH / 'datasets' / 'simulated.py'
-        with pytest.raises(SystemExit, match=r'0'):
-            check_install([
-                str(DUMMY_BENCHMARK_PATH), str(pgd_solver.resolve()), 'Dataset'
-            ], 'benchopt')
-
-    def test_dataset_does_not_exists(self):
-        pgd_solver = DUMMY_BENCHMARK_PATH / 'datasets' / 'invalid.py'
-        with pytest.raises(FileNotFoundError, match=r'invalid.py'):
-            check_install([
-                str(DUMMY_BENCHMARK_PATH), str(pgd_solver.resolve()), 'Dataset'
-            ], 'benchopt')
diff --git a/benchopt/cli/tests/test_helpers.py b/benchopt/cli/tests/test_helpers.py
new file mode 100644
index 0000000..abb31a6
--- /dev/null
+++ b/benchopt/cli/tests/test_helpers.py
@@ -0,0 +1,87 @@
+import pytest
+
+
+from benchopt.tests import DUMMY_BENCHMARK_PATH
+from benchopt.tests.utils import CaptureRunOutput
+from benchopt.utils.temp_benchmark import temp_benchmark
+from benchopt.utils.terminal_output import TICK, CROSS
+
+
+from benchopt.cli.helpers import check_data
+from benchopt.cli.helpers import check_install
+
+
+class TestCheckInstallCmd:
+    def test_solver_installed(self):
+        pgd_solver = DUMMY_BENCHMARK_PATH / 'solvers' / 'python_pgd.py'
+        with pytest.raises(SystemExit, match=r'0'):
+            check_install([
+                str(DUMMY_BENCHMARK_PATH), str(pgd_solver.resolve()), 'Solver'
+            ], 'benchopt')
+
+    def test_solver_does_not_exists(self):
+        pgd_solver = DUMMY_BENCHMARK_PATH / 'solvers' / 'invalid.py'
+        with pytest.raises(FileNotFoundError, match=r'invalid.py'):
+            check_install([
+                str(DUMMY_BENCHMARK_PATH), str(pgd_solver.resolve()), 'Solver'
+            ], 'benchopt')
+
+    def test_dataset_installed(self):
+        pgd_solver = DUMMY_BENCHMARK_PATH / 'datasets' / 'simulated.py'
+        with pytest.raises(SystemExit, match=r'0'):
+            check_install([
+                str(DUMMY_BENCHMARK_PATH), str(pgd_solver.resolve()), 'Dataset'
+            ], 'benchopt')
+
+    def test_dataset_does_not_exists(self):
+        pgd_solver = DUMMY_BENCHMARK_PATH / 'datasets' / 'invalid.py'
+        with pytest.raises(FileNotFoundError, match=r'invalid.py'):
+            check_install([
+                str(DUMMY_BENCHMARK_PATH), str(pgd_solver.resolve()), 'Dataset'
+            ], 'benchopt')
+
+
+class TestCheckDataCmd:
+    def test_download_data(self):
+
+        # Make sure that when the Objective is not installed, due to a missing
+        # dependency, an error is raised.
+        dataset = """from benchopt import BaseDataset
+
+        class Dataset(BaseDataset):
+            name = 'test_data'
+            def get_data(self):
+                print("LOADING DATA")
+                return {'X': 1, 'y': 2}
+        """
+        with temp_benchmark(datasets=dataset) as benchmark:
+            with CaptureRunOutput() as out:
+                check_data(
+                    [str(benchmark.benchmark_dir), '-d', 'test_data'],
+                    'benchopt', standalone_mode=False
+                )
+
+            out.check_output("LOADING DATA", repetition=1)
+            out.check_output(TICK, repetition=1)
+
+    def test_fail_download_data(self):
+
+        # Make sure that when the Objective is not installed, due to a missing
+        # dependency, an error is raised.
+        dataset = """from benchopt import BaseDataset
+
+        class Dataset(BaseDataset):
+            name = 'test_data'
+            def get_data(self):
+                raise ValueError("Failed to load data")
+                return {'X': 1, 'y': 2}
+        """
+        with temp_benchmark(datasets=dataset) as benchmark:
+            with CaptureRunOutput() as out:
+                check_data(
+                    [str(benchmark.benchmark_dir), '-d', 'test_data'],
+                    'benchopt', standalone_mode=False
+                )
+
+            out.check_output(CROSS, repetition=1)
+            out.check_output("ValueError: Failed to load data", repetition=1)