Repository: benchopt/benchopt
Pull request #323 (2022-04-07 12:27:52)

allowed objective to skip some datasets
This solves #321 

However, I don't really know how I should test this. I understand there is a dummy benchmark, but should I add a case where I skip a dataset i the objective? Like create a fake dataset and always skip it?

zaccharieramzi: Yup makes total sense, just added in your suggestions @tomMoral 

Patch:
diff --git a/benchopt/base.py b/benchopt/base.py
index d174de4..9aaf449 100644
--- a/benchopt/base.py
+++ b/benchopt/base.py
@@ -352,7 +352,33 @@ class BaseObjective(ParametrizedNameMixin, DependenciesMixin):
     def set_dataset(self, dataset):
         self._dataset = dataset
         self._dimension, data = dataset._get_data()
-        return self.set_data(**data)
+        # Check if the dataset is compatible with the objective
+        skip, reason = self.skip(**data)
+        if skip:
+            return skip, reason
+
+        self.set_data(**data)
+        return False,  None
+
+    def skip(self, **data):
+        """Used to decide if the ``Objective`` is compatible with the data.
+
+        Parameters
+        ----------
+        **data: dict
+            Extra parameters of the objective. This dictionary is retrieved
+            by calling ``data = Dataset.get_data()``.
+
+        Returns
+        -------
+        skip : bool
+            Whether this objective should be skipped or not for this data
+            (accessible in the objective attributes).
+        reason : str | None
+            The reason why it should be skipped for display purposes.
+            If skip is False, the reason should be None.
+        """
+        return False, None
 
     def get_one_beta(self):
         """Return one point in which the objective function can be evaluated.
diff --git a/benchopt/runner.py b/benchopt/runner.py
index cd6cf82..e39228b 100644
--- a/benchopt/runner.py
+++ b/benchopt/runner.py
@@ -399,8 +399,16 @@ def run_benchmark(benchmark, solver_names=None, forced_solvers=None,
                 objective = objective_class.get_instance(**obj_parameters)
                 if not is_matched(str(objective), objective_filters):
                     continue
-                print_normalize(f"  |--{objective}")
-                objective.set_dataset(dataset)
+                tag = f"  |--{objective}"
+                skip, reason = objective.set_dataset(dataset)
+                if skip:
+                    print_normalize(
+                        f"{tag} - {colorify('skip', YELLOW)}"
+                    )
+                    if reason is not None:
+                        print(f'Reason: {reason}')
+                    continue
+                print_normalize(tag)
 
                 for solver_class in solver_classes:
 


Test patch:
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/dataset_test.py b/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/dataset_test.py
index 0451daf..46afdac 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/dataset_test.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/dataset_test.py
@@ -16,18 +16,29 @@ class Dataset(BaseDataset):
         ], 'rho': [0]
     }
 
-    def __init__(self, n_samples=10, n_features=50,  rho=0.6, random_state=27):
+    def __init__(
+        self,
+        n_samples=10,
+        n_features=50,
+        rho=0.6,
+        random_state=27,
+        skip=False,
+    ):
         # Store the parameters of the dataset
         self.n_samples = n_samples
         self.n_features = n_features
         self.random_state = random_state
         self.rho = rho
+        self.skip = skip
 
     def get_data(self):
         rng = np.random.RandomState(self.random_state)
 
         X, y, _ = make_correlated_data(self.n_samples, self.n_features,
                                        rho=self.rho, random_state=rng)
+        if self.skip:
+            X = np.zeros_like(X)
+            y = np.zeros_like(y)
 
         data = dict(X=X, y=y)
 
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/objective.py b/benchopt/tests/test_benchmarks/dummy_benchmark/objective.py
index 1b5d7db..e6f3845 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/objective.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/objective.py
@@ -1,4 +1,8 @@
-from benchopt.base import BaseObjective
+from benchopt import BaseObjective, safe_import_context
+
+
+with safe_import_context() as import_ctx:
+    import numpy as np
 
 
 class Objective(BaseObjective):
@@ -16,6 +20,11 @@ class Objective(BaseObjective):
         self.X, self.y = X, y
         self.lmbd = self.reg * self._get_lambda_max()
 
+    def skip(self, X, y):
+        if np.all(X == 0):
+            return True, 'X is all zeros'
+        return False, None
+
     def compute(self, beta):
         diff = self.y - self.X.dot(beta)
         objective_value = .5 * diff.dot(diff) + self.lmbd * abs(beta).sum()
diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
index 0beb735..b5fc735 100644
--- a/benchopt/tests/test_runner.py
+++ b/benchopt/tests/test_runner.py
@@ -21,3 +21,9 @@ def test_skip_api(capsys):
     skip, reason = solver._set_objective(objective)
     assert not skip
     assert reason is None
+
+    dataset = TEST_DATASET.get_instance(skip=True)
+    objective = TEST_OBJECTIVE.get_instance()
+    skip, reason = objective.set_dataset(dataset)
+    assert skip
+    assert reason == 'X is all zeros'