Repository: benchopt/benchopt
Pull request #350 (2022-05-05 14:39:21)

ENH smaller simulated dataset for faster tests


Patch:
diff --git a/README.rst b/README.rst
index 4904bb5..4268c95 100644
--- a/README.rst
+++ b/README.rst
@@ -22,10 +22,12 @@ be **as simple as doing**:
 
 .. code-block::
 
+    conda create -n benchopt python=3.8
+    conda activate benchopt
     pip install benchopt
     git clone https://github.com/benchopt/benchmark_logreg_l2
-    benchopt install -e benchmark_logreg_l2
-    benchopt run --env ./benchmark_logreg_l2
+    benchopt install -e benchmark_logreg_l2 -s cd -s sklearn
+    benchopt run -e ./benchmark_logreg_l2 -s cd -s sklearn
 
 Running this command will give you a benchmark plot on l2-regularized
 logistic regression:
@@ -43,7 +45,14 @@ using the `benchmark template <https://github.com/benchopt/template_benchmark>`_
 Install
 --------
 
-The command line tool to run the benchmarks can be installed through `pip`.
+The command line tool to run the benchmarks can be installed through `pip`. In order to allow `benchopt`
+to automatically install solvers dependencies, the install needs to be done in a `conda` environment.
+
+
+.. code-block::
+    conda create -n benchopt python=3.9
+    conda activate benchopt
+
 To get the **latest release**, use:
 
 .. code-block::
diff --git a/benchopt/base.py b/benchopt/base.py
index b7d6412..fbe2bd8 100644
--- a/benchopt/base.py
+++ b/benchopt/base.py
@@ -220,13 +220,10 @@ class BaseDataset(ParametrizedNameMixin, DependenciesMixin, ABC):
 
     @abstractmethod
     def get_data(self):
-        """Return the problem's dimension as well as the objective parameters.
+        """Return the data to feed to the objective .
 
         Returns
         -------
-        dimension: int or tuple
-            Dimension of the optimized parameter. The solvers should return a
-            parameter of shape ``(dimension,)`` or ``*dimension``.
         data: dict
             Extra parameters of the objective. The objective will be
             instanciated by calling ``Objective.set_data(**data)``.
@@ -236,14 +233,29 @@ class BaseDataset(ParametrizedNameMixin, DependenciesMixin, ABC):
     def _get_data(self):
         "Wrapper to make sure the returned results are correctly formated."
 
+        # Automatically cache the _data to avoid reloading it.s
         if not hasattr(self, '_data') or self._data is None:
-            self._dimension, self._data = self.get_data()
-
-        # Make sure dimension is a tuple
-        if isinstance(self._dimension, numbers.Integral):
-            self._dimension = (self._dimension,)
-
-        return self._dimension, self._data
+            self._data = self.get_data()
+
+            # XXX - Remove in version 1.2
+            if type(self._data) != dict:
+                import warnings
+                warnings.warn(
+                    "`get_data` should return a dict containing the data. "
+                    "The dimension should not be returned anymore and "
+                    "Objective should have a method `get_one_solution` "
+                    "to provide one feasible point. This will cause an "
+                    "error starting version 1.2.",
+                    FutureWarning
+                )
+                dimension, data = self._data
+
+                # Make sure dimension is a tuple
+                if isinstance(dimension, numbers.Integral):
+                    dimension = (dimension,)
+                self._data = (dimension, data)
+
+        return self._data
 
     # Reduce the pickling and hashing burden by only pickling class parameters.
     @staticmethod
@@ -360,7 +372,12 @@ class BaseObjective(ParametrizedNameMixin, DependenciesMixin):
     # hashing the data directly.
     def set_dataset(self, dataset):
         self._dataset = dataset
-        self._dimension, data = dataset._get_data()
+        data = dataset._get_data()
+
+        # XXX - Remove in version 1.2
+        if type(data) != dict:
+            self._dimension, data = data
+
         # Check if the dataset is compatible with the objective
         skip, reason = self.skip(**data)
         if skip:
@@ -389,7 +406,7 @@ class BaseObjective(ParametrizedNameMixin, DependenciesMixin):
         """
         return False, None
 
-    def get_one_beta(self):
+    def get_one_solution(self):
         """Return one point in which the objective function can be evaluated.
 
         This method is mainly for testing purposes, to check that the return
@@ -401,12 +418,22 @@ class BaseObjective(ParametrizedNameMixin, DependenciesMixin):
         point. When `Dataset.get_data` returns "object", this method must be
         overwritten.
         """
-        if self._dimension == 'object':
+        if not hasattr(self, '_dimension'):
             raise NotImplementedError(
-                "If solvers return objects, `Objective.get_one_beta` should "
-                "be overriden to return an object compatible with `compute`."
+                "If solvers return objects, `Objective.get_one_solution` "
+                "should be overriden to return an object compatible with "
+                "`compute`."
             )
 
+        # XXX - make this an abstract class in version 1.2
+        import warnings
+        warnings.warn(
+            "Objective should have a method `get_one_solution` "
+            "to provide one feasible point. This will cause an "
+            "error starting version 1.2.",
+            FutureWarning
+        )
+
         import numpy as np
         return np.zeros(self._dimension)
 
diff --git a/benchopt/cli/helpers.py b/benchopt/cli/helpers.py
index 79227ea..e9b0950 100644
--- a/benchopt/cli/helpers.py
+++ b/benchopt/cli/helpers.py
@@ -123,12 +123,12 @@ def print_info(cls_name_list, cls_list, env_name=None, verbose=False):
 
     # select objects to print info from
     include_cls = []
-    cls_name_list = [item.upper() for item in cls_name_list]
+    cls_name_list = [item.lower() for item in cls_name_list]
     if 'all' in cls_name_list:
         include_cls = cls_list
     else:
         include_cls = [
-            item for item in cls_list if item.name.upper() in cls_name_list
+            item for item in cls_list if item.name.lower() in cls_name_list
         ]
     if not verbose:
         # short output
@@ -237,7 +237,6 @@ def info(benchmark, solver_names, dataset_names, env_name='False',
     # get solvers and datasets in the benchmark
     all_solvers = benchmark.get_solvers()
     all_datasets = benchmark.get_datasets()
-
     # enable verbosity if any environment was provided
     if env_name is not None and env_name != 'False':
         verbose = True
@@ -278,7 +277,6 @@ def info(benchmark, solver_names, dataset_names, env_name='False',
     if not dataset_names and not solver_names:
         dataset_names = ['all']
         solver_names = ['all']
-
     if dataset_names:
         print("# DATASETS", flush=True)
         print_info(dataset_names, all_datasets, env_name, verbose)
diff --git a/examples/plot_run_benchmark_python_R.py b/examples/plot_run_benchmark_python_R.py
index 123ed77..db9b2f3 100644
--- a/examples/plot_run_benchmark_python_R.py
+++ b/examples/plot_run_benchmark_python_R.py
@@ -9,7 +9,6 @@ from pathlib import Path
 import matplotlib.pyplot as plt
 from benchopt import run_benchmark
 from benchopt.benchmark import Benchmark
-from benchopt.tests import SELECT_ONE_SIMULATED
 from benchopt.plotting import plot_benchmark, PLOT_KINDS
 
 
@@ -26,7 +25,7 @@ if not BENCHMARK_PATH.exists():
 save_file = run_benchmark(
     Benchmark(BENCHMARK_PATH),
     ['Python-PGD[use_acceleration=False]', 'R-PGD'],
-    dataset_names=[SELECT_ONE_SIMULATED],
+    dataset_names=["Simulated[n_features=5000,n_samples=100,rho=0]"],
     objective_filters=['*fit_intercept=False,reg=0.5'],
     max_runs=100, timeout=100, n_repetitions=5,
     plot_result=False, show_progress=False


Test patch:
diff --git a/benchopt/tests/__init__.py b/benchopt/tests/__init__.py
index 99c6993..e03a660 100644
--- a/benchopt/tests/__init__.py
+++ b/benchopt/tests/__init__.py
@@ -12,7 +12,7 @@ DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark'
 REQUIREMENT_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'requirement_benchmark'
 
 # Pattern to select specific datasets or solvers.
-SELECT_ONE_SIMULATED = r'simulated*500*rho=0]'
+SELECT_ONE_SIMULATED = r'simulated*200*rho=0]'
 SELECT_ONE_PGD = r'python-pgd*step_size=1]'
 SELECT_ONE_OBJECTIVE = r'dummy*reg=0.1]'
 
diff --git a/benchopt/tests/test_benchmarks.py b/benchopt/tests/test_benchmarks.py
index 8bb71df..ed23afd 100644
--- a/benchopt/tests/test_benchmarks.py
+++ b/benchopt/tests/test_benchmarks.py
@@ -20,7 +20,7 @@ def test_benchmark_objective(benchmark, dataset_simu):
     # check that the reported dimension is correct and that the result of
     # the objective function is a dictionary containing a scalar value for
     # `objective_value`.
-    beta_hat = objective.get_one_beta()
+    beta_hat = objective.get_one_solution()
     objective_dict = objective(beta_hat)
 
     assert 'objective_value' in objective_dict, (
@@ -66,27 +66,30 @@ def test_dataset_get_data(benchmark, dataset_class):
     if dataset_class.name.lower() == 'finance':
         pytest.skip("Do not download finance.")
 
-    res = dataset._get_data()
-    assert isinstance(res, tuple), (
-        "Output of get_data should be a 2-tuple"
-    )
-    assert len(res) == 2, (
-        "Output of get_data should be a 2-tuple"
+    data = dataset._get_data()
+    assert isinstance(data, (tuple, dict)), (
+        "Output of get_data should be a 2-tuple or a dict."
     )
+    # XXX - remove in version 1.2
+    if isinstance(data, tuple):
+        assert len(data) == 2, (
+            "Output of get_data should be a 2-tuple"
+        )
 
-    dimension, data = res
+        dimension, data = data
 
-    assert isinstance(dimension, tuple) or dimension == 'object', (
-        "First output of get_data should be an integer or a tuple of integers."
-        f" Got {dimension}."
-    )
-    if dimension != 'object':
-        assert all(isinstance(d, numbers.Integral) for d in dimension), (
+        assert isinstance(dimension, tuple) or dimension == 'object', (
             "First output of get_data should be an integer or a tuple of "
             f"integers. Got {dimension}."
         )
+        if dimension != 'object':
+            assert all(isinstance(d, numbers.Integral) for d in dimension), (
+                "First output of get_data should be an integer or a tuple of "
+                f"integers. Got {dimension}."
+            )
+
     assert isinstance(data, dict), (
-        f"Second output of get_data should be a dict. Got {data}."
+        f"The returned data from get_data should be a dict. Got {data}."
     )
 
 
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/dataset_test.py b/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/dataset_test.py
index 46afdac..f04eb2c 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/dataset_test.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/dataset_test.py
@@ -23,6 +23,7 @@ class Dataset(BaseDataset):
         rho=0.6,
         random_state=27,
         skip=False,
+        deprecated_return=False
     ):
         # Store the parameters of the dataset
         self.n_samples = n_samples
@@ -30,6 +31,7 @@ class Dataset(BaseDataset):
         self.random_state = random_state
         self.rho = rho
         self.skip = skip
+        self.deprecated_return = deprecated_return
 
     def get_data(self):
         rng = np.random.RandomState(self.random_state)
@@ -42,4 +44,8 @@ class Dataset(BaseDataset):
 
         data = dict(X=X, y=y)
 
-        return self.n_features, data
+        # XXX - remove in version 1.2
+        if self.deprecated_return:
+            return self.n_features, data
+        else:
+            return data
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/simulated.py b/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/simulated.py
index da6dbe7..d546da0 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/simulated.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/datasets/simulated.py
@@ -13,9 +13,9 @@ class Dataset(BaseDataset):
     # the cross product for each key in the dictionary.
     parameters = {
         'n_samples, n_features': [
-            (100, 5000),
-            (100, 10000)
-        ], 'rho': [0, 0.6]
+            (100, 100),
+            (100, 200)
+        ], 'rho': [0]
     }
 
     def __init__(self, n_samples=10, n_features=50,  rho=0.6, random_state=27):
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/objective.py b/benchopt/tests/test_benchmarks/dummy_benchmark/objective.py
index e6f3845..e93ae83 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/objective.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/objective.py
@@ -12,10 +12,13 @@ class Objective(BaseObjective):
         'reg': [0.05, .1, .5]
     }
 
-    def __init__(self, reg=.1, fit_intercept=False):
+    def __init__(self, reg=.1, fit_intercept=False, deprecated_dataset=False):
         self.reg = reg
         self.fit_intercept = fit_intercept
 
+        # XXX - remove in version 1.2
+        self.deprecated_dataset = deprecated_dataset
+
     def set_data(self, X, y):
         self.X, self.y = X, y
         self.lmbd = self.reg * self._get_lambda_max()
@@ -25,6 +28,13 @@ class Objective(BaseObjective):
             return True, 'X is all zeros'
         return False, None
 
+    def get_one_solution(self):
+        # XXX - remove in version 1.2
+        if self.deprecated_dataset:
+            return super().get_one_solution()
+
+        return np.zeros(self.X.shape[1])
+
     def compute(self, beta):
         diff = self.y - self.X.dot(beta)
         objective_value = .5 * diff.dot(diff) + self.lmbd * abs(beta).sum()
diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
index b5fc735..57c2c06 100644
--- a/benchopt/tests/test_runner.py
+++ b/benchopt/tests/test_runner.py
@@ -1,9 +1,13 @@
+import numpy as np
+
+import pytest
+
 from benchopt.tests import TEST_SOLVER
 from benchopt.tests import TEST_DATASET
 from benchopt.tests import TEST_OBJECTIVE
 
 
-def test_skip_api(capsys):
+def test_skip_api():
 
     dataset = TEST_DATASET.get_instance()
     objective = TEST_OBJECTIVE.get_instance(reg=0)
@@ -27,3 +31,25 @@ def test_skip_api(capsys):
     skip, reason = objective.set_dataset(dataset)
     assert skip
     assert reason == 'X is all zeros'
+
+
+def test_get_one_solution():
+    dataset = TEST_DATASET.get_instance()
+    objective = TEST_OBJECTIVE.get_instance()
+    objective.set_dataset(dataset)
+
+    one_solution = objective.get_one_solution()
+    expected = np.zeros(objective.X.shape[1])
+    assert all(one_solution == expected)
+
+    # XXX - remove this check in version 1.2
+    dataset = TEST_DATASET.get_instance(deprecated_return=True)
+    objective = TEST_OBJECTIVE.get_instance(deprecated_dataset=True)
+
+    with pytest.warns(FutureWarning, match="`get_data` should return a dict"):
+        objective.set_dataset(dataset)
+
+    with pytest.warns(FutureWarning, match="Objective should have a method"):
+        one_solution = objective.get_one_solution()
+    expected = np.zeros(objective.X.shape[1])
+    assert all(one_solution == expected)