Repository: benchopt/benchopt (python)

Issue:
ENH: be able to set arbitrary values of solver/dataset/objective parameters via CLI
I think it would be nice to be able to do `benchopt run . -d simulated[n_samples=30,n_features=50]` even if `simulated.py` does not have the values, 30 and 50, as long as `n_samples` and `n_features` are keys of `Dataset.parameters` 

all unspecified parameters would take all their default values 

Pull request description:
ENH add support for custom parameters in CLI
Fixes #193

This PR makes the following commands work as expected:
- `benchopt run . -d simulated[n_samples=30,n_features=50]`
- `benchopt run . -d simulated[n_features=50, n_samples=30,]`
- `benchopt run . -d simulated[n_samples=[30, 300], n_features=[10, 12]]`
- `benchopt run . -s sklearn[liblinear, lbfgs]`


I added a bunch of unit tests, but I am not super familiar with the test suite, so this might require additional integration tests.


Patch:
diff --git a/README.rst b/README.rst
index 727f92f..bd85db0 100644
--- a/README.rst
+++ b/README.rst
@@ -22,12 +22,13 @@ be **as simple as doing**:
 
 .. code-block::
 
-    conda create -n benchopt python
-    conda activate benchopt
-    pip install benchopt
-    git clone https://github.com/benchopt/benchmark_logreg_l2
-    benchopt install -e benchmark_logreg_l2 -s cd -s sklearn
-    benchopt run -e ./benchmark_logreg_l2 -s cd -s sklearn
+   conda create -n benchopt python
+   conda activate benchopt
+   pip install benchopt
+   git clone https://github.com/benchopt/benchmark_logreg_l2
+   cd benchmark_logreg_l2
+   benchopt install -e . -s lightning -s sklearn
+   benchopt run -e . --config ./config_example.yml
 
 Running this command will give you a benchmark plot on l2-regularized
 logistic regression:
diff --git a/benchopt/benchmark.py b/benchopt/benchmark.py
index 8ab0e9c..bc3ecd7 100644
--- a/benchopt/benchmark.py
+++ b/benchopt/benchmark.py
@@ -391,7 +391,7 @@ class Benchmark:
 
         Parameters
         ----------
-        solver_names : list | None
+        solver_names : list | None
             List of solvers to include in the benchmark. If None
             all solvers available are run.
         forced_solvers : list | None
@@ -449,12 +449,12 @@ class Benchmark:
 
 
 def _check_name_lists(*name_lists):
-    "Normalize name_list ot a list of lowercase str."
+    "Normalize name_list to a list of string."
     res = []
     for name_list in name_lists:
         if name_list is None:
             continue
-        res.extend([str(name).lower() for name in name_list])
+        res.extend([str(name) for name in name_list])
     return res
 
 
@@ -466,16 +466,117 @@ def is_matched(name, include_patterns=None, default=True):
     if include_patterns is None or len(include_patterns) == 0:
         return default
     name = str(name)
-    # we use [] to signal options in patterns, we must escape them for re
-    substitutions = {"*": ".*", "[": r"\[", "]": r"\]"}
+    name = _extract_options(name)[0]
+    substitutions = {"*": ".*"}
     for p in include_patterns:
+        p = _extract_options(p)[0]
         for old, new in substitutions.items():
             p = p.replace(old, new)
-        if re.match(f"^{p}.*", name, flags=re.IGNORECASE) is not None:
+        if re.match(f"^{p}$", name, flags=re.IGNORECASE) is not None:
             return True
     return False
 
 
+def _extract_options(name):
+    """Remove options indicated within '[]' from a name.
+
+    Parameters
+    ----------
+    name : str
+        Input name.
+
+    Returns
+    -------
+    basename : str
+        Name without options.
+    args : list
+        List of unnamed options.
+    kwargs : dict()
+        Dictionary with options.
+
+    Examples
+    --------
+    >>> _extract_options("foo")  # "foo", [], dict()
+    >>> _extract_options("foo[bar=2]")  # "foo", [], dict(bar=2)
+    >>> _extract_options("foo[baz]")  # "foo", ["baz"], dict()
+    """
+    if name.count("[") != name.count("]"):
+        raise ValueError(f"Invalid name (missing bracket): {name}")
+
+    basename = "".join(re.split(r"\[.*\]", name))
+    matches = re.findall(r"\[.*\]", name)
+
+    if len(matches) == 0:
+        return basename, [], {}
+    elif len(matches) > 1:
+        raise ValueError(f"Invalid name (multiple brackets): {name}")
+    else:
+        match = matches[0]
+        match = match[1:-1]  # remove brackets
+
+        result = _extract_parameters(match)
+        if isinstance(result, dict):
+            return basename, [], result
+        elif isinstance(result, list):
+            return basename, result, {}
+        else:
+            raise ValueError(
+                f"Impossible. Please report this bug.\n"
+                f"_extract_parameters returned '{result}'"
+            )
+
+
+def _extract_parameters(string):
+    """Extract parameters from a string.
+
+    If the string contains a "=", returns a dict, otherwise returns a list.
+
+    Examples
+    --------
+    >>> _extract_parameters("foo")  # ["foo"]
+    >>> _extract_parameters("foo, 42, True")  # ["foo", 42, True]
+    >>> _extract_parameters("foo=bar")  # {"foo": "bar"}
+    >>> _extract_parameters("foo=[bar, baz]")  # {"foo": ["bar", "baz"]}
+    >>> _extract_parameters("foo=1, baz=True")  # {"foo": 1, "baz": True}
+    >>> _extract_parameters("'foo, bar'=[(0, 1),(1, 0)]")
+    >>> # {"foo, bar": [(0, 1),(1, 0)]}
+    """
+    import ast
+    original = string
+
+    # First, replace all quoted names with their hashes, to avoid modification.
+    all_matches = re.findall(r"'[^'\"]*'", string)
+    all_matches += re.findall(r'"[^\'"]*"', string)
+    for match in all_matches:
+        string = string.replace(match, str(hash(match)))
+
+    # Second, add quotes to all variable names (foo -> 'foo').
+    string = re.sub(r"[a-zA-Z][a-zA-Z0-9_-]*", r"'\g<0>'", string)
+
+    # Third, change back the hashes to their original names.
+    for match in all_matches:
+        string = string.replace(str(hash(match)), match)
+
+    # Prepare the sequence for AST parsing.
+    # Sequences with "=" are made into a dict expression {'foo': 'bar'}.
+    # Sequences without "=" are made into a list expression ['foo', 'bar'].
+    if "=" in string:
+        string = "{" + string.replace("=", ":") + "}"
+    else:
+        string = "[" + string + "]"
+
+    # Remove quotes for python language tokens
+    for token in ["True", "False", "None"]:
+        string = string.replace(f'"{token}"', token)
+        string = string.replace(f"'{token}'", token)
+
+    # Evaluate the string.
+    try:
+        return ast.literal_eval(string)
+    except (ValueError, SyntaxError):
+        raise ValueError(f"Invalid name '{original}', evaluated as {string}.")
+
+
 def _validate_patterns(all_names, patterns, name_type='dataset'):
     "Check that all provided patterns match at least one name."
     if patterns is None:
@@ -488,33 +589,81 @@ def _validate_patterns(all_names, patterns, name_type='dataset'):
         if not matched:
             invalid_patterns.append(p)
 
-    # If some patterns did not matched any dataset, raise an error
+    # If some patterns did not matched any class, raise an error
     if len(invalid_patterns) > 0:
-        all_names = '- ' + '\n- '.join(all_names)
+        all_names_ = '- ' + '\n- '.join(all_names)
         raise click.BadParameter(
             f"Patterns {invalid_patterns} did not match any {name_type}.\n"
-            f"Available {name_type}s are:\n{all_names}"
+            f"Available {name_type}s are:\n{all_names_}"
         )
 
 
 def _filter_classes(*classes, filters=None):
     """Filter a list of class based on its names."""
     for klass in classes:
-        is_installed = None
-        for parameters in product_param(klass.parameters):
-            obj = klass.get_instance(**parameters)
-            if is_matched(str(obj), filters):
-                if is_installed is None:
-                    is_installed = (
-                        not hasattr(klass, 'is_installed')
-                        or klass.is_installed(
-                            raise_on_not_installed=RAISE_INSTALL_ERROR
-                        )
-                    )
-                if not is_installed:
-                    yield klass.name, False
-                    break
-                yield obj, True
+        if not is_matched(klass.name, filters):
+            continue
+
+        if not klass.is_installed(raise_on_not_installed=RAISE_INSTALL_ERROR):
+            yield klass.name, False
+            continue
+
+        for parameters in _get_used_parameters(klass, filters):
+            yield klass.get_instance(**parameters), True
+
+
+def _get_used_parameters(klass, filters):
+    """Get the list of parameters to use in the class."""
+
+    # Use the default parameters if no filters are provided.
+    if filters is None or len(filters) == 0:
+        return product_param(klass.parameters)
+
+    # If filters are provided, get the list of parameters from the names.
+    update_parameters = []
+    for filt in filters:
+        if is_matched(klass.name, [filt]):
+            # Extract the parameters from the filter name.
+            _, args, kwargs = _extract_options(filt)
+
+            if len(args) > 0:
+                # positional args are allowed only for single-parameter classes
+                if len(klass.parameters) > 1:
+                    raise ValueError(
+                        f"Ambiguous positional parameter in {filt}.")
+                elif len(kwargs) > 0:
+                    raise ValueError(
+                        f"Both positional and keyword parameters in {filt}.")
+                else:
+                    key = list(klass.parameters.keys())[0]
+                    kwargs = {key: args}
+
+            # Use product_param to get all combinations of parameters.
+            kwargs = {
+                key: (val if isinstance(val, (list, tuple)) else [val])
+                for key, val in kwargs.items()
+            }
+            update_parameters.extend(product_param(kwargs))
+
+    # Then, update the default parameters (klass.parameters) with the
+    # parameters extracted from filter names.
+    used_parameters = []
+    for update in update_parameters:
+        for default in product_param(klass.parameters):
+            default = default.copy()  # avoid modifying the original
+
+            # check that all parameters are defined in klass.parameters
+            for key in update:
+                if key not in default:
+                    raise ValueError(
+                        f"Unknown parameter '{key}', parameter must be in "
+                        f"{list(default.keys())}")
+
+            default.update(update)
+            if default not in used_parameters:  # avoid duplicates
+                used_parameters.append(default)
+
+    return used_parameters
 
 
 def buffer_iterator(it):
diff --git a/benchopt/cli/completion.py b/benchopt/cli/completion.py
index 507b9eb..8c111bd 100644
--- a/benchopt/cli/completion.py
+++ b/benchopt/cli/completion.py
@@ -84,6 +84,17 @@ def complete_output_files(ctx, param, incomplete):
     return propose_from_list(candidates, incomplete)
 
 
+def complete_config_files(ctx, param, incomplete):
+    "Auto-completion for configuration files."
+    skip_import()
+    benchmark = find_benchmark_in_args(ctx.args)
+    if benchmark is None:
+        return []
+    benchmark_folder = benchmark.benchmark_dir
+    candidates = list(benchmark_folder.glob('*.yml'))
+    return propose_from_list(candidates, incomplete)
+
+
 def complete_conda_envs(ctx, param, incomplete):
     "Auto-completion for env-names."
     _, all_envs = list_conda_envs()
diff --git a/benchopt/cli/main.py b/benchopt/cli/main.py
index a150612..2f2491e 100644
--- a/benchopt/cli/main.py
+++ b/benchopt/cli/main.py
@@ -8,6 +8,7 @@ from benchopt.cli.completion import complete_solvers
 from benchopt.cli.completion import complete_datasets
 from benchopt.cli.completion import complete_benchmarks
 from benchopt.cli.completion import complete_conda_envs
+from benchopt.cli.completion import complete_config_files
 from benchopt.utils.conda_env_cmd import list_conda_envs
 from benchopt.utils.conda_env_cmd import create_conda_env
 from benchopt.utils.shell_cmd import _run_shell_in_conda_env
@@ -73,8 +74,9 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               help="Deprecated alias for `--objective`.")
 @click.option('--objective', '-o',
               metavar='<objective_filter>', multiple=True, type=str,
-              help="Filter the objective based on its parametrized name. This "
-              "can be used to only include one set of parameters.")
+              help="Select the objective based on its parameters, with the "
+              "syntax `objective[parameter=value]`. This can be used to only "
+              "include one set of parameters. ")
 @click.option('--old-objective-filter', '-p',
               multiple=True, type=str,
               help="Deprecated alias for --objective_filter/-o.")
@@ -82,8 +84,10 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               metavar="<solver_name>", multiple=True, type=str,
               help="Include <solver_name> in the benchmark. By default, all "
               "solvers are included. When `-s` is used, only listed solvers"
-              " are included. To include multiple solvers, "
-              "use multiple `-s` options.", shell_complete=complete_solvers)
+              " are included. Note that <solver_name> can include parameters,"
+              " with the syntax `solver[parameter=value]`. "
+              "To include multiple solvers, use multiple `-s` options.",
+              shell_complete=complete_solvers)
 @click.option('--force-solver', '-f',
               metavar="<solver_name>", multiple=True, type=str,
               help="Force the re-run for <solver_name>. This "
@@ -94,7 +98,8 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               metavar="<dataset_name>", multiple=True, type=str,
               help="Run the benchmark on <dataset_name>. By default, all "
               "datasets are included. When `-d` is used, only listed datasets"
-              " are included. Note that <dataset_name> can be a regexp. "
+              " are included. Note that <dataset_name> can include parameters,"
+              " with the syntax `dataset[parameter=value]`. "
               "To include multiple datasets, use multiple `-d` options.",
               shell_complete=complete_datasets)
 @click.option('--n-jobs', '-j',
@@ -113,6 +118,7 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               metavar="<int>", default=100, show_default=True, type=int,
               help='Timeout a solver when run for more than <timeout> seconds')
 @click.option('--config', 'config_file', default=None,
+              shell_complete=complete_config_files,
               help="YAML configuration file containing benchmark options.")
 @click.option('--plot/--no-plot', default=True,
               help="Whether or not to plot the results. Default is True.")
@@ -237,7 +243,7 @@ def run(config_file=None, **kwargs):
         )
 
     # run the command in the conda env
-    solvers_option = ' '.join(['-s ' + s for s in solver_names])
+    solvers_option = ' '.join([f"-s '{s}'" for s in solver_names])
     forced_solvers_option = ' '.join([f"-f '{s}'" for s in forced_solvers])
     datasets_option = ' '.join([f"-d '{d}'" for d in dataset_names])
     objective_option = ' '.join([f"-o '{o}'" for o in objective_filters])
@@ -286,10 +292,15 @@ def run(config_file=None, **kwargs):
               "datasets are included, except when -s flag is used. "
               "If -s flag is used, then no dataset is included. "
               "When `-d` is used, only listed datasets "
-              "are included. Note that <dataset_name> can be a regexp. "
+              "are included. Note that <dataset_name> can include parameters "
+              "with the syntax `dataset[parameter=value]`. "
               "To include multiple datasets, use multiple `-d` options."
               "To include all datasets, use -d 'all' option.",
               shell_complete=complete_datasets)
+@click.option('--config', 'config_file', default=None,
+              shell_complete=complete_config_files,
+              help="YAML configuration file containing benchmark options, "
+              "whose solvers and datasets will be installed.")
 @click.option('--env', '-e', 'env_name',
               flag_value='True', type=str, default='False',
               help="Install all requirements in a dedicated "
@@ -313,8 +324,20 @@ def run(config_file=None, **kwargs):
               help="If this flag is set, no confirmation will be asked "
               "to the user to install requirements in the current environment."
               " Useless with options `-e/--env` or `--env-name`.")
-def install(benchmark, minimal, solver_names, dataset_names, force=False,
-            recreate=False, env_name='False', confirm=False, quiet=False):
+def install(
+        benchmark, minimal, solver_names, dataset_names, config_file=None,
+        force=False, recreate=False, env_name='False', confirm=False,
+        quiet=False):
+
+    if config_file is not None:
+        with open(config_file, "r") as f:
+            config = yaml.safe_load(f)
+        if not dataset_names:
+            dataset_names = config.get("dataset", tuple())
+        if not solver_names:
+            solver_names = config.get("solver", tuple())
+            forced_solvers = config.get("force-solver", tuple())
+            solver_names = list(set(solver_names).union(set(forced_solvers)))
 
     # Check that the dataset/solver patterns match actual dataset
     benchmark = Benchmark(benchmark)
diff --git a/doc/cli.rst b/doc/cli.rst
index b9853b3..0edd5ad 100644
--- a/doc/cli.rst
+++ b/doc/cli.rst
@@ -19,6 +19,22 @@ For example using BASH shell you need to run:
 The `benchopt` command also comes with tab completion for the solver name
 and the dataset name.
 
+.. note::
+    **Optional parameters syntax:** For some CLI parameters (solver, objective,
+    dataset), additional values can be given with the following syntax:
+
+    .. code-block:: bash
+
+        # Run a particular solver with a particular set of parameters:
+        --solver solver_name[param_1=method_name, param_2=100]
+        # To select a grid of parameters, the following syntax is allowed:
+        --solver solver_name[param_1=[True, False]]
+        # For objects with only one parameter, the name can be omitted:
+        --solver solver_name[True, False]
+        # For more advanced selections over multiple parameters, use:
+        --solver solver_name["param_1,param_2"=[(True, 100), (False, 1000)]]
+
+
 .. click:: benchopt.cli:benchopt
    :prog: benchopt
    :nested: full
diff --git a/examples/plot_run_benchmark.py b/examples/plot_run_benchmark.py
index d3ee105..30b3fe8 100644
--- a/examples/plot_run_benchmark.py
+++ b/examples/plot_run_benchmark.py
@@ -21,7 +21,7 @@ try:
     save_file = run_benchmark(
         Benchmark(BENCHMARK_PATH), ['sklearn[liblinear]', 'sklearn[newton-cg]',
                                     'lightning'],
-        dataset_names=['Simulated*n_features=500,n_samples=200*'],
+        dataset_names=['Simulated*[n_features=500,n_samples=200]'],
         objective_filters=['L2 Logistic Regression[lmbd=1.0]'],
         max_runs=100, timeout=20, n_repetitions=15,
         plot_result=False, show_progress=True
diff --git a/examples/plot_run_benchmark_python_R.py b/examples/plot_run_benchmark_python_R.py
index db9b2f3..0fc612a 100644
--- a/examples/plot_run_benchmark_python_R.py
+++ b/examples/plot_run_benchmark_python_R.py
@@ -26,7 +26,7 @@ save_file = run_benchmark(
     Benchmark(BENCHMARK_PATH),
     ['Python-PGD[use_acceleration=False]', 'R-PGD'],
     dataset_names=["Simulated[n_features=5000,n_samples=100,rho=0]"],
-    objective_filters=['*fit_intercept=False,reg=0.5'],
+    objective_filters=['*[fit_intercept=False,reg=0.5]'],
     max_runs=100, timeout=100, n_repetitions=5,
     plot_result=False, show_progress=False
 )


Test patch:
diff --git a/benchopt/tests/__init__.py b/benchopt/tests/__init__.py
index e03a660..373778b 100644
--- a/benchopt/tests/__init__.py
+++ b/benchopt/tests/__init__.py
@@ -12,9 +12,9 @@ DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark'
 REQUIREMENT_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'requirement_benchmark'
 
 # Pattern to select specific datasets or solvers.
-SELECT_ONE_SIMULATED = r'simulated*200*rho=0]'
-SELECT_ONE_PGD = r'python-pgd*step_size=1]'
-SELECT_ONE_OBJECTIVE = r'dummy*reg=0.1]'
+SELECT_ONE_SIMULATED = r'simulated[n_features=200,rho=0]'
+SELECT_ONE_PGD = r'python-pgd[step_size=1]'
+SELECT_ONE_OBJECTIVE = r'dummy*[reg=0.1]'
 
 try:
     DUMMY_BENCHMARK = Benchmark(DUMMY_BENCHMARK_PATH)
diff --git a/benchopt/tests/test_cli.py b/benchopt/tests/test_cli.py
index f3f6738..3f46d4c 100644
--- a/benchopt/tests/test_cli.py
+++ b/benchopt/tests/test_cli.py
@@ -155,6 +155,29 @@ class TestRunCmd:
         # Make sure the results were saved in a result file
         assert len(out.result_files) == 1, out.output
 
+    def test_benchopt_run_custom_parameters(self):
+        SELECT_DATASETS = r'simulated[n_features=[100, 200]]'
+        SELECT_SOLVERS = r'python-pgd-with-cb[use_acceleration=[True, False]]'
+        SELECT_OBJECTIVES = r'dummy*[0.1, 0.2]'
+
+        with CaptureRunOutput() as out:
+            run([str(DUMMY_BENCHMARK_PATH), '-l', '-d', SELECT_DATASETS,
+                 '-f', SELECT_SOLVERS, '-n', '1', '-r', '1', '-o',
+                 SELECT_OBJECTIVES, '--no-plot'],
+                'benchopt', standalone_mode=False)
+
+        out.check_output(r'Simulated\[n_features=100,', repetition=1)
+        out.check_output(r'Simulated\[n_features=200,', repetition=1)
+        out.check_output(r'Simulated\[n_features=5000,', repetition=0)
+        out.check_output(r'Dummy Sparse Regression\[reg=0.1\]', repetition=2)
+        out.check_output(r'Dummy Sparse Regression\[reg=0.2\]', repetition=2)
+        out.check_output(r'Dummy Sparse Regression\[reg=0.05\]', repetition=0)
+        out.check_output(r'--Python-PGD\[', repetition=0)
+        out.check_output(r'--Python-PGD-with-cb\[use_acceleration=False\]:',
+                         repetition=28)
+        out.check_output(r'--Python-PGD-with-cb\[use_acceleration=True\]:',
+                         repetition=28)
+
     def test_benchopt_run_profile(self):
         with CaptureRunOutput() as out:
             run_cmd = [str(DUMMY_BENCHMARK_PATH),
@@ -187,7 +210,7 @@ class TestRunCmd:
         n-repetitions: 2
         max-runs: 1
         force-solver:
-          - python-pgd[step_size=1.5]
+          - python-pgd[step_size=[2, 3]]
           - Test-Solver
         """
         tmp = tempfile.NamedTemporaryFile(mode="w+")
@@ -201,8 +224,8 @@ class TestRunCmd:
             run(run_cmd, 'benchopt', standalone_mode=False)
 
         out.check_output(r'Test-Solver:', repetition=11)
-        out.check_output(
-            r'Python-PGD\[step_size=1.5\]:', repetition=11)
+        out.check_output(r'Python-PGD\[step_size=2\]:', repetition=11)
+        out.check_output(r'Python-PGD\[step_size=3\]:', repetition=11)
 
         # test that CLI options take precedence
         with CaptureRunOutput() as out:
diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
index 6a0fe26..d6478e3 100644
--- a/benchopt/tests/test_runner.py
+++ b/benchopt/tests/test_runner.py
@@ -5,6 +5,9 @@ import pytest
 from benchopt.tests import TEST_SOLVER
 from benchopt.tests import TEST_DATASET
 from benchopt.tests import TEST_OBJECTIVE
+from benchopt.benchmark import _filter_classes
+from benchopt.benchmark import _extract_options
+from benchopt.benchmark import _extract_parameters
 
 
 def test_skip_api():
@@ -53,3 +56,189 @@ def test_get_one_solution():
         one_solution = objective.get_one_solution()
     expected = np.zeros(objective.X.shape[1])
     assert all(one_solution == expected)
+
+
+def _assert_parameters_equal(instance, parameters):
+    for key, val in parameters.items():
+        assert getattr(instance, key) == val
+
+
+class TEST_DATASET_TWO_PARAMS(TEST_DATASET):
+    """Used to test the selection of datasets by keyword parameters."""
+    parameters = {'n_samples': [10, 11], 'n_features': [20, 21]}
+
+
+def test_filter_classes_two_parameters():
+    # Test the selection of dataset with optional parameters.
+
+    def filt_(filters):
+        return list(_filter_classes(TEST_DATASET_TWO_PARAMS, filters=filters))
+
+    # no selection (default grid)
+    results = filt_(["Test-Dataset"])
+    assert len(results) == 4
+    _assert_parameters_equal(results[0][0], dict(n_samples=10, n_features=20))
+    _assert_parameters_equal(results[1][0], dict(n_samples=10, n_features=21))
+    _assert_parameters_equal(results[2][0], dict(n_samples=11, n_features=20))
+    _assert_parameters_equal(results[3][0], dict(n_samples=11, n_features=21))
+
+    # select one parameter (n_samples)
+    results = filt_(["Test-Dataset[n_samples=42]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=42, n_features=20))
+    _assert_parameters_equal(results[1][0], dict(n_samples=42, n_features=21))
+
+    # select one parameter (n_features)
+    results = filt_(["Test-Dataset[n_features=42]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=10, n_features=42))
+    _assert_parameters_equal(results[1][0], dict(n_samples=11, n_features=42))
+
+    # select two parameters (n_samples, n_features)
+    results = filt_(["Test-Dataset[n_samples=41, n_features=42]"])
+    assert len(results) == 1
+    _assert_parameters_equal(results[0][0], dict(n_samples=41, n_features=42))
+
+    # get grid over one parameter (n_samples)
+    results = filt_(["Test-Dataset[n_samples=[41,42], n_features=19]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=41, n_features=19))
+    _assert_parameters_equal(results[1][0], dict(n_samples=42, n_features=19))
+
+    # get grid over two parameter (n_samples, n_features)
+    results = filt_(["Test-Dataset[n_samples=[41,42], n_features=[19, 20]]"])
+    assert len(results) == 4
+    _assert_parameters_equal(results[0][0], dict(n_samples=41, n_features=19))
+    _assert_parameters_equal(results[1][0], dict(n_samples=41, n_features=20))
+    _assert_parameters_equal(results[2][0], dict(n_samples=42, n_features=19))
+    _assert_parameters_equal(results[3][0], dict(n_samples=42, n_features=20))
+
+    results = filt_(
+        ["Test-Dataset[n_samples=[foo,bar], n_features=[True, False]]"])
+    assert len(results) == 4
+    _assert_parameters_equal(results[0][0],
+                             dict(n_samples="foo", n_features=True))
+    _assert_parameters_equal(results[1][0],
+                             dict(n_samples="foo", n_features=False))
+    _assert_parameters_equal(results[2][0],
+                             dict(n_samples="bar", n_features=True))
+    _assert_parameters_equal(results[3][0],
+                             dict(n_samples="bar", n_features=False))
+
+    # get list of tuples
+    results = filt_(
+        ["Test-Dataset['n_samples, n_features'=[(41, 19), (42, 20)]]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=41, n_features=19))
+    _assert_parameters_equal(results[1][0], dict(n_samples=42, n_features=20))
+
+    # invalid use of positional parameters
+    with pytest.raises(ValueError, match="Ambiguous positional parameter"):
+        filt_(["Test-Dataset[41,42]"])
+
+    # invalid use of unknown parameter
+    with pytest.raises(ValueError, match="Unknown parameter 'n_targets'"):
+        filt_(["Test-Dataset[n_targets=42]"])
+
+
+class TEST_DATASET_ONE_PARAM(TEST_DATASET):
+    """Used to test the selection of dataset with a positional parameter."""
+    parameters = {'n_samples': [10, 11]}
+
+
+def test_filter_classes_one_param():
+    # Test the selection of dataset with only one parameter.
+
+    def filt_(filters):
+        return list(_filter_classes(TEST_DATASET_ONE_PARAM, filters=filters))
+
+    # test positional parameter
+    results = filt_(["Test-Dataset[42]"])
+    assert len(results) == 1
+    _assert_parameters_equal(results[0][0], dict(n_samples=42))
+
+    # test grid of positional parameter
+    results = filt_(["Test-Dataset[41,42]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=41))
+    _assert_parameters_equal(results[1][0], dict(n_samples=42))
+
+    results = filt_(["Test-Dataset[foo, 'bar', None]"])
+    assert len(results) == 3
+    _assert_parameters_equal(results[0][0], dict(n_samples="foo"))
+    _assert_parameters_equal(results[1][0], dict(n_samples="bar"))
+    _assert_parameters_equal(results[2][0], dict(n_samples=None))
+
+    # test grid of keyword parameter
+    results = filt_(["Test-Dataset[n_samples=[foo,True]]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples="foo"))
+    _assert_parameters_equal(results[1][0], dict(n_samples=True))
+
+    # invalid use of both positional and keyword parameter
+    with pytest.raises(ValueError, match="Invalid name"):
+        filt_(["Test-Dataset[41, n_samples=42]"])
+
+
+def test_extract_options():
+    basename, args, kwargs = _extract_options("Dataset")
+    assert basename == "Dataset"
+    assert len(args) == 0
+    assert kwargs == dict()
+
+    basename, args, kwargs = _extract_options("Test-Dataset[n_samples=41]")
+    assert basename == "Test-Dataset"
+    assert len(args) == 0
+    assert kwargs == dict(n_samples=41)
+
+    basename, args, kwargs = _extract_options("n_samples[n_samples=41,]")
+    assert basename == "n_samples"
+    assert len(args) == 0
+    assert kwargs == dict(n_samples=41)
+
+    basename, args, kwargs = _extract_options(
+        "Dataset[n_samples=[41, 42], n_features=123.0]")
+    assert basename == "Dataset"
+    assert len(args) == 0
+    assert kwargs == dict(n_samples=[41, 42], n_features=123.)
+
+    with pytest.raises(ValueError, match="Invalid name"):
+        _extract_options("Dataset[n_samples=42")  # missing "]"
+    with pytest.raises(ValueError, match="Invalid name"):
+        _extract_options("Dataset[n_samples=[41, 42]")  # missing "]"
+
+
+def test_extract_parameters():
+    # Test the conversion of parameters.
+
+    # Convert to a list
+    assert _extract_parameters("42") == [42]
+    assert _extract_parameters("True") == [True]
+    assert _extract_parameters("foo") == ["foo"]
+    assert _extract_parameters("foo-bar") == ["foo-bar"]
+    assert _extract_parameters("1, 2, 3") == [1, 2, 3]
+
+    assert _extract_parameters("42, True, foo") == [42, True, "foo"]
+    assert _extract_parameters("42, (1, 2, 3)") == [42, (1, 2, 3)]
+    assert _extract_parameters("foo,bar ") == ["foo", "bar"]
+    assert _extract_parameters("foo, bar") == ["foo", "bar"]
+    assert _extract_parameters("foo,bar,") == ["foo", "bar"]
+    assert _extract_parameters("'foo, bar'") == ["foo, bar"]
+    assert _extract_parameters('"foo, bar"') == ["foo, bar"]
+    assert _extract_parameters("foo, (bar, baz)") == ["foo", ("bar", "baz")]
+
+    # Convert to a dict
+    assert _extract_parameters("foo=(bar, baz)") == {'foo': ('bar', 'baz')}
+    assert _extract_parameters("foo=(0, 1),bar=2") == {'foo': (0, 1), 'bar': 2}
+    assert _extract_parameters("foo=[100, 200]") == {'foo': [100, 200]}
+
+    # Special case with a list of tuple parameters
+    assert _extract_parameters("'foo, bar'=[(0, 1),(1, 0)]") == \
+        {'foo, bar': [(0, 1), (1, 0)]}
+    assert _extract_parameters('"foo, bar"=[(0, 1),(1, 0)]') == \
+        {'foo, bar': [(0, 1), (1, 0)]}
+
+    for token in [True, False, None]:  # python tokens
+        assert _extract_parameters(f"{token}") == [token]
+        assert _extract_parameters(f"'{token}'") == [token]
+        assert _extract_parameters(f"\"{token}\"") == [token]