Repository: benchopt/benchopt (python)

Issue:
DOC ENH add contribution instructions 
We can reuse content from https://scikit-learn.org/stable/developers/contributing.html#how-to-contribute

Pull request description:
Add contrib to doc 
should fix #346 

Patch:
diff --git a/.gitignore b/.gitignore
index 478b51f..0abac8a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,6 +28,7 @@ benchopt_front_page_convergence.pdf
 
 # IDE specific folders
 .vscode
+.idea
 
 # Config files
 benchopt.ini
diff --git a/README.rst b/README.rst
index 727f92f..bd85db0 100644
--- a/README.rst
+++ b/README.rst
@@ -22,12 +22,13 @@ be **as simple as doing**:
 
 .. code-block::
 
-    conda create -n benchopt python
-    conda activate benchopt
-    pip install benchopt
-    git clone https://github.com/benchopt/benchmark_logreg_l2
-    benchopt install -e benchmark_logreg_l2 -s cd -s sklearn
-    benchopt run -e ./benchmark_logreg_l2 -s cd -s sklearn
+   conda create -n benchopt python
+   conda activate benchopt
+   pip install benchopt
+   git clone https://github.com/benchopt/benchmark_logreg_l2
+   cd benchmark_logreg_l2
+   benchopt install -e . -s lightning -s sklearn
+   benchopt run -e . --config ./config_example.yml
 
 Running this command will give you a benchmark plot on l2-regularized
 logistic regression:
diff --git a/benchopt/benchmark.py b/benchopt/benchmark.py
index 8ab0e9c..aa528d8 100644
--- a/benchopt/benchmark.py
+++ b/benchopt/benchmark.py
@@ -391,7 +391,7 @@ class Benchmark:
 
         Parameters
         ----------
-        solver_names : list | None
+        solver_names : list | None
             List of solvers to include in the benchmark. If None
             all solvers available are run.
         forced_solvers : list | None
@@ -449,12 +449,12 @@ class Benchmark:
 
 
 def _check_name_lists(*name_lists):
-    "Normalize name_list ot a list of lowercase str."
+    "Normalize name_list to a list of string."
     res = []
     for name_list in name_lists:
         if name_list is None:
             continue
-        res.extend([str(name).lower() for name in name_list])
+        res.extend([str(name) for name in name_list])
     return res
 
 
@@ -466,16 +466,116 @@ def is_matched(name, include_patterns=None, default=True):
     if include_patterns is None or len(include_patterns) == 0:
         return default
     name = str(name)
-    # we use [] to signal options in patterns, we must escape them for re
-    substitutions = {"*": ".*", "[": r"\[", "]": r"\]"}
+    name = _extract_options(name)[0]
+    substitutions = {"*": ".*"}
     for p in include_patterns:
+        p = _extract_options(p)[0]
         for old, new in substitutions.items():
             p = p.replace(old, new)
-        if re.match(f"^{p}.*", name, flags=re.IGNORECASE) is not None:
+        if re.match(f"^{p}$", name, flags=re.IGNORECASE) is not None:
             return True
     return False
 
 
+def _extract_options(name):
+    """Remove options indicated within '[]' from a name.
+
+    Parameters
+    ----------
+    name : str
+        Input name.
+
+    Returns
+    -------
+    basename : str
+        Name without options.
+    args : list
+        List of unnamed options.
+    kwargs : dict()
+        Dictionary with options.
+
+    Examples
+    --------
+    >>> _extract_options("foo")  # "foo", [], dict()
+    >>> _extract_options("foo[bar=2]")  # "foo", [], dict(bar=2)
+    >>> _extract_options("foo[baz]")  # "foo", ["baz"], dict()
+    """
+    if name.count("[") != name.count("]"):
+        raise ValueError(f"Invalid name (missing bracket): {name}")
+
+    basename = "".join(re.split(r"\[.*\]", name))
+    matches = re.findall(r"\[.*\]", name)
+
+    if len(matches) == 0:
+        return basename, [], {}
+    elif len(matches) > 1:
+        raise ValueError(f"Invalid name (multiple brackets): {name}")
+    else:
+        match = matches[0]
+        match = match[1:-1]  # remove brackets
+
+        result = _extract_parameters(match)
+        if isinstance(result, dict):
+            return basename, [], result
+        elif isinstance(result, list):
+            return basename, result, {}
+        else:
+            raise ValueError(
+                f"Impossible. Please report this bug.\n"
+                f"_extract_parameters returned '{result}'"
+            )
+
+
+def _extract_parameters(string):
+    """Extract parameters from a string.
+
+    If the string contains a "=", returns a dict, otherwise returns a list.
+
+    Examples
+    --------
+    >>> _extract_parameters("foo")  # ["foo"]
+    >>> _extract_parameters("foo, 42, True")  # ["foo", 42, True]
+    >>> _extract_parameters("foo=bar")  # {"foo": "bar"}
+    >>> _extract_parameters("foo=[bar, baz]")  # {"foo": ["bar", "baz"]}
+    >>> _extract_parameters("foo=1, baz=True")  # {"foo": 1, "baz": True}
+    >>> _extract_parameters("'foo, bar'=[(0, 1),(1, 0)]")
+    >>> # {"foo, bar": [(0, 1),(1, 0)]}
+    """
+    import ast
+    original = string
+
+    # First, replace all quoted names with their hashes, to avoid modification.
+    all_matches = re.findall(r"'[^'\"]*'", string)
+    all_matches += re.findall(r'"[^\'"]*"', string)
+    for match in all_matches:
+        string = string.replace(match, str(hash(match)))
+
+    # Second, add quotes to all variable names (foo -> 'foo').
+    string = re.sub(r"[a-zA-Z][a-zA-Z0-9._-]*", r"'\g<0>'", string)
+
+    # Third, change back the hashes to their original names.
+    for match in all_matches:
+        string = string.replace(str(hash(match)), match)
+    # Prepare the sequence for AST parsing.
+    # Sequences with "=" are made into a dict expression {'foo': 'bar'}.
+    # Sequences without "=" are made into a list expression ['foo', 'bar'].
+    if "=" in string:
+        string = "{" + string.replace("=", ":") + "}"
+    else:
+        string = "[" + string + "]"
+
+    # Remove quotes for python language tokens
+    for token in ["True", "False", "None"]:
+        string = string.replace(f'"{token}"', token)
+        string = string.replace(f"'{token}'", token)
+
+    # Evaluate the string.
+    try:
+        return ast.literal_eval(string)
+    except (ValueError, SyntaxError):
+        raise ValueError(f"Invalid name '{original}', evaluated as {string}.")
+
+
 def _validate_patterns(all_names, patterns, name_type='dataset'):
     "Check that all provided patterns match at least one name."
     if patterns is None:
@@ -488,33 +588,81 @@ def _validate_patterns(all_names, patterns, name_type='dataset'):
         if not matched:
             invalid_patterns.append(p)
 
-    # If some patterns did not matched any dataset, raise an error
+    # If some patterns did not matched any class, raise an error
     if len(invalid_patterns) > 0:
-        all_names = '- ' + '\n- '.join(all_names)
+        all_names_ = '- ' + '\n- '.join(all_names)
         raise click.BadParameter(
             f"Patterns {invalid_patterns} did not match any {name_type}.\n"
-            f"Available {name_type}s are:\n{all_names}"
+            f"Available {name_type}s are:\n{all_names_}"
         )
 
 
 def _filter_classes(*classes, filters=None):
     """Filter a list of class based on its names."""
     for klass in classes:
-        is_installed = None
-        for parameters in product_param(klass.parameters):
-            obj = klass.get_instance(**parameters)
-            if is_matched(str(obj), filters):
-                if is_installed is None:
-                    is_installed = (
-                        not hasattr(klass, 'is_installed')
-                        or klass.is_installed(
-                            raise_on_not_installed=RAISE_INSTALL_ERROR
-                        )
-                    )
-                if not is_installed:
-                    yield klass.name, False
-                    break
-                yield obj, True
+        if not is_matched(klass.name, filters):
+            continue
+
+        if not klass.is_installed(raise_on_not_installed=RAISE_INSTALL_ERROR):
+            yield klass.name, False
+            continue
+
+        for parameters in _get_used_parameters(klass, filters):
+            yield klass.get_instance(**parameters), True
+
+
+def _get_used_parameters(klass, filters):
+    """Get the list of parameters to use in the class."""
+
+    # Use the default parameters if no filters are provided.
+    if filters is None or len(filters) == 0:
+        return product_param(klass.parameters)
+
+    # If filters are provided, get the list of parameters from the names.
+    update_parameters = []
+    for filt in filters:
+        if is_matched(klass.name, [filt]):
+            # Extract the parameters from the filter name.
+            _, args, kwargs = _extract_options(filt)
+
+            if len(args) > 0:
+                # positional args are allowed only for single-parameter classes
+                if len(klass.parameters) > 1:
+                    raise ValueError(
+                        f"Ambiguous positional parameter in {filt}.")
+                elif len(kwargs) > 0:
+                    raise ValueError(
+                        f"Both positional and keyword parameters in {filt}.")
+                else:
+                    key = list(klass.parameters.keys())[0]
+                    kwargs = {key: args}
+
+            # Use product_param to get all combinations of parameters.
+            kwargs = {
+                key: (val if isinstance(val, (list, tuple)) else [val])
+                for key, val in kwargs.items()
+            }
+            update_parameters.extend(product_param(kwargs))
+
+    # Then, update the default parameters (klass.parameters) with the
+    # parameters extracted from filter names.
+    used_parameters = []
+    for update in update_parameters:
+        for default in product_param(klass.parameters):
+            default = default.copy()  # avoid modifying the original
+
+            # check that all parameters are defined in klass.parameters
+            for key in update:
+                if key not in default:
+                    raise ValueError(
+                        f"Unknown parameter '{key}', parameter must be in "
+                        f"{list(default.keys())}")
+
+            default.update(update)
+            if default not in used_parameters:  # avoid duplicates
+                used_parameters.append(default)
+
+    return used_parameters
 
 
 def buffer_iterator(it):
diff --git a/benchopt/cli/completion.py b/benchopt/cli/completion.py
index 507b9eb..8c111bd 100644
--- a/benchopt/cli/completion.py
+++ b/benchopt/cli/completion.py
@@ -84,6 +84,17 @@ def complete_output_files(ctx, param, incomplete):
     return propose_from_list(candidates, incomplete)
 
 
+def complete_config_files(ctx, param, incomplete):
+    "Auto-completion for configuration files."
+    skip_import()
+    benchmark = find_benchmark_in_args(ctx.args)
+    if benchmark is None:
+        return []
+    benchmark_folder = benchmark.benchmark_dir
+    candidates = list(benchmark_folder.glob('*.yml'))
+    return propose_from_list(candidates, incomplete)
+
+
 def complete_conda_envs(ctx, param, incomplete):
     "Auto-completion for env-names."
     _, all_envs = list_conda_envs()
diff --git a/benchopt/cli/main.py b/benchopt/cli/main.py
index a150612..2f2491e 100644
--- a/benchopt/cli/main.py
+++ b/benchopt/cli/main.py
@@ -8,6 +8,7 @@ from benchopt.cli.completion import complete_solvers
 from benchopt.cli.completion import complete_datasets
 from benchopt.cli.completion import complete_benchmarks
 from benchopt.cli.completion import complete_conda_envs
+from benchopt.cli.completion import complete_config_files
 from benchopt.utils.conda_env_cmd import list_conda_envs
 from benchopt.utils.conda_env_cmd import create_conda_env
 from benchopt.utils.shell_cmd import _run_shell_in_conda_env
@@ -73,8 +74,9 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               help="Deprecated alias for `--objective`.")
 @click.option('--objective', '-o',
               metavar='<objective_filter>', multiple=True, type=str,
-              help="Filter the objective based on its parametrized name. This "
-              "can be used to only include one set of parameters.")
+              help="Select the objective based on its parameters, with the "
+              "syntax `objective[parameter=value]`. This can be used to only "
+              "include one set of parameters. ")
 @click.option('--old-objective-filter', '-p',
               multiple=True, type=str,
               help="Deprecated alias for --objective_filter/-o.")
@@ -82,8 +84,10 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               metavar="<solver_name>", multiple=True, type=str,
               help="Include <solver_name> in the benchmark. By default, all "
               "solvers are included. When `-s` is used, only listed solvers"
-              " are included. To include multiple solvers, "
-              "use multiple `-s` options.", shell_complete=complete_solvers)
+              " are included. Note that <solver_name> can include parameters,"
+              " with the syntax `solver[parameter=value]`. "
+              "To include multiple solvers, use multiple `-s` options.",
+              shell_complete=complete_solvers)
 @click.option('--force-solver', '-f',
               metavar="<solver_name>", multiple=True, type=str,
               help="Force the re-run for <solver_name>. This "
@@ -94,7 +98,8 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               metavar="<dataset_name>", multiple=True, type=str,
               help="Run the benchmark on <dataset_name>. By default, all "
               "datasets are included. When `-d` is used, only listed datasets"
-              " are included. Note that <dataset_name> can be a regexp. "
+              " are included. Note that <dataset_name> can include parameters,"
+              " with the syntax `dataset[parameter=value]`. "
               "To include multiple datasets, use multiple `-d` options.",
               shell_complete=complete_datasets)
 @click.option('--n-jobs', '-j',
@@ -113,6 +118,7 @@ def _get_run_args(cli_kwargs, config_file_kwargs):
               metavar="<int>", default=100, show_default=True, type=int,
               help='Timeout a solver when run for more than <timeout> seconds')
 @click.option('--config', 'config_file', default=None,
+              shell_complete=complete_config_files,
               help="YAML configuration file containing benchmark options.")
 @click.option('--plot/--no-plot', default=True,
               help="Whether or not to plot the results. Default is True.")
@@ -237,7 +243,7 @@ def run(config_file=None, **kwargs):
         )
 
     # run the command in the conda env
-    solvers_option = ' '.join(['-s ' + s for s in solver_names])
+    solvers_option = ' '.join([f"-s '{s}'" for s in solver_names])
     forced_solvers_option = ' '.join([f"-f '{s}'" for s in forced_solvers])
     datasets_option = ' '.join([f"-d '{d}'" for d in dataset_names])
     objective_option = ' '.join([f"-o '{o}'" for o in objective_filters])
@@ -286,10 +292,15 @@ def run(config_file=None, **kwargs):
               "datasets are included, except when -s flag is used. "
               "If -s flag is used, then no dataset is included. "
               "When `-d` is used, only listed datasets "
-              "are included. Note that <dataset_name> can be a regexp. "
+              "are included. Note that <dataset_name> can include parameters "
+              "with the syntax `dataset[parameter=value]`. "
               "To include multiple datasets, use multiple `-d` options."
               "To include all datasets, use -d 'all' option.",
               shell_complete=complete_datasets)
+@click.option('--config', 'config_file', default=None,
+              shell_complete=complete_config_files,
+              help="YAML configuration file containing benchmark options, "
+              "whose solvers and datasets will be installed.")
 @click.option('--env', '-e', 'env_name',
               flag_value='True', type=str, default='False',
               help="Install all requirements in a dedicated "
@@ -313,8 +324,20 @@ def run(config_file=None, **kwargs):
               help="If this flag is set, no confirmation will be asked "
               "to the user to install requirements in the current environment."
               " Useless with options `-e/--env` or `--env-name`.")
-def install(benchmark, minimal, solver_names, dataset_names, force=False,
-            recreate=False, env_name='False', confirm=False, quiet=False):
+def install(
+        benchmark, minimal, solver_names, dataset_names, config_file=None,
+        force=False, recreate=False, env_name='False', confirm=False,
+        quiet=False):
+
+    if config_file is not None:
+        with open(config_file, "r") as f:
+            config = yaml.safe_load(f)
+        if not dataset_names:
+            dataset_names = config.get("dataset", tuple())
+        if not solver_names:
+            solver_names = config.get("solver", tuple())
+            forced_solvers = config.get("force-solver", tuple())
+            solver_names = list(set(solver_names).union(set(forced_solvers)))
 
     # Check that the dataset/solver patterns match actual dataset
     benchmark = Benchmark(benchmark)
diff --git a/doc/cli.rst b/doc/cli.rst
index b9853b3..fdd6a55 100644
--- a/doc/cli.rst
+++ b/doc/cli.rst
@@ -11,7 +11,7 @@ completion by following the instructions given in the
 `click documentation <https://click.palletsprojects.com/en/8.0.x/shell-completion/#enabling-completion>`_.
 For example using BASH shell you need to run:
 
-.. code-block::
+.. prompt:: bash $
 
     eval "$(_BENCHOPT_COMPLETE=bash_source benchopt)"
 
@@ -19,6 +19,23 @@ For example using BASH shell you need to run:
 The `benchopt` command also comes with tab completion for the solver name
 and the dataset name.
 
+.. admonition:: Optional parameters syntax
+
+    For some CLI parameters (solver, objective,
+    dataset), additional values can be given with the following syntax:
+
+    .. code-block:: bash
+
+        # Run a particular solver with a particular set of parameters:
+        --solver solver_name[param_1=method_name, param_2=100]
+        # To select a grid of parameters, the following syntax is allowed:
+        --solver solver_name[param_1=[True, False]]
+        # For objects with only one parameter, the name can be omitted:
+        --solver solver_name[True, False]
+        # For more advanced selections over multiple parameters, use:
+        --solver solver_name["param_1,param_2"=[(True, 100), (False, 1000)]]
+
+
 .. click:: benchopt.cli:benchopt
    :prog: benchopt
    :nested: full
diff --git a/doc/conf.py b/doc/conf.py
index 451f6b7..eb70e30 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -33,6 +33,7 @@ sys.path.append(os.path.abspath(os.path.join(curdir, 'sphinxext')))
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'sphinx-prompt',
     'sphinx.ext.autodoc',
     'sphinx.ext.autosummary',
     'sphinx.ext.doctest',
diff --git a/doc/contrib.rst b/doc/contrib.rst
new file mode 100644
index 0000000..e439431
--- /dev/null
+++ b/doc/contrib.rst
@@ -0,0 +1,129 @@
+.. _contrib_doc:
+
+How to contribute
+=================
+
+
+Submitting a bug report or a feature request
+--------------------------------------------
+
+We use benchopt `GitHub repo <https://github.com/benchopt/benchopt/issues>`_ to track all bugs and feature requests; feel free to open
+an issue if you have found a bug or wish to see a feature implemented.
+
+
+
+
+Contributing code
+-----------------
+
+The preferred way to contribute to benchopt is to fork the `main
+repository <https://github.com/benchopt/benchopt/>`__ on GitHub,
+then submit a "Pull Request" (PR).
+
+In the first few steps, we explain how to locally install benchopt, and
+how to set up your git repository:
+
+1. `Create an account <https://github.com/join>`_ on
+   GitHub if you do not already have one.
+
+2. Fork the `project repository
+   <https://github.com/benchopt/benchopt>`__: click on the 'Fork'
+   button near the top of the page. This creates a copy of the code under your
+   account on the GitHub user account. For more details on how to fork a
+   repository see `this guide <https://help.github.com/articles/fork-a-repo/>`_.
+
+3. Clone your fork of the benchopt repo from your GitHub account to your
+   local disk:
+
+   .. prompt:: bash $
+
+      git clone git@github.com:YourLogin/benchopt.git
+      cd benchopt
+
+.. _upstream:
+
+4. Add the ``upstream`` remote. This saves a reference to the main
+   benchopt repository, which you can use to keep your repository
+   synchronized with the latest changes:
+
+   .. prompt:: bash $
+
+        git remote add upstream https://github.com/benchopt/benchopt
+
+5. Check that the `upstream` and `origin` remote aliases are configured correctly
+   by running `git remote -v` which should display::
+
+        origin	git@github.com:YourLogin/benchopt.git (fetch)
+        origin	git@github.com:YourLogin/benchopt.git (push)
+        upstream	https://github.com/benchopt/benchopt (fetch)
+        upstream	https://github.com/benchopt/benchopt (push)
+
+
+You should now have a working installation of benchopt, and your git
+repository properly configured. The next steps now describe the process of
+modifying code and submitting a PR:
+
+6. Synchronize your ``main`` branch with the ``upstream/main`` branch,
+   more details on `GitHub Docs <https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/syncing-a-fork>`_:
+
+   .. prompt:: bash $
+
+        git switch main
+        git fetch upstream
+        git merge upstream/main
+
+7. Create a feature branch to hold your development changes:
+
+   .. prompt:: bash $
+
+        git switch -c my_feature
+
+   and start making changes. Always use a feature branch. It's good
+   practice to never work on the ``main`` branch!
+
+8. Develop the feature on your feature branch on your computer, using Git to
+   do the version control. When you're done editing, add changed files using
+   ``git add`` and then ``git commit``:
+
+   .. prompt:: bash $
+
+      git add modified_files
+      git commit
+
+   to record your changes in Git, then push the changes to your GitHub
+   account with:
+
+   .. prompt:: bash $
+
+      git push -u origin my_feature
+
+9. Follow `these <https://help.github.com/articles/creating-a-pull-request-from-a-fork>`_ instructions
+   to create a pull request from your fork.
+
+.. note::
+
+    It is often helpful to keep your local feature branch synchronized with the latest
+    changes of the main benchopt repository:
+
+    .. prompt:: bash $
+
+        git fetch upstream
+        git merge upstream/main
+
+Documentation
+-------------
+
+We are glad to accept any sort of documentation: function docstrings,
+reStructuredText documents (like this one), tutorials, etc. reStructuredText
+documents live in the source code repository under the ``doc/`` directory.
+
+You can edit the documentation using any text editor, and then generate the
+HTML output by typing, in a shell:
+
+.. prompt:: bash $
+
+    pip install benchopt[doc]
+    cd doc/
+    make html
+    firefox _build/html/index.html
+
diff --git a/doc/index.rst b/doc/index.rst
index b5d899d..b6ab984 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -18,11 +18,11 @@ Benchopt is used through a command line as documented
 in the :ref:`cli_documentation`.
 Once benchopt is installed, running and replicating an optimization benchmark is **as simple as doing**:
 
-.. code-block::
+.. prompt:: bash $
 
-    $ git clone https://github.com/benchopt/benchmark_logreg_l2
-    $ benchopt install --env ./benchmark_logreg_l2
-    $ benchopt run --env ./benchmark_logreg_l2
+    git clone https://github.com/benchopt/benchmark_logreg_l2
+    benchopt install --env ./benchmark_logreg_l2
+    benchopt run --env ./benchmark_logreg_l2
 
 Running these commands will fetch the benchmark files, install the benchmark
 requirements in a dedicated environment called ``benchopt_benchmark_logreg_l2`` and
@@ -38,17 +38,25 @@ Learn how to :ref:`how`.
 Install
 --------
 
-This package can be installed through `pip`. To get the **latest release**, use:
+This package can be installed through `pip`.  In order to allow benchopt to automatically
+install solvers dependencies, the install needs to be done in a `conda` environment.
 
-.. code-block::
+.. prompt:: bash $
 
-    $ pip install benchopt
+    conda create -n benchopt python
+    conda activate benchopt
+
+To get the **latest release**, use:
+
+.. prompt:: bash $
+
+    pip install benchopt
 
 And to get the **latest development version**, you can use:
 
-.. code-block::
+.. prompt:: bash $
 
-    $ pip install -U -i https://test.pypi.org/simple/ benchopt
+    pip install -U -i https://test.pypi.org/simple/ benchopt
 
 This will install the command line tool to run the benchmark. Then, existing
 benchmarks can be retrieved from GitHub or created locally. To discover which
@@ -57,9 +65,9 @@ benchmarks are presently available look for
 such as for `Lasso -- l1-regularized linear regression <https://github.com/benchopt/benchmark_lasso>`_.
 This benchmark can be retrieved locally with:
 
-.. code-block::
+.. prompt:: bash $
 
-    $ git clone https://github.com/benchopt/benchmark_lasso.git
+    git clone https://github.com/benchopt/benchmark_lasso.git
 
 Quickstart: command line usage on the Lasso benchmark
 -----------------------------------------------------
@@ -70,34 +78,34 @@ The ``--env`` flag specifies that everything is run in the ``benchopt_benchmark_
 
 **Installing benchmark dependencies**: to install all requirements of the benchmark, run:
 
-.. code-block::
+.. prompt:: bash $
 
-    $ benchopt install --env ./benchmark_lasso
+    benchopt install --env ./benchmark_lasso
 
 **Run a benchmark**: to run benchmarks on all datasets and with all solvers, run:
 
-.. code-block::
+.. prompt:: bash $
 
-    $ benchopt run --env ./benchmark_lasso
+    benchopt run --env ./benchmark_lasso
 
 **Run only some solvers and datasets**: to run only the ``sklearn`` and ``celer`` solvers, on the ``simulated`` and ``finance`` datasets, run:
 
-.. code-block::
+.. prompt:: bash $
 
-    $ benchopt run --env ./benchmark_lasso -s sklearn -s celer -d simulated -d finance
+    benchopt run --env ./benchmark_lasso -s sklearn -s celer -d simulated -d finance
 
 **Run a solver or dataset with specific parameters**:  some solvers and datasets have parameters; by default all combinations are run.
 If you want to run a specific configuration, pass it explicitly, e.g., to run the ``python-pgd`` solver only with its parameter ``use_acceleration`` set to True, use:
 
-.. code-block::
+.. prompt:: bash $
 
-    $ benchopt run --env ./benchmark_lasso -s python-pgd[use_acceleration=True]
+    benchopt run --env ./benchmark_lasso -s python-pgd[use_acceleration=True]
 
 **Set the number of repetitions**: the benchmark are repeated 5 times by default for greater precision. To run the benchmark 10 times, run:
 
-.. code-block::
+.. prompt:: bash $
 
-    $ benchopt run --env ./benchmark_lasso -r 10
+    benchopt run --env ./benchmark_lasso -r 10
 
 **Passing option through configuration file**: all options of ``benchopt run`` can be passed through a YAML configuration file, together with ``--config <configuration_file_name.yml>``.
 The options are defined using the same name as the CLI options.
@@ -120,9 +128,9 @@ When options are passed both via file and CLI, the CLI takes precedence.
 
 **Getting help**: use
 
-.. code-block::
+.. prompt:: bash $
 
-    $ benchopt run -h
+    benchopt run -h
 
 to get more details about the different options.
 You can also read the :ref:`cli_documentation`.
@@ -218,6 +226,8 @@ All the public benchmark results are available at `Benchopt Benchmarks results <
 
 **Publish results**: You can directly publish the result of a run of ``benchopt`` on `Benchopt Benchmarks results <https://benchopt.github.io/results/>`_. You can have a look at this page to :ref:`publish_doc`.
 
+.. include:: contrib.rst
+
 Contents
 ========
 
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
index 2603853..d22b867 100644
--- a/doc/whats_new.rst
+++ b/doc/whats_new.rst
@@ -7,8 +7,16 @@ What's new
 
 .. _changes_1_3:
 
-Versioni 1.3 - in development
------------------------------
+Version 1.3 - in development
+----------------------------
+
+CLI
+~~~
+
+- Add support for custom parameters in CLI for objectives, datasets, and
+  solvers, through the syntax ``-s solver_name[parameter=value]``. See the `CLI
+  documentation <https://benchopt.github.io/cli.html>`_ for more details on the
+  syntax. By `Tom Dupré la Tour`_ (:gh:`362`).
 
 .. _changes_1_2:
 
@@ -52,11 +60,18 @@ API
 - Add ``X_density`` argument to ``datasets.make_correlated_data`` to simulate
   sparse design matrices, by `Mathurin Massias`_ (:gh:`289`).
 
+- ``Dataset.get_data`` should now return a dict and not a tuple. A point for
+  testing  should be returned by a dedicated method
+  ``Objective.get_one_solution``, by `Thomas Moreau`_ (:gh:`345`).
+
 CLI
 ~~~
 
 - Replace ``-p`` flag by ``-o`` for Objective, by `Mathurin Massias`_ (:gh:`281`).
 
+- Add ``--config`` option to support passing argument with a ``yaml``
+  config file, by `Mathurin Massias`_ (:gh:`325`).
+
 .. _changes_1_1:
 
 Version 1.1 - 22-04-2021
diff --git a/examples/plot_run_benchmark.py b/examples/plot_run_benchmark.py
index d3ee105..30b3fe8 100644
--- a/examples/plot_run_benchmark.py
+++ b/examples/plot_run_benchmark.py
@@ -21,7 +21,7 @@ try:
     save_file = run_benchmark(
         Benchmark(BENCHMARK_PATH), ['sklearn[liblinear]', 'sklearn[newton-cg]',
                                     'lightning'],
-        dataset_names=['Simulated*n_features=500,n_samples=200*'],
+        dataset_names=['Simulated*[n_features=500,n_samples=200]'],
         objective_filters=['L2 Logistic Regression[lmbd=1.0]'],
         max_runs=100, timeout=20, n_repetitions=15,
         plot_result=False, show_progress=True
diff --git a/examples/plot_run_benchmark_python_R.py b/examples/plot_run_benchmark_python_R.py
index db9b2f3..0fc612a 100644
--- a/examples/plot_run_benchmark_python_R.py
+++ b/examples/plot_run_benchmark_python_R.py
@@ -26,7 +26,7 @@ save_file = run_benchmark(
     Benchmark(BENCHMARK_PATH),
     ['Python-PGD[use_acceleration=False]', 'R-PGD'],
     dataset_names=["Simulated[n_features=5000,n_samples=100,rho=0]"],
-    objective_filters=['*fit_intercept=False,reg=0.5'],
+    objective_filters=['*[fit_intercept=False,reg=0.5]'],
     max_runs=100, timeout=100, n_repetitions=5,
     plot_result=False, show_progress=False
 )
diff --git a/setup.cfg b/setup.cfg
index 5d7f818..60f936b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -63,5 +63,7 @@ doc =
     sphinx-bootstrap-theme
     sphinx-click
     sphinx_gallery
+    sphinx-prompt
     pillow
     scikit-learn
+


Test patch:
diff --git a/benchopt/tests/__init__.py b/benchopt/tests/__init__.py
index e03a660..373778b 100644
--- a/benchopt/tests/__init__.py
+++ b/benchopt/tests/__init__.py
@@ -12,9 +12,9 @@ DUMMY_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'dummy_benchmark'
 REQUIREMENT_BENCHMARK_PATH = TEST_BENCHMARK_DIR / 'requirement_benchmark'
 
 # Pattern to select specific datasets or solvers.
-SELECT_ONE_SIMULATED = r'simulated*200*rho=0]'
-SELECT_ONE_PGD = r'python-pgd*step_size=1]'
-SELECT_ONE_OBJECTIVE = r'dummy*reg=0.1]'
+SELECT_ONE_SIMULATED = r'simulated[n_features=200,rho=0]'
+SELECT_ONE_PGD = r'python-pgd[step_size=1]'
+SELECT_ONE_OBJECTIVE = r'dummy*[reg=0.1]'
 
 try:
     DUMMY_BENCHMARK = Benchmark(DUMMY_BENCHMARK_PATH)
diff --git a/benchopt/tests/test_benchmarks/dummy_benchmark/test_config.py b/benchopt/tests/test_benchmarks/dummy_benchmark/test_config.py
index 39c6411..4745f76 100644
--- a/benchopt/tests/test_benchmarks/dummy_benchmark/test_config.py
+++ b/benchopt/tests/test_benchmarks/dummy_benchmark/test_config.py
@@ -11,7 +11,5 @@ def check_test_solver_install(solver_class):
     # Skip test_solver_install for julia in OSX as there is a version
     # conflict with conda packages for R
     # See issue #64
-    # julia=1.6.4 broke on conda, see PR #252
-    if 'julia' in solver_class.name.lower():
-        # pytest.xfail('Julia causes segfault on OSX for now.')
-        pytest.xfail('Julia install via conda is broken for now.')
+    if 'julia' in solver_class.name.lower() and sys.platform == 'darwin':
+        pytest.xfail('Julia causes segfault on OSX for now.')
diff --git a/benchopt/tests/test_cli.py b/benchopt/tests/test_cli.py
index f3f6738..3f46d4c 100644
--- a/benchopt/tests/test_cli.py
+++ b/benchopt/tests/test_cli.py
@@ -155,6 +155,29 @@ class TestRunCmd:
         # Make sure the results were saved in a result file
         assert len(out.result_files) == 1, out.output
 
+    def test_benchopt_run_custom_parameters(self):
+        SELECT_DATASETS = r'simulated[n_features=[100, 200]]'
+        SELECT_SOLVERS = r'python-pgd-with-cb[use_acceleration=[True, False]]'
+        SELECT_OBJECTIVES = r'dummy*[0.1, 0.2]'
+
+        with CaptureRunOutput() as out:
+            run([str(DUMMY_BENCHMARK_PATH), '-l', '-d', SELECT_DATASETS,
+                 '-f', SELECT_SOLVERS, '-n', '1', '-r', '1', '-o',
+                 SELECT_OBJECTIVES, '--no-plot'],
+                'benchopt', standalone_mode=False)
+
+        out.check_output(r'Simulated\[n_features=100,', repetition=1)
+        out.check_output(r'Simulated\[n_features=200,', repetition=1)
+        out.check_output(r'Simulated\[n_features=5000,', repetition=0)
+        out.check_output(r'Dummy Sparse Regression\[reg=0.1\]', repetition=2)
+        out.check_output(r'Dummy Sparse Regression\[reg=0.2\]', repetition=2)
+        out.check_output(r'Dummy Sparse Regression\[reg=0.05\]', repetition=0)
+        out.check_output(r'--Python-PGD\[', repetition=0)
+        out.check_output(r'--Python-PGD-with-cb\[use_acceleration=False\]:',
+                         repetition=28)
+        out.check_output(r'--Python-PGD-with-cb\[use_acceleration=True\]:',
+                         repetition=28)
+
     def test_benchopt_run_profile(self):
         with CaptureRunOutput() as out:
             run_cmd = [str(DUMMY_BENCHMARK_PATH),
@@ -187,7 +210,7 @@ class TestRunCmd:
         n-repetitions: 2
         max-runs: 1
         force-solver:
-          - python-pgd[step_size=1.5]
+          - python-pgd[step_size=[2, 3]]
           - Test-Solver
         """
         tmp = tempfile.NamedTemporaryFile(mode="w+")
@@ -201,8 +224,8 @@ class TestRunCmd:
             run(run_cmd, 'benchopt', standalone_mode=False)
 
         out.check_output(r'Test-Solver:', repetition=11)
-        out.check_output(
-            r'Python-PGD\[step_size=1.5\]:', repetition=11)
+        out.check_output(r'Python-PGD\[step_size=2\]:', repetition=11)
+        out.check_output(r'Python-PGD\[step_size=3\]:', repetition=11)
 
         # test that CLI options take precedence
         with CaptureRunOutput() as out:
diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
index 6a0fe26..2c075d6 100644
--- a/benchopt/tests/test_runner.py
+++ b/benchopt/tests/test_runner.py
@@ -5,6 +5,9 @@ import pytest
 from benchopt.tests import TEST_SOLVER
 from benchopt.tests import TEST_DATASET
 from benchopt.tests import TEST_OBJECTIVE
+from benchopt.benchmark import _filter_classes
+from benchopt.benchmark import _extract_options
+from benchopt.benchmark import _extract_parameters
 
 
 def test_skip_api():
@@ -53,3 +56,190 @@ def test_get_one_solution():
         one_solution = objective.get_one_solution()
     expected = np.zeros(objective.X.shape[1])
     assert all(one_solution == expected)
+
+
+def _assert_parameters_equal(instance, parameters):
+    for key, val in parameters.items():
+        assert getattr(instance, key) == val
+
+
+class TEST_DATASET_TWO_PARAMS(TEST_DATASET):
+    """Used to test the selection of datasets by keyword parameters."""
+    parameters = {'n_samples': [10, 11], 'n_features': [20, 21]}
+
+
+def test_filter_classes_two_parameters():
+    # Test the selection of dataset with optional parameters.
+
+    def filt_(filters):
+        return list(_filter_classes(TEST_DATASET_TWO_PARAMS, filters=filters))
+
+    # no selection (default grid)
+    results = filt_(["Test-Dataset"])
+    assert len(results) == 4
+    _assert_parameters_equal(results[0][0], dict(n_samples=10, n_features=20))
+    _assert_parameters_equal(results[1][0], dict(n_samples=10, n_features=21))
+    _assert_parameters_equal(results[2][0], dict(n_samples=11, n_features=20))
+    _assert_parameters_equal(results[3][0], dict(n_samples=11, n_features=21))
+
+    # select one parameter (n_samples)
+    results = filt_(["Test-Dataset[n_samples=42]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=42, n_features=20))
+    _assert_parameters_equal(results[1][0], dict(n_samples=42, n_features=21))
+
+    # select one parameter (n_features)
+    results = filt_(["Test-Dataset[n_features=42]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=10, n_features=42))
+    _assert_parameters_equal(results[1][0], dict(n_samples=11, n_features=42))
+
+    # select two parameters (n_samples, n_features)
+    results = filt_(["Test-Dataset[n_samples=41, n_features=42]"])
+    assert len(results) == 1
+    _assert_parameters_equal(results[0][0], dict(n_samples=41, n_features=42))
+
+    # get grid over one parameter (n_samples)
+    results = filt_(["Test-Dataset[n_samples=[41,42], n_features=19]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=41, n_features=19))
+    _assert_parameters_equal(results[1][0], dict(n_samples=42, n_features=19))
+
+    # get grid over two parameter (n_samples, n_features)
+    results = filt_(["Test-Dataset[n_samples=[41,42], n_features=[19, 20]]"])
+    assert len(results) == 4
+    _assert_parameters_equal(results[0][0], dict(n_samples=41, n_features=19))
+    _assert_parameters_equal(results[1][0], dict(n_samples=41, n_features=20))
+    _assert_parameters_equal(results[2][0], dict(n_samples=42, n_features=19))
+    _assert_parameters_equal(results[3][0], dict(n_samples=42, n_features=20))
+
+    results = filt_(
+        ["Test-Dataset[n_samples=[foo,bar], n_features=[True, False]]"])
+    assert len(results) == 4
+    _assert_parameters_equal(results[0][0],
+                             dict(n_samples="foo", n_features=True))
+    _assert_parameters_equal(results[1][0],
+                             dict(n_samples="foo", n_features=False))
+    _assert_parameters_equal(results[2][0],
+                             dict(n_samples="bar", n_features=True))
+    _assert_parameters_equal(results[3][0],
+                             dict(n_samples="bar", n_features=False))
+
+    # get list of tuples
+    results = filt_(
+        ["Test-Dataset['n_samples, n_features'=[(41, 19), (42, 20)]]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=41, n_features=19))
+    _assert_parameters_equal(results[1][0], dict(n_samples=42, n_features=20))
+
+    # invalid use of positional parameters
+    with pytest.raises(ValueError, match="Ambiguous positional parameter"):
+        filt_(["Test-Dataset[41,42]"])
+
+    # invalid use of unknown parameter
+    with pytest.raises(ValueError, match="Unknown parameter 'n_targets'"):
+        filt_(["Test-Dataset[n_targets=42]"])
+
+
+class TEST_DATASET_ONE_PARAM(TEST_DATASET):
+    """Used to test the selection of dataset with a positional parameter."""
+    parameters = {'n_samples': [10, 11]}
+
+
+def test_filter_classes_one_param():
+    # Test the selection of dataset with only one parameter.
+
+    def filt_(filters):
+        return list(_filter_classes(TEST_DATASET_ONE_PARAM, filters=filters))
+
+    # test positional parameter
+    results = filt_(["Test-Dataset[42]"])
+    assert len(results) == 1
+    _assert_parameters_equal(results[0][0], dict(n_samples=42))
+
+    # test grid of positional parameter
+    results = filt_(["Test-Dataset[41,42]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples=41))
+    _assert_parameters_equal(results[1][0], dict(n_samples=42))
+
+    results = filt_(["Test-Dataset[foo, 'bar', None]"])
+    assert len(results) == 3
+    _assert_parameters_equal(results[0][0], dict(n_samples="foo"))
+    _assert_parameters_equal(results[1][0], dict(n_samples="bar"))
+    _assert_parameters_equal(results[2][0], dict(n_samples=None))
+
+    # test grid of keyword parameter
+    results = filt_(["Test-Dataset[n_samples=[foo,True]]"])
+    assert len(results) == 2
+    _assert_parameters_equal(results[0][0], dict(n_samples="foo"))
+    _assert_parameters_equal(results[1][0], dict(n_samples=True))
+
+    # invalid use of both positional and keyword parameter
+    with pytest.raises(ValueError, match="Invalid name"):
+        filt_(["Test-Dataset[41, n_samples=42]"])
+
+
+def test_extract_options():
+    basename, args, kwargs = _extract_options("Dataset")
+    assert basename == "Dataset"
+    assert len(args) == 0
+    assert kwargs == dict()
+
+    basename, args, kwargs = _extract_options("Test-Dataset[n_samples=41]")
+    assert basename == "Test-Dataset"
+    assert len(args) == 0
+    assert kwargs == dict(n_samples=41)
+
+    basename, args, kwargs = _extract_options("n_samples[n_samples=41,]")
+    assert basename == "n_samples"
+    assert len(args) == 0
+    assert kwargs == dict(n_samples=41)
+
+    basename, args, kwargs = _extract_options(
+        "Dataset[n_samples=[41, 42], n_features=123.0]")
+    assert basename == "Dataset"
+    assert len(args) == 0
+    assert kwargs == dict(n_samples=[41, 42], n_features=123.)
+
+    with pytest.raises(ValueError, match="Invalid name"):
+        _extract_options("Dataset[n_samples=42")  # missing "]"
+    with pytest.raises(ValueError, match="Invalid name"):
+        _extract_options("Dataset[n_samples=[41, 42]")  # missing "]"
+
+
+def test_extract_parameters():
+    # Test the conversion of parameters.
+
+    # Convert to a list
+    assert _extract_parameters("42") == [42]
+    assert _extract_parameters("True") == [True]
+    assert _extract_parameters("foo") == ["foo"]
+    assert _extract_parameters("foo-bar") == ["foo-bar"]
+    assert _extract_parameters("foo.bar") == ["foo.bar"]
+    assert _extract_parameters("1, 2, 3") == [1, 2, 3]
+
+    assert _extract_parameters("42, True, foo") == [42, True, "foo"]
+    assert _extract_parameters("42, (1, 2, 3)") == [42, (1, 2, 3)]
+    assert _extract_parameters("foo,bar ") == ["foo", "bar"]
+    assert _extract_parameters("foo, bar") == ["foo", "bar"]
+    assert _extract_parameters("foo,bar,") == ["foo", "bar"]
+    assert _extract_parameters("'foo, bar'") == ["foo, bar"]
+    assert _extract_parameters('"foo, bar"') == ["foo, bar"]
+    assert _extract_parameters("foo, (bar, baz)") == ["foo", ("bar", "baz")]
+
+    # Convert to a dict
+    assert _extract_parameters("foo=(bar, baz)") == {'foo': ('bar', 'baz')}
+    assert _extract_parameters("foo=(0, 1),bar=2") == {'foo': (0, 1), 'bar': 2}
+    assert _extract_parameters("foo=[100, 200]") == {'foo': [100, 200]}
+
+    # Special case with a list of tuple parameters
+    assert _extract_parameters("'foo, bar'=[(0, 1),(1, 0)]") == \
+        {'foo, bar': [(0, 1), (1, 0)]}
+    assert _extract_parameters('"foo, bar"=[(0, 1),(1, 0)]') == \
+        {'foo, bar': [(0, 1), (1, 0)]}
+
+    for token in [True, False, None]:  # python tokens
+        assert _extract_parameters(f"{token}") == [token]
+        assert _extract_parameters(f"'{token}'") == [token]
+        assert _extract_parameters(f"\"{token}\"") == [token]