Repository: iterative/dvc

Issue:
exp: basic hydra sweeper support
Hydra also allows for running hyperparameter sweeps. Using Hydra, the same could be supported in DVC (see [https://hydra.cc/docs/intro/#multirun](https://hydra.cc/docs/intro/#multirun)):

```bash
$ dvc exp run --set-param train=random_forest,gradient_boosting
```

_Originally posted by @dberenbaum in https://github.com/iterative/dvc/discussions/7044#discussioncomment-3271855_


Discussion:

Patch:
diff --git a/dvc/repo/experiments/run.py b/dvc/repo/experiments/run.py
--- a/dvc/repo/experiments/run.py
+++ b/dvc/repo/experiments/run.py
@@ -2,6 +2,7 @@
 from typing import Dict, Iterable, Optional
 
 from dvc.dependency.param import ParamsDependency
+from dvc.exceptions import InvalidArgumentError
 from dvc.repo import locked
 from dvc.ui import ui
 from dvc.utils.cli_parse import to_path_overrides
@@ -27,6 +28,8 @@ def run(
     Returns a dict mapping new experiment SHAs to the results
     of `repro` for that experiment.
     """
+    from dvc.utils.hydra import to_hydra_overrides
+
     if run_all:
         entries = list(repo.experiments.celery_queue.iter_queued())
         return repo.experiments.reproduce_celery(entries, jobs=jobs)
@@ -42,19 +45,42 @@ def run(
         # Force `_update_params` even if `--set-param` was not used
         path_overrides[hydra_output_file] = []
 
-    if queue:
-        if not kwargs.get("checkpoint_resume", None):
-            kwargs["reset"] = True
+    hydra_sweep = any(
+        x.is_sweep_override()
+        for param_file in path_overrides
+        for x in to_hydra_overrides(path_overrides[param_file])
+    )
+
+    if hydra_sweep and not queue:
+        raise InvalidArgumentError(
+            "Sweep overrides can't be used without `--queue`"
+        )
+
+    if not queue:
+        return repo.experiments.reproduce_one(
+            targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs
+        )
+
+    if hydra_sweep:
+        from dvc.utils.hydra import get_hydra_sweeps
+
+        sweeps = get_hydra_sweeps(path_overrides)
+    else:
+        sweeps = [path_overrides]
+
+    if not kwargs.get("checkpoint_resume", None):
+        kwargs["reset"] = True
+
+    for sweep_overrides in sweeps:
         queue_entry = repo.experiments.queue_one(
             repo.experiments.celery_queue,
             targets=targets,
-            params=path_overrides,
+            params=sweep_overrides,
             **kwargs,
         )
+        if sweep_overrides:
+            ui.write(f"Queueing with overrides '{sweep_overrides}'.")
         name = queue_entry.name or queue_entry.stash_rev[:7]
         ui.write(f"Queued experiment '{name}' for future execution.")
-        return {}
 
-    return repo.experiments.reproduce_one(
-        targets=targets, params=path_overrides, tmp_dir=tmp_dir, **kwargs
-    )
+    return {}
diff --git a/dvc/utils/hydra.py b/dvc/utils/hydra.py
--- a/dvc/utils/hydra.py
+++ b/dvc/utils/hydra.py
@@ -1,11 +1,13 @@
+from collections import defaultdict
 from pathlib import Path
 from typing import TYPE_CHECKING, List
 
 from hydra import compose, initialize_config_dir
 from hydra._internal.config_loader_impl import ConfigLoaderImpl
+from hydra._internal.core_plugins.basic_sweeper import BasicSweeper
 from hydra.core.override_parser.overrides_parser import OverridesParser
+from hydra.core.override_parser.types import ValueType
 from hydra.errors import ConfigCompositionException, OverrideParseException
-from hydra.types import RunMode
 from omegaconf import OmegaConf
 
 from dvc.exceptions import InvalidArgumentError
@@ -59,11 +61,7 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
     modify_data = MODIFIERS[suffix]
     with modify_data(path) as original_data:
         try:
-            parser = OverridesParser.create()
-            parsed = parser.parse_overrides(overrides=overrides)
-            ConfigLoaderImpl.validate_sweep_overrides_legal(
-                parsed, run_mode=RunMode.RUN, from_shell=True
-            )
+            parsed = to_hydra_overrides(overrides)
 
             new_data = OmegaConf.create(
                 to_omegaconf(original_data),
@@ -78,3 +76,34 @@ def apply_overrides(path: "StrPath", overrides: List[str]) -> None:
 
         merge_dicts(original_data, new_data)
         remove_missing_keys(original_data, new_data)
+
+
+def to_hydra_overrides(path_overrides):
+    parser = OverridesParser.create()
+    return parser.parse_overrides(overrides=path_overrides)
+
+
+def get_hydra_sweeps(path_overrides):
+    merged_overrides = []
+    for path, overrides in path_overrides.items():
+        merged_overrides.extend(
+            [f"{path}___{override}" for override in overrides]
+        )
+
+    hydra_overrides = to_hydra_overrides(merged_overrides)
+    for hydra_override in hydra_overrides:
+        if hydra_override.value_type == ValueType.GLOB_CHOICE_SWEEP:
+            raise InvalidArgumentError(
+                f"Glob override '{hydra_override.input_line}' "
+                "is not supported."
+            )
+
+    splits = BasicSweeper.split_arguments(hydra_overrides, None)[0]
+    sweeps = []
+    for split in splits:
+        sweep_overrides = defaultdict(list)
+        for merged_override in split:
+            path, override = merged_override.split("___")
+            sweep_overrides[path].append(override)
+        sweeps.append(dict(sweep_overrides))
+    return sweeps


Test patch:
diff --git a/tests/func/experiments/test_set_params.py b/tests/func/experiments/test_set_params.py
--- a/tests/func/experiments/test_set_params.py
+++ b/tests/func/experiments/test_set_params.py
@@ -2,6 +2,8 @@
 
 import pytest
 
+from dvc.exceptions import InvalidArgumentError
+
 from ..utils.test_hydra import hydra_setup
 
 
@@ -85,3 +87,57 @@ def test_hydra_compose_and_dump(
             "goo": {"bag": 3.0},
             "lorem": False,
         }
+
+
+@pytest.mark.parametrize(
+    "hydra_enabled,overrides,expected",
+    [
+        (
+            pytest.param(
+                True,
+                marks=pytest.mark.xfail(
+                    sys.version_info >= (3, 11), reason="unsupported on 3.11"
+                ),
+            ),
+            ["db=mysql,postgresql"],
+            [
+                {"params.yaml": ["db=mysql"]},
+                {"params.yaml": ["db=postgresql"]},
+            ],
+        ),
+        (
+            False,
+            ["foo=bar,baz"],
+            [{"params.yaml": ["foo=bar"]}, {"params.yaml": ["foo=baz"]}],
+        ),
+    ],
+)
+def test_hydra_sweep(
+    tmp_dir, params_repo, dvc, mocker, hydra_enabled, overrides, expected
+):
+    patched = mocker.patch.object(dvc.experiments, "queue_one")
+
+    if hydra_enabled:
+        hydra_setup(
+            tmp_dir,
+            config_dir="conf",
+            config_name="config",
+        )
+        with dvc.config.edit() as conf:
+            conf["hydra"]["enabled"] = True
+
+    dvc.experiments.run(params=overrides, queue=True)
+
+    assert patched.call_count == len(expected)
+    for e in expected:
+        patched.assert_any_call(
+            mocker.ANY,
+            params=e,
+            reset=True,
+            targets=None,
+        )
+
+
+def test_hydra_sweep_requires_queue(params_repo, dvc):
+    with pytest.raises(InvalidArgumentError):
+        dvc.experiments.run(params=["db=mysql,postgresql"])
diff --git a/tests/func/utils/test_hydra.py b/tests/func/utils/test_hydra.py
--- a/tests/func/utils/test_hydra.py
+++ b/tests/func/utils/test_hydra.py
@@ -178,3 +178,54 @@ def test_compose_and_dump(tmp_dir, suffix, overrides, expected):
     output_file = tmp_dir / f"params.{suffix}"
     compose_and_dump(output_file, config_dir, config_name, overrides)
     assert output_file.parse() == expected
+
+
+@pytest.mark.parametrize(
+    "overrides, expected",
+    [
+        (
+            {"params.yaml": ["foo=1,2", "bar=3,4"]},
+            [
+                {"params.yaml": ["foo=1", "bar=3"]},
+                {"params.yaml": ["foo=1", "bar=4"]},
+                {"params.yaml": ["foo=2", "bar=3"]},
+                {"params.yaml": ["foo=2", "bar=4"]},
+            ],
+        ),
+        (
+            {"params.yaml": ["foo=choice(1,2)"]},
+            [{"params.yaml": ["foo=1"]}, {"params.yaml": ["foo=2"]}],
+        ),
+        (
+            {"params.yaml": ["foo=range(1, 3)"]},
+            [{"params.yaml": ["foo=1"]}, {"params.yaml": ["foo=2"]}],
+        ),
+        (
+            {"params.yaml": ["foo=1,2"], "others.yaml": ["bar=3"]},
+            [
+                {"params.yaml": ["foo=1"], "others.yaml": ["bar=3"]},
+                {"params.yaml": ["foo=2"], "others.yaml": ["bar=3"]},
+            ],
+        ),
+        (
+            {"params.yaml": ["foo=1,2"], "others.yaml": ["bar=3,4"]},
+            [
+                {"params.yaml": ["foo=1"], "others.yaml": ["bar=3"]},
+                {"params.yaml": ["foo=1"], "others.yaml": ["bar=4"]},
+                {"params.yaml": ["foo=2"], "others.yaml": ["bar=3"]},
+                {"params.yaml": ["foo=2"], "others.yaml": ["bar=4"]},
+            ],
+        ),
+    ],
+)
+def test_hydra_sweeps(overrides, expected):
+    from dvc.utils.hydra import get_hydra_sweeps
+
+    assert get_hydra_sweeps(overrides) == expected
+
+
+def test_invalid_sweep():
+    from dvc.utils.hydra import get_hydra_sweeps
+
+    with pytest.raises(InvalidArgumentError):
+        get_hydra_sweeps({"params.yaml": ["foo=glob(*)"]})