Repository: facebookresearch/hydra

Issue:
Instantiate: Support annotation of passthrough values
When instantiating PyTorch optimizers, the params cannot be a part of the configuration.
It is a tensor that is only available at runtime.
To enable instantiating of optimizers, instantiate supports passthrough:
```python
optimize = instantiate(cfg.optimizer, params=model.parameters())
```

Since it's not possible to annotate params as a Tensor in dataclasses used for Structured Configs, there is no way to declare that params is a required value.

The idea is to add support for marking such parameters as MISSING. if the type is incompatible the can be annotated as Any:
```python
@dataclass
class AdamConf:
  params: Any = MISSING
  lr: float = 1.0
```

Parameters annotated as Any that are overridden in the instantiate call site are not type checked and are just passed through to the underlying function.


Discussion:

Resolving patch:
diff --git a/hydra/_internal/utils.py b/hydra/_internal/utils.py
--- a/hydra/_internal/utils.py
+++ b/hydra/_internal/utils.py
@@ -10,17 +10,7 @@
 from os.path import dirname, join, normpath, realpath
 from traceback import print_exc, print_exception
 from types import FrameType
-from typing import (
-    Any,
-    Callable,
-    List,
-    MutableMapping,
-    Optional,
-    Sequence,
-    Tuple,
-    Type,
-    Union,
-)
+from typing import Any, Callable, List, Optional, Sequence, Tuple, Type, Union
 
 from omegaconf import DictConfig, OmegaConf, read_write
 from omegaconf.errors import OmegaConfBaseException
@@ -586,19 +576,12 @@ def _get_kwargs(config: Union[ObjectConf, DictConfig], **kwargs: Any) -> Any:
             warnings.warn(category=UserWarning, message=msg)
             params = config.params
         else:
-            params = OmegaConf.create()
-            for k, v in config.items():
-                if k != "_target_":
-                    params[k] = v
+            params = config
 
     assert isinstance(
-        params, MutableMapping
+        params, DictConfig
     ), f"Input config params are expected to be a mapping, found {type(config.params).__name__}"
 
-    if isinstance(config, DictConfig):
-        assert isinstance(params, DictConfig)
-        params._set_parent(config)
-
     config_overrides = {}
     passthrough = {}
     for k, v in kwargs.items():
@@ -611,8 +594,12 @@ def _get_kwargs(config: Union[ObjectConf, DictConfig], **kwargs: Any) -> Any:
     with read_write(params):
         params.merge_with(config_overrides)
 
-    for k, v in params.items():
-        final_kwargs[k] = v
+    for k in params.keys():
+        if k == "_target_":
+            continue
+        if OmegaConf.is_missing(params, k) and k in passthrough:
+            continue
+        final_kwargs[k] = params[k]
 
     for k, v in passthrough.items():
         final_kwargs[k] = v
diff --git a/tools/configen/configen/configen.py b/tools/configen/configen/configen.py
--- a/tools/configen/configen/configen.py
+++ b/tools/configen/configen/configen.py
@@ -145,9 +145,9 @@ def generate_module(cfg: ConfigenConf, module: ModuleConf) -> str:
                 if type_ == str:
                     default_ = f'"{default_}"'
                 elif isinstance(default_, list):
-                    default_ = f"field(default_factory={default_})"
+                    default_ = f"field(default_factory=lambda: {default_})"
                 elif isinstance(default_, dict):
-                    default_ = f"field(default_factory={default_})"
+                    default_ = f"field(default_factory=lambda: {default_})"
 
             # fields that are incompatible with the config are flagged as passthrough and are added as a comment
             passthrough = _is_passthrough(type_) or passthrough_value


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -39,10 +39,24 @@ def static_method(z: int) -> int:
         return z
 
 
+@dataclass
+class UntypedPassthroughConf:
+    _target_: str = "tests.UntypedPassthroughClass"
+    a: Any = MISSING
+
+
+class UntypedPassthroughClass:
+    def __init__(self, a: Any) -> None:
+        self.a = a
+
+    def __eq__(self, other: Any) -> Any:
+        return self.a == other.a
+
+
 # Type not legal in a config
 class IllegalType:
     def __eq__(self, other: Any) -> Any:
-        return isinstance(other, AnotherClass)
+        return isinstance(other, IllegalType)
 
 
 @dataclass
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -23,6 +23,8 @@
     IllegalType,
     NestingClass,
     Parameters,
+    UntypedPassthroughClass,
+    UntypedPassthroughConf,
 )
 
 objectconf_depreacted = (
@@ -139,6 +141,12 @@ def test_get_class(path: str, expected_type: type) -> None:
             AClass(a=10, b=20, c=30, d={"x": IllegalType()}),
             id="passthrough",
         ),
+        pytest.param(
+            UntypedPassthroughConf,
+            {"a": IllegalType()},
+            UntypedPassthroughClass(a=IllegalType()),
+            id="missing_passthrough",
+        ),
     ],
 )
 def test_class_instantiate(
diff --git a/tools/configen/tests/gen-test-expected/configen.yaml b/tools/configen/tests/gen-test-expected/configen.yaml
--- a/tools/configen/tests/gen-test-expected/configen.yaml
+++ b/tools/configen/tests/gen-test-expected/configen.yaml
@@ -1,6 +1,6 @@
 configen:
   # output directory
-  output_dir: ${hydra:runtime.cwd}/tests/test_modules/expected
+  output_dir: ${hydra:runtime.cwd}/
 
   header: |
     # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
@@ -11,7 +11,7 @@ configen:
     # flake8: noqa
 
   # The directory of each module under output_dir
-  module_dir: ''
+  module_path_pattern: '{{module_path}}/generated.py'
 
   # list of modules to generate configs for
   modules:
@@ -23,3 +23,8 @@ configen:
         - UnionArg
         - WithLibraryClassArg
         - IncompatibleDataclassArg
+        - WithStringDefault
+        - ListValues
+        - DictValues
+        - PeskySentinelUsage
+        - Tuples
diff --git a/tools/configen/tests/test_generate.py b/tools/configen/tests/test_generate.py
--- a/tools/configen/tests/test_generate.py
+++ b/tools/configen/tests/test_generate.py
@@ -1,18 +1,37 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 from textwrap import dedent
 
-import subprocess
-import sys
 from difflib import unified_diff
 from pathlib import Path
 from typing import Any
 
 import pytest
+
+from hydra.utils import get_class, instantiate
 from omegaconf import OmegaConf
 
 from configen.config import ConfigenConf, ModuleConf
 from configen.configen import generate_module
 from hydra.test_utils.test_utils import chdir_hydra_root, get_run_output
+from tests.test_modules import (
+    User,
+    Color,
+    Empty,
+    UntypedArg,
+    IntArg,
+    UnionArg,
+    WithLibraryClassArg,
+    LibraryClass,
+    IncompatibleDataclassArg,
+    IncompatibleDataclass,
+    WithStringDefault,
+    ListValues,
+    DictValues,
+    PeskySentinelUsage,
+    Tuples,
+)
+
+from tests.test_modules.generated import PeskySentinelUsageConf
 
 chdir_hydra_root(subdir="tools/configen")
 
@@ -35,9 +54,11 @@
 )
 
 
-@pytest.mark.parametrize(
-    "class_name",
-    [
+MODULE_NAME = "tests.test_modules"
+
+
+def test_generated_code() -> None:
+    classes = [
         "Empty",
         "UntypedArg",
         "IntArg",
@@ -49,20 +70,15 @@
         "DictValues",
         "PeskySentinelUsage",
         "Tuples",
-    ],
-)
-def test_generated_code(class_name: str) -> None:
-    module_name = "tests.test_modules"
-    expected_file = (
-        Path(module_name.replace(".", "/")) / "expected" / f"{class_name}.py"
-    )
+    ]
+    expected_file = Path(MODULE_NAME.replace(".", "/")) / "generated.py"
     expected = expected_file.read_text()
 
     generated = generate_module(
         cfg=conf,
         module=ModuleConf(
-            name=module_name,
-            classes=[class_name],
+            name=MODULE_NAME,
+            classes=classes,
         ),
     )
 
@@ -82,6 +98,119 @@ def test_generated_code(class_name: str) -> None:
         assert False, f"Mismatch between {expected_file} and generated code"
 
 
+@pytest.mark.parametrize(
+    "classname, params, args, kwargs, expected",
+    [
+        pytest.param("Empty", {}, [], {}, Empty(), id="Empty"),
+        pytest.param(
+            "UntypedArg", {"param": 11}, [], {}, UntypedArg(param=11), id="UntypedArg"
+        ),
+        pytest.param(
+            "UntypedArg",
+            {},
+            [],
+            {"param": LibraryClass()},
+            UntypedArg(param=LibraryClass()),
+            id="UntypedArg_passthrough_lib_class",
+        ),
+        pytest.param("IntArg", {"param": 1}, [], {}, IntArg(param=1), id="IntArg"),
+        pytest.param(
+            "UnionArg", {"param": 1}, [], {}, UnionArg(param=1), id="UnionArg"
+        ),
+        pytest.param(
+            "UnionArg", {"param": 3.14}, [], {}, UnionArg(param=3.14), id="UnionArg"
+        ),
+        # This is okay because Union is not supported and is treated as Any
+        pytest.param(
+            "UnionArg",
+            {"param": "str"},
+            [],
+            {},
+            UnionArg(param="str"),
+            id="UnionArg:illegal_but_ok_arg",
+        ),
+        pytest.param(
+            "WithLibraryClassArg",
+            {"num": 10},
+            [],
+            {"param": LibraryClass()},
+            WithLibraryClassArg(num=10, param=LibraryClass()),
+            id="WithLibraryClassArg",
+        ),
+        pytest.param(
+            "IncompatibleDataclassArg",
+            {"num": 10},
+            [],
+            {"incompat": IncompatibleDataclass()},
+            IncompatibleDataclassArg(num=10, incompat=IncompatibleDataclass()),
+            id="IncompatibleDataclassArg",
+        ),
+        pytest.param(
+            "WithStringDefault",
+            {"no_default": "foo"},
+            [],
+            {},
+            WithStringDefault(no_default="foo"),
+            id="WithStringDefault",
+        ),
+        pytest.param(
+            "ListValues",
+            {
+                "lst": ["1"],
+                "enum_lst": ["RED"],
+                "dataclass_val": [{"name": "Bond", "age": 7}],
+            },
+            [],
+            {"passthrough_list": [LibraryClass()]},
+            ListValues(
+                lst=["1"],
+                enum_lst=[Color.RED],
+                passthrough_list=[LibraryClass()],
+                dataclass_val=[User(name="Bond", age=7)],
+            ),
+            id="ListValues",
+        ),
+        pytest.param(
+            "DictValues",
+            {
+                "dct": {"foo": "bar"},
+                "enum_key": {"RED": "red"},
+                "dataclass_val": {"007": {"name": "Bond", "age": 7}},
+            },
+            [],
+            {"passthrough_dict": {"lib": LibraryClass()}},
+            DictValues(
+                dct={"foo": "bar"},
+                enum_key={Color.RED: "red"},
+                dataclass_val={"007": User(name="Bond", age=7)},
+                passthrough_dict={"lib": LibraryClass()},
+            ),
+            id="DictValues",
+        ),
+        pytest.param(
+            "Tuples", {"t1": [1.0, 2.1]}, [], {}, Tuples(t1=(1.0, 2.1)), id="Tuples"
+        ),
+    ],
+)
+def test_instantiate_classes(
+    classname: str, params: Any, args: Any, kwargs: Any, expected: Any
+) -> None:
+    full_class = f"{MODULE_NAME}.generated.{classname}Conf"
+    schema = OmegaConf.structured(get_class(full_class))
+    cfg = OmegaConf.merge(schema, params)
+    obj = instantiate(config=cfg, *args, **kwargs)
+    assert obj == expected
+
+
+def test_pesky_sentinel() -> None:
+    # Tests that pesky sentinels are not actually working when going through config objects
+    full_class = f"{MODULE_NAME}.generated.PeskySentinelUsageConf"
+    schema = OmegaConf.structured(get_class(full_class))
+    cfg = OmegaConf.merge(schema, {})
+    obj = instantiate(config=cfg)
+    assert obj != PeskySentinelUsageConf()
+
+
 def test_example_application(monkeypatch: Any, tmpdir: Path):
     monkeypatch.chdir("example")
     cmd = [
diff --git a/tools/configen/tests/test_modules/__init__.py b/tools/configen/tests/test_modules/__init__.py
--- a/tools/configen/tests/test_modules/__init__.py
+++ b/tools/configen/tests/test_modules/__init__.py
@@ -26,41 +26,75 @@ class LibraryClass:
     def __init__(self):
         pass
 
+    def __eq__(self, other):
+        return isinstance(other, type(self))
+
 
 class Empty:
     def __init__(self):
         ...
 
+    def __eq__(self, other):
+        return isinstance(other, type(self))
+
 
 class UntypedArg:
     def __init__(self, param):
-        ...
+        self.param = param
+
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and other.param == self.param
 
 
 class IntArg:
     def __init__(self, param: int):
-        ...
+        self.param = param
+
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and other.param == self.param
 
 
 class UnionArg:
     # Union is not currently supported by OmegaConf, it will be typed as Any
     def __init__(self, param: Union[int, float]):
-        ...
+        self.param = param
+
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and other.param == self.param
 
 
 class WithLibraryClassArg:
     def __init__(self, num: int, param: LibraryClass):
-        ...
+        self.num = num
+        self.param = param
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, type(self))
+            and other.num == self.num
+            and other.param == self.param
+        )
 
 
 @dataclass
-class Incompatible:
+class IncompatibleDataclass:
     library: LibraryClass = LibraryClass()
 
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and other.library == self.library
+
 
 class IncompatibleDataclassArg:
-    def __init__(self, num: int, incompat: Incompatible):
-        ...
+    def __init__(self, num: int, incompat: IncompatibleDataclass):
+        self.num = num
+        self.incompat = incompat
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, type(self))
+            and self.num == other.num
+            and self.incompat == other.incompat
+        )
 
 
 class WithStringDefault:
@@ -70,7 +104,17 @@ def __init__(
         default_str: str = "Bond, James Bond",
         none_str: Optional[str] = None,
     ):
-        ...
+        self.no_default = no_default
+        self.default_str = default_str
+        self.none_str = none_str
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, type(self))
+            and self.no_default == other.no_default
+            and self.default_str == other.default_str
+            and self.none_str == other.none_str
+        )
 
 
 class ListValues:
@@ -82,7 +126,21 @@ def __init__(
         dataclass_val: List[User],
         def_value: List[str] = [],
     ):
-        ...
+        self.lst = lst
+        self.enum_lst = enum_lst
+        self.passthrough_list = passthrough_list
+        self.dataclass_val = dataclass_val
+        self.def_value = def_value
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, type(self))
+            and self.lst == other.lst
+            and self.enum_lst == other.enum_lst
+            and self.passthrough_list == other.passthrough_list
+            and self.dataclass_val == other.dataclass_val
+            and self.def_value == other.def_value
+        )
 
 
 class DictValues:
@@ -94,7 +152,21 @@ def __init__(
         passthrough_dict: Dict[str, LibraryClass],
         def_value: Dict[str, str] = {},
     ):
-        ...
+        self.dct = dct
+        self.enum_key = enum_key
+        self.dataclass_val = dataclass_val
+        self.passthrough_dict = passthrough_dict
+        self.def_value = def_value
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, type(self))
+            and self.dct == other.dct
+            and self.enum_key == other.enum_key
+            and self.dataclass_val == other.dataclass_val
+            and self.passthrough_dict == other.passthrough_dict
+            and self.def_value == other.def_value
+        )
 
 
 class PeskySentinel(object):
@@ -107,7 +179,7 @@ def __repr__(self):
 
 class PeskySentinelUsage:
     def __init__(self, foo=pesky):
-        ...
+        self.foo = foo
 
 
 class Tuples:
@@ -117,4 +189,14 @@ def __init__(
         t2=(1, 2, 3),
         t3: Tuple[float, ...] = (0.1, 0.2, 0.3),
     ):
-        ...
+        self.t1 = t1
+        self.t2 = t2
+        self.t3 = t3
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, type(self))
+            and self.t1 == other.t1
+            and self.t2 == other.t2
+            and self.t3 == other.t3
+        )
diff --git a/tools/configen/tests/test_modules/expected/DictValues.py b/tools/configen/tests/test_modules/expected/DictValues.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/DictValues.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-from tests.test_modules import Color
-from tests.test_modules import User
-from typing import Dict
-
-
-@dataclass
-class DictValuesConf:
-    _target_: str = "tests.test_modules.DictValues"
-    dct: Dict[str, str] = MISSING
-    enum_key: Dict[Color, str] = MISSING
-    dataclass_val: Dict[str, User] = MISSING
-    # [passthrough] passthrough_dict: Dict[str, LibraryClass]
-    def_value: Dict[str, str] = field(default_factory={})
diff --git a/tools/configen/tests/test_modules/expected/Empty.py b/tools/configen/tests/test_modules/expected/Empty.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/Empty.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-
-
-@dataclass
-class EmptyConf:
-    _target_: str = "tests.test_modules.Empty"
diff --git a/tools/configen/tests/test_modules/expected/IncompatibleDataclassArg.py b/tools/configen/tests/test_modules/expected/IncompatibleDataclassArg.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/IncompatibleDataclassArg.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-
-
-@dataclass
-class IncompatibleDataclassArgConf:
-    _target_: str = "tests.test_modules.IncompatibleDataclassArg"
-    num: int = MISSING
-    # [passthrough] incompat: Incompatible
diff --git a/tools/configen/tests/test_modules/expected/IntArg.py b/tools/configen/tests/test_modules/expected/IntArg.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/IntArg.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-
-
-@dataclass
-class IntArgConf:
-    _target_: str = "tests.test_modules.IntArg"
-    param: int = MISSING
diff --git a/tools/configen/tests/test_modules/expected/ListValues.py b/tools/configen/tests/test_modules/expected/ListValues.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/ListValues.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-from tests.test_modules import Color
-from tests.test_modules import User
-from typing import List
-
-
-@dataclass
-class ListValuesConf:
-    _target_: str = "tests.test_modules.ListValues"
-    lst: List[str] = MISSING
-    enum_lst: List[Color] = MISSING
-    # [passthrough] passthrough_list: List[LibraryClass]
-    dataclass_val: List[User] = MISSING
-    def_value: List[str] = field(default_factory=[])
diff --git a/tools/configen/tests/test_modules/expected/PeskySentinelUsage.py b/tools/configen/tests/test_modules/expected/PeskySentinelUsage.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/PeskySentinelUsage.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-
-
-@dataclass
-class PeskySentinelUsageConf:
-    _target_: str = "tests.test_modules.PeskySentinelUsage"
-    # [passthrough] foo: Any = <I am a pesky sentinel>
diff --git a/tools/configen/tests/test_modules/expected/Tuples.py b/tools/configen/tests/test_modules/expected/Tuples.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/Tuples.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-from typing import Any
-from typing import Tuple
-
-
-@dataclass
-class TuplesConf:
-    _target_: str = "tests.test_modules.Tuples"
-    t1: Tuple[float, float] = MISSING
-    t2: Any = (1, 2, 3)
-    t3: Tuple[float, ...] = (0.1, 0.2, 0.3)
diff --git a/tools/configen/tests/test_modules/expected/UnionArg.py b/tools/configen/tests/test_modules/expected/UnionArg.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/UnionArg.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-from typing import Any
-
-
-@dataclass
-class UnionArgConf:
-    _target_: str = "tests.test_modules.UnionArg"
-    param: Any = MISSING
diff --git a/tools/configen/tests/test_modules/expected/UntypedArg.py b/tools/configen/tests/test_modules/expected/UntypedArg.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/UntypedArg.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-from typing import Any
-
-
-@dataclass
-class UntypedArgConf:
-    _target_: str = "tests.test_modules.UntypedArg"
-    param: Any = MISSING
diff --git a/tools/configen/tests/test_modules/expected/WithLibraryClassArg.py b/tools/configen/tests/test_modules/expected/WithLibraryClassArg.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/WithLibraryClassArg.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-
-
-@dataclass
-class WithLibraryClassArgConf:
-    _target_: str = "tests.test_modules.WithLibraryClassArg"
-    num: int = MISSING
-    # [passthrough] param: LibraryClass
diff --git a/tools/configen/tests/test_modules/expected/WithStringDefault.py b/tools/configen/tests/test_modules/expected/WithStringDefault.py
deleted file mode 100644
--- a/tools/configen/tests/test_modules/expected/WithStringDefault.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-# Generated by configen, do not edit.
-# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
-# fmt: off
-# isort:skip_file
-# flake8: noqa
-
-from dataclasses import dataclass, field
-from omegaconf import MISSING
-from typing import Optional
-
-
-@dataclass
-class WithStringDefaultConf:
-    _target_: str = "tests.test_modules.WithStringDefault"
-    no_default: str = MISSING
-    default_str: str = "Bond, James Bond"
-    none_str: Optional[str] = None
diff --git a/tools/configen/tests/test_modules/generated.py b/tools/configen/tests/test_modules/generated.py
new file mode 100644
--- /dev/null
+++ b/tools/configen/tests/test_modules/generated.py
@@ -0,0 +1,95 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Generated by configen, do not edit.
+# See https://github.com/facebookresearch/hydra/tree/master/tools/configen
+# fmt: off
+# isort:skip_file
+# flake8: noqa
+
+from dataclasses import dataclass, field
+from omegaconf import MISSING
+from tests.test_modules import Color
+from tests.test_modules import User
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Tuple
+
+
+@dataclass
+class EmptyConf:
+    _target_: str = "tests.test_modules.Empty"
+
+
+@dataclass
+class UntypedArgConf:
+    _target_: str = "tests.test_modules.UntypedArg"
+    param: Any = MISSING
+
+
+@dataclass
+class IntArgConf:
+    _target_: str = "tests.test_modules.IntArg"
+    param: int = MISSING
+
+
+@dataclass
+class UnionArgConf:
+    _target_: str = "tests.test_modules.UnionArg"
+    param: Any = MISSING
+
+
+@dataclass
+class WithLibraryClassArgConf:
+    _target_: str = "tests.test_modules.WithLibraryClassArg"
+    num: int = MISSING
+    # [passthrough] param: LibraryClass
+
+
+@dataclass
+class IncompatibleDataclassArgConf:
+    _target_: str = "tests.test_modules.IncompatibleDataclassArg"
+    num: int = MISSING
+    # [passthrough] incompat: IncompatibleDataclass
+
+
+@dataclass
+class WithStringDefaultConf:
+    _target_: str = "tests.test_modules.WithStringDefault"
+    no_default: str = MISSING
+    default_str: str = "Bond, James Bond"
+    none_str: Optional[str] = None
+
+
+@dataclass
+class ListValuesConf:
+    _target_: str = "tests.test_modules.ListValues"
+    lst: List[str] = MISSING
+    enum_lst: List[Color] = MISSING
+    # [passthrough] passthrough_list: List[LibraryClass]
+    dataclass_val: List[User] = MISSING
+    def_value: List[str] = field(default_factory=lambda: [])
+
+
+@dataclass
+class DictValuesConf:
+    _target_: str = "tests.test_modules.DictValues"
+    dct: Dict[str, str] = MISSING
+    enum_key: Dict[Color, str] = MISSING
+    dataclass_val: Dict[str, User] = MISSING
+    # [passthrough] passthrough_dict: Dict[str, LibraryClass]
+    def_value: Dict[str, str] = field(default_factory=lambda: {})
+
+
+@dataclass
+class PeskySentinelUsageConf:
+    _target_: str = "tests.test_modules.PeskySentinelUsage"
+    # [passthrough] foo: Any = <I am a pesky sentinel>
+
+
+@dataclass
+class TuplesConf:
+    _target_: str = "tests.test_modules.Tuples"
+    t1: Tuple[float, float] = MISSING
+    t2: Any = (1, 2, 3)
+    t3: Tuple[float, ...] = (0.1, 0.2, 0.3)