Repository: facebookresearch/hydra

Issue:
[Bug] speed regression when using ListConfig in place of tuple when configuring Adam with instantiate
#1011 reports performance regression in Adam when using instantiate.
The root cause appears to be that ListConfig is used to configure the betas where tuple or list would be more performant.

Adam is accessing the betas twice per layer during the optimization, which - in some scenarios - is enough to cause a speed regression.



Discussion:

Resolving patch:
diff --git a/examples/instantiate/object/my_app.py b/examples/instantiate/object/my_app.py
--- a/examples/instantiate/object/my_app.py
+++ b/examples/instantiate/object/my_app.py
@@ -2,6 +2,7 @@
 from omegaconf import DictConfig
 
 import hydra
+from hydra.utils import instantiate
 
 
 class DBConnection:
@@ -32,7 +33,7 @@ def connect(self) -> None:
 
 @hydra.main(config_path="conf", config_name="config")
 def my_app(cfg: DictConfig) -> None:
-    connection = hydra.utils.instantiate(cfg.db)
+    connection = instantiate(cfg.db)
     connection.connect()
 
 
diff --git a/hydra/_internal/utils.py b/hydra/_internal/utils.py
--- a/hydra/_internal/utils.py
+++ b/hydra/_internal/utils.py
@@ -10,7 +10,7 @@
 from types import FrameType
 from typing import Any, Callable, List, Optional, Sequence, Tuple, Union
 
-from omegaconf import DictConfig, ListConfig, OmegaConf
+from omegaconf import DictConfig, ListConfig, OmegaConf, open_dict
 from omegaconf._utils import type_str
 from omegaconf.errors import OmegaConfBaseException
 
@@ -614,8 +614,39 @@ def _get_target_type(config: Any, kwargs: Any) -> Union[type, Callable[..., Any]
         raise InstantiationException(f"Unsupported target type : {type(target)}")
 
 
+def _pop_convert_mode(d: Any) -> Any:
+    from hydra.utils import ConvertMode
+
+    # default value is to not convert OmegaConf containers
+    ret = ConvertMode.NONE
+
+    if "_convert_" in d:
+        with open_dict(d):
+            convert = d.pop("_convert_")
+            if convert is not None:
+                if isinstance(convert, str):
+                    if convert == "none":
+                        ret = ConvertMode.NONE
+                    elif convert == "partial":
+                        ret = ConvertMode.PARTIAL
+                    elif convert == "all":
+                        ret = ConvertMode.ALL
+                    else:
+                        raise InstantiationException(
+                            f"Unsupported _convert_ value: {convert}"
+                        )
+                elif isinstance(convert, ConvertMode):
+                    ret = convert
+                else:
+                    raise InstantiationException(
+                        f"_convert_ must be a string or ConvertMode enum (got `{type(convert).__name__}`)"
+                    )
+    return ret
+
+
 def _get_kwargs(
     config: Union[DictConfig, ListConfig],
+    root: bool = True,
     **kwargs: Any,
 ) -> Any:
     from hydra.utils import instantiate
@@ -624,32 +655,36 @@ def _get_kwargs(
 
     if OmegaConf.is_list(config):
         assert isinstance(config, ListConfig)
-        return [_get_kwargs(x) if OmegaConf.is_config(x) else x for x in config]
+        return [
+            _get_kwargs(x, root=False) if OmegaConf.is_config(x) else x for x in config
+        ]
 
     assert OmegaConf.is_dict(config), "Input config is not an OmegaConf DictConfig"
 
-    final_kwargs = {}
-
     recursive = _is_recursive(config, kwargs)
     overrides = OmegaConf.create(kwargs, flags={"allow_objects": True})
     config.merge_with(overrides)
 
-    for k, v in config.items():
-        final_kwargs[k] = v
-
+    final_kwargs = OmegaConf.create(flags={"allow_objects": True})
+    final_kwargs._set_parent(config._get_parent())
+    final_kwargs._set_flag("readonly", False)
+    final_kwargs._set_flag("struct", False)
     if recursive:
-        for k, v in final_kwargs.items():
-            if _is_target(v):
+        for k, v in config.items_ex(resolve=False):
+            if OmegaConf.is_none(v):
+                final_kwargs[k] = v
+            elif _is_target(v):
                 final_kwargs[k] = instantiate(v)
-            elif OmegaConf.is_dict(v) and not OmegaConf.is_none(v):
+            elif OmegaConf.is_dict(v):
                 d = OmegaConf.create({}, flags={"allow_objects": True})
-                for key, value in v.items():
+                for key, value in v.items_ex(resolve=False):
                     if _is_target(value):
                         d[key] = instantiate(value)
                     elif OmegaConf.is_config(value):
-                        d[key] = _get_kwargs(value)
+                        d[key] = _get_kwargs(value, root=False)
                     else:
                         d[key] = value
+                d._metadata.object_type = v._metadata.object_type
                 final_kwargs[k] = d
             elif OmegaConf.is_list(v):
                 lst = OmegaConf.create([], flags={"allow_objects": True})
@@ -657,15 +692,25 @@ def _get_kwargs(
                     if _is_target(x):
                         lst.append(instantiate(x))
                     elif OmegaConf.is_config(x):
-                        lst.append(_get_kwargs(x))
+                        lst.append(_get_kwargs(x, root=False))
+                        lst[-1]._metadata.object_type = x._metadata.object_type
                     else:
                         lst.append(x)
                 final_kwargs[k] = lst
             else:
-                if OmegaConf.is_none(v):
-                    v = None
                 final_kwargs[k] = v
-
+    else:
+        for k, v in config.items_ex(resolve=False):
+            final_kwargs[k] = v
+
+    final_kwargs._set_flag("readonly", None)
+    final_kwargs._set_flag("struct", None)
+    final_kwargs._set_flag("allow_objects", None)
+    if not root:
+        # This is tricky, since the root kwargs is exploded anyway we can treat is as an untyped dict
+        # the motivation is that the object type is used as an indicator to treat the object differently during
+        # conversion to a primitive container in some cases
+        final_kwargs._metadata.object_type = config._metadata.object_type
     return final_kwargs
 
 
diff --git a/hydra/utils.py b/hydra/utils.py
--- a/hydra/utils.py
+++ b/hydra/utils.py
@@ -1,6 +1,8 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import logging.config
 import os
+import sys
+from enum import Enum
 from pathlib import Path
 from typing import Any, Callable
 
@@ -13,6 +15,7 @@
     _get_kwargs,
     _get_target_type,
     _locate,
+    _pop_convert_mode,
 )
 from hydra.core.hydra_config import HydraConfig
 from hydra.errors import HydraException, InstantiationException
@@ -21,15 +24,31 @@
 log = logging.getLogger(__name__)
 
 
+class ConvertMode(Enum):
+    # Use DictConfig/ListConfig
+    NONE = 0
+    # Convert the OmegaConf config to primitive container, Structured Configs are preserved
+    PARTIAL = 1
+    # Fully convert the OmegaConf config to primitive containers (dict, list and primitives).
+    ALL = 2
+
+
 def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any:
     """
     :param config: An config object describing what to call and what params to use.
                    In addition to the parameters, the config must contain:
                    _target_ : target class or callable name (str)
+                   And may contain:
                    _recursive_: Construct nested objects as well (bool).
                                 True by default.
                                 may be overridden via a _recursive_ key in
                                 the kwargs
+                   _convert_: Conversion strategy
+                        none    : Passed objects are DictConfig and ListConfig, default
+                        partial : Passed objects are converted to dict and list, with
+                                  the exception of Structured Configs (and their fields).
+                        all     : Passed objects are dicts, lists and primitives without
+                                  a trace of OmegaConf containers
     :param args: Optional positional parameters pass-through
     :param kwargs: Optional named parameters to override
                    parameters in the config object. Parameters not present
@@ -77,11 +96,24 @@ def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any:
     try:
         config._set_flag("allow_objects", True)
         final_kwargs = _get_kwargs(config, **kwargs)
-        return target(*args, **final_kwargs)
+        convert = _pop_convert_mode(final_kwargs)
+        if convert == ConvertMode.PARTIAL:
+            final_kwargs = OmegaConf.to_container(
+                final_kwargs, resolve=True, exclude_structured_configs=True
+            )
+            return target(*args, **final_kwargs)
+        elif convert == ConvertMode.ALL:
+            final_kwargs = OmegaConf.to_container(final_kwargs, resolve=True)
+            return target(*args, **final_kwargs)
+        elif convert == ConvertMode.NONE:
+            return target(*args, **final_kwargs)
+        else:
+            assert False
     except Exception as e:
+        # preserve the original exception backtrace
         raise type(e)(
             f"Error instantiating/calling '{_convert_target_to_string(target)}' : {e}"
-        )
+        ).with_traceback(sys.exc_info()[2])
 
 
 # Alias for instantiate


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -267,3 +267,53 @@ class MappingConf:
 
     def __init__(self, dictionary: Optional[Dict[str, "MappingConf"]] = None):
         self.dictionary = dictionary
+
+
+@dataclass
+class SimpleDataClass:
+    a: Any = None
+    b: Any = None
+
+
+class SimpleClass:
+    a: Any = None
+    b: Any = None
+
+    def __init__(self, a: Any, b: Any) -> None:
+        self.a = a
+        self.b = b
+
+    def __eq__(self, other: Any) -> Any:
+        if isinstance(other, SimpleClass):
+            return self.a == other.a and self.b == other.b
+        return False
+
+
+@dataclass
+class SimpleClassPrimitiveConf:
+    _target_: str = "tests.SimpleClass"
+    _convert_: str = "partial"
+    a: Any = None
+    b: Any = None
+
+
+@dataclass
+class SimpleClassNonPrimitiveConf:
+    _target_: str = "tests.SimpleClass"
+    _convert_: str = "none"
+    a: Any = None
+    b: Any = None
+
+
+@dataclass
+class SimpleClassDefaultPrimitiveConf:
+    _target_: str = "tests.SimpleClass"
+    a: Any = None
+    b: Any = None
+
+
+@dataclass
+class NestedConf:
+    _target_: str = "tests.SimpleClass"
+    a: Any = User(name="a", age=1)
+    b: Any = User(name="b", age=2)
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -2,11 +2,12 @@
 import copy
 import os
 import re
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional
 
 import pytest
-from omegaconf import DictConfig, OmegaConf
+from omegaconf import MISSING, DictConfig, ListConfig, OmegaConf
 
 from hydra import utils
 from hydra._internal.utils import _convert_container_targets_to_strings
@@ -14,6 +15,7 @@
 from hydra.core.hydra_config import HydraConfig
 from hydra.errors import InstantiationException
 from hydra.types import TargetConf
+from hydra.utils import ConvertMode
 from tests import (
     AClass,
     Adam,
@@ -29,14 +31,21 @@
     IllegalType,
     Mapping,
     MappingConf,
+    NestedConf,
     NestingClass,
     Parameters,
     Rotation,
     RotationConf,
+    SimpleClass,
+    SimpleClassDefaultPrimitiveConf,
+    SimpleClassNonPrimitiveConf,
+    SimpleClassPrimitiveConf,
+    SimpleDataClass,
     Tree,
     TreeConf,
     UntypedPassthroughClass,
     UntypedPassthroughConf,
+    User,
     module_function,
 )
 
@@ -211,6 +220,34 @@ def test_interpolation_accessing_parent(
     assert obj == expected
 
 
+def test_interpolation_is_live_in_instantiated_object() -> None:
+    """
+    Interpolations in instantiated objects are live config objects but not for primitive objects.
+    """
+    cfg = OmegaConf.create(
+        {
+            "node": {
+                "_target_": "tests.AClass",
+                "a": "${value}",
+                "b": {"x": "${value}"},
+                "c": 30,
+                "d": 40,
+            },
+            "value": 99,
+        }
+    )
+    obj = utils.instantiate(cfg.node)
+    assert obj.a == 99
+    assert obj.b.x == 99
+
+    cfg.value = 3.14
+
+    # interpolation is not live for primitive objects
+    assert obj.a == 99  # unchanged
+    # but is live for config objects
+    assert obj.b.x == 3.14
+
+
 def test_class_instantiate_omegaconf_node() -> Any:
     conf = OmegaConf.structured(
         {
@@ -292,7 +329,13 @@ def test_instantiate_adam_conf() -> None:
 
     adam_params = Parameters([1, 2, 3])
     res = utils.instantiate(AdamConf(lr=0.123), params=adam_params)
-    assert res == Adam(lr=0.123, params=adam_params)
+    expected = Adam(lr=0.123, params=adam_params)
+    assert res.params == expected.params
+    assert res.lr == expected.lr
+    assert list(res.betas) == list(expected.betas)  # OmegaConf converts tuples to lists
+    assert res.eps == expected.eps
+    assert res.weight_decay == expected.weight_decay
+    assert res.amsgrad == expected.amsgrad
 
 
 def test_targetconf_deprecated() -> None:
@@ -784,3 +827,316 @@ def test_override_target(input_conf: Any, passthrough: Any, expected: Any) -> No
 def test_convert_target_to_string(input_: Any, expected: Any) -> None:
     _convert_container_targets_to_strings(input_)
     assert input_ == expected
+
+
+@pytest.mark.parametrize(  # type: ignore
+    "primitive,expected_primitive",
+    [
+        pytest.param(None, False, id="unspecified"),
+        pytest.param(ConvertMode.NONE, False, id="none"),
+        pytest.param(ConvertMode.PARTIAL, True, id="partial"),
+        pytest.param(ConvertMode.ALL, True, id="all"),
+    ],
+)
+@pytest.mark.parametrize(  # type: ignore
+    "input_,expected",
+    [
+        pytest.param(
+            {
+                "obj": {
+                    "_target_": "tests.AClass",
+                    "a": {"foo": "bar"},
+                    "b": OmegaConf.create({"foo": "bar"}),
+                    "c": [1, 2, 3],
+                    "d": OmegaConf.create([1, 2, 3]),
+                },
+            },
+            AClass(
+                a={"foo": "bar"},
+                b={"foo": "bar"},
+                c=[1, 2, 3],
+                d=[1, 2, 3],
+            ),
+            id="simple",
+        ),
+        pytest.param(
+            {
+                "value": 99,
+                "obj": {
+                    "_target_": "tests.AClass",
+                    "a": {"foo": "${value}"},
+                    "b": OmegaConf.create({"foo": "${value}"}),
+                    "c": [1, "${value}"],
+                    "d": OmegaConf.create([1, "${value}"]),
+                },
+            },
+            AClass(
+                a={"foo": 99},
+                b={"foo": 99},
+                c=[1, 99],
+                d=[1, 99],
+            ),
+            id="interpolation",
+        ),
+    ],
+)
+def test_convert_params_override(
+    primitive: Optional[bool], expected_primitive: bool, input_: Any, expected: Any
+) -> None:
+    input_cfg = OmegaConf.create(input_)
+    if primitive is not None:
+        ret = utils.instantiate(input_cfg.obj, _convert_=primitive)
+    else:
+        ret = utils.instantiate(input_cfg.obj)
+
+    expected_list: Any
+    expected_dict: Any
+    if expected_primitive is True:
+        expected_list = list
+        expected_dict = dict
+    else:
+        expected_list = ListConfig
+        expected_dict = DictConfig
+
+    assert ret == expected
+    assert isinstance(ret.a, expected_dict)
+    assert isinstance(ret.b, expected_dict)
+    assert isinstance(ret.c, expected_list)
+    assert isinstance(ret.d, expected_list)
+
+
+@pytest.mark.parametrize(  # type: ignore
+    "convert_mode",
+    [  # type: ignore
+        pytest.param(None, id="none"),
+        pytest.param(ConvertMode.NONE, id="none"),
+        pytest.param(ConvertMode.PARTIAL, id="partial"),
+        pytest.param(ConvertMode.ALL, id="all"),
+    ],
+)
+@pytest.mark.parametrize(  # type: ignore
+    "input_,expected",
+    [
+        pytest.param(
+            {
+                "value": 99,
+                "obj": {
+                    "_target_": "tests.SimpleClass",
+                    "a": {
+                        "_target_": "tests.SimpleClass",
+                        "a": {"foo": "${value}"},
+                        "b": [1, "${value}"],
+                    },
+                    "b": None,
+                },
+            },
+            SimpleClass(a={"foo": 99}, b=[1, 99]),
+            id="simple",
+        ),
+    ],
+)
+def test_convert_params(input_: Any, expected: Any, convert_mode: Any):
+    cfg = OmegaConf.create(input_)
+    ret = utils.instantiate(
+        cfg.obj,
+        **{
+            "a": {"_convert_": convert_mode},
+        },
+    )
+
+    if convert_mode in (ConvertMode.PARTIAL, ConvertMode.ALL):
+        assert isinstance(ret.a.a, dict)
+        assert isinstance(ret.a.b, list)
+    elif convert_mode in (None, ConvertMode.NONE):
+        assert isinstance(ret.a.a, DictConfig)
+        assert isinstance(ret.a.b, ListConfig)
+    else:
+        assert False
+
+    assert ret.a == expected
+
+
+@pytest.mark.parametrize(  # type: ignore
+    "convert",
+    [  # type: ignore
+        pytest.param(None, id="p=unspecified"),
+        pytest.param(ConvertMode.NONE, id="none"),
+        pytest.param(ConvertMode.PARTIAL, id="partial"),
+        pytest.param(ConvertMode.ALL, id="all"),
+    ],
+)
+@pytest.mark.parametrize(  # type: ignore
+    "input_,expected",
+    [
+        pytest.param(
+            {
+                "value": 99,
+                "obj": {
+                    "_target_": "tests.SimpleDataClass",
+                    "a": {
+                        "_target_": "tests.SimpleDataClass",
+                        "a": {"foo": "${value}"},
+                        "b": [1, "${value}"],
+                    },
+                    "b": None,
+                },
+            },
+            SimpleDataClass(a={"foo": 99}, b=[1, 99]),
+            id="simple",
+        ),
+    ],
+)
+def test_convert_params_with_dataclass_obj(input_: Any, expected: Any, convert: Any):
+    # Instantiated dataclasses are never converted to primitives.
+    # This is due to the ambiguity between dataclass as an object and as
+    # an input for creating a config object (Structured Configs)
+    # Even in ConvertMode.ALL - the underlying dict and list are converted to DictConfig and ListConfig
+    # because the parent DictConfig cannot hold unwrapped list and dict.
+    # This behavior is unique to instantiated objects that are dataclasses.
+
+    cfg = OmegaConf.create(input_)
+    kwargs = {"a": {"_convert_": convert}}
+    ret = utils.instantiate(cfg.obj, **kwargs)
+
+    assert ret.a == expected
+    # as close as it gets for dataclasses:
+    # Object is a DictConfig and the underlying type is the expected type.
+    assert isinstance(ret.a, DictConfig) and OmegaConf.get_type(ret.a) == type(expected)
+    # Since these are nested in , they are not getting converted. not the greatest behavior.
+    # Can potentially be solved if this is causing issues.
+    assert isinstance(ret.a.a, DictConfig)
+    assert isinstance(ret.a.b, ListConfig)
+
+
+@pytest.mark.parametrize(  # type: ignore
+    "input_,is_primitive,expected",
+    [
+        pytest.param(
+            {
+                "value": 99,
+                "obj": SimpleClassPrimitiveConf(
+                    a={"foo": "${value}"}, b=[1, "${value}"]
+                ),
+            },
+            True,
+            SimpleClass(a={"foo": 99}, b=[1, 99]),
+            id="primitive_specified_true",
+        ),
+        pytest.param(
+            {
+                "value": 99,
+                "obj": SimpleClassNonPrimitiveConf(
+                    a={"foo": "${value}"}, b=[1, "${value}"]
+                ),
+            },
+            False,
+            SimpleClass(a={"foo": 99}, b=[1, 99]),
+            id="primitive_specified_false",
+        ),
+        pytest.param(
+            {
+                "value": 99,
+                "obj": SimpleClassDefaultPrimitiveConf(
+                    a={"foo": "${value}"}, b=[1, "${value}"]
+                ),
+            },
+            False,
+            SimpleClass(a={"foo": 99}, b=[1, 99]),
+            id="default_behavior",
+        ),
+    ],
+)
+def test_convert_in_config(input_: Any, is_primitive: bool, expected: Any) -> None:
+    cfg = OmegaConf.create(input_)
+    ret = utils.instantiate(cfg.obj)
+    assert ret == expected
+
+    if is_primitive:
+        assert isinstance(ret.a, dict)
+        assert isinstance(ret.b, list)
+    else:
+        assert isinstance(ret.a, DictConfig)
+        assert isinstance(ret.b, ListConfig)
+
+
+def test_nested_dataclass_with_partial_convert() -> None:
+    # dict
+    cfg = OmegaConf.structured(NestedConf)
+    ret = utils.instantiate(cfg, _convert_="partial")
+    assert isinstance(ret.a, DictConfig) and OmegaConf.get_type(ret.a) == User
+    assert isinstance(ret.b, DictConfig) and OmegaConf.get_type(ret.b) == User
+    expected = SimpleClass(a=User(name="a", age=1), b=User(name="b", age=2))
+    assert ret == expected
+
+    # list
+    lst = [User(name="a", age=1)]
+    cfg = OmegaConf.structured(NestedConf(a=lst))
+    ret = utils.instantiate(cfg, _convert_="partial")
+    assert isinstance(ret.a, list) and OmegaConf.get_type(ret.a[0]) == User
+    assert isinstance(ret.b, DictConfig) and OmegaConf.get_type(ret.b) == User
+    expected = SimpleClass(a=lst, b=User(name="b", age=2))
+    assert ret == expected
+
+
+class DictValues:
+    def __init__(self, d: Dict[str, User]):
+        self.d = d
+
+
+class ListValues:
+    def __init__(self, d: List[User]):
+        self.d = d
+
+
+def test_dict_with_structured_config() -> None:
+    @dataclass
+    class DictValuesConf:
+        _target_: str = "tests.test_utils.DictValues"
+        d: Dict[str, User] = MISSING
+
+    schema = OmegaConf.structured(DictValuesConf)
+    cfg = OmegaConf.merge(schema, {"d": {"007": {"name": "Bond", "age": 7}}})
+    obj = utils.instantiate(config=cfg, _convert_="none")
+    assert OmegaConf.is_dict(obj.d)
+    assert OmegaConf.get_type(obj.d["007"]) == User
+
+    obj = utils.instantiate(config=cfg, _convert_="partial")
+    assert isinstance(obj.d, dict)
+    assert OmegaConf.get_type(obj.d["007"]) == User
+
+    obj = utils.instantiate(config=cfg, _convert_="all")
+    assert isinstance(obj.d, dict)
+    assert isinstance(obj.d["007"], dict)
+
+
+def test_list_with_structured_config() -> None:
+    @dataclass
+    class ListValuesConf:
+        _target_: str = "tests.test_utils.ListValues"
+        d: List[User] = MISSING
+
+    schema = OmegaConf.structured(ListValuesConf)
+    cfg = OmegaConf.merge(schema, {"d": [{"name": "Bond", "age": 7}]})
+
+    obj = utils.instantiate(config=cfg, _convert_="none")
+    assert isinstance(obj.d, ListConfig)
+    assert OmegaConf.get_type(obj.d[0]) == User
+
+    obj = utils.instantiate(config=cfg, _convert_="partial")
+    assert isinstance(obj.d, list)
+    assert OmegaConf.get_type(obj.d[0]) == User
+
+    obj = utils.instantiate(config=cfg, _convert_="all")
+    assert isinstance(obj.d, list)
+    assert isinstance(obj.d[0], dict)
+
+
+def test_list_as_none() -> None:
+    @dataclass
+    class ListValuesConf:
+        _target_: str = "tests.test_utils.ListValues"
+        d: Optional[List[User]] = None
+
+    cfg = OmegaConf.structured(ListValuesConf)
+    obj = utils.instantiate(config=cfg)
+    assert obj.d is None