Repository: facebookresearch/hydra

Issue:
[Bug] Cannot Instantiate a nested interpolated target multiple times
# 🐛 Bug
## Description
The recent push to master fixed the original bug referenced in #1410 and #1420. This is essentially the same, but attempting it twice. If the interpolation is instantiated more than once, it doesn't work
## Checklist
- [X] I checked on the latest version of Hydra
- [X] I created a minimal repro (See [this](https://stackoverflow.com/help/minimal-reproducible-example) for tips).

## To reproduce

#### config_test.yaml
```yaml
child:
    _target_: config_test.Child
parent:
    _target_: config_test.Parent
    child: ${child}
```

##### config_test.py
```python
import hydra
from hydra.utils import instantiate


class Child:
    pass

class Parent:
    def __init__(self, child):
        self.child = child

@hydra.main(config_name="config_test")
def run(cfg):
    parent = instantiate(cfg.parent)
    parent = instantiate(cfg.parent)

    print(parent.child)


if __name__ == "__main__":
    run()
```

```
{}
```

## Expected Behavior
The child object should be correctly instantiated both times.
## System information
- **Hydra Version** :  master (1.1.dev4)
- **Python version** :  3.8
- **Operating system** :  Ubuntu 20.04

## Additional context
Deep copying the dict object allows this to work


Discussion:

Resolving patch:
diff --git a/hydra/_internal/instantiate/_instantiate2.py b/hydra/_internal/instantiate/_instantiate2.py
--- a/hydra/_internal/instantiate/_instantiate2.py
+++ b/hydra/_internal/instantiate/_instantiate2.py
@@ -5,7 +5,7 @@
 from enum import Enum
 from typing import Any, Callable, Union
 
-from omegaconf import Container, OmegaConf
+from omegaconf import OmegaConf, SCMode
 from omegaconf._utils import is_structured_config
 
 from hydra._internal.utils import _locate
@@ -48,16 +48,25 @@ def _convert_target_to_string(t: Any) -> Any:
         return t
 
 
-def _convert_container_targets_to_strings(d: Any) -> None:
+def _prepare_input_dict(d: Any) -> Any:
+    res: Any
     if isinstance(d, dict):
-        if "_target_" in d:
-            d["_target_"] = _convert_target_to_string(d["_target_"])
-        for _, v in d.items():
-            _convert_container_targets_to_strings(v)
+        res = {}
+        for k, v in d.items():
+            if k == "_target_":
+                v = _convert_target_to_string(d["_target_"])
+            elif isinstance(v, (dict, list)):
+                v = _prepare_input_dict(v)
+            res[k] = v
     elif isinstance(d, list):
-        for e in d:
-            if isinstance(e, (list, dict)):
-                _convert_container_targets_to_strings(e)
+        res = []
+        for v in d:
+            if isinstance(v, (list, dict)):
+                v = _prepare_input_dict(v)
+            res.append(v)
+    else:
+        assert False
+    return res
 
 
 def _resolve_target(
@@ -71,7 +80,7 @@ def _resolve_target(
     if callable(target):
         return target
     raise InstantiationException(
-        f"Unsupported target type : {type(target)} (target = {target})"
+        f"Unsupported target type: {type(target).__name__}. value: {target}"
     )
 
 
@@ -113,8 +122,10 @@ def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any:
             f"\nA common problem is forgetting to annotate _target_ as a string : '_target_: str = ...'"
         )
 
-    _convert_container_targets_to_strings(config)
-    _convert_container_targets_to_strings(kwargs)
+    if isinstance(config, dict):
+        config = _prepare_input_dict(config)
+
+    kwargs = _prepare_input_dict(kwargs)
 
     # Structured Config always converted first to OmegaConf
     if is_structured_config(config) or isinstance(config, dict):
@@ -142,82 +153,77 @@ def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any:
         )
 
 
-def _convert_conf(config: Container, convert: Union[ConvertMode, str]) -> Any:
-    if convert == ConvertMode.ALL:
-        return OmegaConf.to_container(config, resolve=True)
-    if convert == ConvertMode.PARTIAL:
-        return OmegaConf.to_container(
-            config, resolve=True, exclude_structured_configs=True
-        )
-    else:
-        return config
+def _convert_node(node: Any, convert: Union[ConvertMode, str]) -> Any:
+    if OmegaConf.is_config(node):
+        if convert == ConvertMode.ALL:
+            node = OmegaConf.to_container(node, resolve=True)
+        elif convert == ConvertMode.PARTIAL:
+            node = OmegaConf.to_container(
+                node, resolve=True, structured_config_mode=SCMode.DICT_CONFIG
+            )
+    return node
 
 
 def instantiate_node(
-    config: Any,
+    node: Any,
     *args: Any,
     convert: Union[str, ConvertMode] = ConvertMode.NONE,
     recursive: bool = True,
 ) -> Any:
     # Return None if config is None
-    if config is None or OmegaConf.is_none(config):
+    if node is None or OmegaConf.is_none(node):
         return None
 
-    if OmegaConf.is_config(config):
-        # Turn off struct flag on node to enable popping fields from Structured Configs.
-        OmegaConf.set_struct(config, False)
-    else:
-        # probably a primitive node, return as is.
-        return config
+    if not OmegaConf.is_config(node):
+        return node
 
     # Override parent modes from config if specified
-    if OmegaConf.is_dict(config):
-        convert = config.pop(_Keys.CONVERT) if _Keys.CONVERT in config else convert
-        recursive = (
-            config.pop(_Keys.RECURSIVE) if _Keys.RECURSIVE in config else recursive
-        )
-        if not isinstance(recursive, bool):
-            raise TypeError(f"_recursive_ flag must be a bool, got {type(recursive)}")
+    if OmegaConf.is_dict(node):
+        # using getitem instead of get(key, default) because OmegaConf will raise an exception
+        # if the key type is incompatible on get.
+        convert = node[_Keys.CONVERT] if _Keys.CONVERT in node else convert
+        recursive = node[_Keys.RECURSIVE] if _Keys.RECURSIVE in node else recursive
+
+    if not isinstance(recursive, bool):
+        raise TypeError(f"_recursive_ flag must be a bool, got {type(recursive)}")
 
     # If OmegaConf list, create new list of instances if recursive
-    if OmegaConf.is_list(config):
-        if recursive:
-            items = [
-                instantiate_node(item, convert=convert, recursive=recursive)
-                for item in config._iter_ex(resolve=True)
-            ]
-
-            if convert in (ConvertMode.ALL, ConvertMode.PARTIAL):
-                # If ALL or PARTIAL, use plain list as container
-                return items
-            else:
-                # Otherwise, use ListConfig as container
-                lst = OmegaConf.create(items, flags={"allow_objects": True})
-                lst._set_parent(config)
-                return lst
+    if OmegaConf.is_list(node):
+        items = [
+            instantiate_node(item, convert=convert, recursive=recursive)
+            for item in node._iter_ex(resolve=True)
+        ]
+
+        if convert in (ConvertMode.ALL, ConvertMode.PARTIAL):
+            # If ALL or PARTIAL, use plain list as container
+            return items
+        else:
+            # Otherwise, use ListConfig as container
+            lst = OmegaConf.create(items, flags={"allow_objects": True})
+            lst._set_parent(node)
+            return lst
+
+    elif OmegaConf.is_dict(node):
+        exclude_keys = set({"_target_", "_convert_", "_recursive_"})
+        if _is_target(node):
+            target = _resolve_target(node.get(_Keys.TARGET))
+            kwargs = {}
+            for key, value in node.items_ex(resolve=True):
+                if key not in exclude_keys:
+                    if recursive:
+                        value = instantiate_node(
+                            value, convert=convert, recursive=recursive
+                        )
+                    kwargs[key] = _convert_node(value, convert)
+            return _call_target(target, *args, **kwargs)
         else:
-            return _convert_conf(config, convert)
-
-    elif OmegaConf.is_dict(config):
-        if _is_target(config):
-            target = _resolve_target(config.pop(_Keys.TARGET))
-            if recursive:
-                final_kwargs = {}
-                for key, value in config.items_ex(resolve=True):
-                    final_kwargs[key] = instantiate_node(
-                        value, convert=convert, recursive=recursive
-                    )
-            else:
-                final_kwargs = config
-            return _call_target(target, *args, **final_kwargs)
-
-        if recursive:
             # If ALL or PARTIAL non structured, instantiate in dict and resolve interpolations eagerly.
             if convert == ConvertMode.ALL or (
-                convert == ConvertMode.PARTIAL and config._metadata.object_type is None
+                convert == ConvertMode.PARTIAL and node._metadata.object_type is None
             ):
                 dict_items = {}
-                for key, value in config.items_ex(resolve=True):
+                for key, value in node.items_ex(resolve=True):
+                    # list items inherits recursive flag from the containing dict.
                     dict_items[key] = instantiate_node(
                         value, convert=convert, recursive=recursive
                     )
@@ -225,15 +231,13 @@ def instantiate_node(
             else:
                 # Otherwise use DictConfig and resolve interpolations lazily.
                 cfg = OmegaConf.create({}, flags={"allow_objects": True})
-                for key, value in config.items_ex(resolve=False):
+                for key, value in node.items_ex(resolve=False):
                     cfg[key] = instantiate_node(
                         value, convert=convert, recursive=recursive
                     )
-                cfg._set_parent(config)
-                cfg._metadata.object_type = config._metadata.object_type
+                cfg._set_parent(node)
+                cfg._metadata.object_type = node._metadata.object_type
                 return cfg
-        else:
-            return _convert_conf(config, convert)
 
     else:
-        assert False, f"Unexpected config type : {type(config).__name__}"
+        assert False, f"Unexpected config type : {type(node).__name__}"


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -12,9 +12,6 @@
 import hydra._internal.instantiate._instantiate2
 import tests
 from hydra import utils
-from hydra._internal.instantiate._instantiate2 import (
-    _convert_container_targets_to_strings,
-)
 from hydra.conf import HydraConf, RuntimeConf
 from hydra.core.hydra_config import HydraConfig
 from hydra.errors import InstantiationException
@@ -49,7 +46,6 @@
     UntypedPassthroughClass,
     UntypedPassthroughConf,
     User,
-    module_function,
     recisinstance,
 )
 
@@ -66,6 +62,23 @@ def instantiate_func(request: Any) -> Any:
     return request.param
 
 
+@fixture(
+    params=[
+        lambda cfg: copy.deepcopy(cfg),
+        lambda cfg: OmegaConf.create(cfg),
+    ],
+    ids=[
+        "dict",
+        "dict_config",
+    ],
+)
+def config(request: Any, src: Any) -> Any:
+    config = request.param(src)
+    cfg_copy = copy.deepcopy(config)
+    yield config
+    assert config == cfg_copy
+
+
 @mark.parametrize("path,expected_type", [("tests.AClass", AClass)])
 def test_get_class(path: str, expected_type: type) -> None:
     assert utils.get_class(path) == expected_type
@@ -75,7 +88,7 @@ def test_get_class(path: str, expected_type: type) -> None:
     "recursive", [param(False, id="not_recursive"), param(True, id="recursive")]
 )
 @mark.parametrize(
-    "input_conf, passthrough, expected",
+    "src, passthrough, expected",
     [
         param(
             {"_target_": "tests.AClass", "a": 10, "b": 20, "c": 30, "d": 40},
@@ -184,21 +197,14 @@ def test_get_class(path: str, expected_type: type) -> None:
 )
 def test_class_instantiate(
     instantiate_func: Any,
-    input_conf: Any,
+    config: Any,
     passthrough: Dict[str, Any],
     expected: Any,
     recursive: bool,
 ) -> Any:
-    def test(conf: Any) -> None:
-        passthrough["_recursive_"] = recursive
-        conf_copy = copy.deepcopy(conf)
-        obj = instantiate_func(conf, **passthrough)
-        assert obj == expected
-        # make sure config is not modified by instantiate
-        assert conf == conf_copy
-
-    test(input_conf)
-    test(OmegaConf.create(input_conf))
+    passthrough["_recursive_"] = recursive
+    obj = instantiate_func(config, **passthrough)
+    assert obj == expected
 
 
 @mark.parametrize(
@@ -233,9 +239,11 @@ def test(conf: Any) -> None:
 def test_interpolation_accessing_parent(
     instantiate_func: Any, input_conf: Any, passthrough: Dict[str, Any], expected: Any
 ) -> Any:
+    cfg_copy = OmegaConf.create(input_conf)
     input_conf = OmegaConf.create(input_conf)
     obj = instantiate_func(input_conf.node, **passthrough)
     assert obj == expected
+    assert input_conf == cfg_copy
 
 
 def test_interpolation_is_live_in_instantiated_object(instantiate_func: Any) -> None:
@@ -266,15 +274,12 @@ def test_interpolation_is_live_in_instantiated_object(instantiate_func: Any) ->
     assert obj.b.x == 3.14
 
 
-def test_class_instantiate_omegaconf_node(instantiate_func: Any) -> Any:
-    conf = OmegaConf.structured(
-        {
-            "_target_": "tests.AClass",
-            "b": 200,
-            "c": {"x": 10, "y": "${b}"},
-        }
-    )
-    obj = instantiate_func(conf, a=10, d=AnotherClass(99))
+@mark.parametrize(
+    "src",
+    [({"_target_": "tests.AClass", "b": 200, "c": {"x": 10, "y": "${b}"}})],
+)
+def test_class_instantiate_omegaconf_node(instantiate_func: Any, config: Any) -> Any:
+    obj = instantiate_func(config, a=10, d=AnotherClass(99))
     assert obj == AClass(a=10, b=200, c={"x": 10, "y": 200}, d=AnotherClass(99))
     assert OmegaConf.is_config(obj.c)
 
@@ -330,13 +335,14 @@ def test_to_absolute_path_without_hydra(
     assert utils.to_absolute_path(path) == expected
 
 
-def test_instantiate_adam(instantiate_func: Any) -> None:
+@mark.parametrize("src", [{"_target_": "tests.Adam"}])
+def test_instantiate_adam(instantiate_func: Any, config: Any) -> None:
     with raises(Exception):
         # can't instantiate without passing params
-        instantiate_func({"_target_": "tests.Adam"})
+        instantiate_func(config)
 
     adam_params = Parameters([1, 2, 3])
-    res = instantiate_func({"_target_": "tests.Adam"}, params=adam_params)
+    res = instantiate_func(config, params=adam_params)
     assert res == Adam(params=adam_params)
 
 
@@ -406,7 +412,7 @@ def test_pass_extra_variables(instantiate_func: Any) -> None:
 
 
 @mark.parametrize(
-    "cfg, passthrough, expected",
+    "src, passthrough, expected",
     [
         # direct
         param(
@@ -592,16 +598,16 @@ def test_pass_extra_variables(instantiate_func: Any) -> None:
 )
 def test_recursive_instantiation(
     instantiate_func: Any,
-    cfg: Any,
+    config: Any,
     passthrough: Dict[str, Any],
     expected: Any,
 ) -> None:
-    obj = instantiate_func(cfg, **passthrough)
+    obj = instantiate_func(config, **passthrough)
     assert obj == expected
 
 
 @mark.parametrize(
-    "cfg, passthrough, expected",
+    ("src", "passthrough", "expected"),
     [
         param(
             {
@@ -738,16 +744,16 @@ def test_recursive_instantiation(
 )
 def test_recursive_override(
     instantiate_func: Any,
-    cfg: Any,
+    config: Any,
     passthrough: Any,
     expected: Any,
 ) -> None:
-    obj = instantiate_func(cfg, **passthrough)
+    obj = instantiate_func(config, **passthrough)
     assert obj == expected
 
 
 @mark.parametrize(
-    "input_conf, passthrough, expected",
+    ("src", "passthrough", "expected"),
     [
         param(
             {"_target_": "tests.AClass", "a": 10, "b": 20, "c": 30, "d": 40},
@@ -813,76 +819,52 @@ def test_recursive_override(
     ],
 )
 def test_override_target(
-    instantiate_func: Any, input_conf: Any, passthrough: Any, expected: Any
+    instantiate_func: Any, config: Any, passthrough: Any, expected: Any
 ) -> None:
-    conf_copy = copy.deepcopy(input_conf)
-    obj = instantiate_func(input_conf, **passthrough)
+    obj = instantiate_func(config, **passthrough)
     assert obj == expected
-    # make sure config is not modified by instantiate
-    assert input_conf == conf_copy
 
 
 @mark.parametrize(
-    "input_, expected",
+    "config, passthrough, expected",
     [
-        param({"a": 10}, {"a": 10}),
-        param([1, 2, 3], [1, 2, 3]),
-        param({"_target_": "abc", "a": 10}, {"_target_": "abc", "a": 10}),
-        param({"_target_": AClass, "a": 10}, {"_target_": "tests.AClass", "a": 10}),
-        param(
-            {"_target_": AClass.static_method, "a": 10},
-            {"_target_": "tests.AClass.static_method", "a": 10},
-        ),
-        param(
-            {"_target_": ASubclass.class_method, "a": 10},
-            {"_target_": "tests.ASubclass.class_method", "a": 10},
-        ),
-        param(
-            {"_target_": module_function, "a": 10},
-            {"_target_": "tests.module_function", "a": 10},
-        ),
         param(
-            {
-                "_target_": AClass,
-                "a": {"_target_": AClass, "b": 10},
-            },
-            {
-                "_target_": "tests.AClass",
-                "a": {"_target_": "tests.AClass", "b": 10},
-            },
-        ),
-        param(
-            [1, {"_target_": AClass, "a": 10}],
-            [1, {"_target_": "tests.AClass", "a": 10}],
+            {"_target_": tests.AnotherClass, "x": 10},
+            {},
+            AnotherClass(10),
+            id="class_in_config_dict",
         ),
     ],
 )
-def test_convert_target_to_string(input_: Any, expected: Any) -> None:
-    _convert_container_targets_to_strings(input_)
-    assert input_ == expected
+def test_instantiate_from_class_in_dict(
+    instantiate_func: Any, config: Any, passthrough: Any, expected: Any
+) -> None:
+    config_copy = copy.deepcopy(config)
+    assert instantiate_func(config, **passthrough) == expected
+    assert config == config_copy
 
 
 @mark.parametrize(
-    "input_conf, passthrough, expected",
+    "config, passthrough, expected",
     [
         param(
-            {"_target_": tests.AClass, "a": 10, "b": 20, "c": 30, "d": 40},
+            OmegaConf.create({"_target_": tests.AClass}),
             {},
             AClass(10, 20, 30, 40),
             id="class_in_config_dict",
         ),
-        param(
-            {"_target_": tests.AClass, "a": 10, "b": 20, "c": 30, "d": 40},
-            {"_target_": tests.BClass},
-            BClass(10, 20, 30, 40),
-            id="class_target_in_passthrough",
-        ),
     ],
 )
-def test_instantiate_from_class_in_dict(
-    instantiate_func: Any, input_conf: Any, passthrough: Any, expected: Any
+def test_instantiate_from_dataclass_in_dict_fails(
+    instantiate_func: Any, config: Any, passthrough: Any, expected: Any
 ) -> None:
-    assert instantiate_func(input_conf, **passthrough) == expected
+    # not the best error, but it will get the user to check their input config.
+    msg = "Unsupported target type: DictConfig. value: {'a': '???', 'b': '???', 'c': '???', 'd': 'default_value'}"
+    with raises(
+        InstantiationException,
+        match=re.escape(msg),
+    ):
+        assert instantiate_func(config, **passthrough) == expected
 
 
 @mark.parametrize(
@@ -1017,8 +999,29 @@ def test_convert_params(
     assert ret.a == expected
 
 
+@mark.parametrize("nested_recursive", [True, False])
+def test_convert_and_recursive_node(
+    instantiate_func: Any, nested_recursive: bool
+) -> None:
+    cfg = {
+        "_target_": "tests.SimpleClass",
+        "a": {
+            "_target_": "tests.SimpleClass",
+            "_convert_": "all",
+            "_recursive_": nested_recursive,
+            "a": {},
+            "b": [],
+        },
+        "b": None,
+    }
+
+    obj = instantiate_func(cfg)
+    assert isinstance(obj.a.a, dict)
+    assert isinstance(obj.a.b, list)
+
+
 @mark.parametrize(
-    "config,expected",
+    "src,expected",
     [
         param(
             {
@@ -1161,15 +1164,20 @@ def test_instantiated_regular_class_container_types_partial2(
     assert OmegaConf.get_type(ret.a[1]) is User
 
 
+@mark.parametrize(
+    "src",
+    [
+        {
+            "_target_": "tests.SimpleClass",
+            "a": {"_target_": "tests.SimpleClass", "a": {}, "b": User},
+            "b": None,
+        }
+    ],
+)
 def test_instantiated_regular_class_container_types_partial__recursive(
-    instantiate_func: Any,
+    instantiate_func: Any, config: Any
 ) -> None:
-    cfg = {
-        "_target_": "tests.SimpleClass",
-        "a": {"_target_": "tests.SimpleClass", "a": {}, "b": User},
-        "b": None,
-    }
-    ret = instantiate_func(cfg, _convert_=ConvertMode.PARTIAL)
+    ret = instantiate_func(config, _convert_=ConvertMode.PARTIAL)
     assert isinstance(ret.a, SimpleClass)
     assert isinstance(ret.a.a, dict)
     assert isinstance(ret.a.b, DictConfig)