Repository: omry/omegaconf

Issue:
Variables not resolved when accessing through iterator
When accessing `ListConfig` through the iterator, the variables are not resolved. A similar issue is #21 regarding DictConfig.

config.yaml:
```
var1: 1
var2: 2
var3:
  - ${var1}
  - ${var2}
```

test.py:
```
import hydra

from omegaconf import DictConfig


@hydra.main('config.yaml')
def main(cfg: DictConfig) -> None:
    print(cfg.var1)
    print(cfg.var2)
    print(cfg.var3)
    print([x for x in cfg.var3])
    print([cfg.var3[i] for i in range(len(cfg.var3))])
    print(next(iter(cfg.var3)))

    return


if __name__ == '__main__':
    main()
```

Output:
```
$ python test.py
1
2
['${var1}', '${var2}']
['${var1}', '${var2}']
[1, 2]
${var1}
```

Hydra version: facebookresearch/hydra@b9df40a2
Omegaconf version: 4c5be9a

Discussion:

Patch:
diff --git a/omegaconf/_utils.py b/omegaconf/_utils.py
index 1a5b762..5879d6d 100644
--- a/omegaconf/_utils.py
+++ b/omegaconf/_utils.py
@@ -477,19 +477,12 @@ def format_and_raise(
         KEY_TYPE=f"{type(key).__name__}",
     )
 
-    full_backtrace = "OC_CAUSE" in os.environ and os.environ["OC_CAUSE"] == "1"
     template = """$MSG
 \tfull_key: $FULL_KEY
 \treference_type=$REF_TYPE
 \tobject_type=$OBJECT_TYPE
 """
 
-    if not full_backtrace:
-        template += """
-
-Set env OC_CAUSE=1 to include causing exception
-    """
-
     s = string.Template(template=template)
 
     message = s.substitute(
@@ -515,6 +508,9 @@ Set env OC_CAUSE=1 to include causing exception
         ex.ref_type = ref_type
         ex.ref_type_str = ref_type_str
 
+    # Set the environment variable OC_CAUSE=1 to get a stacktrace that includes the
+    # causing exception.
+    full_backtrace = "OC_CAUSE" in os.environ and os.environ["OC_CAUSE"] == "1"
     if full_backtrace:
         ex.__cause__ = cause
     else:
diff --git a/omegaconf/base.py b/omegaconf/base.py
index fd26d11..f1916d5 100644
--- a/omegaconf/base.py
+++ b/omegaconf/base.py
@@ -115,7 +115,7 @@ class Node(ABC):
             key = self._key()
             if value_kind == ValueKind.INTERPOLATION:
                 assert parent is not None
-                v = parent._resolve_interpolation(
+                v = parent._resolve_simple_interpolation(
                     key=key,
                     inter_type=match.group(1),
                     inter_key=match.group(2),
@@ -125,7 +125,7 @@ class Node(ABC):
                 return v
             elif value_kind == ValueKind.STR_INTERPOLATION:
                 assert parent is not None
-                ret = parent._resolve_str_interpolation(
+                ret = parent._resolve_interpolation(
                     key=key,
                     value=self,
                     throw_on_missing=throw_on_missing,
@@ -279,7 +279,7 @@ class Container(Node):
         )
         if value is None:
             return root, last_key, value
-        value = root._resolve_str_interpolation(
+        value = root._resolve_interpolation(
             key=last_key,
             value=value,
             throw_on_missing=False,
@@ -287,7 +287,7 @@ class Container(Node):
         )
         return root, last_key, value
 
-    def _resolve_interpolation(
+    def _resolve_simple_interpolation(
         self,
         key: Any,
         inter_type: str,
@@ -309,7 +309,8 @@ class Container(Node):
                 throw_on_resolution_failure=throw_on_resolution_failure,
             )
 
-            if parent is None or (value is None and last_key not in parent):  # type: ignore
+            # if parent is None or (value is None and last_key not in parent):  # type: ignore
+            if parent is None or value is None:
                 if throw_on_resolution_failure:
                     raise ConfigKeyError(
                         f"{inter_type} interpolation key '{inter_key}' not found"
@@ -341,7 +342,7 @@ class Container(Node):
                 else:
                     return None
 
-    def _resolve_str_interpolation(
+    def _resolve_interpolation(
         self,
         key: Any,
         value: "Node",
@@ -357,7 +358,7 @@ class Container(Node):
         if value_kind == ValueKind.INTERPOLATION:
             # simple interpolation, inherit type
             match = match_list[0]
-            return self._resolve_interpolation(
+            return self._resolve_simple_interpolation(
                 key=key,
                 inter_type=match.group(1),
                 inter_key=match.group(2),
@@ -371,7 +372,7 @@ class Container(Node):
             new = ""
             last_index = 0
             for match in match_list:
-                new_val = self._resolve_interpolation(
+                new_val = self._resolve_simple_interpolation(
                     key=key,
                     inter_type=match.group(1),
                     inter_key=match.group(2),
diff --git a/omegaconf/basecontainer.py b/omegaconf/basecontainer.py
index 47baea4..7acbdb9 100644
--- a/omegaconf/basecontainer.py
+++ b/omegaconf/basecontainer.py
@@ -56,7 +56,7 @@ class BaseContainer(Container, ABC):
         if has_default and (value is None or is_mandatory_missing(value)):
             return default_value
 
-        resolved = self._resolve_str_interpolation(
+        resolved = self._resolve_interpolation(
             key=key,
             value=value,
             throw_on_missing=not has_default,
@@ -179,9 +179,7 @@ class BaseContainer(Container, ABC):
             return retdict
         elif isinstance(conf, ListConfig):
             retlist: List[Any] = []
-            for index, item in enumerate(conf):
-                if resolve:
-                    item = conf[index]
+            for index, item in enumerate(conf._iter_ex(resolve=resolve)):
                 item = convert(item)
                 if isinstance(item, Container):
                     item = BaseContainer._to_content(
diff --git a/omegaconf/listconfig.py b/omegaconf/listconfig.py
index cb21c94..7a57951 100644
--- a/omegaconf/listconfig.py
+++ b/omegaconf/listconfig.py
@@ -16,6 +16,7 @@ from typing import (
 
 from ._utils import (
     ValueKind,
+    _get_value,
     format_and_raise,
     get_list_element_type,
     get_value_kind,
@@ -32,7 +33,6 @@ from .errors import (
     ReadonlyConfigError,
     ValidationError,
 )
-from .nodes import ValueNode
 
 
 class ListConfig(BaseContainer, MutableSequence[Any]):
@@ -388,6 +388,9 @@ class ListConfig(BaseContainer, MutableSequence[Any]):
         return hash(str(self))
 
     def __iter__(self) -> Iterator[Any]:
+        return self._iter_ex(resolve=True)
+
+    def _iter_ex(self, resolve: bool) -> Iterator[Any]:
         try:
             if self._is_none():
                 raise TypeError("Cannot iterate on ListConfig object representing None")
@@ -395,21 +398,24 @@ class ListConfig(BaseContainer, MutableSequence[Any]):
                 raise MissingMandatoryValue("Cannot iterate on a missing ListConfig")
 
             class MyItems(Iterator[Any]):
-                def __init__(self, lst: List[Any]) -> None:
+                def __init__(self, lst: ListConfig) -> None:
                     self.lst = lst
-                    self.iterator = iter(lst)
+                    self.index = 0
 
                 def __next__(self) -> Any:
-                    return self.next()
-
-                def next(self) -> Any:
-                    v = next(self.iterator)
-                    if isinstance(v, ValueNode):
-                        v = v._value()
+                    if self.index == len(self.lst):
+                        raise StopIteration()
+                    if resolve:
+                        v = self.lst[self.index]
+                    else:
+                        v = self.lst.__dict__["_content"][self.index]
+                        if v is not None:
+                            v = _get_value(v)
+                    self.index = self.index + 1
                     return v
 
             assert isinstance(self.__dict__["_content"], list)
-            return MyItems(self.__dict__["_content"])
+            return MyItems(self)
         except (ReadonlyConfigError, TypeError, MissingMandatoryValue) as e:
             self._format_and_raise(key=None, value=None, cause=e)
             assert False
@@ -453,8 +459,12 @@ class ListConfig(BaseContainer, MutableSequence[Any]):
             if isinstance(value, ListConfig):
                 self.__dict__["_metadata"] = copy.deepcopy(value._metadata)
                 self.__dict__["_metadata"].flags = {}
-            for item in value:
-                self.append(item)
+                for item in value._iter_ex(resolve=False):
+                    self.append(item)
+                self.__dict__["_metadata"].flags = copy.deepcopy(value._metadata.flags)
+            elif is_primitive_list(value):
+                for item in value:
+                    self.append(item)
 
             if isinstance(value, ListConfig):
                 self.__dict__["_metadata"].flags = value._metadata.flags


Test patch:
diff --git a/tests/test_base_config.py b/tests/test_base_config.py
index 3b8ad4b..a0c6a0d 100644
--- a/tests/test_base_config.py
+++ b/tests/test_base_config.py
@@ -434,7 +434,7 @@ def test_not_implemented() -> None:
 def test_resolve_str_interpolation(query: str, result: Any) -> None:
     cfg = OmegaConf.create({"foo": 10, "bar": "${foo}"})
     assert (
-        cfg._resolve_str_interpolation(
+        cfg._resolve_interpolation(
             key=None,
             value=StringNode(value=query),
             throw_on_missing=False,
diff --git a/tests/test_basic_ops_dict.py b/tests/test_basic_ops_dict.py
index b8ee516..8145d7b 100644
--- a/tests/test_basic_ops_dict.py
+++ b/tests/test_basic_ops_dict.py
@@ -237,13 +237,23 @@ def test_pickle_get_root() -> None:
 
 
 def test_iterate_dictionary() -> None:
-    c = OmegaConf.create(dict(a=1, b=2))
+    c = OmegaConf.create({"a": 1, "b": 2})
     m2 = {}
     for key in c:
         m2[key] = c[key]
     assert m2 == c
 
 
+def test_iterate_dict_with_interpolation() -> None:
+    c = OmegaConf.create({"a": "${b}", "b": 2})
+    expected = [("a", 2), ("b", 2)]
+    i = 0
+    for k, v in c.items():
+        assert k == expected[i][0]
+        assert v == expected[i][1]
+        i = i + 1
+
+
 @pytest.mark.parametrize(  # type: ignore
     "cfg, key, default_, expected",
     [
diff --git a/tests/test_basic_ops_list.py b/tests/test_basic_ops_list.py
index 1566ce3..5af36e3 100644
--- a/tests/test_basic_ops_list.py
+++ b/tests/test_basic_ops_list.py
@@ -6,6 +6,7 @@ import pytest
 
 from omegaconf import AnyNode, ListConfig, OmegaConf
 from omegaconf.errors import (
+    ConfigKeyError,
     ConfigTypeError,
     KeyValidationError,
     MissingMandatoryValue,
@@ -57,11 +58,11 @@ def test_list_get_with_default() -> None:
 
 
 @pytest.mark.parametrize(  # type: ignore
-    "input_, list_key",
+    "input_, expected, list_key",
     [
-        ([1, 2], None),
-        (["${1}", 2], None),
-        (
+        pytest.param([1, 2], [1, 2], None, id="simple"),
+        pytest.param(["${1}", 2], [2, 2], None, id="interpolation"),
+        pytest.param(
             {
                 "defaults": [
                     {"optimizer": "adam"},
@@ -69,25 +70,36 @@ def test_list_get_with_default() -> None:
                     {"foo": "${defaults.0.optimizer}_${defaults.1.dataset}"},
                 ]
             },
+            [{"optimizer": "adam"}, {"dataset": "imagenet"}, {"foo": "adam_imagenet"}],
             "defaults",
+            id="str_interpolation",
         ),
-        ([1, 2, "${missing}"], None),
     ],
 )
-def test_iterate_list(input_: Any, list_key: str) -> None:
+def test_iterate_list(input_: Any, expected: Any, list_key: str) -> None:
     c = OmegaConf.create(input_)
     if list_key is not None:
         lst = c.get(list_key)
     else:
         lst = c
     items = [x for x in lst]
-    assert items == lst
+    assert items == expected
+
+
+def test_iterate_list_with_missing_interpolation() -> None:
+    c = OmegaConf.create([1, "${10}"])
+    itr = iter(c)
+    assert 1 == next(itr)
+    with pytest.raises(ConfigKeyError):
+        next(itr)
 
 
-def test_iterate_interpolated_list() -> None:
-    cfg = OmegaConf.create({"inter": "${list}", "list": [1, 2, 3]})
-    items = [x for x in cfg.inter]
-    assert items == [1, 2, 3]
+def test_iterate_list_with_missing() -> None:
+    c = OmegaConf.create([1, "???"])
+    itr = iter(c)
+    assert 1 == next(itr)
+    with pytest.raises(MissingMandatoryValue):
+        next(itr)
 
 
 def test_items_with_interpolation() -> None:
@@ -198,7 +210,8 @@ def test_list_append() -> None:
 def test_pretty_without_resolve() -> None:
     c = OmegaConf.create([100, "${0}"])
     # without resolve, references are preserved
-    c2 = OmegaConf.create(c.pretty(resolve=False))
+    yaml_str = c.pretty(resolve=False)
+    c2 = OmegaConf.create(yaml_str)
     assert isinstance(c2, ListConfig)
     c2[0] = 1000
     assert c2[1] == 1000