Repository: childsish/dynamic-yaml

Issue:
Problems with recursive=True
I have the following .yml file:
```
prefix: /opt/ml

input_path: '{prefix}/input'
model_path: '{prefix}/model'
output_path: '{prefix}/output'

failure_path: '{output_path}/failure'

# Training
training:
  training_data_path: '{input_path}/data/training'
#  mia_path: '{training_data_path}/MiaPds.json'
  metro_path: '{training.training_data_path}/MetroPds.json'
  test_size: 0.8
  random_seed: 42
```
which I'm reading in using the following snippet:
```
with open(CONFIG_PATH) as inp:
    config = dynamic_yaml.load(inp, recursive=True)
```
if I enter here with a debugger and type in just `config` I see:
```
{'prefix': '/opt/ml',
 'input_path': '/opt/ml/input',
 'model_path': '/opt/ml/model',
 'output_path': '/opt/ml/output',
 'failure_path': '/opt/ml/output/failure',
 'training': {'training_data_path': '/opt/ml/input/data/training',
  'metro_path': '/opt/ml/input/data/training/MetroPds.json',
  'test_size': 0.8,
  'random_seed': 42}}
```
which is awesome, that was my intention.

But if I type in `print(config)` I get:
```
{'prefix': '/opt/ml', 'input_path': '{prefix}/input', 'model_path': '{prefix}/model', 'output_path': '{prefix}/output', 'failure_path': '{output_path}/failure', 'training': {'training_data_path': '{input_path}/data/training', 'metro_path': '{training.training_data_path}/MetroPds.json', 'test_size': 0.8, 'random_seed': 42}}
```
(I know it's long, I just wanted to copy it as-is.)

And now the fun part:
* if I ask for `print(config['training']['metro_path'])` I get what I expected: `/opt/ml/input/data/training/MetroPds.json`
* but when I pass on `config` to a function `train` like so:
   ```
        train(**config['training'])
        # where the train function's signature is:
        # def train(metro_path, def train(metro_path, training_data_path, test_size, random_seed, **kwargs):
   ```
    within that function I ask for `metro_path` I get:
    ```
        '{training.training_data_path}/MetroPds.json'
    ```
     which is not what I was expecting :\

I would appreciate any help / hints / suggestions / whatever ;)

Discussion:

Patch:
diff --git a/dynamic_yaml/__init__.py b/dynamic_yaml/__init__.py
index 8e77e89..1b505aa 100644
--- a/dynamic_yaml/__init__.py
+++ b/dynamic_yaml/__init__.py
@@ -27,5 +27,5 @@ add_wrappers(DynamicYamlLoader)
 
 def load(stream, loader=DynamicYamlLoader, recursive=False):
     result = yaml.load(stream, Loader=loader)
-    result.set_as_root(recursive=recursive)
+    result._set_as_root(recursive=recursive)
     return result
diff --git a/dynamic_yaml/yaml_wrappers.py b/dynamic_yaml/yaml_wrappers.py
index 4ed1322..14c5ecc 100644
--- a/dynamic_yaml/yaml_wrappers.py
+++ b/dynamic_yaml/yaml_wrappers.py
@@ -1,81 +1,85 @@
 import re
 
-from collections import Sequence, Mapping
+from collections.abc import Collection, Mapping, MutableMapping, MutableSequence, Sequence
 
 
-class YamlDict(dict):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        super().__setattr__('_root', self)
+class DynamicYamlObject(Collection):
+    def __init__(self):
+        super().__setattr__('_collection', {})
+        super().__setattr__('_root', {})
         super().__setattr__('_recursive', False)
         super().__setattr__('_regx', re.compile('.*{.*}.*'))
 
-    def __getattr__(self, key):
-        if key in self:
-            return self[key]
-        return super().__getattribute__(key)
+    def __iter__(self):
+        yield from super().__getattribute__('_collection')
 
-    def __setattr__(self, key, value):
-        self[key] = value
+    def __contains__(self, value: object) -> bool:
+        return value in super().__getattribute__('_collection')
+
+    def __len__(self) -> int:
+        return len(super().__getattribute__('_collection'))
 
     def __getitem__(self, key):
-        v = super().__getitem__(key)
-        if isinstance(v, str):
-            while self._regx.match(v) is not None:
-                v = v.format(**super().__getattribute__('_root'))
-                if not self._recursive:
+        value = super().__getattribute__('_collection')[key]
+        if isinstance(value, str):
+            while super().__getattribute__('_regx').match(value) is not None:
+                value = value.format(**super().__getattribute__('_root')._collection)
+                if not super().__getattribute__('_recursive'):
                     break
-        return v
+        return value
 
     def __setitem__(self, key, value):
         if isinstance(value, Mapping) and not isinstance(value, YamlDict):
             value = YamlDict(value)
-            value.set_as_root(super().__getattribute__('_root'))
+            value._set_as_root(super().__getattribute__('_root'))
         elif isinstance(value, Sequence) and not isinstance(value, (str, YamlList)):
             value = YamlList(value)
-            value.set_as_root(super().__getattribute__('_root'))
-        super().__setitem__(key, value)
+            value._set_as_root(super().__getattribute__('_root'))
+        super().__getattribute__('_collection')[key] = value
+
+    def __delitem__(self, key):
+        del super().__getattribute__('_collection')[key]
 
-    def set_as_root(self, root=None, *, recursive=False):
+    def _values(self):
+        raise NotImplementedError('This method must be implemented in the child class')
+
+    def _set_as_root(self, root=None, *, recursive=False):
         super().__setattr__('_recursive', recursive)
         if root is not None:
             super().__setattr__('_root', root)
-        for k, v in self.items():
-            if hasattr(v, 'set_as_root'):
-                v.set_as_root(super().__getattribute__('_root'), recursive=recursive)
+        for value in self._values():
+            if isinstance(value, DynamicYamlObject):
+                value._set_as_root(super().__getattribute__('_root'), recursive=recursive)
+
+
+class YamlDict(DynamicYamlObject, MutableMapping):
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+        super().__setattr__('_collection', dict(*args, **kwargs))
+        super().__setattr__('_root', self)
 
+    def __getattr__(self, key):
+        if key in self:
+            return self[key]
+        return super().__getattribute__(key)
 
-class YamlList(list):
+    def __setattr__(self, key, value):
+        self[key] = value
+
+    def _values(self):
+        return super().__getattribute__('_collection').values()
+
+
+class YamlList(DynamicYamlObject, MutableSequence):
     ROOT_NAME = 'root'
 
     def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        super().__setattr__('_root', {YamlList.ROOT_NAME: self})
-        super().__setattr__('_recursive', False)
-        super().__setattr__('_regx', re.compile('.*{.*}.*'))
-
-    def __getitem__(self, key):
-        v = super().__getitem__(key)
-        if isinstance(v, str):
-            while self._regx.match(v) is not None:
-                v = v.format(**super().__getattribute__('_root'))
-                if not self._recursive:
-                    break
-        return v
+        super().__init__()
+        super().__setattr__('_collection', list(*args, **kwargs))
+        super().__setattr__('_root', YamlDict([(YamlList.ROOT_NAME, self)]))
 
-    def __setitem__(self, key, value):
-        if isinstance(value, Mapping) and not isinstance(value, YamlDict):
-            value = YamlDict(value)
-            value.set_as_root(super().__getattribute__('_root'))
-        elif isinstance(value, Sequence) and not isinstance(value, (str, YamlList)):
-            value = YamlList(value)
-            value.set_as_root(super().__getattribute__('_root'))
-        super().__setitem__(key, value)
+    def insert(self, index: int, object):
+        super().__getattribute__('_collection').insert(index, object)
 
-    def set_as_root(self, root=None, *, recursive=False):
-        super().__setattr__('_recursive', recursive)
-        if root is not None:
-            super().__setattr__('_root', root)
-        for v in self:
-            if hasattr(v, 'set_as_root'):
-                v.set_as_root(super().__getattribute__('_root'), recursive=recursive)
+    def _values(self):
+        return super().__getattribute__('_collection')


Test patch:
diff --git a/tests/test_yaml_dict.py b/tests/test_dynamic_yaml.py
similarity index 75%
rename from tests/test_yaml_dict.py
rename to tests/test_dynamic_yaml.py
index 3d7990d..c666952 100644
--- a/tests/test_yaml_dict.py
+++ b/tests/test_dynamic_yaml.py
@@ -4,7 +4,29 @@ from unittest import TestCase, main
 from dynamic_yaml import load
 
 
-class TestYamlDict(TestCase):
+class TestDynamicYaml(TestCase):
+    def test_list(self):
+        config = '''
+        - a
+        - b
+        - c
+        '''
+        res = load(config)
+        self.assertEqual('a', res[0])
+        self.assertEqual('b', res[1])
+        self.assertEqual('c', res[2])
+
+    def test_list_resolution(self):
+        config = '''
+        - a
+        - b
+        - '{root[0]}'
+        '''
+        res = load(config)
+        self.assertEqual('a', res[0])
+        self.assertEqual('b', res[1])
+        self.assertEqual('a', res[2])
+
     def test_dict(self):
         config = '''
         a: 1
@@ -12,7 +34,9 @@ class TestYamlDict(TestCase):
         c: a
         '''
         res = load(config)
-        self.assertDictEqual({'a': 1, 'b': 2, 'c': 'a'}, res)
+        self.assertEqual(1, res.a)
+        self.assertEqual(2, res.b)
+        self.assertEqual('a', res.c)
     
     def test_nested_dict(self):
         config = '''
@@ -23,7 +47,10 @@ class TestYamlDict(TestCase):
           e: 'a'
         '''
         res = load(config)
-        self.assertDictEqual({'a': 1, 'b': {'c': 3, 'd': 4, 'e': 'a'}}, res)
+        self.assertEqual(1, res.a)
+        self.assertEqual(3, res.b.c)
+        self.assertEqual(4, res.b.d)
+        self.assertEqual('a', res.b.e)
 
     def test_deeply_nested_dict(self):
         config = '''
@@ -37,7 +64,11 @@ class TestYamlDict(TestCase):
               h: 5
         '''
         res = load(config)
-        self.assertDictEqual({'a': 1, 'b': {'c': 2, 'd': 3, 'e': {'f': 4, 'g': {'h': 5}}}}, res)
+        self.assertEqual(1, res.a)
+        self.assertEqual(2, res.b.c)
+        self.assertEqual(3, res.b.d)
+        self.assertEqual(4, res.b.e.f)
+        self.assertEqual(5, res.b.e.g.h)
 
     def test_resolve_simple(self):
         config = '''
@@ -144,13 +175,28 @@ class TestYamlDict(TestCase):
     def test_recursive(self):
         config = '''
         prefix: /opt/ml
-        input_path: "{prefix}/input"
+        input_path: '{prefix}/input'
         training_data_path: '{input_path}/data/training'
         '''
 
         res = load(config, recursive=True)
+        self.assertEqual('/opt/ml/input', res.input_path)
         self.assertEqual('/opt/ml/input/data/training', res.training_data_path)
 
+    def test_keyword_args(self):
+        config = '''
+        prefix: /opt/ml
+        input_path: '{prefix}/input'
+        training_data_path: '{input_path}/data/training'
+        '''
+
+        def inner_test(input_path, training_data_path, **kwargs):
+            self.assertEqual('/opt/ml/input', input_path)
+            self.assertEqual('/opt/ml/input/data/training', training_data_path)
+
+        res = load(config, recursive=True)
+        inner_test(**res)
+
 
 if __name__ == '__main__':
     import sys