Repository: mahmoud/glom

Issue:
deeply nested setting
I work on a project that flings around deeply nested Python structures with wild abandon. glom nicely handles the "get something from this structure even if all the branches of the path aren't there" and now I can replace some code I wrote. Yay!

The other side of things that I need to handle is setting a value in a deeply nested structure where the branches of the path may not be there.

For example, maybe something like this which uses dicts:

```
>>> from glom import glom_set
>>> foo = {}
>>> glom_set(foo, 'a.b.c', value=5)
>>> foo
{'a': {'b': {'c': 5}}}
```

There are more complex tree manipulations that could be done, but at the moment I'm thinking about setting a single leaf value.

Is manipulating deeply nested data structures in place in-scope for glom?

Discussion:

Patch:
diff --git a/glom/mutable.py b/glom/mutable.py
index d84eda6..ee605b7 100644
--- a/glom/mutable.py
+++ b/glom/mutable.py
@@ -4,7 +4,7 @@ This module contains Specs that perform mutations.
 import operator
 from pprint import pprint
 
-from .core import Path, T, S, Spec, glom, UnregisteredTarget, GlomError
+from .core import Path, T, S, Spec, glom, UnregisteredTarget, GlomError, PathAccessError
 from .core import TType, register_op, TargetRegistry
 
 try:
@@ -88,9 +88,21 @@ class Assign(object):
     :func:`~glom.register()` using the ``"assign"`` operation name.
 
     Attempting to assign to an immutable structure, like a
-    :class:`tuple`, will result in a :class:`~glom.PathAssignError`.
+    :class:`tuple`, will result in a
+    :class:`~glom.PathAssignError`. Attempting to assign to a path
+    that doesn't exist will raise a :class:`~PathAccessError`.
+
+    To automatically backfill missing structures, you can pass a
+    callable to the *missing* argument. This callable will be called
+    for each path segment along the assignment which is not
+    present.
+
+       >>> target = {}
+       >>> assign(target, 'a.b.c', 'hi', missing=dict)
+       {'a': {'b': {'c': 'hi'}}}
+
     """
-    def __init__(self, path, val):
+    def __init__(self, path, val, missing=None):
         # TODO: an option like require_preexisting or something to
         # ensure that a value is mutated, not just added. Current
         # workaround is to do a Check().
@@ -105,6 +117,7 @@ class Assign(object):
             self.op, self.arg = path.items()[-1]
         except IndexError:
             raise ValueError('path must have at least one element')
+        self._orig_path = path
         self.path = path[:-1]
 
         if self.op not in '[.P':
@@ -112,32 +125,51 @@ class Assign(object):
             raise ValueError('last part of path must be setattr or setitem')
         self.val = val
 
+        if missing is not None:
+            if not callable(missing):
+                raise TypeError('expected missing to be callable, not %r' % (missing,))
+        self.missing = missing
+
     def glomit(self, target, scope):
         if type(self.val) is Spec:
             val = scope[glom](target, self.val, scope)
         else:
             val = self.val
-        dest = scope[glom](target, self.path, scope)
+
+        op, arg, path = self.op, self.arg, self.path
+        try:
+            dest = scope[glom](target, self.path, scope)
+        except PathAccessError as pae:
+            if not self.missing:
+                raise
+
+            remaining_path = self._orig_path[pae.part_idx + 1:]
+            val = scope[glom](self.missing(), Assign(remaining_path, val, missing=self.missing), scope)
+
+            op, arg = self._orig_path.items()[pae.part_idx]
+            path = self._orig_path[:pae.part_idx]
+            dest = scope[glom](target, path, scope)
+
         # TODO: forward-detect immutable dest?
-        if self.op == '[':
-            dest[self.arg] = val
-        elif self.op == '.':
-            setattr(dest, self.arg, val)
-        elif self.op == 'P':
-            assign = scope[TargetRegistry].get_handler('assign', dest)
-            if not assign:
+        if op == '[':
+            dest[arg] = val
+        elif op == '.':
+            setattr(dest, arg, val)
+        elif op == 'P':
+            _assign = scope[TargetRegistry].get_handler('assign', dest)
+            if not _assign:
                 raise UnregisteredTarget('assign', type(dest),
                                          scope[TargetRegistry].get_type_map('assign'),
                                          path=scope[Path])
             try:
-                assign(dest, self.arg, val)
+                _assign(dest, arg, val)
             except Exception as e:
-                raise PathAssignError(e, self.path, self.arg)
+                raise PathAssignError(e, path, arg)
 
         return target
 
 
-def assign(obj, path, val):
+def assign(obj, path, val, missing=None):
     """The ``assign()`` function provides convenient "deep set"
     functionality, modifying nested data structures in-place::
 
@@ -146,10 +178,11 @@ def assign(obj, path, val):
       >>> pprint(target)
       {'a': [{'b': 'c'}, {'d': 'e'}]}
 
-    For more information and examples, see the :class:`~glom.Assign`
-    specifier type, which this function wraps.
+    Missing structures can also be automatically created with the
+    *missing* parameter. For more information and examples, see the
+    :class:`~glom.Assign` specifier type, which this function wraps.
     """
-    return glom(obj, Assign(path, val))
+    return glom(obj, Assign(path, val, missing=missing))
 
 
 _ALL_BUILTIN_TYPES = [v for v in __builtins__.values() if isinstance(v, type)]
diff --git a/glom/tutorial.py b/glom/tutorial.py
index 28ee336..66d3d42 100644
--- a/glom/tutorial.py
+++ b/glom/tutorial.py
@@ -76,7 +76,7 @@ Let's follow some astronomers on their journey exploring the solar system.
   >>> glom(target, spec)
   'jupiter'
 
-Our astronomers want to focus in on the Solar system, and represent planets as a list.
+Our astronomers want to focus in on the Solar system, and represent planets as a list. 
 Let's restructure the data to make a list of names:
 
   >>> target = {'system': {'planets': [{'name': 'earth'}, {'name': 'jupiter'}]}}
@@ -95,34 +95,14 @@ And let's say we want to capture a parallel list of moon counts with the names a
 We can react to changing data requirements as fast as the data itself can change, naturally restructuring our results,
 despite the input's nested nature. Like a list comprehension, but for nested data, our code mirrors our output.
 
-Handling Nested Lists
-=====================
-In the example above we introduced a new wrinkle: the target for ``planets`` has multiple
-entries stored in a list. Previously our targets were all nested dictionaries.
-
-To handle this we use a new *spec* pattern: ``(path, [subpath])``. In this pattern ``path`` is the path
-to the list, and ``subpath`` is the path within each element of the list. What's that? You need to handle
-lists within lists (within lists ...)? Then just repeat the pattern, replacing ``subpath`` with another
-``(path, [subpath])`` tuple. For example, say we have information about each planet's moons like so:
-
-  >>> target = {'system': {'planets': [{'name': 'earth', 'moons': [{'name': 'luna'}]},
-  ...                                  {'name': 'jupiter', 'moons': [{'name': 'io'},
-  ...                                                                {'name': 'europa'}]}]}}
-
-We can get the names of each moon from our nested lists by nesting our subpath specs:
-
-  >>> spec = {'planet_names': ('system.planets', ['name']),
-  ...         'moon_names': ('system.planets', [('moons', ['name'])])}
-  >>> pprint(glom(target, spec))
-  {'moon_names': [['luna'], ['io', 'europa']], 'planet_names': ['earth', 'jupiter']}
 
 Changing Requirements
 =====================
 
-Unfortunately, data in the real world is messy. You might be expecting a certain format and end up getting something
-completely different. No worries, glom to the rescue.
+Unfortunately, data in the real world is messy. You might be expecting a certain format and end up getting something 
+completely different. No worries, glom to the rescue. 
 
-Coalesce is a glom construct that allows you to specify fallback behavior for a list of subspecs.
+Coalesce is a glom construct that allows you to specify fallback behavior for a list of subspecs. 
 Subspecs are passed as positional arguments, while defaults can be set using keyword arguments.
 
 Let's say our astronomers recently got a new update in their systems, and sometimes ``system`` will contain
@@ -160,7 +140,7 @@ Going back to our example, let's say we wanted to get an aggregate moon count:
   >>> pprint(glom(target, {'moon_count': ('system.planets', ['moons'], sum)}))
   {'moon_count': 70}
 
-With glom, you have full access to Python at any given moment.
+With glom, you have full access to Python at any given moment. 
 Pass values to functions, whether built-in, imported, or defined inline with lambda.
 
 


Test patch:
diff --git a/glom/test/test_mutable.py b/glom/test/test_mutable.py
index 843a3fc..97f12ad 100644
--- a/glom/test/test_mutable.py
+++ b/glom/test/test_mutable.py
@@ -1,6 +1,6 @@
 import pytest
 
-from glom import glom, Path, T, Spec, Glommer, PathAssignError
+from glom import glom, Path, T, Spec, Glommer, PathAssignError, PathAccessError
 from glom.core import UnregisteredTarget
 from glom.mutable import Assign, assign
 
@@ -60,6 +60,9 @@ def test_bad_assign_target():
 
     with pytest.raises(PathAssignError, match='could not assign'):
         glom(BadTarget(), spec)
+
+    with pytest.raises(PathAccessError, match='could not access'):
+        assign({}, 'a.b.c', 'moot')
     return
 
 
@@ -88,3 +91,96 @@ def test_invalid_assign_op_target():
 
     with pytest.raises(ValueError):
         assign(target, spec, None)
+    return
+
+
+def test_assign_missing_signature():
+    # test signature (non-callable missing hook)
+    with pytest.raises(TypeError, match='callable'):
+        assign({}, 'a.b.c', 'lol', missing='invalidbcnotcallable')
+    return
+
+
+def test_assign_missing_dict():
+    target = {}
+    val = object()
+
+    from itertools import count
+    counter = count()
+    def debugdict():
+        ret = dict()
+        #ret['id'] = id(ret)
+        #ret['inc'] = counter.next()
+        return ret
+
+    assign(target, 'a.b.c.d', val, missing=debugdict)
+
+    assert target == {'a': {'b': {'c': {'d': val}}}}
+
+
+def test_assign_missing_object():
+    val = object()
+    class Container(object):
+        pass
+
+    target = Container()
+    target.a = extant_a = Container()
+    assign(target, 'a.b.c.d', val, missing=Container)
+
+    assert target.a.b.c.d is val
+    assert target.a is extant_a  # make sure we didn't overwrite anything on the path
+
+
+def test_assign_missing_with_extant_keys():
+    """This test ensures that assign with missing doesn't overwrite
+    perfectly fine extant keys that are along the path it needs to
+    assign to. call count is also checked to make sure missing() isn't
+    invoked too many times.
+
+    """
+    target = {}
+    value = object()
+    default_struct = {'b': {'c': {}}}
+
+    call_count = [0]
+
+    def _get_default_struct():
+        call_count[0] += 1  # make sure this is only called once
+        return default_struct
+
+    assign(target, 'a.b.c', value, missing=_get_default_struct)
+
+    assert target['a']['b']['c'] is value
+    assert target['a']['b'] is default_struct['b']
+    assert call_count == [1]
+
+
+def test_assign_missing_unassignable():
+    """Check that the final assignment to the target object comes last,
+    ensuring that failed assignments don't leave targets in a bad
+    state.
+
+    """
+
+    class Tarjay(object):
+        init_count = 0
+        def __init__(self):
+            self.__class__.init_count += 1
+
+        @property
+        def unassignable(self):
+            return
+
+    value = object()
+    target = {"preexisting": "ok"}
+
+    with pytest.raises(PathAssignError):
+        assign(target, 'tarjay.unassignable.a.b.c', value, missing=Tarjay)
+
+    assert target == {'preexisting': 'ok'}
+
+    # why 3? "c" gets the value of "value", while "b", "a", and
+    # "tarjay" all succeed and are set to Tarjay instances. Then
+    # unassignable is already present, but not possible to assign to,
+    # raising the PathAssignError.
+    assert Tarjay.init_count == 3