Repository: jsonpickle/jsonpickle (python)

Issue:
Nested dictionaries in `py/state` incorrectly restored when tracking object references
Repro:
```python
import jsonpickle


class MyClass:
    def __init__(self, o):
        self.o = o

    def __getstate__(self):
        return self.__dict__


i1 = MyClass({"a": "a"})
i2 = MyClass({"b": "b"})
original = [i1, i1, i2, i2]

assert original[0] is original[1]  # ok
assert original[2] is original[3]  # ok

json = jsonpickle.encode(original)
# Correctly encoded as:
# [
#   {"py/object": "__main__.MyClass", "py/state": {"o": {"a": "a"}}},
#   {"py/id": 1},
#   {"py/object": "__main__.MyClass", "py/state": {"o": {"b": "b"}}},
#   {"py/id": 4}
# ]

decoded = jsonpickle.decode(json)
assert decoded[0] is decoded[1]  # ok
assert decoded[2] is decoded[3]  # fail
```

The assertion fails because `decoded[3]` is incorrectly set to the dictionary `{'a': 'a'}`, instead of the second instance of `MyClass`.

I think this only occurs for types that define `__getstate__` without also defining `__setstate__`.

This occurs because `Unpickler._restore_state` will first restore the dict under `py/state`, which tracks references of any objects under it. But then it goes into `_restore_from_dict`, which restores the same dict _again_, tracking the references of any objects under it _again_. 


Pull request:
Fix reference tracking when using nested dictionaries/objects under `py/state`
Fixes #500

This PR causes a dict returned from `__getstate__` (and stored in `py/state`) to no longer be restored twice, which previously caused any objects in it to be incorrectly duplicated in the Unpickler's list of object instances.

Patch:
diff --git a/jsonpickle/unpickler.py b/jsonpickle/unpickler.py
index 8a50dde..6b8b5e4 100644
--- a/jsonpickle/unpickler.py
+++ b/jsonpickle/unpickler.py
@@ -607,23 +607,26 @@ class Unpickler(object):
 
         return restore_key
 
-    def _restore_from_dict(self, obj, instance, ignorereserved=True):
+    def _restore_from_dict(self, obj, instance, restore_dict_items=True):
         restore_key = self._restore_key_fn()
         method = _obj_setattr
         deferred = {}
 
         for k, v in util.items(obj):
             # ignore the reserved attribute
-            if ignorereserved and k in tags.RESERVED:
+            if restore_dict_items and k in tags.RESERVED:
                 continue
             if isinstance(k, numeric_types):
                 str_k = k.__str__()
             else:
                 str_k = k
             self._namestack.append(str_k)
-            k = restore_key(k)
-            # step into the namespace
-            value = self._restore(v)
+            if restore_dict_items:
+                k = restore_key(k)
+                # step into the namespace
+                value = self._restore(v)
+            else:
+                value = v
             if util.is_noncomplex(instance) or util.is_dictionary_subclass(instance):
                 try:
                     if k == '__dict__':
@@ -690,12 +693,12 @@ class Unpickler(object):
             # implements described default handling
             # of state for object with instance dict
             # and no slots
-            instance = self._restore_from_dict(state, instance, ignorereserved=False)
+            instance = self._restore_from_dict(state, instance, restore_dict_items=False)
         elif has_slots:
-            instance = self._restore_from_dict(state[1], instance, ignorereserved=False)
+            instance = self._restore_from_dict(state[1], instance, restore_dict_items=False)
             if has_slots_and_dict:
                 instance = self._restore_from_dict(
-                    state[0], instance, ignorereserved=False
+                    state[0], instance, restore_dict_items=False
                 )
         elif not hasattr(instance, '__getnewargs__') and not hasattr(
             instance, '__getnewargs_ex__'


Test patch:
diff --git a/tests/jsonpickle_test.py b/tests/jsonpickle_test.py
index 41e238f..012fd08 100644
--- a/tests/jsonpickle_test.py
+++ b/tests/jsonpickle_test.py
@@ -1102,6 +1102,9 @@ class PickleProtocol2GetStateDict(PickleProtocol2Thing):
     def __getstate__(self):
         return {'magic': True}
 
+class PickleProtocol2GetStateNestedDict(PickleProtocol2Thing):
+    def __getstate__(self):
+        return {'nested': {'magic': True}}
 
 class PickleProtocol2GetStateSlots(PickleProtocol2Thing):
     def __getstate__(self):
@@ -1555,6 +1558,20 @@ class PicklingProtocol2TestCase(SkippableTest):
         decoded = jsonpickle.decode(encoded)
         self.assertTrue(decoded.magic)
 
+    def test_restore_nested_dict_state_with_references_preserved(self):
+        """
+        Ensure that nested dicts in "py/state" are not duplicated when tracking
+        objects to preserve references
+        """
+        instance1 = PickleProtocol2GetStateNestedDict('whatevs')
+        instance2 = PickleProtocol2GetStateNestedDict('different')
+        encoded = jsonpickle.encode([instance1, instance1, instance2, instance2])
+        decoded = jsonpickle.decode(encoded)
+        self.assertTrue(decoded[0].nested['magic'])
+        self.assertIs(decoded[1], decoded[0])
+        self.assertTrue(decoded[2].nested['magic'])
+        self.assertIs(decoded[3], decoded[2])
+
     def test_restore_slots_state(self):
         """
         Ensure that if getstate returns a 2-tuple with a dict in the second