Repository: uqfoundation/dill

Issue:
pickling nested namedtuples
While dill pickles nested classes without problem, the same cannot be said of namedtuples defined within another class:

```
In [1]: import dill

In [2]: from collections import namedtuple

In [3]: class T:
    class A: pass
    B = namedtuple("B", "")
   ...: 

In [4]: dill.dumps(T.A)
Out[4]: <elided>

In [5]: dill.dumps(T.B)
<oops!>
```

I would be OK if fixing this required manually setting some attribute on `B` to indicate where it lives, if that could help.


Discussion:
mmckerns: There is also an approach like this quick hack… it should be general, but it's messy… and as is, it breaks the namespace.

```
>>> import m
>>> import dill
>>> x = dill.dumps(compile(dill.source.importable(m.A.B, source=True), '<string>', 'exec'))
>>> exec(dill.loads(x))
>>> B
<class __main__.B at 0x10860c9a8>
```
mmckerns: Note (as a follow-up from issue #132):
```
>>> import collections
>>> X = collections.namedtuple('Y', ['a','b'])
>>> Xi = X(0,1)
>>> import dill
>>> dill.detect.errors(Xi)
PicklingError("Can't pickle <class '__main__.Y'>: it's not found as __main__.Y",)
```
Should this be the expected behavior?  As demonstrated in `test_classdef.py`, one could conceivably pickle a badly named `namedtuple` by messing with the `__name__` (or `__qualname__`).  However, one shouldn't need to do that.  I believe that the PR for this issue will enable pickling of a badly named `namedtuple`... and is a good solution.
matsjoyce: Do you want me to rebase this?
mmckerns: Yes, if you don't mind.
matsjoyce: OK, updated. I've also changed the behaviour so that it will serialize assuming that the name is bad but on deserialize it will try to find the type, and if it fails recreate it.
```py3
>>> import collections, dill
>>> X=collections.namedtuple("Y", ("a", "b"))
>>> Z=collections.namedtuple("Z", ("a", "b"))
>>> x=X(1,2)
>>> z=Z(1,2)
>>> x2=dill.copy(x)
>>> z2=dill.copy(z)
>>> type(x2)
<class '__main__.Y'>
>>> type(z2)
<class '__main__.Z'>
>>> type(x2) is X
False
>>> type(z2) is Z
True
```
matsjoyce: Ooo, CI works. I'm looking at the failure.

Patch:
diff --git a/dill/dill.py b/dill/dill.py
index e4f2b56..e941c7d 100644
--- a/dill/dill.py
+++ b/dill/dill.py
@@ -717,6 +717,18 @@ def _create_array(f, args, state, npdict=None):
         array.__dict__.update(npdict)
     return array
 
+def _create_namedtuple(name, fieldnames, modulename):
+    mod = _import_module(modulename, safe=True)
+    if mod is not None:
+        try:
+            return getattr(mod, name)
+        except:
+            pass
+    import collections
+    t = collections.namedtuple(name, fieldnames)
+    t.__module__ = modulename
+    return t
+
 def _getattr(objclass, name, repr_str):
     # hack to grab the reference directly
     try: #XXX: works only for __builtin__ ?
@@ -1087,7 +1099,7 @@ def _proxy_helper(obj): # a dead proxy returns a reference to None
         return id(None)
     if _str == _repr: return id(obj) # it's a repr
     try: # either way, it's a proxy from here
-        address = int(_str.rstrip('>').split(' at ')[-1], base=16)   
+        address = int(_str.rstrip('>').split(' at ')[-1], base=16)
     except ValueError: # special case: proxy of a 'type'
         if not IS_PYPY:
             address = int(_repr.rstrip('>').split(' at ')[-1], base=16)
@@ -1198,15 +1210,13 @@ def save_type(pickler, obj):
         log.info("T1: %s" % obj)
         pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj)
         log.info("# T1")
+    elif issubclass(obj, tuple) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]):
+        # special case: namedtuples
+        log.info("T6: %s" % obj)
+        pickler.save_reduce(_create_namedtuple, (getattr(obj, "__qualname__", obj.__name__), obj._fields, obj.__module__), obj=obj)
+        log.info("# T6")
+        return
     elif obj.__module__ == '__main__':
-        try: # use StockPickler for special cases [namedtuple,]
-            [getattr(obj, attr) for attr in ('_fields','_asdict',
-                                             '_make','_replace')]
-            log.info("T6: %s" % obj)
-            StockPickler.save_global(pickler, obj)
-            log.info("# T6")
-            return
-        except AttributeError: pass
         if issubclass(type(obj), type):
         #   try: # used when pickling the class as code (or the interpreter)
             if is_dill(pickler) and not pickler._byref:


Test patch:
diff --git a/tests/test_classdef.py b/tests/test_classdef.py
index 0e47473..21a99c9 100644
--- a/tests/test_classdef.py
+++ b/tests/test_classdef.py
@@ -93,20 +93,24 @@ if hex(sys.hexversion) >= '0x20600f0':
     Z = namedtuple("Z", ['a','b'])
     Zi = Z(0,1)
     X = namedtuple("Y", ['a','b'])
-    if hex(sys.hexversion) >= '0x30500f0':
+    X.__name__ = "X"
+    if hex(sys.hexversion) >= '0x30300f0':
         X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle
-    else:
-        X.__name__ = "X"
     Xi = X(0,1)
+    Bad = namedtuple("FakeName", ['a','b'])
+    Badi = Bad(0,1)
 else:
-    Z = Zi = X = Xi = None
+    Z = Zi = X = Xi = Bad = Badi = None
 
 # test namedtuple
 def test_namedtuple():
-    assert Z == dill.loads(dill.dumps(Z))
+    assert Z is dill.loads(dill.dumps(Z))
     assert Zi == dill.loads(dill.dumps(Zi))
-    assert X == dill.loads(dill.dumps(X))
+    assert X is dill.loads(dill.dumps(X))
     assert Xi == dill.loads(dill.dumps(Xi))
+    assert Bad is not dill.loads(dill.dumps(Bad))
+    assert Bad._fields == dill.loads(dill.dumps(Bad))._fields
+    assert tuple(Badi) == tuple(dill.loads(dill.dumps(Badi)))
 
 def test_array_subclass():
     try:
@@ -115,7 +119,7 @@ def test_array_subclass():
         class TestArray(np.ndarray):
             def __new__(cls, input_array, color):
                 obj = np.asarray(input_array).view(cls)
-                obj.color = color 
+                obj.color = color
                 return obj
             def __array_finalize__(self, obj):
                 if obj is None: