Repository: uqfoundation/dill

Issue:
Regression caused by #449?
Pickling of a rather complicated object in one of my projects broke when updating dill from 0.3.4 to 0.3.5. I've constructed a MWE that I think shows the problem.
```
import dill
import numpy as np
from scipy.interpolate import RectBivariateSpline

#dill.detect.trace(True)

class Foo:
    def __init__(self):
        x = np.linspace(0,1,10)[:,None]
        y = np.linspace(2,3,9)[None,:]
        self.f = RectBivariateSpline(x,y,x*y)
        def f2(self,x,y):
            return self.f(x,y)
        self.f2 = f2.__get__(self)

foo = Foo()
pickled = dill.dumps(foo)
unpickled = dill.loads(pickled)

print(unpickled.f2(0.5, 0.6))
```

Up to e2831d0, this script ran without errors, and just printed `[[1.]]`, but at 914d47f (where #449 was merged), instead I get an error
```
Traceback (most recent call last):
  File ".../dill-test.py", line 20, in <module>
    unpickled = dill.loads(pickled)
  File ".../dill/dill/_dill.py", line 387, in loads
    return load(file, ignore, **kwds)
  File ".../dill/dill/_dill.py", line 373, in load
    return Unpickler(file, ignore=ignore, **kwds).load()
  File ".../dill/dill/_dill.py", line 606, in load
    obj = StockUnpickler.load(self)
AttributeError: 'Foo' object has no attribute 'f2'
```

---------------------------
In case it's helpful, here is the output with tracing enabled. For the successful version (e2831d0):
```
T2: <class '__main__.Foo'>
F2: <function _create_type at 0x7fd58af151f0>
# F2
T1: <class 'type'>
F2: <function _load_type at 0x7fd58af15160>
# F2
# T1
T1: <class 'object'>
# T1
D2: <dict object at 0x7fd56cb7f600>
F1: <function Foo.__init__ at 0x7fd56e42bca0>
F2: <function _create_function at 0x7fd58af15280>
# F2
Co: <code object __init__ at 0x7fd58b309c90, file "/home/jomotani/temp/dill-test/./dill-test.py", line 10>
F2: <function _create_code at 0x7fd58af15310>
# F2
Co: <code object f2 at 0x7fd58b309920, file "/home/jomotani/temp/dill-test/./dill-test.py", line 14>
# Co
# Co
D1: <dict object at 0x7fd58b3a9500>
# D1
D2: <dict object at 0x7fd56cb7fb00>
# D2
# F1
# D2
# T2
D2: <dict object at 0x7fd58af32940>
T4: <class 'scipy.interpolate._fitpack2.RectBivariateSpline'>
# T4
D2: <dict object at 0x7fd58b3a9680>
B3: <built-in function _reconstruct>
F2: <function _get_attr at 0x7fd58af15dc0>
# F2
M2: <module 'numpy.core._multiarray_umath' from '/home/jomotani/miniconda3/lib/python3.9/site-packages/numpy/core/_multiarray_umath.cpython-39-x86_64-linux-gnu.so'>
F2: <function _import_module at 0x7fd58af15ee0>
# F2
# M2
# B3
T4: <class 'numpy.ndarray'>
# T4
T4: <class 'numpy.dtype'>
# T4
# D2
Me: <bound method Foo.__init__.<locals>.f2 of <__main__.Foo object at 0x7fd58b114be0>>
T1: <class 'method'>
# T1
F1: <function Foo.__init__.<locals>.f2 at 0x7fd58b1f5f70>
D1: <dict object at 0x7fd58b3a9500>
# D1
D2: <dict object at 0x7fd56cb7fa80>
# D2
# F1
# Me
# D2
[[1.]]
```

and for the failing version (914d47f):
```
T2: <class '__main__.Foo'>
F2: <function _create_type at 0x7fe74f16b1f0>
# F2
T1: <class 'type'>
F2: <function _load_type at 0x7fe74f16b160>
# F2
# T1
T1: <class 'object'>
# T1
D2: <dict object at 0x7fe730d4b500>
F1: <function Foo.__init__ at 0x7fe7325f7e50>
F2: <function _create_function at 0x7fe74f16b280>
# F2
Co: <code object __init__ at 0x7fe74f4d4c90, file "/home/jomotani/temp/dill-test/./dill-test.py", line 10>
F2: <function _create_code at 0x7fe74f16b310>
# F2
Co: <code object f2 at 0x7fe74f4d4920, file "/home/jomotani/temp/dill-test/./dill-test.py", line 14>
# Co
# Co
D1: <dict object at 0x7fe74f574500>
# D1
D2: <dict object at 0x7fe730d4ba40>
# D2
# F1
# D2
# T2
D2: <dict object at 0x7fe74f0fe9c0>
T4: <class 'scipy.interpolate._fitpack2.RectBivariateSpline'>
# T4
D2: <dict object at 0x7fe74f574680>
T4: <class 'numpy.ndarray'>
# T4
T4: <class 'numpy.dtype'>
# T4
# D2
# D2
Traceback (most recent call last):
  File ".../dill-test.py", line 20, in <module>
    unpickled = dill.loads(pickled)
  File ".../dill/dill/_dill.py", line 387, in loads
    return load(file, ignore, **kwds)
  File ".../dill/dill/_dill.py", line 373, in load
    return Unpickler(file, ignore=ignore, **kwds).load()
  File ".../dill/dill/_dill.py", line 606, in load
    obj = StockUnpickler.load(self)
AttributeError: 'Foo' object has no attribute 'f2'
```

Discussion:
johnomotani: Thanks for the rapid fix @anivegesana! Can confirm that this does completly fix my original issue too (https://github.com/boutproject/hypnotoad/issues/122).
:heart: :tada:

Patch:
diff --git a/dill/_dill.py b/dill/_dill.py
index 89f2c46..becd1b6 100644
--- a/dill/_dill.py
+++ b/dill/_dill.py
@@ -1767,15 +1767,29 @@ if OLDER or not PY3:
             log.info("# B2")
         return
 
-    @register(MethodType) #FIXME: fails for 'hidden' or 'name-mangled' classes
-    def save_instancemethod0(pickler, obj):# example: cStringIO.StringI
-        log.info("Me: %s" % obj) #XXX: obj.__dict__ handled elsewhere?
-        if PY3:
-            pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
-        else:
-            pickler.save_reduce(MethodType, (obj.im_func, obj.im_self,
-                                             obj.im_class), obj=obj)
-        log.info("# Me")
+if IS_PYPY:
+    @register(MethodType)
+    def save_instancemethod0(pickler, obj):
+        code = getattr(obj.__func__, '__code__', None)
+        if code is not None and type(code) is not CodeType \
+              and getattr(obj.__self__, obj.__name__) == obj:
+            # Some PyPy builtin functions have no module name
+            log.info("Me2: %s" % obj)
+            # TODO: verify that this works for all PyPy builtin methods
+            pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
+            log.info("# Me2")
+            return
+
+        log.info("Me1: %s" % obj)
+        pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
+        log.info("# Me1")
+        return
+else:
+    @register(MethodType)
+    def save_instancemethod0(pickler, obj):
+        log.info("Me1: %s" % obj)
+        pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
+        log.info("# Me1")
         return
 
 if sys.hexversion >= 0x20500f0:
@@ -1801,17 +1815,6 @@ if sys.hexversion >= 0x20500f0:
             log.info("# Wr")
             return
 
-    @register(MethodWrapperType)
-    def save_instancemethod(pickler, obj):
-        log.info("Mw: %s" % obj)
-        if IS_PYPY2 and obj.__self__ is None and obj.im_class:
-            # Can be a class method in PYPY2 if __self__ is none
-            pickler.save_reduce(getattr, (obj.im_class, obj.__name__), obj=obj)
-            return
-        pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
-        log.info("# Mw")
-        return
-
 elif not IS_PYPY:
     @register(MethodDescriptorType)
     @register(WrapperDescriptorType)
@@ -2139,6 +2142,27 @@ def save_classmethod(pickler, obj):
 @register(FunctionType)
 def save_function(pickler, obj):
     if not _locate_function(obj, pickler):
+        if type(obj.__code__) is not CodeType:
+            # Some PyPy builtin functions have no module name, and thus are not
+            # able to be located
+            module_name = getattr(obj, '__module__', None)
+            if module_name is None:
+                module_name = __builtin__.__name__
+            module = _import_module(module_name, safe=True)
+            _pypy_builtin = False
+            try:
+                found, _ = _getattribute(module, obj.__qualname__)
+                if getattr(found, '__func__', None) is obj:
+                    _pypy_builtin = True
+            except:
+                pass
+
+            if _pypy_builtin:
+                log.info("F3: %s" % obj)
+                pickler.save_reduce(getattr, (found, '__func__'), obj=obj)
+                log.info("# F3")
+                return
+
         log.info("F1: %s" % obj)
         _recurse = getattr(pickler, '_recurse', None)
         _postproc = getattr(pickler, '_postproc', None)


Test patch:
diff --git a/tests/test_functions.py b/tests/test_functions.py
index ec9670e..a86cbde 100644
--- a/tests/test_functions.py
+++ b/tests/test_functions.py
@@ -54,6 +54,21 @@ def function_with_unassigned_variable():
     return (lambda: value)
 
 
+def test_issue_510():
+    # A very bizzare use of functions and methods that pickle doesn't get
+    # correctly for odd reasons.
+    class Foo:
+        def __init__(self):
+                def f2(self):
+                        return self
+                self.f2 = f2.__get__(self)
+
+    import dill, pickletools
+    f = Foo()
+    f1 = dill.copy(f)
+    assert f1.f2() is f1
+
+
 def test_functions():
     dumped_func_a = dill.dumps(function_a)
     assert dill.loads(dumped_func_a)(0) == 0
@@ -104,3 +119,4 @@ assert dill.loads(dumped_func_e)(1, 2, 3, e2=4, e3=5) == 15''')
 
 if __name__ == '__main__':
     test_functions()
+    test_issue_510()