Repository: numba/numba

Issue:
Regression: guvectorized functions can no longer be pickled in numba 0.53
This is a showstopper regression for users of guvectorized functions on dask.distributed:

```python
import numba
import pickle


@numba.guvectorize(["f8,f8[:]"], "()->()")
def double(x, out):
    out[:] = x * 2


print(pickle.dumps(double))
```

On Linux x64:
```bash
$ conda create -n test52 -c conda-forge python=3.8 numba=0.52
$ conda activate test52
$ python demo.py
b'\x80\x04\x95\x1a\x00\x00\x00\x00\x00\x00\x00\x8c\x0b__mp_main__\x94\x8c\x06double\x94\x93\x94.'
$ conda create -n test53 -c conda-forge python=3.8 numba=0.53
$ conda activate test53
$ python demo.py
Traceback (most recent call last):
  File "demo.py", line 10, in <module>
    print(pickle.dumps(double))
_pickle.PicklingError: Can't pickle <ufunc 'double'>: it's not the same object as __main__.double
```

Regression: guvectorized functions can no longer be pickled in numba 0.53
This is a showstopper regression for users of guvectorized functions on dask.distributed:

```python
import numba
import pickle


@numba.guvectorize(["f8,f8[:]"], "()->()")
def double(x, out):
    out[:] = x * 2


print(pickle.dumps(double))
```

On Linux x64:
```bash
$ conda create -n test52 -c conda-forge python=3.8 numba=0.52
$ conda activate test52
$ python demo.py
b'\x80\x04\x95\x1a\x00\x00\x00\x00\x00\x00\x00\x8c\x0b__mp_main__\x94\x8c\x06double\x94\x93\x94.'
$ conda create -n test53 -c conda-forge python=3.8 numba=0.53
$ conda activate test53
$ python demo.py
Traceback (most recent call last):
  File "demo.py", line 10, in <module>
    print(pickle.dumps(double))
_pickle.PicklingError: Can't pickle <ufunc 'double'>: it's not the same object as __main__.double
```



Discussion:
@crusaderky thank you for submitting this! I can confirm, I am able to reproduce this with `b1258c82c0ed2845fce52508f439e29f729384a9` (current master) on OSX.
@crusaderky thank you again for reporting this. During today's issue triage we concluded fixing this is important and we are planning a 0.53.1 release in two weeks from now. I have created a 0.53.1 milestone and will now add this issue to that.
@crusaderky thank you for submitting this! I can confirm, I am able to reproduce this with `b1258c82c0ed2845fce52508f439e29f729384a9` (current master) on OSX.
@crusaderky thank you again for reporting this. During today's issue triage we concluded fixing this is important and we are planning a 0.53.1 release in two weeks from now. I have created a 0.53.1 milestone and will now add this issue to that.

Resolving patch:
diff --git a/numba/np/ufunc/gufunc.py b/numba/np/ufunc/gufunc.py
--- a/numba/np/ufunc/gufunc.py
+++ b/numba/np/ufunc/gufunc.py
@@ -3,9 +3,10 @@
 from numba.np.ufunc.ufuncbuilder import GUFuncBuilder
 from numba.np.ufunc.sigparse import parse_signature
 from numba.np.numpy_support import ufunc_find_matching_loop
+from numba.core import serialize
 
 
-class GUFunc(object):
+class GUFunc(serialize.ReduceMixin):
     """
     Dynamic generalized universal function (GUFunc)
     intended to act like a normal Numpy gufunc, but capable
@@ -18,6 +19,7 @@ def __init__(self, py_func, signature, identity=None, cache=None,
         self.ufunc = None
         self._frozen = False
         self._is_dynamic = is_dynamic
+        self._identity = identity
 
         # GUFunc cannot inherit from GUFuncBuilder because "identity"
         # is a property of GUFunc. Thus, we hold a reference to a GUFuncBuilder
@@ -26,6 +28,32 @@ def __init__(self, py_func, signature, identity=None, cache=None,
             py_func, signature, identity, cache, targetoptions)
         self.__name__ = self.gufunc_builder.py_func.__name__
 
+    def _reduce_states(self):
+        gb = self.gufunc_builder
+        dct = dict(
+            py_func=gb.py_func,
+            signature=gb.signature,
+            identity=self._identity,
+            cache=gb.cache,
+            is_dynamic=self._is_dynamic,
+            targetoptions=gb.targetoptions,
+            typesigs=gb._sigs,
+            frozen=self._frozen,
+        )
+        return dct
+
+    @classmethod
+    def _rebuild(cls, py_func, signature, identity, cache, is_dynamic,
+                 targetoptions, typesigs, frozen):
+        self = cls(py_func=py_func, signature=signature, identity=identity,
+                   cache=cache, is_dynamic=is_dynamic,
+                   targetoptions=targetoptions)
+        for sig in typesigs:
+            self.add(sig)
+        self.build_ufunc()
+        self._frozen = frozen
+        return self
+
     def __repr__(self):
         return f"<numba._GUFunc '{self.__name__}'>"
 


Test patch:
diff --git a/numba/tests/npyufunc/test_gufunc.py b/numba/tests/npyufunc/test_gufunc.py
--- a/numba/tests/npyufunc/test_gufunc.py
+++ b/numba/tests/npyufunc/test_gufunc.py
@@ -1,10 +1,12 @@
+import unittest
+import pickle
+
 import numpy as np
 import numpy.core.umath_tests as ut
 
 from numba import void, float32, jit, guvectorize
 from numba.np.ufunc import GUVectorize
 from numba.tests.support import tag, TestCase
-import unittest
 
 
 def matmulcore(A, B, C):
@@ -281,5 +283,119 @@ class TestGUVectorizeScalarParallel(TestGUVectorizeScalar):
     target = 'parallel'
 
 
+class TestGUVectorizePickling(TestCase):
+    def test_pickle_gufunc_non_dyanmic(self):
+        """Non-dynamic gufunc.
+        """
+        @guvectorize(["f8,f8[:]"], "()->()")
+        def double(x, out):
+            out[:] = x * 2
+
+        # pickle
+        ser = pickle.dumps(double)
+        cloned = pickle.loads(ser)
+
+        # attributes carried over
+        self.assertEqual(cloned._frozen, double._frozen)
+        self.assertEqual(cloned.identity, double.identity)
+        self.assertEqual(cloned.is_dynamic, double.is_dynamic)
+        self.assertEqual(cloned.gufunc_builder._sigs,
+                         double.gufunc_builder._sigs)
+        # expected value of attributes
+        self.assertTrue(cloned._frozen)
+
+        cloned.disable_compile()
+        self.assertTrue(cloned._frozen)
+
+        # scalar version
+        self.assertPreciseEqual(double(0.5), cloned(0.5))
+        # array version
+        arr = np.arange(10)
+        self.assertPreciseEqual(double(arr), cloned(arr))
+
+    def test_pickle_gufunc_dyanmic_null_init(self):
+        """Dynamic gufunc w/o prepopulating before pickling.
+        """
+        @guvectorize("()->()", identity=1)
+        def double(x, out):
+            out[:] = x * 2
+
+        # pickle
+        ser = pickle.dumps(double)
+        cloned = pickle.loads(ser)
+
+        # attributes carried over
+        self.assertEqual(cloned._frozen, double._frozen)
+        self.assertEqual(cloned.identity, double.identity)
+        self.assertEqual(cloned.is_dynamic, double.is_dynamic)
+        self.assertEqual(cloned.gufunc_builder._sigs,
+                         double.gufunc_builder._sigs)
+        # expected value of attributes
+        self.assertFalse(cloned._frozen)
+
+        # scalar version
+        expect = np.zeros(1)
+        got = np.zeros(1)
+        double(0.5, out=expect)
+        cloned(0.5, out=got)
+        self.assertPreciseEqual(expect, got)
+        # array version
+        arr = np.arange(10)
+        expect = np.zeros_like(arr)
+        got = np.zeros_like(arr)
+        double(arr, out=expect)
+        cloned(arr, out=got)
+        self.assertPreciseEqual(expect, got)
+
+    def test_pickle_gufunc_dyanmic_initialized(self):
+        """Dynamic gufunc prepopulated before pickling.
+
+        Once unpickled, we disable compilation to verify that the gufunc
+        compilation state is carried over.
+        """
+        @guvectorize("()->()", identity=1)
+        def double(x, out):
+            out[:] = x * 2
+
+        # prepopulate scalar
+        expect = np.zeros(1)
+        got = np.zeros(1)
+        double(0.5, out=expect)
+        # prepopulate array
+        arr = np.arange(10)
+        expect = np.zeros_like(arr)
+        got = np.zeros_like(arr)
+        double(arr, out=expect)
+
+        # pickle
+        ser = pickle.dumps(double)
+        cloned = pickle.loads(ser)
+
+        # attributes carried over
+        self.assertEqual(cloned._frozen, double._frozen)
+        self.assertEqual(cloned.identity, double.identity)
+        self.assertEqual(cloned.is_dynamic, double.is_dynamic)
+        self.assertEqual(cloned.gufunc_builder._sigs,
+                         double.gufunc_builder._sigs)
+        # expected value of attributes
+        self.assertFalse(cloned._frozen)
+
+        # disable compilation
+        cloned.disable_compile()
+        self.assertTrue(cloned._frozen)
+        # scalar version
+        expect = np.zeros(1)
+        got = np.zeros(1)
+        double(0.5, out=expect)
+        cloned(0.5, out=got)
+        self.assertPreciseEqual(expect, got)
+        # array version
+        expect = np.zeros_like(arr)
+        got = np.zeros_like(arr)
+        double(arr, out=expect)
+        cloned(arr, out=got)
+        self.assertPreciseEqual(expect, got)
+
+
 if __name__ == '__main__':
     unittest.main()