Repository: numba/numba

Issue:
Issue with new ufunc cache capability
```
import numba as nb
@nb.vectorize('f8(f8,f8)', cache=True)
def test(x,y):
    return x + y
```
The initial compile works, but subsequent runs do this:
```
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/targets/base.py in get_constant_generic(self, builder, ty, val)
    440         try:
--> 441             impl = self._get_constants.find((ty,))
    442             return impl(self, builder, ty, val)

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/targets/base.py in find(self, sig)
     48         if out is None:
---> 49             out = self._find(sig)
     50             self._cache[sig] = out

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/targets/base.py in _find(self, sig)
     57         else:
---> 58             raise NotImplementedError(self, sig)
     59 

NotImplementedError: (<numba.targets.base.OverloadSelector object at 0x10da50780>, (int64,))

During handling of the above exception, another exception occurred:

NotImplementedError                       Traceback (most recent call last)
<ipython-input-2-3eeed12626d5> in <module>()
----> 1 @nb.vectorize('f8(f8,f8)', cache=True)
      2 def test(x,y):
      3     return x + y

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/npyufunc/decorators.py in wrap(func)
    118         vec = Vectorize(func, **kws)
    119         for sig in ftylist:
--> 120             vec.add(sig)
    121         if len(ftylist) > 0:
    122             vec.disable_compile()

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/npyufunc/dufunc.py in add(self, sig)
    134         """
    135         args, return_type = sigutils.normalize_signature(sig)
--> 136         return self._compile_for_argtys(args, return_type)
    137 
    138     def _compile_for_args(self, *args, **kws):

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/npyufunc/dufunc.py in _compile_for_argtys(self, argtys, return_type)
    184             cres, argtys, return_type)
    185         dtypenums, ptr, env = ufuncbuilder._build_element_wise_ufunc_wrapper(
--> 186             cres, actual_sig)
    187         self._add_loop(utils.longint(ptr), dtypenums)
    188         self._keepalive.append((ptr, cres.library, env))

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/npyufunc/ufuncbuilder.py in _build_element_wise_ufunc_wrapper(cres, signature)
    145 
    146     ptr = build_ufunc_wrapper(library, ctx, fname, signature,
--> 147                               cres.objectmode, envptr, env)
    148 
    149     # Get dtypes

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/npyufunc/wrappers.py in build_ufunc_wrapper(library, context, fname, signature, objmode, envptr, env)
    167     arrays = []
    168     for i, typ in enumerate(signature.args):
--> 169         arrays.append(UArrayArg(context, builder, arg_args, arg_steps, i, typ))
    170 
    171     # Prepare output

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/npyufunc/wrappers.py in __init__(self, context, builder, args, steps, i, fe_type)
    231         self.builder = builder
    232         self.fe_type = fe_type
--> 233         offset = self.context.get_constant(types.intp, i)
    234         offseted_args = self.builder.load(builder.gep(args, [offset]))
    235         data_type = context.get_data_type(fe_type)

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/targets/base.py in get_constant(self, ty, val)
    450         """
    451         # HACK: pass builder=None to preserve get_constant() API
--> 452         return self.get_constant_generic(None, ty, val)
    453 
    454     def get_constant_undef(self, ty):

/Users/nbm442/miniconda3/envs/xxx/lib/python3.5/site-packages/numba/targets/base.py in get_constant_generic(self, builder, ty, val)
    442             return impl(self, builder, ty, val)
    443         except NotImplementedError:
--> 444             raise NotImplementedError("cannot lower constant of type '%s'" % (ty,))
    445 
    446     def get_constant(self, ty, val):

NotImplementedError: cannot lower constant of type 'int64'
```


Discussion:
cc: @jcrist

This seems to happen when loading a cached guvectorize is the first thing done in a script.  For some reason, the cache loading bypassed some initialization for target context.


Resolving patch:
diff --git a/numba/caching.py b/numba/caching.py
--- a/numba/caching.py
+++ b/numba/caching.py
@@ -344,6 +344,8 @@ def load_overload(self, sig, target_context):
         Load and recreate the cached CompileResult for the given signature,
         using the *target_context*.
         """
+        # Refresh the context to ensure it is initialized
+        target_context.refresh()
         with self._guard_against_spurious_io_errors():
             return self._load_overload(sig, target_context)
         # None returned if the `with` block swallows an exception


Test patch:
diff --git a/numba/tests/npyufunc/test_caching.py b/numba/tests/npyufunc/test_caching.py
--- a/numba/tests/npyufunc/test_caching.py
+++ b/numba/tests/npyufunc/test_caching.py
@@ -1,8 +1,10 @@
 from __future__ import print_function, absolute_import, division
 
+import sys
 import os.path
 import re
 from contextlib import contextmanager
+import subprocess
 
 import numpy as np
 
@@ -164,5 +166,46 @@ def test_indirect_gufunc_cache_parallel(self, **kwargs):
         self.test_indirect_gufunc_cache(target='parallel')
 
 
+class TestCacheSpecificIssue(UfuncCacheTest):
+
+    def run_in_separate_process(self, runcode):
+        # Based on the same name util function in test_dispatcher but modified
+        # to allow user to define what to run.
+        code = """if 1:
+            import sys
+
+            sys.path.insert(0, %(tempdir)r)
+            mod = __import__(%(modname)r)
+            mod.%(runcode)s
+            """ % dict(tempdir=self.tempdir, modname=self.modname,
+                       runcode=runcode)
+
+        popen = subprocess.Popen([sys.executable, "-c", code],
+                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        out, err = popen.communicate()
+        if popen.returncode != 0:
+            raise AssertionError("process failed with code %s: stderr follows\n%s\n"
+                                 % (popen.returncode, err.decode()))
+
+    #
+    # The following test issue #2198 that loading cached (g)ufunc first
+    # bypasses some target context initialization.
+    #
+
+    def test_first_load_cached_ufunc(self):
+        # ensure function is cached
+        self.run_in_separate_process('direct_ufunc_cache_usecase()')
+        # use the cached function
+        # this will fail if the target context is not init'ed
+        self.run_in_separate_process('direct_ufunc_cache_usecase()')
+
+    def test_first_load_cached_gufunc(self):
+        # ensure function is cached
+        self.run_in_separate_process('direct_gufunc_cache_usecase()')
+        # use the cached function
+        # this will fail out if the target context is not init'ed
+        self.run_in_separate_process('direct_gufunc_cache_usecase()')
+
+
 if __name__ == '__main__':
     unittest.main()