Repository: numba/numba

Issue:
Error when linking a function in a library with cache=True in another numba function
I've started using numba's caching feature and it has been fantastic in terms of removing the overhead from jit-ing the code (especially important when you have to run the same script many times). I've run into an issue where I get a linking problem, and I think I've been able to reproduce it with a minimal example, which can be found in the following gist:

https://gist.github.com/synapticarbors/22022c166e3776bb3e0b

and copied below:

`lib1.py`

``` python
import numba as nb

@nb.jit(nopython=True, cache=True)
def func(y):
    s = 0.0

    for k in range(y.shape[0]):
        s += y[k]

    return s
```

and `app.py`

``` python
import numpy as np
import numba as nb

import lib1


@nb.jit(nopython=True)
def fcalc(x):
    s = lib1.func(x)
    return s * 0.5 - 1.0


def run(N):
    x = np.linspace(-1.0, 1.0, N)
    res = fcalc(x)

    print res

if __name__ == '__main__':
    N = 1000
    run(N)
```

If first time I run it (`python app.py`), it works as expected, but the second time I run it, I get the following error:

``` python
$ python app.py
Traceback (most recent call last):
  File "app.py", line 21, in <module>
    run(N)
  File "app.py", line 15, in run
    res = fcalc(x)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/dispatcher.py", line 172, in _compile_for_args
    return self.compile(sig)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/dispatcher.py", line 350, in compile
    flags=flags, locals=self.locals)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 644, in compile_extra
    return pipeline.compile_extra(func)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 361, in compile_extra
    return self.compile_bytecode(bc, func_attr=self.func_attr)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 370, in compile_bytecode
    return self._compile_bytecode()
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 631, in _compile_bytecode
    return pm.run(self.status)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 251, in run
    raise patched_exception
RuntimeError: Caused By:
Traceback (most recent call last):
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 243, in run
    res = stage()
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 587, in stage_nopython_backend
    return self._backend(lowerfn, objectmode=False)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 540, in _backend
    lowered = lowerfn()
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 527, in backend_nopython_mode
    self.flags)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/compiler.py", line 786, in native_lowering_stage
    cfunc = targetctx.get_executable(library, fndesc, env)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/targets/cpu.py", line 147, in get_executable
    baseptr = library.get_pointer_to_function(fndesc.llvm_func_name)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/targets/codegen.py", line 377, in get_pointer_to_function
    self._ensure_finalized()
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/targets/codegen.py", line 67, in _ensure_finalized
    self.finalize()
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/targets/codegen.py", line 170, in finalize
    library._get_module_for_linking(), preserve=True)
  File "/Users/userx/anaconda/lib/python2.7/site-packages/numba/targets/codegen.py", line 113, in _get_module_for_linking
    % (self,))
RuntimeError: library unfit for linking: no available functions in <Library 'func' at 0x10b73ef10>

Failed at nopython (nopython mode backend)
library unfit for linking: no available functions in <Library 'func' at 0x10b73ef10>
```

I get this error in the `0.22.1` release as well as the newest conda build from the numba channel (`0.22.1 np110py27_283`). 

The contents of `__pycache__` are:

``` bash
$ ls -l __pycache__/
total 24
-rw-r--r--  1 userx  staff  4599 Jan  5 11:59 lib1.func-3.py27.1.nbc
-rw-r--r--  1 userx  staff   398 Jan  5 11:59 lib1.func-3.py27.nbi
```

It's unclear to me why caching isn't working here and whether this is expected or a possible bug. Any assistance would be appreciated.



Discussion:

Resolving patch:
diff --git a/numba/targets/codegen.py b/numba/targets/codegen.py
--- a/numba/targets/codegen.py
+++ b/numba/targets/codegen.py
@@ -95,6 +95,7 @@ def _get_module_for_linking(self):
 
         See discussion in https://github.com/numba/numba/pull/890
         """
+        self._ensure_finalized()
         if self._shared_module is not None:
             return self._shared_module
         mod = self._final_module
@@ -315,11 +316,13 @@ def serialize_using_bitcode(self):
     def serialize_using_object_code(self):
         """
         Serialize this library using its object code as the cached
-        representation.
+        representation.  We also include its bitcode for further inlining
+        with other libraries.
         """
         self._ensure_finalized()
-        ll_module = self._final_module
-        return (self._name, 'object', self._get_compiled_object())
+        data = (self._get_compiled_object(),
+                self._get_module_for_linking().as_bitcode())
+        return (self._name, 'object', data)
 
     @classmethod
     def _unserialize(cls, codegen, state):
@@ -332,8 +335,10 @@ def _unserialize(cls, codegen, state):
             self._finalize_final_module()
             return self
         elif kind == 'object':
+            object_code, shared_bitcode = data
             self.enable_object_caching()
-            self._set_compiled_object(data)
+            self._set_compiled_object(object_code)
+            self._shared_module = ll.parse_bitcode(shared_bitcode)
             self._finalize_final_module()
             return self
         else:


Test patch:
diff --git a/numba/tests/cache_usecases.py b/numba/tests/cache_usecases.py
--- a/numba/tests/cache_usecases.py
+++ b/numba/tests/cache_usecases.py
@@ -37,6 +37,10 @@ def inner(x, y):
 def outer(x, y):
     return inner(-y, x)
 
+@jit(cache=False, nopython=True)
+def outer_uncached(x, y):
+    return inner(-y, x)
+
 
 @jit(cache=True, forceobj=True)
 def looplifted(n):
diff --git a/numba/tests/test_dispatcher.py b/numba/tests/test_dispatcher.py
--- a/numba/tests/test_dispatcher.py
+++ b/numba/tests/test_dispatcher.py
@@ -402,7 +402,9 @@ def tearDown(self):
         shutil.rmtree(self.tempdir)
 
     def import_module(self):
-        # Import a fresh version of the test module
+        # Import a fresh version of the test module.  All jitted functions
+        # in the test module will start anew and load overloads from
+        # the on-disk cache if possible.
         old = sys.modules.pop(self.modname, None)
         if old is not None:
             # Make sure cached bytecode is removed
@@ -444,7 +446,9 @@ def dummy_test(self):
         pass
 
     def run_in_separate_process(self):
-        # Cached functions can be run from a distinct process
+        # Cached functions can be run from a distinct process.
+        # Also stresses issue #1603: uncached function calling cached function
+        # shouldn't fail compiling.
         code = """if 1:
             import sys
 
@@ -452,6 +456,7 @@ def run_in_separate_process(self):
             mod = __import__(%(modname)r)
             assert mod.add_usecase(2, 3) == 6
             assert mod.add_objmode_usecase(2, 3) == 6
+            assert mod.outer_uncached(3, 2) == 2
             assert mod.outer(3, 2) == 2
             packed_rec = mod.record_return(mod.packed_arr, 1)
             assert tuple(packed_rec) == (2, 43.5), packed_rec
@@ -513,6 +518,15 @@ def test_inner_then_outer(self):
         mod = self.import_module()
         self.assertPreciseEqual(mod.inner(3, 2), 6)
         self.check_cache(2)  # 1 index, 1 data
+        # Uncached outer function shouldn't fail (issue #1603)
+        f = mod.outer_uncached
+        self.assertPreciseEqual(f(3, 2), 2)
+        self.check_cache(2)  # 1 index, 1 data
+        mod = self.import_module()
+        f = mod.outer_uncached
+        self.assertPreciseEqual(f(3, 2), 2)
+        self.check_cache(2)  # 1 index, 1 data
+        # Cached outer will create new cache entries
         f = mod.outer
         self.assertPreciseEqual(f(3, 2), 2)
         self.check_cache(4)  # 2 index, 2 data
@@ -524,8 +538,12 @@ def test_outer_then_inner(self):
         mod = self.import_module()
         self.assertPreciseEqual(mod.outer(3, 2), 2)
         self.check_cache(4)  # 2 index, 2 data
+        self.assertPreciseEqual(mod.outer_uncached(3, 2), 2)
+        self.check_cache(4)  # same
+        mod = self.import_module()
         f = mod.inner
         self.assertPreciseEqual(f(3, 2), 6)
+        self.check_cache(4)  # same
         self.assertPreciseEqual(f(3.5, 2), 6.5)
         self.check_cache(5)  # 2 index, 3 data
 
@@ -589,6 +607,7 @@ def test_cache_reuse(self):
         mod = self.import_module()
         mod.add_usecase(2, 3)
         mod.add_objmode_usecase(2, 3)
+        mod.outer_uncached(2, 3)
         mod.outer(2, 3)
         mod.record_return(mod.packed_arr, 0)
         mod.record_return(mod.aligned_arr, 1)