Repository: numba/numba

Issue:
Cryptic error when compiling a function obtained via `exec` with bad `__name__`
Possibly connected to these issues (found while searching for snippets from the stack trace):
 - https://github.com/numba/numba/issues/2547
 - https://github.com/numba/numba/issues/3236

## Reporting a bug

- [x] I am using the latest released version of Numba (most recent is visible in
 the change log (https://github.com/numba/numba/blob/master/CHANGE_LOG).
- [x] I have included below a minimal working reproducer (if you are unsure how
 to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).

## Minimal reproducer

```python
filename = 'test.py'
name = 'mypackage'
code = """
def f(x):
    return x
"""

objs = dict(__file__=filename, __name__=name)
compiled = compile(code, filename, 'exec')
exec(compiled, objs)

from numba import njit
compiled_f = njit(objs['f'])
print(compiled_f(0))
```

<details>
  <summary>Click for full stack trace!</summary>

```
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/lowering.py in from_fndesc(cls, fndesc)
     33             # Avoid creating new Env
---> 34             return cls._memo[fndesc.env_name]
     35         except KeyError:

~/anaconda3/envs/nengo-numba/lib/python3.7/weakref.py in __getitem__(self, key)
    136             self._commit_removals()
--> 137         o = self.data[key]()
    138         if o is None:

KeyError: '_ZN08NumbaEnv9mypackage6f$2433Ex'

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
<ipython-input-57-67f415c0b452> in <module>
     12 from numba import njit
     13 compiled_f = njit(objs['f'])
---> 14 print(compiled_f(0))

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    418                     e.patch_message('\n'.join((str(e).rstrip(), help_msg)))
    419             # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 420             raise e
    421 
    422     def inspect_llvm(self, signature=None):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    351                 argtypes.append(self.typeof_pyval(a))
    352         try:
--> 353             return self.compile(tuple(argtypes))
    354         except errors.ForceLiteralArg as e:
    355             # Received request for compiler re-entry with the list of arguments

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in compile(self, sig)
    766             self._cache_misses[sig] += 1
    767             try:
--> 768                 cres = self._compiler.compile(args, return_type)
    769             except errors.ForceLiteralArg as e:
    770                 def folded(args, kws):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in compile(self, args, return_type)
     75 
     76     def compile(self, args, return_type):
---> 77         status, retval = self._compile_cached(args, return_type)
     78         if status:
     79             return retval

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in _compile_cached(self, args, return_type)
     89 
     90         try:
---> 91             retval = self._compile_core(args, return_type)
     92         except errors.TypingError as e:
     93             self._failed_cache[key] = e

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in _compile_core(self, args, return_type)
    107                                       args=args, return_type=return_type,
    108                                       flags=flags, locals=self.locals,
--> 109                                       pipeline_class=self.pipeline_class)
    110         # Check typing error if object mode is used
    111         if cres.typing_error is not None and not flags.enable_pyobject:

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library, pipeline_class)
    526     pipeline = pipeline_class(typingctx, targetctx, library,
    527                               args, return_type, flags, locals)
--> 528     return pipeline.compile_extra(func)
    529 
    530 

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in compile_extra(self, func)
    324         self.state.lifted = ()
    325         self.state.lifted_from = None
--> 326         return self._compile_bytecode()
    327 
    328     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in _compile_bytecode(self)
    383         """
    384         assert self.state.func_ir is None
--> 385         return self._compile_core()
    386 
    387     def _compile_ir(self):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in _compile_core(self)
    363                 self.state.status.fail_reason = e
    364                 if is_final_pipeline:
--> 365                     raise e
    366         else:
    367             raise CompilerError("All available pipelines exhausted")

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in _compile_core(self)
    354             res = None
    355             try:
--> 356                 pm.run(self.state)
    357                 if self.state.cr is not None:
    358                     break

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_machinery.py in run(self, state)
    326                     (self.pipeline_name, pass_desc)
    327                 patched_exception = self._patch_error(msg, e)
--> 328                 raise patched_exception
    329 
    330     def dependency_analysis(self):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_machinery.py in run(self, state)
    317                 pass_inst = _pass_registry.get(pss).pass_inst
    318                 if isinstance(pass_inst, CompilerPass):
--> 319                     self._runPass(idx, pass_inst, state)
    320                 else:
    321                     raise BaseException("Legacy pass in use")

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_machinery.py in _runPass(self, index, pss, internal_state)
    279             mutated |= check(pss.run_initialization, internal_state)
    280         with SimpleTimer() as pass_time:
--> 281             mutated |= check(pss.run_pass, internal_state)
    282         with SimpleTimer() as finalize_time:
    283             mutated |= check(pss.run_finalizer, internal_state)

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_machinery.py in check(func, compiler_state)
    266 
    267         def check(func, compiler_state):
--> 268             mangled = func(compiler_state)
    269             if mangled not in (True, False):
    270                 msg = ("CompilerPass implementations should return True/False. "

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/typed_passes.py in run_pass(self, state)
    378             state.library.enable_object_caching()
    379 
--> 380         NativeLowering().run_pass(state) # TODO: Pull this out into the pipeline
    381         lowered = state['cr']
    382         signature = typing.signature(state.return_type, *state.args)

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/typed_passes.py in run_pass(self, state)
    322             with targetctx.push_code_library(library):
    323                 lower = lowering.Lower(targetctx, library, fndesc, interp,
--> 324                                        metadata=metadata)
    325                 lower.lower()
    326                 if not flags.no_cpython_wrapper:

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/lowering.py in __init__(self, context, library, fndesc, func_ir, metadata)
     95         # Python execution environment (will be available to the compiled
     96         # function).
---> 97         self.env = Environment.from_fndesc(self.fndesc)
     98 
     99         # Internal states

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/lowering.py in from_fndesc(cls, fndesc)
     34             return cls._memo[fndesc.env_name]
     35         except KeyError:
---> 36             inst = cls(fndesc.lookup_globals())
     37             inst.env_name = fndesc.env_name
     38             cls._memo[fndesc.env_name] = inst

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/funcdesc.py in lookup_globals(self)
     80         dynamically (i.e. exec)
     81         """
---> 82         return self.global_dict or self.lookup_module().__dict__
     83 
     84     def lookup_module(self):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/funcdesc.py in lookup_module(self)
     91             return _dynamic_module
     92         else:
---> 93             return sys.modules[self.modname]
     94 
     95     def lookup_function(self):

KeyError: "Failed in nopython mode pipeline (step: nopython mode backend)\n'mypackage'"
```  
</details>

## Expected behavio[u]r

I would expect this to either correctly print `0` (as it would without the decorator), or a better error message. This occurs in both 'nopython' and 'object' modes.

## Context

This occurred when trying to Numba-compile a function that another library constructed in the above way (via `exec`). This might be considered a mistake with that library (i.e., poor control of `__name__`)?

## Workarounds

Changing `__name__` to one of (`'__main__'`, `None`, `'numba'`, `'numpy'`) or another module name that is already in the namespace, appears to fix the issue. If `name` cannot be changed, then the following hack also appears to work (inserted after the `exec` statement):
```python
import imp
import sys
sys.modules[name] = imp.new_module(name)
```


Discussion:
Thanks for the report, I can reproduce. The issue here is because Numba expects the module from which the function is being compiled to exist in the namespace, which it clearly does not.

This sort of thing works:
```pycon
In [1]: z="""def foo(): 
   ...:     return 10 
   ...: """                                                                                                                     

In [2]: l={}; exec(z, {}, l)                                                                                                    

In [3]: from numba import njit                                                                                                  

In [4]: f=njit(l['foo'])                                                                                                        

In [5]: f()                                                                                                                     
Out[5]: 10
```
because Numba can see from the bytecode that this is a dynamically generated module, similarly:
```python
filename = 'test.py'
code = """
def f(x):
    return x
"""

objs = dict(__file__=filename)
compiled = compile(code, filename, 'exec')
exec(compiled, objs)

from numba import njit
compiled_f = njit(objs['f'])
print(compiled_f(0))
```
would work fine.

The logic employed is here:
https://github.com/numba/numba/blob/ceab10620764d1ff29e0c0e550d1d5975674819b/numba/bytecode.py#L334-L337
with `utils._dynamic_module` being the magic token that permits compilation when the module is declared as `None`.
It is common for python code to expect `sys.modules[__name__]` to resolve to the current module.  For instance, the following use of pickle will fail as well:

```python

In [1]: filename = 'test.py'
   ...: name = 'mypackage'
   ...: code = """
   ...: def f(x):
   ...:     return x
   ...: import pickle
   ...: pickle.dumps(f)
   ...: """
   ...:
   ...: objs = dict(__file__=filename, __name__=name)
   ...: compiled = compile(code, filename, 'exec')
   ...: exec(compiled, objs)
---------------------------------------------------------------------------
PicklingError                             Traceback (most recent call last)
<ipython-input-1-d8cb7c1bf98a> in <module>
     10 objs = dict(__file__=filename, __name__=name)
     11 compiled = compile(code, filename, 'exec')
---> 12 exec(compiled, objs)

test.py in <module>

PicklingError: Can't pickle <function f at 0x110ebd050>: import of module 'mypackage' failed
```

Python doesn't enforce that a code object is stored in a valid and resolvable Module object, but users should make sure that because many code expects it. In addition, the function object do not keep a reference back to the parent module, as the module is referenced by name and resolved by `sys.modules[func.__module__]`.  Therefore, it is the programmers responsibility to make sure the parent module is stored somewhere.
Thanks for looking into this quickly. It sounds fair then to say that this is more of an issue with the library that I am using than it is with Numba, and so I'm trying to get this corrected in the other library. Nevertheless if this is indeed connected to other issues such as #2547, #3236 then it might help people who are trying to use Numba within other tools to have another fallback or a better error message for these edge cases where the library is doing something non-conventional.
@arvoelke agreed, patches welcome :) In the weekly Numba issue triage meeting the outcome for this issue was to make the error message match that of `pickle` such that it hopefully makes it clear that it's the same problem.
The "good first issue" task is to run the above reproducer, locate in the code base where the problem arises, catch it and raise an error message like that from `pickle`, then add a test to demonstrate it works :)

Resolving patch:
diff --git a/numba/core/funcdesc.py b/numba/core/funcdesc.py
--- a/numba/core/funcdesc.py
+++ b/numba/core/funcdesc.py
@@ -84,12 +84,17 @@ def lookup_module(self):
         """
         Return the module in which this function is supposed to exist.
         This may be a dummy module if the function was dynamically
-        generated.
+        generated. Raise exception if the module can't be found.
         """
         if self.modname == _dynamic_modname:
             return _dynamic_module
         else:
-            return sys.modules[self.modname]
+            try:
+                return sys.modules[self.modname]
+            except:
+                raise ModuleNotFoundError(
+                    f"can't compile {self.qualname}: "
+                    f"import of module {self.modname} failed")
 
     def lookup_function(self):
         """


Test patch:
diff --git a/numba/tests/test_funcdesc.py b/numba/tests/test_funcdesc.py
new file mode 100644
--- /dev/null
+++ b/numba/tests/test_funcdesc.py
@@ -0,0 +1,31 @@
+
+import unittest
+from numba import njit
+
+
+class TestModule(unittest.TestCase):
+    def test_module_not_in_namespace(self):
+        """ Test of trying to run a compiled function
+        where the module from which the function is being compiled
+        doesn't exist in the namespace.
+        """
+        filename = 'test.py'
+        name = 'mypackage'
+        code = """
+def f(x):
+    return x
+"""
+
+        objs = dict(__file__=filename, __name__=name)
+        compiled = compile(code, filename, 'exec')
+        exec(compiled, objs)
+
+        compiled_f = njit(objs['f'])
+        with self.assertRaises(ModuleNotFoundError) as raises:
+            compiled_f(0)
+        msg = "can't compile f: import of module mypackage failed"
+        self.assertIn(msg, str(raises.exception))
+
+
+if __name__ == '__main__':
+    unittest.main()