Repository: numba/numba

Issue:
Cryptic error when compiling a function obtained via `exec` with bad `__name__`
Possibly connected to these issues (found while searching for snippets from the stack trace):
 - https://github.com/numba/numba/issues/2547
 - https://github.com/numba/numba/issues/3236

## Reporting a bug

- [x] I am using the latest released version of Numba (most recent is visible in
 the change log (https://github.com/numba/numba/blob/master/CHANGE_LOG).
- [x] I have included below a minimal working reproducer (if you are unsure how
 to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).

## Minimal reproducer

```python
filename = 'test.py'
name = 'mypackage'
code = """
def f(x):
    return x
"""

objs = dict(__file__=filename, __name__=name)
compiled = compile(code, filename, 'exec')
exec(compiled, objs)

from numba import njit
compiled_f = njit(objs['f'])
print(compiled_f(0))
```

<details>
  <summary>Click for full stack trace!</summary>

```
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/lowering.py in from_fndesc(cls, fndesc)
     33             # Avoid creating new Env
---> 34             return cls._memo[fndesc.env_name]
     35         except KeyError:

~/anaconda3/envs/nengo-numba/lib/python3.7/weakref.py in __getitem__(self, key)
    136             self._commit_removals()
--> 137         o = self.data[key]()
    138         if o is None:

KeyError: '_ZN08NumbaEnv9mypackage6f$2433Ex'

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
<ipython-input-57-67f415c0b452> in <module>
     12 from numba import njit
     13 compiled_f = njit(objs['f'])
---> 14 print(compiled_f(0))

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    418                     e.patch_message('\n'.join((str(e).rstrip(), help_msg)))
    419             # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 420             raise e
    421 
    422     def inspect_llvm(self, signature=None):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    351                 argtypes.append(self.typeof_pyval(a))
    352         try:
--> 353             return self.compile(tuple(argtypes))
    354         except errors.ForceLiteralArg as e:
    355             # Received request for compiler re-entry with the list of arguments

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in compile(self, sig)
    766             self._cache_misses[sig] += 1
    767             try:
--> 768                 cres = self._compiler.compile(args, return_type)
    769             except errors.ForceLiteralArg as e:
    770                 def folded(args, kws):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in compile(self, args, return_type)
     75 
     76     def compile(self, args, return_type):
---> 77         status, retval = self._compile_cached(args, return_type)
     78         if status:
     79             return retval

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in _compile_cached(self, args, return_type)
     89 
     90         try:
---> 91             retval = self._compile_core(args, return_type)
     92         except errors.TypingError as e:
     93             self._failed_cache[key] = e

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/dispatcher.py in _compile_core(self, args, return_type)
    107                                       args=args, return_type=return_type,
    108                                       flags=flags, locals=self.locals,
--> 109                                       pipeline_class=self.pipeline_class)
    110         # Check typing error if object mode is used
    111         if cres.typing_error is not None and not flags.enable_pyobject:

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library, pipeline_class)
    526     pipeline = pipeline_class(typingctx, targetctx, library,
    527                               args, return_type, flags, locals)
--> 528     return pipeline.compile_extra(func)
    529 
    530 

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in compile_extra(self, func)
    324         self.state.lifted = ()
    325         self.state.lifted_from = None
--> 326         return self._compile_bytecode()
    327 
    328     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in _compile_bytecode(self)
    383         """
    384         assert self.state.func_ir is None
--> 385         return self._compile_core()
    386 
    387     def _compile_ir(self):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in _compile_core(self)
    363                 self.state.status.fail_reason = e
    364                 if is_final_pipeline:
--> 365                     raise e
    366         else:
    367             raise CompilerError("All available pipelines exhausted")

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler.py in _compile_core(self)
    354             res = None
    355             try:
--> 356                 pm.run(self.state)
    357                 if self.state.cr is not None:
    358                     break

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_machinery.py in run(self, state)
    326                     (self.pipeline_name, pass_desc)
    327                 patched_exception = self._patch_error(msg, e)
--> 328                 raise patched_exception
    329 
    330     def dependency_analysis(self):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_machinery.py in run(self, state)
    317                 pass_inst = _pass_registry.get(pss).pass_inst
    318                 if isinstance(pass_inst, CompilerPass):
--> 319                     self._runPass(idx, pass_inst, state)
    320                 else:
    321                     raise BaseException("Legacy pass in use")

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_machinery.py in _runPass(self, index, pss, internal_state)
    279             mutated |= check(pss.run_initialization, internal_state)
    280         with SimpleTimer() as pass_time:
--> 281             mutated |= check(pss.run_pass, internal_state)
    282         with SimpleTimer() as finalize_time:
    283             mutated |= check(pss.run_finalizer, internal_state)

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/compiler_machinery.py in check(func, compiler_state)
    266 
    267         def check(func, compiler_state):
--> 268             mangled = func(compiler_state)
    269             if mangled not in (True, False):
    270                 msg = ("CompilerPass implementations should return True/False. "

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/typed_passes.py in run_pass(self, state)
    378             state.library.enable_object_caching()
    379 
--> 380         NativeLowering().run_pass(state) # TODO: Pull this out into the pipeline
    381         lowered = state['cr']
    382         signature = typing.signature(state.return_type, *state.args)

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/typed_passes.py in run_pass(self, state)
    322             with targetctx.push_code_library(library):
    323                 lower = lowering.Lower(targetctx, library, fndesc, interp,
--> 324                                        metadata=metadata)
    325                 lower.lower()
    326                 if not flags.no_cpython_wrapper:

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/lowering.py in __init__(self, context, library, fndesc, func_ir, metadata)
     95         # Python execution environment (will be available to the compiled
     96         # function).
---> 97         self.env = Environment.from_fndesc(self.fndesc)
     98 
     99         # Internal states

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/lowering.py in from_fndesc(cls, fndesc)
     34             return cls._memo[fndesc.env_name]
     35         except KeyError:
---> 36             inst = cls(fndesc.lookup_globals())
     37             inst.env_name = fndesc.env_name
     38             cls._memo[fndesc.env_name] = inst

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/funcdesc.py in lookup_globals(self)
     80         dynamically (i.e. exec)
     81         """
---> 82         return self.global_dict or self.lookup_module().__dict__
     83 
     84     def lookup_module(self):

~/anaconda3/envs/nengo-numba/lib/python3.7/site-packages/numba/funcdesc.py in lookup_module(self)
     91             return _dynamic_module
     92         else:
---> 93             return sys.modules[self.modname]
     94 
     95     def lookup_function(self):

KeyError: "Failed in nopython mode pipeline (step: nopython mode backend)\n'mypackage'"
```  
</details>

## Expected behavio[u]r

I would expect this to either correctly print `0` (as it would without the decorator), or a better error message. This occurs in both 'nopython' and 'object' modes.

## Context

This occurred when trying to Numba-compile a function that another library constructed in the above way (via `exec`). This might be considered a mistake with that library (i.e., poor control of `__name__`)?

## Workarounds

Changing `__name__` to one of (`'__main__'`, `None`, `'numba'`, `'numpy'`) or another module name that is already in the namespace, appears to fix the issue. If `name` cannot be changed, then the following hack also appears to work (inserted after the `exec` statement):
```python
import imp
import sys
sys.modules[name] = imp.new_module(name)
```

Discussion:

Patch:
diff --git a/numba/core/funcdesc.py b/numba/core/funcdesc.py
index 6c1f8c0bc..4a9213077 100644
--- a/numba/core/funcdesc.py
+++ b/numba/core/funcdesc.py
@@ -84,12 +84,17 @@ class FunctionDescriptor(object):
         """
         Return the module in which this function is supposed to exist.
         This may be a dummy module if the function was dynamically
-        generated.
+        generated. Raise exception if the module can't be found.
         """
         if self.modname == _dynamic_modname:
             return _dynamic_module
         else:
-            return sys.modules[self.modname]
+            try:
+                return sys.modules[self.modname]
+            except:
+                raise ModuleNotFoundError(
+                    f"can't compile {self.qualname}: "
+                    f"import of module {self.modname} failed")
 
     def lookup_function(self):
         """


Test patch:
diff --git a/numba/tests/test_funcdesc.py b/numba/tests/test_funcdesc.py
new file mode 100644
index 000000000..4ea5f4949
--- /dev/null
+++ b/numba/tests/test_funcdesc.py
@@ -0,0 +1,31 @@
+
+import unittest
+from numba import njit
+
+
+class TestModule(unittest.TestCase):
+    def test_module_not_in_namespace(self):
+        """ Test of trying to run a compiled function
+        where the module from which the function is being compiled
+        doesn't exist in the namespace.
+        """
+        filename = 'test.py'
+        name = 'mypackage'
+        code = """
+def f(x):
+    return x
+"""
+
+        objs = dict(__file__=filename, __name__=name)
+        compiled = compile(code, filename, 'exec')
+        exec(compiled, objs)
+
+        compiled_f = njit(objs['f'])
+        with self.assertRaises(ModuleNotFoundError) as raises:
+            compiled_f(0)
+        msg = "can't compile f: import of module mypackage failed"
+        self.assertIn(msg, str(raises.exception))
+
+
+if __name__ == '__main__':
+    unittest.main()