Repository: numba/numba

Issue:
Lowering error with Numpy.fromiter
Error message:

```
numba.errors.LoweringError: Failed in object mode pipeline (step: object mode backend)
'NoneType' object has no attribute 'yield_points'

File "code/utility.py", line 96:
def error():
    <source elided>
    sampled = np.zeros(10, dtype=types)
    res = np.fromiter((func(x) for x in sampled), dtype=int)
    ^

[1] During: lowering "$4.5.117 = yield $4.4" at code/utility.py (96)
```


Reproducer:

```
def func(x):
    return x[0]

@jit
def error():
    topo = [chr(ord('a') + i) for i in range(5)]
    types = [(var_name, "int") for var_name in topo]
    sampled = np.zeros(10, dtype=types)
    res = np.fromiter((func(x) for x in sampled), dtype=int)

error()
```


Discussion:
Similar error, another reproducer:

```
@jit
def error():
    topo = [chr(ord('a') + i) for i in range(5)]
    types = [(var_name, "int") for var_name in topo]
    sampled = np.zeros(10, dtype=types)
    issue = tuple(sampled[i] for i in top[:3])

error()
```
@yujianll thanks for reporting this! We appreciate you using Numba and taking the time to open issues. I will reformat your comments for legibility and queue this for inspection.

I actually get a slightly different error:

```
vhaenel@parraline [numba] [numba:typed_list:★ ★ ★ ] ~/git/numba
💥 zsh» python foo11.py
Traceback (most recent call last):
  File "foo11.py", line 17, in <module>
    error()
  File "/Users/vhaenel/git/numba/numba/dispatcher.py", line 351, in _compile_for_args
    error_rewrite(e, 'typing')
  File "/Users/vhaenel/git/numba/numba/dispatcher.py", line 316, in error_rewrite
    raise e
  File "/Users/vhaenel/git/numba/numba/dispatcher.py", line 327, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "/Users/vhaenel/git/numba/numba/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/Users/vhaenel/git/numba/numba/dispatcher.py", line 659, in compile
    cres = self._compiler.compile(args, return_type)
  File "/Users/vhaenel/git/numba/numba/dispatcher.py", line 83, in compile
    pipeline_class=self.pipeline_class)
  File "/Users/vhaenel/git/numba/numba/compiler.py", line 955, in compile_extra
    return pipeline.compile_extra(func)
  File "/Users/vhaenel/git/numba/numba/compiler.py", line 377, in compile_extra
    return self._compile_bytecode()
  File "/Users/vhaenel/git/numba/numba/compiler.py", line 886, in _compile_bytecode
    return self._compile_core()
  File "/Users/vhaenel/git/numba/numba/compiler.py", line 873, in _compile_core
    res = pm.run(self.status)
  File "/Users/vhaenel/git/numba/numba/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/Users/vhaenel/git/numba/numba/compiler.py", line 254, in run
    raise patched_exception
  File "/Users/vhaenel/git/numba/numba/compiler.py", line 245, in run
    stage()
  File "/Users/vhaenel/git/numba/numba/compiler.py", line 501, in stage_nopython_frontend
    self.locals)
  File "/Users/vhaenel/git/numba/numba/compiler.py", line 1104, in type_inference_stage
    infer.build_constraint()
  File "/Users/vhaenel/git/numba/numba/typeinfer.py", line 877, in build_constraint
    self.constrain_statement(inst)
  File "/Users/vhaenel/git/numba/numba/typeinfer.py", line 1156, in constrain_statement
    self.typeof_assign(inst)
  File "/Users/vhaenel/git/numba/numba/typeinfer.py", line 1227, in typeof_assign
    self.typeof_global(inst, inst.target, value)
  File "/Users/vhaenel/git/numba/numba/typeinfer.py", line 1324, in typeof_global
    typ = self.resolve_value_type(inst, gvar.value)
  File "/Users/vhaenel/git/numba/numba/typeinfer.py", line 1248, in resolve_value_type
    raise TypingError(msg, loc=inst.loc)
numba.errors.TypingError: Failed in nopython mode pipeline (step: nopython frontend)
Untyped global name 'chr': cannot determine Numba type of <class 'builtin_function_or_method'>

File "foo11.py", line 11:
def error():
    topo = [chr(ord('a') + i) for i in range(5)]
```

But this is using `nopython` mode.
Using plain `@jit` I can reproduce the error you mentioned.
Yeah I tried both modes. I want to use fromiter to apply function to each row of Numpy array in the first case, and construct a tuple in the second case.
Thanks for the report. The issue in this function (and the other reproducer):
```python
@jit
def error():
    topo = [chr(ord('a') + i) for i in range(5)]
    types = [(var_name, "int") for var_name in topo]
    sampled = np.zeros(10, dtype=types)
    res = np.fromiter((func(x) for x in sampled), dtype=int)

error()
```
is that Numba does not support generator comprehensions, switching the arg to `np.fromiter` to a list comprehension fixes it, this works fine:
```python
from numba import jit
import numpy as np

def func(x):
    return x[0]

@jit
def error():
    topo = [chr(ord('a') + i) for i in range(5)]
    types = [(var_name, "int") for var_name in topo]
    sampled = np.zeros(10, dtype=types)
    res = np.fromiter([func(x) for x in sampled], dtype=int)

error()
```
am going to note this as a bug because the lowering error should not have happened as the unsupported functionality should have been caught earlier.
Thanks! So I cannot use generator comprehensions even in plain `@jit` mode?
@yujianll running the following suggests that you are correct:

```
from numba import jit


@jit
def error():
    gen = (i**2 for i in range(10))
    print(gen)


error.py_func()
error()
```

The first---the original python function---works and prints something about the generator, whereas the second one fails.
Thanks!

Resolving patch:
diff --git a/numba/inline_closurecall.py b/numba/inline_closurecall.py
--- a/numba/inline_closurecall.py
+++ b/numba/inline_closurecall.py
@@ -1,7 +1,8 @@
 import types as pytypes  # avoid confusion with numba.types
 import ctypes
 import numba
-from numba import config, ir, ir_utils, utils, prange, rewrites, types, typing
+from numba import (config, ir, ir_utils, utils, prange, rewrites, types, typing,
+                   errors)
 from numba.parfor import internal_prange
 from numba.ir_utils import (
     mk_unique_var,
@@ -40,6 +41,17 @@
 """
 enable_inline_arraycall = True
 
+
+def callee_ir_validator(func_ir):
+    """Checks the IR of a callee is supported for inlining
+    """
+    for blk in func_ir.blocks.values():
+        for stmt in blk.find_insts(ir.Assign):
+            if isinstance(stmt.value, ir.Yield):
+                msg = "The use of yield in a closure is unsupported."
+                raise errors.UnsupportedError(msg, loc=stmt.loc)
+
+
 class InlineClosureCallPass(object):
     """InlineClosureCallPass class looks for direct calls to locally defined
     closures, and inlines the body of the closure function to the call site.
@@ -132,7 +144,8 @@ def reduce_func(f, A, v):
             return s
         inline_closure_call(self.func_ir,
                         self.func_ir.func_id.func.__globals__,
-                        block, i, reduce_func, work_list=work_list)
+                        block, i, reduce_func, work_list=work_list,
+                        callee_validator=callee_ir_validator)
         return True
 
     def _inline_stencil(self, instr, call_name, func_def):
@@ -212,7 +225,8 @@ def _inline_closure(self, work_list, block, i, func_def):
                 func_def.op == "make_function")
         inline_closure_call(self.func_ir,
                             self.func_ir.func_id.func.__globals__,
-                            block, i, func_def, work_list=work_list)
+                            block, i, func_def, work_list=work_list,
+                            callee_validator=callee_ir_validator)
         return True
 
 def check_reduce_func(func_ir, func_var):
@@ -232,7 +246,7 @@ def check_reduce_func(func_ir, func_var):
 
 def inline_closure_call(func_ir, glbls, block, i, callee, typingctx=None,
                         arg_typs=None, typemap=None, calltypes=None,
-                        work_list=None):
+                        work_list=None, callee_validator=None):
     """Inline the body of `callee` at its callsite (`i`-th instruction of `block`)
 
     `func_ir` is the func_ir object of the caller function and `glbls` is its
@@ -242,6 +256,9 @@ def inline_closure_call(func_ir, glbls, block, i, callee, typingctx=None,
     make_function node. `typingctx`, `typemap` and `calltypes` are typing
     data structures of the caller, available if we are in a typed pass.
     `arg_typs` includes the types of the arguments at the callsite.
+    `callee_validator` is an optional callable which can be used to validate the
+    IR of the callee to ensure that it contains IR supported for inlining, it
+    takes one argument, the func_ir of the callee
 
     Returns IR blocks of the callee and the variable renaming dictionary used
     for them to facilitate further processing of new blocks.
@@ -257,6 +274,11 @@ def inline_closure_call(func_ir, glbls, block, i, callee, typingctx=None,
     callee_closure = callee.closure if hasattr(callee, 'closure') else callee.__closure__
     # first, get the IR of the callee
     callee_ir = get_ir_of_code(glbls, callee_code)
+    # check that the contents of the callee IR is something that can be inlined
+    # if a validator is supplied
+    if callee_validator is not None:
+        callee_validator(callee_ir)
+
     callee_blocks = callee_ir.blocks
 
     # 1. relabel callee_ir by adding an offset
diff --git a/numba/ir_utils.py b/numba/ir_utils.py
--- a/numba/ir_utils.py
+++ b/numba/ir_utils.py
@@ -1926,6 +1926,12 @@ def raise_on_unsupported_feature(func_ir, typemap):
                     isinstance(ty, types.DictType)):
                     raise TypingError(msg % (ty, stmt.value.name, ty), loc=stmt.loc)
 
+            # checks for generator expressions (yield in use when func_ir has
+            # not been identified as a generator).
+            if isinstance(stmt.value, ir.Yield) and not func_ir.is_generator:
+                msg = "The use of generator expressions is unsupported."
+                raise UnsupportedError(msg, loc=stmt.loc)
+
     # There is more than one call to function gdb/gdb_init
     if len(gdb_calls) > 1:
         msg = ("Calling either numba.gdb() or numba.gdb_init() more than once "
diff --git a/numba/utils.py b/numba/utils.py
--- a/numba/utils.py
+++ b/numba/utils.py
@@ -15,6 +15,7 @@
 import numpy as np
 
 from .six import *
+from .errors import UnsupportedError
 try:
     # preferred over pure-python StringIO due to threadsafety
     # note: parallel write to StringIO could cause data to go missing
@@ -77,11 +78,19 @@ def erase_traceback(exc_value):
     from inspect import Parameter as pyParameter
 except ImportError:
     try:
-        from funcsigs import signature as pysignature
+        from funcsigs import signature as _pysignature
         from funcsigs import Signature as pySignature
         from funcsigs import Parameter as pyParameter
         from funcsigs import BoundArguments
 
+        def pysignature(*args, **kwargs):
+            try:
+                return _pysignature(*args, **kwargs)
+            except ValueError as e:
+                msg = ("Cannot obtain a signature for: %s. The error message "
+                       "from funcsigs was: '%s'." % (args,  e.message))
+                raise UnsupportedError(msg)
+
         # monkey patch `apply_defaults` onto `BoundArguments` cf inspect in py3
         # This patch is from https://github.com/aliles/funcsigs/pull/30/files
         # with minor modifications, and thanks!


Test patch:
diff --git a/numba/tests/test_closure.py b/numba/tests/test_closure.py
--- a/numba/tests/test_closure.py
+++ b/numba/tests/test_closure.py
@@ -6,7 +6,7 @@
 
 import numba.unittest_support as unittest
 from numba import njit, jit, testing, utils
-from numba.errors import (TypingError, LoweringError, UnsupportedError)
+from numba.errors import UnsupportedError
 from .support import TestCase, tag
 from numba.six import exec_
 
@@ -429,22 +429,22 @@ def inner(x):
         msg = "make_function"
         self.assertIn(msg, str(raises.exception))
 
-        with self.assertRaises(TypingError) as raises:
+        with self.assertRaises(UnsupportedError) as raises:
             cfunc = jit(nopython=True)(outer16)
             cfunc(var)
-        msg = "with argument(s) of type(s): (none)"
+        msg = "The use of yield in a closure is unsupported."
         self.assertIn(msg, str(raises.exception))
 
-        with self.assertRaises(LoweringError) as raises:
+        with self.assertRaises(UnsupportedError) as raises:
             cfunc = jit(nopython=True)(outer17)
             cfunc(var)
-        msg = "'NoneType' object has no attribute 'yield_points'"
+        msg = "The use of yield in a closure is unsupported."
         self.assertIn(msg, str(raises.exception))
 
-        with self.assertRaises(TypingError) as raises:
+        with self.assertRaises(UnsupportedError) as raises:
             cfunc = jit(nopython=True)(outer18)
             cfunc(var)
-        msg = "Invalid use of getiter with parameters (none)"
+        msg = "The use of yield in a closure is unsupported."
         self.assertIn(msg, str(raises.exception))
 
 
diff --git a/numba/tests/test_errorhandling.py b/numba/tests/test_errorhandling.py
--- a/numba/tests/test_errorhandling.py
+++ b/numba/tests/test_errorhandling.py
@@ -168,6 +168,21 @@ def foo():
                     "has it been imported")
         self.assertIn(expected, str(raises.exception))
 
+    def test_handling_unsupported_generator_expression(self):
+        def foo():
+            y = (x for x in range(10))
+
+        if utils.IS_PY3:
+            expected = "The use of yield in a closure is unsupported."
+        else:
+            # funcsigs falls over on py27
+            expected = "Cannot obtain a signature for"
+
+        for dec in jit(forceobj=True), njit:
+            with self.assertRaises(errors.UnsupportedError) as raises:
+                dec(foo)()
+            self.assertIn(expected, str(raises.exception))
+
 
 class TestConstantInferenceErrorHandling(unittest.TestCase):