Repository: numba/numba

Issue:
Multiple return types of `yield` are not detected if one of them is `None`
When trying to compile some code, I got an error message asking me to submit a bug report, so I am doing so. A minimal reproducer is:

```
import numba as nb

@nb.njit
def test():
	yield None
	yield 1

g = test()
```

I guess the only bug here is in the error message, since if I change `None` to (for example) `'a'`, I get a message saying that the return types of yield are incompatible. With the code quoted above, I get the error message

```
Traceback (most recent call last):
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/errors.py", line 617, in new_error_context
    yield
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/lowering.py", line 259, in lower_block
    self.lower_inst(inst)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/lowering.py", line 308, in lower_inst
    val = self.lower_assign(ty, inst)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/lowering.py", line 482, in lower_assign
    res = self.lower_yield(ty, value)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/lowering.py", line 505, in lower_yield
    self.call_conv.return_value(self.builder, retval)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/targets/callconv.py", line 351, in return_value
    (str(retval.type), str(retptr.type.pointee))
AssertionError: ('{i64, i1}', 'i64')

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "minrep.py", line 8, in <module>
    g = test()
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/dispatcher.py", line 367, in _compile_for_args
    raise e
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/dispatcher.py", line 324, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/dispatcher.py", line 655, in compile
    cres = self._compiler.compile(args, return_type)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/dispatcher.py", line 82, in compile
    pipeline_class=self.pipeline_class)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 926, in compile_extra
    return pipeline.compile_extra(func)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 374, in compile_extra
    return self._compile_bytecode()
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 857, in _compile_bytecode
    return self._compile_core()
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 844, in _compile_core
    res = pm.run(self.status)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 255, in run
    raise patched_exception
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 246, in run
    stage()
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 717, in stage_nopython_backend
    self._backend(lowerfn, objectmode=False)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 666, in _backend
    lowered = lowerfn()
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 653, in backend_nopython_mode
    self.metadata)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/compiler.py", line 1048, in native_lowering_stage
    lower.lower()
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/lowering.py", line 184, in lower
    self.genlower.lower_next_func(self)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/generators.py", line 153, in lower_next_func
    entry_block_tail = lower.lower_function_body()
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/lowering.py", line 244, in lower_function_body
    self.lower_block(block)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/lowering.py", line 259, in lower_block
    self.lower_inst(inst)
  File "/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/errors.py", line 625, in new_error_context
    six.reraise(type(newerr), newerr, tb)
  File "/Users/ndv/Library/Python/3.6/lib/python/site-packages/numba/six.py", line 659, in reraise
    raise value
numba.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend)
('{i64, i1}', 'i64')

File "minrep.py", line 5:
def test():
	yield None
 ^

[1] During: lowering "$0.2 = yield $const0.1" at minrep.py (5)
-------------------------------------------------------------------------------
This should not have happened, a problem has occurred in Numba's internals.

Please report the error message and traceback, along with a minimal reproducer
at: https://github.com/numba/numba/issues/new

If more help is needed please feel free to speak to the Numba core developers
directly at: https://gitter.im/numba/numba

Thanks in advance for your help in improving Numba!
```

(I thought this was a different problem at first - I have edited this post to give a better minimal reproducer.)


Discussion:

Resolving patch:
diff --git a/numba/typeinfer.py b/numba/typeinfer.py
--- a/numba/typeinfer.py
+++ b/numba/typeinfer.py
@@ -27,11 +27,14 @@
 from .funcdesc import qualifying_prefix
 
 
-class NOTSET: pass
+class NOTSET:
+    pass
+
 
 # terminal color markup
 _termcolor = termcolor()
 
+
 class TypeVar(object):
     def __init__(self, context, var):
         self.context = context
@@ -954,13 +957,13 @@ def check_var(name):
                 val = getattr(offender, 'value', 'unknown operation')
                 loc = getattr(offender, 'loc', ir.unknown_loc)
                 msg = ("Type of variable '%s' cannot be determined, operation:"
-                      " %s, location: %s")
+                       " %s, location: %s")
                 raise TypingError(msg % (var, val, loc), loc)
             tp = tv.getone()
             if not tp.is_precise():
                 offender = find_offender(name, exhaustive=True)
                 msg = ("Cannot infer the type of variable '%s'%s, "
-                      "have imprecise type: %s. %s")
+                       "have imprecise type: %s. %s")
                 istmp = " (temporary variable)" if var.startswith('$') else ""
                 loc = getattr(offender, 'loc', ir.unknown_loc)
                 # is this an untyped list? try and provide help
@@ -999,9 +1002,28 @@ def get_generator_type(self, typdict, retty):
             msg = "Cannot type generator: it does not yield any value"
             raise TypingError(msg)
         yield_type = self.context.unify_types(*yield_types)
-        if yield_type is None:
+        if yield_type is None or isinstance(yield_type, types.Optional):
             msg = "Cannot type generator: cannot unify yielded types %s"
-            raise TypingError(msg % (yield_types,))
+            yp_highlights = []
+            for y in gi.get_yield_points():
+                msg = (_termcolor.errmsg("Yield of: IR '%s', type '%s', "
+                                         "location: %s"))
+                yp_highlights.append(msg % (str(y.inst),
+                                            typdict[y.inst.value.name],
+                                            y.inst.loc.strformat()))
+
+            explain_ty = set()
+            for ty in yield_types:
+                if isinstance(ty, types.Optional):
+                    explain_ty.add(ty.type)
+                    explain_ty.add(types.NoneType('none'))
+                else:
+                    explain_ty.add(ty)
+            raise TypingError("Can't unify yield type from the "
+                              "following types: %s"
+                              % ", ".join(sorted(map(str, explain_ty))) +
+                              "\n\n" + "\n".join(yp_highlights))
+
         return types.Generator(self.func_id.func, yield_type, arg_types,
                                state_types, has_finalizer=True)
 
@@ -1304,9 +1326,8 @@ def typeof_expr(self, inst, target, expr):
             self.constraints.append(constraint)
             self.calls.append((inst.value, constraint))
         elif expr.op == 'getitem':
-            self.typeof_intrinsic_call(
-                inst, target, operator.getitem, expr.value, expr.index,
-                )
+            self.typeof_intrinsic_call(inst, target, operator.getitem,
+                                       expr.value, expr.index,)
         elif expr.op == 'getattr':
             constraint = GetAttrConstraint(target.name, attr=expr.attr,
                                            value=expr.value, loc=inst.loc,


Test patch:
diff --git a/numba/tests/test_generators.py b/numba/tests/test_generators.py
--- a/numba/tests/test_generators.py
+++ b/numba/tests/test_generators.py
@@ -116,6 +116,20 @@ def gen_ndenumerate(arr):
 def gen_bool():
     yield True
 
+
+def gen_unification_error():
+    yield None
+    yield 1j
+
+
+def gen_optional_and_type_unification_error():
+    # yields complex and optional(literalint)
+    i = 0
+    yield 1j
+    while True:
+        i = yield i
+
+
 class TestGenerators(MemoryLeakMixin, TestCase):
     def check_generator(self, pygen, cgen):
         self.assertEqual(next(cgen), next(pygen))
@@ -185,7 +199,7 @@ def test_gen4_objmode(self):
 
     def test_gen5(self):
         with self.assertTypingError() as cm:
-            cr = compile_isolated(gen5, ())
+            compile_isolated(gen5, ())
         self.assertIn("Cannot type generator: it does not yield any value",
                       str(cm.exception))
 
@@ -319,6 +333,24 @@ def test_ndenumerate(self):
     def test_ndenumerate_objmode(self):
         self.check_np_flat(gen_ndenumerate, flags=forceobj_flags)
 
+    def test_type_unification_error(self):
+        pyfunc = gen_unification_error
+        with self.assertTypingError() as e:
+            compile_isolated(pyfunc, (), flags=no_pyobj_flags)
+
+        msg = ("Can't unify yield type from the following types: complex128, "
+               "none")
+        self.assertIn(msg, str(e.exception))
+
+    def test_optional_expansion_type_unification_error(self):
+        pyfunc = gen_optional_and_type_unification_error
+        with self.assertTypingError() as e:
+            compile_isolated(pyfunc, (), flags=no_pyobj_flags)
+
+        msg = ("Can't unify yield type from the following types: complex128, "
+               "int%s, none")
+        self.assertIn(msg % types.intp.bitwidth, str(e.exception))
+
 
 class TestGenExprs(MemoryLeakMixin, TestCase):
     @testing.allow_interpreter_mode