Repository: hylang/hy

Issue:
py2 `yield-from` does not propagate exceptions *into* the inner generators.
I've found a concrete example of the difference in semantics between the `yield-from` macro and [PEP-380](https://www.python.org/dev/peps/pep-0380#formal-semantics). When using `generator.throw` in Python 2, `yield-from` does not rethrow the exception in the inner generator:

``` hy
hy 0.10.1 using CPython(default) 2.7.8 on Linux
=> (defn f []
...   (yield 1)
...   (try
...     (yield 2)
...     (catch [e ValueError] (yield 3))))
=> (defn g []
...   (yield-from (f))
...   (assert true))  ; see issue #691
=> (setv x (g))
=> (next x)
1L
=> (next x)
2L
=> (x.throw ValueError)
Traceback (most recent call last):
  File "<input>", line 1, in <module>
  File "<input>", line 257, in g  ; note that ValueError didn't even reach `f`
ValueError
```

In Python 3, `ValueError` reaches `f` and is handled there as expected:

``` hy
hy 0.10.1 using CPython(default) 3.4.2 on Linux
...
=> (next x)
2
=> (x.throw ValueError)
3
```



Discussion:
So, this is a real bug, but i'm not sure how we can sove this for 0.10.2 yet.

Testcase for future hypsters:

``` diff
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index 9066f9b..e58c9cd 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -241,6 +241,26 @@
     (yield-from [1 2 3]))
   (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))

+(defn test-yield-from-exception-handling-up []
+  "NATIVE: Ensure exception handling in yield from works right going up"
+  ; testcase taken from https://github.com/hylang/hy/issues/692
+  ; thanks, @pyos!
+
+  (defn f []
+     (yield 1)
+     (try
+       (yield 2)
+     (catch [e ValueError] (yield 3))))
+
+  (defn g []
+     (yield-from (f))
+     (assert true))  ; see issue #691
+  (setv x (g))
+  (next x)
+  (next x)
+  (assert (= (x.throw ValueError) 3)))
+
+
 (defn test-yield-from-exception-handling []
   "NATIVE: Ensure exception handling in yield from works right"
   (defn yield-from-subgenerator-test []
```

This still hasn't been fixed? PEP-380, mentioned above defines the `yield from` statement in equivalent Python. It's not simple, but it should be straightforward to directly translate this into Hy and put it in the macro. The relevant code, in Python:

``` Python
# The statement
# RESULT = yield from EXPR
# is semantically equivalent to

_i = iter(EXPR)
try:
    _y = next(_i)
except StopIteration as _e:
    _r = _e.value
else:
    while 1:
        try:
            _s = yield _y
        except GeneratorExit as _e:
            try:
                _m = _i.close
            except AttributeError:
                pass
            else:
                _m()
            raise _e
        except BaseException as _e:
            _x = sys.exc_info()
            try:
                _m = _i.throw
            except AttributeError:
                raise _e
            else:
                try:
                    _y = _m(*_x)
                except StopIteration as _e:
                    _r = _e.value
                    break
        else:
            try:
                if _s is None:
                    _y = next(_i)
                else:
                    _y = _i.send(_s)
            except StopIteration as _e:
                _r = _e.value
                break
RESULT = _r
```

(Deleted my comment, I should read the bug thread before replying via email)

@gilch send a PR! :D 

A proof of concept in Python 2:

``` Python
import sys

def f():
    yield 1
    try:
        yield 2
    except ValueError as e:
        yield 3

def g():
    EXPR = f()
    # RESULT = yield from EXPR
    _i = iter(EXPR)
    try:
        _y = next(_i)
    except StopIteration as _e:
        _r = _e.value
    else:
        while 1:
            try:
                _s = yield _y
            except GeneratorExit as _e:
                try:
                    _m = _i.close
                except AttributeError:
                    pass
                else:
                    _m()
                raise _e
            except BaseException as _e:
                _x = sys.exc_info()
                try:
                    _m = _i.throw
                except AttributeError:
                    raise _e
                else:
                    try:
                        _y = _m(*_x)
                    except StopIteration as _e:
                        _r = _e.value
                        break
            else:
                try:
                    if _s is None:
                        _y = next(_i)
                    else:
                        _y = _i.send(_s)
                except StopIteration as _e:
                    _r = _e.value
                    break
    RESULT = _r

x = g()
print next(x)
print next(x)
print x.throw(ValueError)
```

``` Python
>>> ================================ RESTART ================================
>>> 
1
2
3
>>> 
```

It actually does seem to work. The macro would probably replace the `_x` (for various `x`) with gensyms. Also, Python 3 didn't get `yield from` until version 3.3. Are we still supporting Python 3 older than that?

A pretty direct translation to Hy.

``` Hy
(import sys)

(defn f []
  (yield 1)
  (try
    (yield 2)
    (except [e ValueError]
            (yield 3))))

(defn g []
  (setv EXPR (f))
  ;; (setv RESULT (yield-from EXPR))
  (setv _i (iter EXPR))
  (try
    (setv _y (next _i))
    (except [_e StopIteration]
            (setv _r _e.value))
    (else
      (while 1
        (try
          (setv _s (yield _y))
          (except [_e GeneratorExit]
                  (try
                    (setv _m _i.close)
                    (except [AttributeError] pass)
                    (else (_m)))
                  (raise _e))
          (except [_e BaseException]
                  (setv _x (sys.exc_info))
                  (try
                    (setv _m _i.throw)
                    (except [AttributeError] (raise _e))
                    (else
                      (try
                        (setv _y (apply _m _x))
                        (except [_e StopIteration]
                                (setv _r _e.value)
                                (break))))))
          (else
            (try
              (if (is _s nil)
                (setv _y (next _i))
                (setv _y (_i.send _s)))
              (except [_e StopIteration]
                      (setv _r _e.value)
                      (break))))))))
  (setv RESULT _r))

(def x (g))
(print (next x))
(print (next x))
(print (x.throw ValueError))
```

It seems to work, but I might have missed something. @hylang/core More eyeballs please.

This will not be simple macro, but it should be a simple matter of inserting gensyms and such now that we have a Hy translation.

The macro version.

(EDIT: see new version below)

Again, eyeballs. I could probably drop this into Hy proper. How well are we testing this already? Do we need to support Python3 < 3.3?

> Do we need to support Python3 < 3.3?

No; we don't support 3.2 (602f392fe7, 5004e754d3).

I hesitate to suggest, but with this macro, we could probably support < 3.3 again. Unless there's something else that breaks.

@gilch The strategy of translating the PEP 380 example might still be vulnerable to #691; at least, I think that's what I observed when [I tried this the other year](https://github.com/zackmdavis/hy/commit/2e0553c9fe), according to the commit message.

Fixed more errors, new proposed drop-in:
EDIT: actually this one.

``` Hy
(if-python2
  (defmacro/g! yield-from* [EXPR]
    `(do
       ;; (setv RESULT (yield-from EXPR))
       (setv ~g!i (iter ~EXPR))
       (try
         (setv ~g!y (next ~g!i))
         (except [~g!e StopIteration]
                 (setv ~g!r (. ~g!e value)))
         (else
           (while 1
             (try
               (setv ~g!s (yield ~g!y))
               (except [~g!e GeneratorExit]
                       (try
                         (setv ~g!m (. ~g!i close))
                         (except [AttributeError] pass)
                         (else (~g!m)))
                       (raise ~g!e))
               (except [~g!e BaseException]
                       (setv ~g!x (sys.exc_info))
                       (try
                         (setv ~g!m (. ~g!i throw))
                         (except [AttributeError] (raise ~g!e))
                         (else
                           (try
                             (setv ~g!y (apply ~g!m ~g!x))
                             (except [~g!e StopIteration]
                                     (setv ~g!r (. ~g!e value))
                                     (break))))))
               (else
                 (try
                   (if (is ~g!s nil)
                     (setv ~g!y (next ~g!i))
                     (setv ~g!y (~g!i.send ~g!s)))
                   (except [~g!e StopIteration]
                           (setv ~g!r (. ~g!e value))
                           (break))))))))
       ~g!r))
  nil)
```

@zackmdavis I can't reproduce #691 with my macro. Maybe it's working now?


Resolving patch:
diff --git a/hy/compiler.py b/hy/compiler.py
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1128,13 +1128,9 @@ def compile_yield_expression(self, expr):
 
         return ret
 
-    @builds("yield_from")
+    @builds_if("yield_from", PY3)
     @checkargs(max=1)
     def compile_yield_from_expression(self, expr):
-        if not PY3:
-            raise HyCompileError(
-                "yield-from only supported in python 3.3+!")
-
         expr.pop(0)
         ret = Result(contains_yield=True)
 


Test patch:
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -306,31 +306,6 @@
   (assert (= (lif-not 0 "false" "true") "true")))
 
 
-(defn test-yield-from []
-  "NATIVE: testing yield from"
-  (defn yield-from-test []
-    (for* [i (range 3)]
-      (yield i))
-    (yield-from [1 2 3]))
-  (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))
-
-(defn test-yield-from-exception-handling []
-  "NATIVE: Ensure exception handling in yield from works right"
-  (defn yield-from-subgenerator-test []
-    (yield 1)
-    (yield 2)
-    (yield 3)
-    (assert 0))
-  (defn yield-from-test []
-    (for* [i (range 3)]
-       (yield i))
-    (try
-     (yield-from (yield-from-subgenerator-test))
-     (except [e AssertionError]
-       (yield 4))))
-  (assert (= (list (yield-from-test)) [0 1 2 1 2 3 4])))
-
-
 (defn test-defmain []
   "NATIVE: make sure defmain is clean"
   (global --name--)
diff --git a/tests/native_tests/py3_only_tests.hy b/tests/native_tests/py3_only_tests.hy
--- a/tests/native_tests/py3_only_tests.hy
+++ b/tests/native_tests/py3_only_tests.hy
@@ -35,3 +35,29 @@
   (assert (= (apply function-of-various-args
                     [1 2 3 4] {"foo" 5 "bar" 6 "quux" 7})
              (, 1 2 (, 3 4)  5 {"bar" 6 "quux" 7}))))
+
+
+(defn test-yield-from []
+  "NATIVE: testing yield from"
+  (defn yield-from-test []
+    (for* [i (range 3)]
+      (yield i))
+    (yield-from [1 2 3]))
+  (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))
+
+
+(defn test-yield-from-exception-handling []
+  "NATIVE: Ensure exception handling in yield from works right"
+  (defn yield-from-subgenerator-test []
+    (yield 1)
+    (yield 2)
+    (yield 3)
+    (assert 0))
+  (defn yield-from-test []
+    (for* [i (range 3)]
+       (yield i))
+    (try
+     (yield-from (yield-from-subgenerator-test))
+     (except [e AssertionError]
+       (yield 4))))
+  (assert (= (list (yield-from-test)) [0 1 2 1 2 3 4])))