Repository: hylang/hy

Issue:
(yield-from) isn't a syntax error
Noticed by @vodik.

It's compiled to `yield from None`, which is never useful.


Discussion:

Resolving patch:
diff --git a/conftest.py b/conftest.py
--- a/conftest.py
+++ b/conftest.py
@@ -1,7 +1,7 @@
 import _pytest
 import hy
 import os
-from hy._compat import PY3, PY35
+from hy._compat import PY3, PY35, PY36
 
 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
 
@@ -10,7 +10,8 @@ def pytest_collect_file(parent, path):
             and NATIVE_TESTS in path.dirname + os.sep
             and path.basename != "__init__.hy"
             and not ("py3_only" in path.basename and not PY3)
-            and not ("py35_only" in path.basename and not PY35)):
+            and not ("py35_only" in path.basename and not PY35)
+            and not ("py36_only" in path.basename and not PY36)):
         m = _pytest.python.pytest_pycollect_makemodule(path, parent)
         # Spoof the module name to avoid hitting an assertion in pytest.
         m.name = m.name[:-len(".hy")] + ".py"
diff --git a/hy/_compat.py b/hy/_compat.py
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -22,6 +22,7 @@ def wr_long(f, x):
 
 PY3 = sys.version_info[0] >= 3
 PY35 = sys.version_info >= (3, 5)
+PY36 = sys.version_info >= (3, 6)
 
 str_type     = str   if PY3 else unicode      # NOQA
 bytes_type   = bytes if PY3 else str          # NOQA
diff --git a/hy/compiler.py b/hy/compiler.py
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -76,6 +76,9 @@ def _is_hy_builtin(name, module_name):
 
 
 _compile_table = {}
+_decoratables = (ast.FunctionDef, ast.ClassDef)
+if PY35:
+    _decoratables += (ast.AsyncFunctionDef,)
 
 
 def ast_str(foobar):
@@ -250,6 +253,8 @@ def rename(self, new_name):
                 var.arg = new_name
             elif isinstance(var, ast.FunctionDef):
                 var.name = new_name
+            elif PY35 and isinstance(var, ast.AsyncFunctionDef):
+                var.name = new_name
             else:
                 raise TypeError("Don't know how to rename a %s!" % (
                     var.__class__.__name__))
@@ -1130,13 +1135,19 @@ def compile_global_or_nonlocal(self, expr):
         return node(expr, names=names)
 
     @builds("yield")
-    @builds("yield_from", iff=PY3)
     @checkargs(max=1)
     def compile_yield_expression(self, expr):
         ret = Result(contains_yield=(not PY3))
         if len(expr) > 1:
             ret += self.compile(expr[1])
-        node = asty.Yield if expr[0] == "yield" else asty.YieldFrom
+        return ret + asty.Yield(expr, value=ret.force_expr)
+
+    @builds("yield_from", iff=PY3)
+    @builds("await", iff=PY35)
+    @checkargs(1)
+    def compile_yield_from_or_await_expression(self, expr):
+        ret = Result() + self.compile(expr[1])
+        node = asty.YieldFrom if expr[0] == "yield_from" else asty.Await
         return ret + node(expr, value=ret.force_expr)
 
     @builds("import")
@@ -1293,25 +1304,26 @@ def compile_cut_expression(self, expr):
     def compile_decorate_expression(self, expr):
         expr.pop(0)  # with-decorator
         fn = self.compile(expr.pop())
-        if not fn.stmts or not isinstance(fn.stmts[-1], (ast.FunctionDef,
-                                                         ast.ClassDef)):
+        if not fn.stmts or not isinstance(fn.stmts[-1], _decoratables):
             raise HyTypeError(expr, "Decorated a non-function")
         decorators, ret, _ = self._compile_collect(expr)
         fn.stmts[-1].decorator_list = decorators + fn.stmts[-1].decorator_list
         return ret + fn
 
     @builds("with*")
+    @builds("with/a*", iff=PY35)
     @checkargs(min=2)
     def compile_with_expression(self, expr):
-        expr.pop(0)  # with*
+        root = expr.pop(0)
 
         args = expr.pop(0)
         if not isinstance(args, HyList):
             raise HyTypeError(expr,
-                              "with expects a list, received `{0}'".format(
-                                  type(args).__name__))
+                              "{0} expects a list, received `{1}'".format(
+                                  root, type(args).__name__))
         if len(args) not in (1, 2):
-            raise HyTypeError(expr, "with needs [arg (expr)] or [(expr)]")
+            raise HyTypeError(expr,
+                              "{0} needs [arg (expr)] or [(expr)]".format(root))
 
         thing = None
         if len(args) == 2:
@@ -1330,10 +1342,11 @@ def compile_with_expression(self, expr):
             expr, targets=[name], value=asty.Name(
                 expr, id=ast_str("None"), ctx=ast.Load()))
 
-        the_with = asty.With(expr,
-                             context_expr=ctx.force_expr,
-                             optional_vars=thing,
-                             body=body.stmts)
+        node = asty.With if root == "with*" else asty.AsyncWith
+        the_with = node(expr,
+                        context_expr=ctx.force_expr,
+                        optional_vars=thing,
+                        body=body.stmts)
 
         if PY3:
             the_with.items = [ast.withitem(context_expr=ctx.force_expr,
@@ -1819,16 +1832,16 @@ def _compile_assign(self, name, result):
         return result
 
     @builds("for*")
+    @builds("for/a*", iff=PY35)
     @checkargs(min=1)
     def compile_for_expression(self, expression):
-        expression.pop(0)  # for
+        root = expression.pop(0)
 
         args = expression.pop(0)
-
         if not isinstance(args, HyList):
             raise HyTypeError(expression,
-                              "`for` expects a list, received `{0}`".format(
-                                  type(args).__name__))
+                              "`{0}` expects a list, received `{1}`".format(
+                                  root, type(args).__name__))
 
         try:
             target_name, iterable = args
@@ -1853,11 +1866,12 @@ def compile_for_expression(self, expression):
         body = self._compile_branch(expression)
         body += body.expr_as_stmt()
 
-        ret += asty.For(expression,
-                        target=target,
-                        iter=ret.force_expr,
-                        body=body.stmts,
-                        orelse=orel.stmts)
+        node = asty.For if root == 'for*' else asty.AsyncFor
+        ret += node(expression,
+                    target=target,
+                    iter=ret.force_expr,
+                    body=body.stmts,
+                    orelse=orel.stmts)
 
         ret.contains_yield = body.contains_yield
 
@@ -1890,12 +1904,15 @@ def compile_while_expression(self, expr):
         return ret
 
     @builds("fn", "fn*")
+    @builds("fn/a", iff=PY35)
     # The starred version is for internal use (particularly, in the
     # definition of `defn`). It ensures that a FunctionDef is
     # produced rather than a Lambda.
     @checkargs(min=1)
     def compile_function_def(self, expression):
-        force_functiondef = expression.pop(0) == "fn*"
+        root = expression.pop(0)
+        force_functiondef = root in ("fn*", "fn/a")
+        asyncdef = root == "fn/a"
 
         arglist = expression.pop(0)
         docstring = None
@@ -1904,7 +1921,7 @@ def compile_function_def(self, expression):
 
         if not isinstance(arglist, HyList):
             raise HyTypeError(expression,
-                              "First argument to `fn' must be a list")
+                              "First argument to `{}' must be a list".format(root))
 
         (ret, args, defaults, stararg,
          kwonlyargs, kwonlydefaults, kwargs) = self._parse_lambda_list(arglist)
@@ -1980,11 +1997,12 @@ def compile_function_def(self, expression):
 
         name = self.get_anon_var()
 
-        ret += asty.FunctionDef(expression,
-                                name=name,
-                                args=args,
-                                body=body.stmts,
-                                decorator_list=[])
+        node = asty.AsyncFunctionDef if asyncdef else asty.FunctionDef
+        ret += node(expression,
+                    name=name,
+                    args=args,
+                    body=body.stmts,
+                    decorator_list=[])
 
         ast_name = asty.Name(expression, id=name, ctx=ast.Load())
 


Test patch:
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -14,6 +14,7 @@
 from hy._compat import PY3
 
 import ast
+import pytest
 
 
 def _ast_spotcheck(arg, root, secondary):
@@ -651,3 +652,15 @@ def test_compiler_macro_tag_try():
     # https://github.com/hylang/hy/issues/1350
     can_compile("(defmacro foo [] (try None (except [] None)) `())")
     can_compile("(deftag foo [] (try None (except [] None)) `())")
+
+
+@pytest.mark.skipif(not PY3, reason="Python 3 required")
+def test_ast_good_yield_from():
+    "Make sure AST can compile valid yield-from"
+    can_compile("(yield-from [1 2])")
+
+
+@pytest.mark.skipif(not PY3, reason="Python 3 required")
+def test_ast_bad_yield_from():
+    "Make sure AST can't compile invalid yield-from"
+    cant_compile("(yield-from)")
diff --git a/tests/native_tests/py35_only_tests.hy b/tests/native_tests/py35_only_tests.hy
--- a/tests/native_tests/py35_only_tests.hy
+++ b/tests/native_tests/py35_only_tests.hy
@@ -5,6 +5,8 @@
 ;; Tests where the emitted code relies on Python ≥3.5.
 ;; conftest.py skips this file when running on Python <3.5.
 
+(import [asyncio [get-event-loop sleep]])
+
 
 (defn test-unpacking-pep448-1star []
   (setv l [1 2 3])
@@ -24,3 +26,78 @@
   (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"}))
   (defn fun [&optional a b c d e f] [a b c d e f])
   (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None])))
+
+
+(defn run-coroutine [coro]
+  "Run a coroutine until its done in the default event loop."""
+  (.run_until_complete (get-event-loop) (coro)))
+
+
+(defn test-fn/a []
+  (assert (= (run-coroutine (fn/a [] (await (sleep 0)) [1 2 3]))
+             [1 2 3])))
+
+
+(defn test-defn/a []
+  (defn/a coro-test []
+    (await (sleep 0))
+    [1 2 3])
+  (assert (= (run-coroutine coro-test) [1 2 3])))
+
+
+(defn test-decorated-defn/a []
+  (defn decorator [func] (fn/a [] (/ (await (func)) 2)))
+
+  #@(decorator
+      (defn/a coro-test []
+        (await (sleep 0))
+        42))
+  (assert (= (run-coroutine coro-test) 21)))
+
+
+(defclass AsyncWithTest []
+  (defn --init-- [self val]
+    (setv self.val val)
+    None)
+
+  (defn/a --aenter-- [self]
+    self.val)
+
+  (defn/a --aexit-- [self tyle value traceback]
+    (setv self.val None)))
+
+
+(defn test-single-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t (AsyncWithTest 1)]
+        (assert (= t 1))))))
+
+(defn test-two-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)]
+        (assert (= t1 1))
+        (assert (= t2 2))))))
+
+(defn test-thrice-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)
+               t3 (AsyncWithTest 3)]
+        (assert (= t1 1))
+        (assert (= t2 2))
+        (assert (= t3 3))))))
+
+(defn test-quince-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)
+               t3 (AsyncWithTest 3)
+               _ (AsyncWithTest 4)]
+        (assert (= t1 1))
+        (assert (= t2 2))
+        (assert (= t3 3))))))
diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy
new file mode 100644
--- /dev/null
+++ b/tests/native_tests/py36_only_tests.hy
@@ -0,0 +1,39 @@
+;; Copyright 2017 the authors.
+;; This file is part of Hy, which is free software licensed under the Expat
+;; license. See the LICENSE.
+
+;; Tests where the emitted code relies on Python ≥3.6.
+;; conftest.py skips this file when running on Python <3.6.
+
+(import [asyncio [get-event-loop sleep]])
+
+
+(defn run-coroutine [coro]
+  "Run a coroutine until its done in the default event loop."""
+  (.run_until_complete (get-event-loop) (coro)))
+
+
+(defn test-for/a []
+  (defn/a numbers []
+    (for [i [1 2]]
+      (yield i)))
+
+  (run-coroutine
+    (fn/a []
+      (setv x 0)
+      (for/a [a (numbers)]
+        (setv x (+ x a)))
+      (assert (= x 3)))))
+
+(defn test-for/a-else []
+  (defn/a numbers []
+    (for [i [1 2]]
+      (yield i)))
+
+  (run-coroutine
+    (fn/a []
+      (setv x 0)
+      (for/a [a (numbers)]
+        (setv x (+ x a))
+        (else (setv x (+ x 50))))
+      (assert (= x 53)))))