Repository: hylang/hy

Issue:
Collapse some similar core macros
Consider the following substitutions:

| Old syntax       | New syntax            |
| ---------------- | --------------------- |
| `(fn/a …)`       | `(fn :async …)`       |
| `(defn/a …)`     | `(defn :async …)`     |
| `(with/a […] …)` | `(with [:async …] …)` |
| `(yield-from …)` | `(yield :from …)`     |

The purpose is to cut down a bit on the number of distinct core macros. There are a lot of them.

For consistency with `for`, notice that the new `with` applies `async` per context manager, not to all context managers in the same `with`.

@scauligi What do you think of this?


Discussion:
Ooh I like these! I may want to play around a little with the placement of `:async` for `with`, since we usually don't have keywords as the first element of a list, but everything else looks great to me (particularly `yield :from` which feels much more natural now)
Cool. Let's talk about `with`, since that's the only part that I think might take some real work to implement. 

My first thought was something like

```
(with :async [db (open-database)]
  …)
```

, but I suspected it would be confusing that `for` looks different, because `for` uses comprehension loop clauses:

```
(for [:async row (db)  :if (get row "n")]
  …)
```

So we get

```
(with [:async db (open-database)]
  …)
```

What do you think? I'm not very invested either way because I rarely use asynchronous stuff.
@scauligi `(with :async …)` would probably be easier, but I suspect I'd regret it and have to change it to `(with [:async …] …)` later, so let's decide before I implement either.
(woops sorry, I'm still on vacation + between jobs at the moment)

Aha, I forgot that we already have `(for [:async ...`, so in that case `(with [:async ...` fits in just fine 👍.

As an aside, we could consider having (both) `for` (and `with`) use the syntax `(for [row :async (db)])` instead of `(for [:async row (db)])`; this makes a little more sense to me when reading it, but I'm not beholden to this syntax.
I see what you mean. I think `:async row (db)` is a little better than `row :async (db)` because it's more consistent with the other uses of keywords in the comprehension macros, where the keyword heads the clause.

Resolving patch:
diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py
--- a/hy/core/result_macros.py
+++ b/hy/core/result_macros.py
@@ -1068,21 +1068,15 @@ def compile_break_or_continue_expression(compiler, expr, root):
 # ------------------------------------------------
 
 
-@pattern_macro(
-    ["with", "with/a"],
-    [
-        brackets(oneplus(FORM + FORM))
-        | brackets(FORM >> (lambda x: [(Symbol("_"), x)])),
-        many(FORM),
-    ],
-)
+@pattern_macro("with", [
+    brackets(oneplus(maybe(keepsym(":async")) + FORM + FORM)) |
+        brackets((maybe(keepsym(":async")) + FORM) >> (lambda x: [(x[0], Symbol("_"), x[1])])),
+    many(FORM)])
 def compile_with_expression(compiler, expr, root, args, body):
-    body = compiler._compile_branch(body)
 
-    # Store the result of the body in a tempvar
+    # We'll store the result of the body in a tempvar
     temp_var = compiler.get_anon_var()
     name = asty.Name(expr, id=mangle(temp_var), ctx=ast.Store())
-    body += asty.Assign(expr, targets=[name], value=body.force_expr)
     # Initialize the tempvar to None in case the `with` exits
     # early with an exception.
     initial_assign = asty.Assign(
@@ -1091,7 +1085,18 @@ def compile_with_expression(compiler, expr, root, args, body):
 
     ret = Result(stmts=[initial_assign])
     items = []
-    for variable, ctx in args[0]:
+    was_async = None
+    cbody = None
+    for i, (is_async, variable, ctx) in enumerate(args[0]):
+        is_async = bool(is_async)
+        if was_async is None:
+            was_async = is_async
+        elif is_async != was_async:
+            # We're compiling a `with` that mixes synchronous and
+            # asynchronous context managers. Python doesn't support
+            # this directly, so start a new `with` inside the body.
+            cbody = compile_with_expression(compiler, expr, root, [args[0][i:]], body)
+            break
         ctx = compiler.compile(ctx)
         ret += ctx
         variable = (
@@ -1103,8 +1108,12 @@ def compile_with_expression(compiler, expr, root, args, body):
             asty.withitem(expr, context_expr=ctx.force_expr, optional_vars=variable)
         )
 
-    node = asty.With if root == "with" else asty.AsyncWith
-    ret += node(expr, body=body.stmts, items=items)
+    if not cbody:
+        cbody = compiler._compile_branch(body)
+        cbody += asty.Assign(expr, targets=[name], value=cbody.force_expr)
+
+    node = asty.AsyncWith if was_async else asty.With
+    ret += node(expr, body=cbody.stmts, items=items)
 
     # And make our expression context our temp variable
     expr_name = asty.Name(expr, id=mangle(temp_var), ctx=ast.Load())
@@ -1454,10 +1463,12 @@ def compile_try_expression(compiler, expr, root, body, catchers, orelse, finalbo
 )
 
 
-@pattern_macro(["fn", "fn/a"],
-    [maybe(type_params), maybe_annotated(lambda_list), many(FORM)])
-def compile_function_lambda(compiler, expr, root, tp, params, body):
-    is_async = root == "fn/a"
+@pattern_macro("fn", [
+    maybe(keepsym(":async")),
+    maybe(type_params),
+    maybe_annotated(lambda_list),
+    many(FORM)])
+def compile_function_lambda(compiler, expr, root, is_async, tp, params, body):
     params, returns = params
     posonly, args, rest, kwonly, kwargs = params
     has_annotations = returns is not None or any(
@@ -1483,12 +1494,14 @@ def compile_function_lambda(compiler, expr, root, tp, params, body):
     return ret + Result(expr=ret.temp_variables[0])
 
 
-@pattern_macro(
-    ["defn", "defn/a"],
-    [maybe(brackets(many(FORM))), maybe(type_params), maybe_annotated(SYM), lambda_list, many(FORM)],
-)
-def compile_function_def(compiler, expr, root, decorators, tp, name, params, body):
-    is_async = root == "defn/a"
+@pattern_macro("defn", [
+    maybe(keepsym(":async")),
+    maybe(brackets(many(FORM))),
+    maybe(type_params),
+    maybe_annotated(SYM),
+    lambda_list,
+    many(FORM)])
+def compile_function_def(compiler, expr, root, is_async, decorators, tp, name, params, body):
     name, returns = name
     node = asty.AsyncFunctionDef if is_async else asty.FunctionDef
     decorators, ret, _ = compiler._compile_collect(decorators[0] if decorators else [])
@@ -1672,23 +1685,27 @@ def compile_return(compiler, expr, root, arg):
     return ret + asty.Return(expr, value=ret.force_expr)
 
 
-@pattern_macro("yield", [maybe(FORM)])
-def compile_yield_expression(compiler, expr, root, arg):
+@pattern_macro("yield", [times(0, 2, FORM)])
+def compile_yield_expression(compiler, expr, root, args):
     if is_inside_function_scope(compiler.scope):
         nearest_python_scope(compiler.scope).has_yield = True
+    yield_from = False
+    if len(args) == 2:
+        from_kw, x = args
+        if from_kw != Keyword("from"):
+            raise compiler._syntax_error(from_kw, "two-argument `yield` requires `:from`")
+        yield_from = True
+        args = [x]
     ret = Result()
-    if arg is not None:
-        ret += compiler.compile(arg)
-    return ret + asty.Yield(expr, value=ret.force_expr)
+    if args:
+        ret += compiler.compile(args[0])
+    return ret + (asty.YieldFrom if yield_from else asty.Yield)(expr, value=ret.force_expr)
 
 
-@pattern_macro(["yield-from", "await"], [FORM])
+@pattern_macro("await", [FORM])
 def compile_yield_from_or_await_expression(compiler, expr, root, arg):
-    if root == "yield-from" and is_inside_function_scope(compiler.scope):
-        nearest_python_scope(compiler.scope).has_yield = True
     ret = Result() + compiler.compile(arg)
-    node = asty.YieldFrom if root == "yield-from" else asty.Await
-    return ret + node(expr, value=ret.force_expr)
+    return ret + asty.Await(expr, value=ret.force_expr)
 
 
 # ------------------------------------------------


Test patch:
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -550,11 +550,12 @@ def test_compiler_macro_tag_try():
 
 
 def test_ast_good_yield_from():
-    can_compile("(yield-from [1 2])")
+    can_compile("(yield :from [1 2])")
+    can_compile("(yield :from)")
 
 
 def test_ast_bad_yield_from():
-    cant_compile("(yield-from)")
+    cant_compile("(yield :ploopy [1 2])")
 
 
 def test_eval_generator_with_return():
diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy
--- a/tests/native_tests/comprehensions.hy
+++ b/tests/native_tests/comprehensions.hy
@@ -387,12 +387,12 @@
 
 
 (defn [async-test] test-for-async []
-  (defn/a numbers []
+  (defn :async numbers []
     (for [i [1 2]]
       (yield i)))
 
   (asyncio.run
-    ((fn/a []
+    ((fn :async []
       (setv x 0)
       (for [:async a (numbers)]
         (setv x (+ x a)))
@@ -400,12 +400,12 @@
 
 
 (defn [async-test] test-for-async-else []
-  (defn/a numbers []
+  (defn :async numbers []
     (for [i [1 2]]
       (yield i)))
 
   (asyncio.run
-    ((fn/a []
+    ((fn :async []
       (setv x 0)
       (for [:async a (numbers)]
         (setv x (+ x a))
diff --git a/tests/native_tests/decorators.hy b/tests/native_tests/decorators.hy
--- a/tests/native_tests/decorators.hy
+++ b/tests/native_tests/decorators.hy
@@ -53,10 +53,10 @@
   (assert (= l ["dec" "arg" "foo" "foo fn" "bar body" 1])))
 
 
-(defn [async-test] test-decorated-defn/a []
-  (defn decorator [func] (fn/a [] (/ (await (func)) 2)))
+(defn [async-test] test-decorated-defn-a []
+  (defn decorator [func] (fn :async [] (/ (await (func)) 2)))
 
-  (defn/a [decorator] coro-test []
+  (defn :async [decorator] coro-test []
     (await (asyncio.sleep 0))
     42)
   (assert (= (asyncio.run (coro-test)) 21)))
diff --git a/tests/native_tests/functions.hy b/tests/native_tests/functions.hy
--- a/tests/native_tests/functions.hy
+++ b/tests/native_tests/functions.hy
@@ -25,8 +25,8 @@
   (assert (= (fn-test) None)))
 
 
-(defn [async-test] test-fn/a []
-  (assert (= (asyncio.run ((fn/a [] (await (asyncio.sleep 0)) [1 2 3])))
+(defn [async-test] test-fn-async []
+  (assert (= (asyncio.run ((fn :async [] (await (asyncio.sleep 0)) [1 2 3])))
              [1 2 3])))
 
 
@@ -182,8 +182,8 @@
     (setv x [#* spam]  y 1)))
 
 
-(defn [async-test] test-defn/a []
-  (defn/a coro-test []
+(defn [async-test] test-defn-async []
+  (defn :async coro-test []
     (await (asyncio.sleep 0))
     [1 2 3])
   (assert (= (asyncio.run (coro-test)) [1 2 3])))
@@ -191,15 +191,15 @@
 
 (defn [async-test] test-no-async-gen-return []
   ; https://github.com/hylang/hy/issues/2523
-  (defn/a runner [gen]
+  (defn :async runner [gen]
     (setv vals [])
     (for [:async val (gen)]
       (.append vals val))
     vals)
-  (defn/a naysayer []
+  (defn :async naysayer []
     (yield "nope"))
   (assert (= (asyncio.run (runner naysayer)) ["nope"]))
-  (assert (= (asyncio.run (runner (fn/a [] (yield "dope!")) ["dope!"])))))
+  (assert (= (asyncio.run (runner (fn :async [] (yield "dope!")) ["dope!"])))))
 
 
 (defn test-root-set-correctly []
@@ -271,30 +271,6 @@
   (assert (= accum [2])))
 
 
-(defn test-yield-from []
-  (defn yield-from-test []
-    (for [i (range 3)]
-      (yield i))
-    (yield-from [1 2 3]))
-  (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))
-
-
-(defn test-yield-from-exception-handling []
-  (defn yield-from-subgenerator-test []
-    (yield 1)
-    (yield 2)
-    (yield 3)
-    (/ 1 0))
-  (defn yield-from-test []
-    (for [i (range 3)]
-      (yield i))
-    (try
-      (yield-from (yield-from-subgenerator-test))
-      (except [e ZeroDivisionError]
-        (yield 4))))
-  (assert (= (list (yield-from-test)) [0 1 2 1 2 3 4])))
-
-
 (defn test-yield []
   (defn gen [] (for [x [1 2 3 4]] (yield x)))
   (setv ret 0)
@@ -355,3 +331,34 @@
     (yield "a")
     (yield "end"))
   (assert (= (list (multi-yield)) [0 1 2 "a" "end"])))
+
+
+(defn test-yield-from []
+  (defn yield-from-test []
+    (for [i (range 3)]
+      (yield i))
+    (yield :from [1 2 3]))
+  (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))
+
+
+(defn test-yield-from-exception-handling []
+  (defn yield-from-subgenerator-test []
+    (yield 1)
+    (yield 2)
+    (yield 3)
+    (/ 1 0))
+  (defn yield-from-test []
+    (for [i (range 3)]
+      (yield i))
+    (try
+      (yield :from (yield-from-subgenerator-test))
+      (except [e ZeroDivisionError]
+        (yield 4))))
+  (assert (= (list (yield-from-test)) [0 1 2 1 2 3 4])))
+
+
+(defn test-yield-from-notreally []
+  (defn f []
+    (yield :from)
+    (yield :from))
+  (assert (= (list (f)) [:from :from])))
diff --git a/tests/native_tests/with.hy b/tests/native_tests/with.hy
--- a/tests/native_tests/with.hy
+++ b/tests/native_tests/with.hy
@@ -25,65 +25,44 @@
 
 (defn test-single-with []
   (with [t (WithTest 1)]
-        (assert (= t 1))))
-
-(defn test-twice-with []
-  (with [t1 (WithTest 1)
-         t2 (WithTest 2)]
-        (assert (= t1 1))
-        (assert (= t2 2))))
-
-(defn test-thrice-with []
-  (with [t1 (WithTest 1)
-         t2 (WithTest 2)
-         t3 (WithTest 3)]
-        (assert (= t1 1))
-        (assert (= t2 2))
-        (assert (= t3 3))))
+    (setv out t))
+  (assert (= out 1)))
 
 (defn test-quince-with []
-  (with [t1 (WithTest 1)
-         t2 (WithTest 2)
-         t3 (WithTest 3)
-         _ (WithTest 4)]
-        (assert (= t1 1))
-        (assert (= t2 2))
-        (assert (= t3 3))))
-
-(defn [async-test] test-single-with/a []
-  (asyncio.run
-    ((fn/a []
-      (with/a [t (AsyncWithTest 1)]
-        (assert (= t 1)))))))
+  (with [t1 (WithTest 1)  t2 (WithTest 2)  t3 (WithTest 3)  _ (WithTest 4)]
+    (setv out [t1 t2 t3]))
+  (assert (= out [1 2 3])))
 
-(defn [async-test] test-two-with/a []
+(defn [async-test] test-single-with-async []
+  (setv out [])
   (asyncio.run
-    ((fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)]
-        (assert (= t1 1))
-        (assert (= t2 2)))))))
+    ((fn :async []
+      (with [:async t (AsyncWithTest 1)]
+        (.append out t)))))
+  (assert (= out [1])))
 
-(defn [async-test] test-thrice-with/a []
+(defn [async-test] test-quince-with-async []
+  (setv out [])
   (asyncio.run
-    ((fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)
-               t3 (AsyncWithTest 3)]
-        (assert (= t1 1))
-        (assert (= t2 2))
-        (assert (= t3 3)))))))
-
-(defn [async-test] test-quince-with/a []
+    ((fn :async []
+      (with [
+          :async t1 (AsyncWithTest 1)
+          :async t2 (AsyncWithTest 2)
+          :async t3 (AsyncWithTest 3)
+          :async _ (AsyncWithTest 4)]
+        (.extend out [t1 t2 t3])))))
+  (assert (= out [1 2 3])))
+
+(defn [async-test] test-with-mixed-async []
+  (setv out [])
   (asyncio.run
-    ((fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)
-               t3 (AsyncWithTest 3)
-               _ (AsyncWithTest 4)]
-        (assert (= t1 1))
-        (assert (= t2 2))
-        (assert (= t3 3)))))))
+    ((fn :async []
+      (with [:async t1 (AsyncWithTest 1)
+             t2 (WithTest 2)
+             :async t3 (AsyncWithTest 3)
+             _ (WithTest 4)]
+        (.extend out [t1 t2 t3])))))
+  (assert (= out [1 2 3])))
 
 (defn test-unnamed-context-with []
   "`_` get compiled to unnamed context"
diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy
--- a/tests/resources/pydemo.hy
+++ b/tests/resources/pydemo.hy
@@ -173,12 +173,12 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little
       pys_accum.append(i)")
 (setv py-accum (py "''.join(map(str, pys_accum))"))
 
-(defn/a coro []
+(defn :async coro []
   (import asyncio
           tests.resources [AsyncWithTest async-loop])
   (await (asyncio.sleep 0))
   (setv values ["a"])
-  (with/a [t (AsyncWithTest "b")]
+  (with [:async t (AsyncWithTest "b")]
     (.append values t))
   (for [:async item (async-loop ["c" "d"])]
     (.append values item))