Repository: ipython/ipython

Issue:
`async with` doesn't allow newlines
When writing a `with` statement, pressing enter goes onto a new line, rather than immediately running the code:
```py
In [26]: with example:
    ...:     do_1()[enter]
    ...:     [<-pointer goes here]
```
However, when using `async with`, this does not apply to any lines except for the first after the colon:
```py
In [26]: async with example:
    ...:     do_1()[enter]
[code is run]
```

(Tested in latest PyPI: `IPython.__version__ == "7.23.1"`)


Discussion:
Indeed, it's still the case on master (`8.0.0.dev`) w/ Python 3.9.1. Thanks for reporting @hrmorley34.
this should be handled in IPython/terminal/shortcuts.py:newline_or_execute_outer  which itself will rely on IPython/core/inputtransformer2.py:check_complete, this is likely where the logic should be tested as to why it eecutes and does not insert a new line.
PR: https://github.com/ipython/ipython/pull/13090
I've just come back to this issue, and it doesn't seem to be fixed; I've tested with IPython 7.31.0 in Python 3.8.10, 3.9.9, and 3.10.1, with IPython 7.28.0 (when the backport was introduced) in Python 3.10.1, and with IPython 8.0.0.dev (`pip install git+https://github.com/ipython/ipython`) in Python 3.10.1; all of these versions have the same problem as in the original example.

Resolving patch:
diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py
--- a/IPython/core/inputtransformer2.py
+++ b/IPython/core/inputtransformer2.py
@@ -507,9 +507,12 @@ def make_tokens_by_line(lines:List[str]):
 
     #   reexported from token on 3.7+
     NEWLINE, NL = tokenize.NEWLINE, tokenize.NL  # type: ignore
-    tokens_by_line:List[List[Any]] = [[]]
-    if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
-        warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
+    tokens_by_line: List[List[Any]] = [[]]
+    if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
+        warnings.warn(
+            "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
+            stacklevel=2,
+        )
     parenlev = 0
     try:
         for token in tokenize.generate_tokens(iter(lines).__next__):
@@ -782,9 +785,6 @@ def __init__(self, extra_flags=0):
         super().__init__()
         self.flags |= extra_flags
 
-    def __call__(self, *args, **kwds):
-        return compile(*args, **kwds)
-
 
 class MaybeAsyncCommandCompiler(CommandCompiler):
     def __init__(self, extra_flags=0):


Test patch:
diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py
--- a/IPython/core/tests/test_inputtransformer2.py
+++ b/IPython/core/tests/test_inputtransformer2.py
@@ -4,6 +4,7 @@
 more complex. See test_inputtransformer2_line for tests for line-based
 transformations.
 """
+import platform
 import string
 import sys
 from textwrap import dedent
@@ -291,6 +292,7 @@ def test_check_complete_param(code, expected, number):
     assert cc(code) == (expected, number)
 
 
+@pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
 @pytest.mark.xfail(
     reason="Bug in python 3.9.8 – bpo 45738",
     condition=sys.version_info in [(3, 9, 8, "final", 0), (3, 11, 0, "alpha", 2)],
@@ -319,7 +321,16 @@ def test_check_complete():
     assert cc("def f():\n  x=0\n  \\\n  ") == ("incomplete", 2)
 
 
-def test_check_complete_II():
+@pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
+@pytest.mark.parametrize(
+    "value, expected",
+    [
+        ('''def foo():\n    """''', ("incomplete", 4)),
+        ("""async with example:\n    pass""", ("incomplete", 4)),
+        ("""async with example:\n    pass\n    """, ("complete", None)),
+    ],
+)
+def test_check_complete_II(value, expected):
     """
     Test that multiple line strings are properly handled.
 
@@ -327,25 +338,31 @@ def test_check_complete_II():
 
     """
     cc = ipt2.TransformerManager().check_complete
-    assert cc('''def foo():\n    """''') == ("incomplete", 4)
-
-
-def test_check_complete_invalidates_sunken_brackets():
+    assert cc(value) == expected
+
+
+@pytest.mark.parametrize(
+    "value, expected",
+    [
+        (")", ("invalid", None)),
+        ("]", ("invalid", None)),
+        ("}", ("invalid", None)),
+        (")(", ("invalid", None)),
+        ("][", ("invalid", None)),
+        ("}{", ("invalid", None)),
+        ("]()(", ("invalid", None)),
+        ("())(", ("invalid", None)),
+        (")[](", ("invalid", None)),
+        ("()](", ("invalid", None)),
+    ],
+)
+def test_check_complete_invalidates_sunken_brackets(value, expected):
     """
     Test that a single line with more closing brackets than the opening ones is
     interpreted as invalid
     """
     cc = ipt2.TransformerManager().check_complete
-    assert cc(")") == ("invalid", None)
-    assert cc("]") == ("invalid", None)
-    assert cc("}") == ("invalid", None)
-    assert cc(")(") == ("invalid", None)
-    assert cc("][") == ("invalid", None)
-    assert cc("}{") == ("invalid", None)
-    assert cc("]()(") == ("invalid", None)
-    assert cc("())(") == ("invalid", None)
-    assert cc(")[](") == ("invalid", None)
-    assert cc("()](") == ("invalid", None)
+    assert cc(value) == expected
 
 
 def test_null_cleanup_transformer():