Repository: ipython/ipython

Issue:
Capturing shell command output after indentation causes SyntaxError in IPython console.
To reproduce, type this in console:

```python
def test():
  for i in range(1):
    print(i)
  res =! ls
```

But if something is put above the line, everything will seem to be fine.
```python
def test():
  for i in range(1):
    print(i)
  'just a separate line'
  res =! ls
```
Find this in IPython 7.1.0.dev

Comments:

Patch:
diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py
index e2dd2d087..b73d70121 100644
--- a/IPython/core/inputtransformer2.py
+++ b/IPython/core/inputtransformer2.py
@@ -13,7 +13,7 @@
 from codeop import compile_command
 import re
 import tokenize
-from typing import List, Tuple
+from typing import List, Tuple, Union
 import warnings
 
 _indent_re = re.compile(r'^[ \t]+')
@@ -87,7 +87,7 @@ def cell_magic(lines):
             % (magic_name, first_line, body)]
 
 
-def _find_assign_op(token_line):
+def _find_assign_op(token_line) -> Union[int, None]:
     """Get the index of the first assignment in the line ('=' not inside brackets)
 
     Note: We don't try to support multiple special assignment (a = b = %foo)
@@ -97,9 +97,9 @@ def _find_assign_op(token_line):
         s = ti.string
         if s == '=' and paren_level == 0:
             return i
-        if s in '([{':
+        if s in {'(','[','{'}:
             paren_level += 1
-        elif s in ')]}':
+        elif s in {')', ']', '}'}:
             if paren_level > 0:
                 paren_level -= 1
 
@@ -449,11 +449,14 @@ def transform(self, lines):
 
         return lines_before + [new_line] + lines_after
 
-def make_tokens_by_line(lines):
+def make_tokens_by_line(lines:List[str]):
     """Tokenize a series of lines and group tokens by line.
 
-    The tokens for a multiline Python string or expression are
-    grouped as one line.
+    The tokens for a multiline Python string or expression are grouped as one
+    line. All lines except the last lines should keep their line ending ('\\n',
+    '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
+    for example when passing block of text to this function.
+
     """
     # NL tokens are used inside multiline expressions, but also after blank
     # lines or comments. This is intentional - see https://bugs.python.org/issue17061
@@ -461,6 +464,8 @@ def make_tokens_by_line(lines):
     # track parentheses level, similar to the internals of tokenize.
     NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
     tokens_by_line = [[]]
+    if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
+        warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
     parenlev = 0
     try:
         for token in tokenize.generate_tokens(iter(lines).__next__):
diff --git a/docs/environment.yml b/docs/environment.yml
index 3ccce04ca..f5076d15e 100644
--- a/docs/environment.yml
+++ b/docs/environment.yml
@@ -4,8 +4,11 @@ dependencies:
 - setuptools>=18.5
 - sphinx>=1.8
 - sphinx_rtd_theme
+- numpy
+- nose
+- testpath
+- matplotlib
 - pip:
     - docrepr
     - prompt_toolkit
-    - ipython
     - ipykernel
diff --git a/docs/source/conf.py b/docs/source/conf.py
index edb93094d..0df8b44c9 100755
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -66,11 +66,6 @@
     'configtraits',
 ]
 
-if ON_RTD:
-    # Remove extensions not currently supported on RTD
-    extensions.remove('IPython.sphinxext.ipython_directive')
-    extensions.remove('IPython.sphinxext.ipython_console_highlighting')
-
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -124,6 +119,7 @@ def is_stable(extra):
 # numpydoc config
 numpydoc_show_class_members = False # Otherwise Sphinx emits thousands of warnings
 numpydoc_class_members_toctree = False
+warning_is_error = True
 
 # The default replacements for |version| and |release|, also used in various
 # other places throughout the built documents.
diff --git a/docs/source/whatsnew/pr/video-width-height.rst b/docs/source/whatsnew/pr/video-width-height.rst
deleted file mode 100644
index 84757f1b4..000000000
--- a/docs/source/whatsnew/pr/video-width-height.rst
+++ /dev/null
@@ -1,1 +0,0 @@
-``IPython.display.Video`` now supports ``width`` and ``height`` arguments, allowing a custom width and height to be set instead of using the video's width and height
\ No newline at end of file
diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst
index 19352cc0d..17b327514 100644
--- a/docs/source/whatsnew/version7.rst
+++ b/docs/source/whatsnew/version7.rst
@@ -2,6 +2,79 @@
  7.x Series
 ============
 
+.. _whatsnew710:
+
+IPython 7.1.0
+=============
+
+
+IPython 7.1.0 is the first minor release after 7.0.0 and mostly bring fixes to
+new feature, internal refactor and regressions that happen during the 6.x->7.x
+transition. It also bring **Compatibility with Python 3.7.1**, as were
+unwillingly relying on a bug in CPython.
+
+New Core Dev:
+
+ - We welcome Jonathan Slenders to the commiters. Jonathan has done a fantastic
+   work on Prompt toolkit, and we'd like to recognise his impact by giving him
+   commit rights. :ghissue:`11397`
+
+Notable New Features:
+
+ - Restore functionality and documentation of the **sphinx directive**, which is
+   now stricter (fail on error by default), gained configuration options, have a
+   brand new documentation page :ref:`ipython_directive`, which need some cleanup.
+   It is also now *tested* so we hope to have less regressions.
+   :ghpull:`11402`
+
+ - ``IPython.display.Video`` now supports ``width`` and ``height`` arguments,
+   allowing a custom width and height to be set instead of using the video's
+   width and height. :ghpull:`11353`
+
+ - Warn when using ``HTML('<iframe>')`` instead of ``IFrame`` :ghpull:`11350`
+
+ - Allow Dynamic switching of editing mode between vi/emacs and show
+   normal/input mode in prompt when using vi. :ghpull:`11390`. Use ``%config
+   TerminalInteractiveShell.editing_mode = 'vi'`` or ``%config
+   TerminalInteractiveShell.editing_mode = 'emacs'`` to dynamically spwitch
+
+
+Notable Fixes:
+
+ - Fix entering of **multi-line block in terminal** IPython, and various crashes
+   in the new input transformation machinery :ghpull:`11354`, :ghpull:`11356`, :ghpull:`11358`, these
+   ones also fix a **Compatibility but with Python 3.7.1**.
+
+ - Fix moving through generator stack in ipdb :ghpull:`11266`
+
+ - Magics arguments now support quoting. :ghpull:`11330`
+
+ - Re-add ``rprint`` and ``rprinte`` aliases. :ghpull:`11331`
+
+ - Remove implicit dependency to ``ipython_genutils`` :ghpull:`11317`
+
+ - Make ``nonlocal`` raise ``SyntaxError`` instead of silently failing in async
+   mode. :ghpull:`11382`
+
+
+Notable Internals improvements:
+
+ - Use of ``os.scandir`` (Python 3 only) to speedup some file system operations.
+   :ghpull:`11365`
+
+ - use ``perf_counter`` instead of ``clock`` for more precise
+   timing result with ``%time`` :ghpull:`11376`
+
+Many thanks to all the contributors and in particular to ``bartskowron``, and
+``tonyfast`` who handled a pretty complicated bugs in the input machinery. We
+had a number of first time contributors and maybe hacktoberfest participant that
+made significant contributions, and helped us free some time to focus on more
+complicated bugs.
+
+You
+can see all the closed issues and Merged PR, new features and fixes `here
+<https://github.com/ipython/ipython/issues?utf8=%E2%9C%93&q=+is%3Aclosed+milestone%3A7.1+>`_.
+
 .. _whatsnew700:
 
 IPython 7.0.0


Test patch:
diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py
index d6c2fa3bd..9c92c394e 100644
--- a/IPython/core/tests/test_inputtransformer2.py
+++ b/IPython/core/tests/test_inputtransformer2.py
@@ -8,7 +8,7 @@
 import string
 
 from IPython.core import inputtransformer2 as ipt2
-from IPython.core.inputtransformer2 import make_tokens_by_line
+from IPython.core.inputtransformer2 import make_tokens_by_line, _find_assign_op
 
 from textwrap import dedent
 
@@ -53,6 +53,22 @@
 g()
 """.splitlines(keepends=True))
 
+#####
+
+MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\
+def test():
+  for i in range(1):
+    print(i)
+  res =! ls
+""".splitlines(keepends=True), (4, 7), '''\
+def test():
+  for i in range(1):
+    print(i)
+  res =get_ipython().getoutput(\' ls\')
+'''.splitlines(keepends=True))
+
+######
+
 AUTOCALL_QUOTE = (
     [",f 1 2 3\n"], (1, 0),
     ['f("1", "2", "3")\n']
@@ -103,6 +119,7 @@
 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
 )
 
+
 def null_cleanup_transformer(lines):
     """
     A cleanup transform that returns an empty list.
@@ -144,18 +161,21 @@ def test_continued_line():
 def test_find_assign_magic():
     check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
     check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
+    check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)
 
 def test_transform_assign_magic():
     check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
 
 def test_find_assign_system():
     check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
+    check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
     check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
     check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
     check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
 
 def test_transform_assign_system():
     check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
+    check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
 
 def test_find_magic_escape():
     check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
@@ -203,6 +223,17 @@ def test_transform_help():
     tf = ipt2.HelpEnd((1, 0), (2, 8))
     nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
 
+def test_find_assign_op_dedent():
+    """
+    be carefull that empty token like dedent are not counted as parens
+    """
+    class Tk:
+        def __init__(self, s):
+            self.string = s
+
+    nt.assert_equal(_find_assign_op([Tk(s) for s in ('','a','=','b')]), 2)
+    nt.assert_equal(_find_assign_op([Tk(s) for s in ('','(', 'a','=','b', ')', '=' ,'5')]), 6)
+
 def test_check_complete():
     cc = ipt2.TransformerManager().check_complete
     nt.assert_equal(cc("a = 1"), ('complete', None))