Repository: ipython/ipython

Issue:
Wrong autoindent for  class method. 
```
class F:
    def a(self):<enter>
```

Only 4 spaces are inserted. 


Discussion:

Patch:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 179d45e56..576f1ae78 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,10 +1,10 @@
 ## Triaging Issues
 
-On the IPython repository,  we strive to trust users and give them responsibility.
-By using one of our bot, any user can close issues or add and remove
+On the IPython repository we strive to trust users and give them responsibility.
+By using one of our bot, any user can close issues, add and remove
 labels by mentioning the bot and asking it to do things on your behalf.
 
-To close an issue (or PR), even if you did not create it, use the following:
+To close and issue (or PR), even if you did not create it, use the following:
 
 > @meeseeksdev close
 
@@ -16,7 +16,7 @@ tags to add:
 
 > @meeseeksdev tag windows, documentation
 
-Only already pre-created tags can be added.  So far, the list is limited to:
+Only already pre-created tags can be added, and the list is so far limited to
 `async/await`, `backported`, `help wanted`, `documentation`, `notebook`,
 `tab-completion`, `windows`
 
diff --git a/IPython/__init__.py b/IPython/__init__.py
index 043a946ab..7bd0ee3c4 100644
--- a/IPython/__init__.py
+++ b/IPython/__init__.py
@@ -75,7 +75,7 @@ def embed_kernel(module=None, local_ns=None, **kwargs):
     
     Parameters
     ----------
-    module : types.ModuleType, optional
+    module : ModuleType, optional
         The module to load into IPython globals (default: caller)
     local_ns : dict, optional
         The namespace to load into IPython user namespace (default: caller)
diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py
index b73d70121..7402b8a39 100644
--- a/IPython/core/inputtransformer2.py
+++ b/IPython/core/inputtransformer2.py
@@ -13,7 +13,7 @@
 from codeop import compile_command
 import re
 import tokenize
-from typing import List, Tuple, Union
+from typing import List, Tuple
 import warnings
 
 _indent_re = re.compile(r'^[ \t]+')
@@ -87,7 +87,7 @@ def cell_magic(lines):
             % (magic_name, first_line, body)]
 
 
-def _find_assign_op(token_line) -> Union[int, None]:
+def _find_assign_op(token_line):
     """Get the index of the first assignment in the line ('=' not inside brackets)
 
     Note: We don't try to support multiple special assignment (a = b = %foo)
@@ -97,9 +97,9 @@ def _find_assign_op(token_line) -> Union[int, None]:
         s = ti.string
         if s == '=' and paren_level == 0:
             return i
-        if s in {'(','[','{'}:
+        if s in '([{':
             paren_level += 1
-        elif s in {')', ']', '}'}:
+        elif s in ')]}':
             if paren_level > 0:
                 paren_level -= 1
 
@@ -449,14 +449,11 @@ def transform(self, lines):
 
         return lines_before + [new_line] + lines_after
 
-def make_tokens_by_line(lines:List[str]):
+def make_tokens_by_line(lines):
     """Tokenize a series of lines and group tokens by line.
 
-    The tokens for a multiline Python string or expression are grouped as one
-    line. All lines except the last lines should keep their line ending ('\\n',
-    '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
-    for example when passing block of text to this function.
-
+    The tokens for a multiline Python string or expression are
+    grouped as one line.
     """
     # NL tokens are used inside multiline expressions, but also after blank
     # lines or comments. This is intentional - see https://bugs.python.org/issue17061
@@ -464,8 +461,6 @@ def make_tokens_by_line(lines:List[str]):
     # track parentheses level, similar to the internals of tokenize.
     NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
     tokens_by_line = [[]]
-    if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
-        warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
     parenlev = 0
     try:
         for token in tokenize.generate_tokens(iter(lines).__next__):
@@ -604,7 +599,7 @@ def check_complete(self, cell: str):
             else:
                 continue
 
-        if ends_with_newline:
+        if not ends_with_newline:
             # Append an newline for consistent tokenization
             # See https://bugs.python.org/issue33899
             cell += '\n'
@@ -649,10 +644,13 @@ def check_complete(self, cell: str):
 
         newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
 
-        # Remove newline_types for the list of tokens
-        while len(tokens_by_line) > 1 and len(tokens_by_line[-1]) == 1 \
-                and tokens_by_line[-1][-1].type in newline_types:
-            tokens_by_line.pop()
+        # Pop the last line which only contains DEDENTs and ENDMARKER
+        last_token_line = None
+        if {t.type for t in tokens_by_line[-1]} in [
+            {tokenize.DEDENT, tokenize.ENDMARKER},
+            {tokenize.ENDMARKER}
+        ] and len(tokens_by_line) > 1:
+            last_token_line = tokens_by_line.pop()
 
         while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
             tokens_by_line[-1].pop()
@@ -685,7 +683,7 @@ def check_complete(self, cell: str):
             if res is None:
                 return 'incomplete', find_last_indent(lines)
 
-        if tokens_by_line[-1][-1].type == tokenize.DEDENT:
+        if last_token_line and last_token_line[0].type == tokenize.DEDENT:
             if ends_with_newline:
                 return 'complete', None
             return 'incomplete', find_last_indent(lines)
diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py
index f264340f6..72d3d4f7a 100644
--- a/IPython/sphinxext/ipython_directive.py
+++ b/IPython/sphinxext/ipython_directive.py
@@ -579,7 +579,7 @@ def process_input(self, data, input_prompt, lineno):
                                          w.filename, w.lineno, w.line)
                 sys.stdout.write(s)
                 sys.stdout.write('<<<' + ('-' * 73) + '\n')
-                if self.warning_is_error:
+                if self.shell.warning_is_error:
                     raise RuntimeError('Non Expected warning in `{}` line {}'.format(filename, lineno))
 
         self.cout.truncate(0)
@@ -1009,15 +1009,6 @@ def run(self):
 
                 if figure is not None:
                     figures.append(figure)
-            else:
-                message = 'Code input with no code at {}, line {}'\
-                            .format(
-                                self.state.document.current_source,
-                                self.state.document.current_line)
-                if self.shell.warning_is_error:
-                    raise RuntimeError(message)
-                else:
-                    warnings.warn(message)
 
         for figure in figures:
             lines.append('')
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 21906b088..0df8b44c9 100755
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -233,8 +233,7 @@ def is_stable(extra):
                        'ipykernel': ('https://ipykernel.readthedocs.io/en/latest/', None),
                        'prompt_toolkit' : ('https://python-prompt-toolkit.readthedocs.io/en/stable/', None),
                        'ipywidgets': ('https://ipywidgets.readthedocs.io/en/stable/', None),
-                       'ipyparallel': ('https://ipyparallel.readthedocs.io/en/stable/', None),
-                       'pip': ('https://pip.pypa.io/en/stable/', None)
+                       'ipyparallel': ('https://ipyparallel.readthedocs.io/en/stable/', None)
                       }
 
 # Options for LaTeX output
diff --git a/docs/source/config/details.rst b/docs/source/config/details.rst
index 6685b14d8..31e26094f 100644
--- a/docs/source/config/details.rst
+++ b/docs/source/config/details.rst
@@ -216,35 +216,15 @@ a :ref:`startup file <startup_files>`::
     def insert_unexpected(event):
         buf = event.current_buffer
         buf.insert_text('The Spanish Inquisition')
+
     # Register the shortcut if IPython is using prompt_toolkit
-    if getattr(ip, 'pt_app', None):
-        registry = ip.pt_app.key_bindings
+    if getattr(ip, 'pt_cli'):
+        registry = ip.pt_cli.application.key_bindings_registry
         registry.add_binding(Keys.ControlN,
                          filter=(HasFocus(DEFAULT_BUFFER)
                                  & ~HasSelection()
                                  & insert_mode))(insert_unexpected)
 
-
-Here is a second example that bind the key sequence ``j``, ``k`` to switch to
-VI input mode to ``Normal`` when in insert mode::
-
-   from IPython import get_ipython
-   from prompt_toolkit.enums import DEFAULT_BUFFER
-   from prompt_toolkit.filters import HasFocus, ViInsertMode
-   from prompt_toolkit.key_binding.vi_state import InputMode
-
-   ip = get_ipython()
-
-   def switch_to_navigation_mode(event):
-      vi_state = event.cli.vi_state
-      vi_state.input_mode = InputMode.NAVIGATION
-
-   if getattr(ip, 'pt_app', None):
-      registry = ip.pt_app.key_bindings
-      registry.add_binding(u'j',u'k',
-                           filter=(HasFocus(DEFAULT_BUFFER)
-                                    & ViInsertMode()))(switch_to_navigation_mode)
-
 For more information on filters and what you can do with the ``event`` object,
 `see the prompt_toolkit docs
 <http://python-prompt-toolkit.readthedocs.io/en/latest/pages/building_prompts.html#adding-custom-key-bindings>`__.
diff --git a/docs/source/whatsnew/pr/video-width-height.rst b/docs/source/whatsnew/pr/video-width-height.rst
new file mode 100644
index 000000000..84757f1b4
--- /dev/null
+++ b/docs/source/whatsnew/pr/video-width-height.rst
@@ -0,0 +1,1 @@
+``IPython.display.Video`` now supports ``width`` and ``height`` arguments, allowing a custom width and height to be set instead of using the video's width and height
\ No newline at end of file
diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst
index dc5a4d0e5..19352cc0d 100644
--- a/docs/source/whatsnew/version7.rst
+++ b/docs/source/whatsnew/version7.rst
@@ -2,79 +2,6 @@
  7.x Series
 ============
 
-.. _whatsnew710:
-
-IPython 7.1.0
-=============
-
-
-IPython 7.1.0 is the first minor release after 7.0.0 and mostly bring fixes to
-new feature, internal refactor and regressions that happen during the 6.x->7.x
-transition. It also bring **Compatibility with Python 3.7.1**, as were
-unwillingly relying on a bug in CPython.
-
-New Core Dev:
-
- - We welcome Jonathan Slenders to the commiters. Jonathan has done a fantastic
-   work on Prompt toolkit, and we'd like to recognise his impact by giving him
-   commit rights. :ghissue:`11397`
-
-Notable New Features:
-
- - Restore functionality and documentation of the **sphinx directive**, which is
-   now stricter (fail on error by default), gained configuration options, have a
-   brand new documentation page :ref:`ipython_directive`, which need some cleanup.
-   It is also now *tested* so we hope to have less regressions.
-   :ghpull:`11402`
-
- - ``IPython.display.Video`` now supports ``width`` and ``height`` arguments,
-   allowing a custom width and height to be set instead of using the video's
-   width and height. :ghpull:`11353`
-
- - Warn when using ``HTML('<iframe>')`` instead of ``IFrame`` :ghpull:`11350`
-
- - Allow Dynamic switching of editing mode between vi/emacs and show
-   normal/input mode in prompt when using vi. :ghpull:`11390`. Use ``%config
-   TerminalInteractiveShell.editing_mode = 'vi'`` or ``%config
-   TerminalInteractiveShell.editing_mode = 'emacs'`` to dynamically spwitch
-
-
-Notable Fixes:
-
- - Fix entering of **multi-line block in terminal** IPython, and various crashes
-   in the new input transformation machinery :ghpull:`11354`, :ghpull:`11356`, :ghpull:`11358`, these
-   ones also fix a **Compatibility but with Python 3.7.1**.
-
- - Fix moving through generator stack in ipdb :ghpull:`11266`
-
- - Magics arguments now support quoting. :ghpull:`11330`
-
- - Re-add ``rprint`` and ``rprinte`` aliases. :ghpull:`11331`
-
- - Remove implicit dependency to ``ipython_genutils`` :ghpull:`11317`
-
- - Make ``nonlocal`` raise ``SyntaxError`` instead of silently failing in async
-   mode. :ghpull:`11382`
-
-
-Notable Internals improvements:
-
- - Use of ``os.scandir`` (Python 3 only) to speedup some file system operations.
-   :ghpull:`11365`
-
- - use ``perf_counter`` instead of ``clock`` for more precise
-   timing result with ``%time`` :ghpull:`11376`
-
-Many thanks to all the contributors and in particular to ``bartskowron``, and
-``tonyfast`` who handled a pretty complicated bugs in the input machinery. We
-had a number of first time contributors and maybe hacktoberfest participant that
-made significant contributions, and helped us free some time to focus on more
-complicated bugs.
-
-You
-can see all the closed issues and Merged PR, new features and fixes `here
-<https://github.com/ipython/ipython/issues?utf8=%E2%9C%93&q=+is%3Aclosed+milestone%3A7.1+>`_.
-
 .. _whatsnew700:
 
 IPython 7.0.0
@@ -279,8 +206,8 @@ Make :magic:`%run -n -i ... <run>` work correctly. Earlier, if :magic:`%run` was
 passed both arguments, ``-n`` would be silently ignored. See :ghpull:`10308`
 
 
-The :cellmagic:`%%script` (as well as :cellmagic:`%%bash`,
-:cellmagic:`%%ruby`... ) cell magics now raise by default if the return code of
+The :cellmagic:`%%script`` (as well as :cellmagic:`%%bash``,
+:cellmagic:`%%ruby``... ) cell magics now raise by default if the return code of
 the given code is non-zero (thus halting execution of further cells in a
 notebook). The behavior can be disable by passing the ``--no-raise-error`` flag.
 


Test patch:
diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py
index 9c92c394e..ea0645238 100644
--- a/IPython/core/tests/test_inputtransformer2.py
+++ b/IPython/core/tests/test_inputtransformer2.py
@@ -8,7 +8,7 @@
 import string
 
 from IPython.core import inputtransformer2 as ipt2
-from IPython.core.inputtransformer2 import make_tokens_by_line, _find_assign_op
+from IPython.core.inputtransformer2 import make_tokens_by_line
 
 from textwrap import dedent
 
@@ -53,22 +53,6 @@
 g()
 """.splitlines(keepends=True))
 
-#####
-
-MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT = ("""\
-def test():
-  for i in range(1):
-    print(i)
-  res =! ls
-""".splitlines(keepends=True), (4, 7), '''\
-def test():
-  for i in range(1):
-    print(i)
-  res =get_ipython().getoutput(\' ls\')
-'''.splitlines(keepends=True))
-
-######
-
 AUTOCALL_QUOTE = (
     [",f 1 2 3\n"], (1, 0),
     ['f("1", "2", "3")\n']
@@ -119,7 +103,6 @@ def test():
 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
 )
 
-
 def null_cleanup_transformer(lines):
     """
     A cleanup transform that returns an empty list.
@@ -161,21 +144,18 @@ def test_continued_line():
 def test_find_assign_magic():
     check_find(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
     check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN, match=False)
-    check_find(ipt2.MagicAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT, match=False)
 
 def test_transform_assign_magic():
     check_transform(ipt2.MagicAssign, MULTILINE_MAGIC_ASSIGN)
 
 def test_find_assign_system():
     check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
-    check_find(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
     check_find(ipt2.SystemAssign, (["a =  !ls\n"], (1, 5), None))
     check_find(ipt2.SystemAssign, (["a=!ls\n"], (1, 2), None))
     check_find(ipt2.SystemAssign, MULTILINE_MAGIC_ASSIGN, match=False)
 
 def test_transform_assign_system():
     check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN)
-    check_transform(ipt2.SystemAssign, MULTILINE_SYSTEM_ASSIGN_AFTER_DEDENT)
 
 def test_find_magic_escape():
     check_find(ipt2.EscapedCommand, MULTILINE_MAGIC)
@@ -223,21 +203,11 @@ def test_transform_help():
     tf = ipt2.HelpEnd((1, 0), (2, 8))
     nt.assert_equal(tf.transform(HELP_MULTILINE[0]), HELP_MULTILINE[2])
 
-def test_find_assign_op_dedent():
-    """
-    be carefull that empty token like dedent are not counted as parens
-    """
-    class Tk:
-        def __init__(self, s):
-            self.string = s
-
-    nt.assert_equal(_find_assign_op([Tk(s) for s in ('','a','=','b')]), 2)
-    nt.assert_equal(_find_assign_op([Tk(s) for s in ('','(', 'a','=','b', ')', '=' ,'5')]), 6)
-
 def test_check_complete():
     cc = ipt2.TransformerManager().check_complete
     nt.assert_equal(cc("a = 1"), ('complete', None))
     nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4))
+    nt.assert_equal(cc("for a in range(5):\n    if a > 0:"), ('incomplete', 8))
     nt.assert_equal(cc("raise = 2"), ('invalid', None))
     nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0))
     nt.assert_equal(cc(")"), ('incomplete', 0))