Repository: ipython/ipython

Issue:
single close parens kills IPython.
```
In [1]: )<enter>
```
```
Unhandled exception in event loop:
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/prompt_toolkit/eventloop/posix.py", line 154, in _run_task
    t()
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/prompt_toolkit/eventloop/context.py", line 115, in new_func
    return func(*a, **kw)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/prompt_toolkit/application/application.py", line 548, in read_from_input
    self.key_processor.process_keys()
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/prompt_toolkit/key_binding/key_processor.py", line 272, in process_keys
    self._process_coroutine.send(key_press)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/prompt_toolkit/key_binding/key_processor.py", line 179, in _process
    self._call_handler(matches[-1], key_sequence=buffer[:])
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/prompt_toolkit/key_binding/key_processor.py", line 321, in _call_handler
    handler.call(event)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/prompt_toolkit/key_binding/key_bindings.py", line 78, in call
    return self.handler(event)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/IPython/terminal/shortcuts.py", line 109, in newline_or_execute
    status, indent = shell.check_complete(check_text)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3307, in check_complete
    status, nspaces = self.input_transformer_manager.check_complete(code)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/IPython/core/inputtransformer2.py", line 589, in check_complete
    lines = self.do_token_transforms(lines)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/IPython/core/inputtransformer2.py", line 531, in do_token_transforms
    changed, lines = self.do_one_token_transform(lines)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/IPython/core/inputtransformer2.py", line 518, in do_one_token_transform
    transformer = transformer_cls.find(tokens_by_line)
  File "/Users/bussonniermatthias/anaconda/lib/python3.6/site-packages/IPython/core/inputtransformer2.py", line 354, in find
    while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:

Exception list index out of range
```


Discussion:
Narrowed it down to 

```python
from IPython.core.inputtransformer2 import EscapedCommand, make_tokens_by_line
# \n is added unconditionally so all cells ends with a \n
toks = make_tokens_by_line(')\n')
EscapedCommand.find(toks)
```
So in above case `make_tokens_by_line()`, last line is an empty list, that we never populate with anything. 

I believe that is not what was intended, so I thing in this special case we could just filter it out with a : 

```
    if not tokens_by_line[-1]:
        tokens_by_line.pop()
```

It fails also on `\x0c` as sole input (FORM-FEED, in string.printable), and we can get some fun with 

```
import string
from IPython.core.inputtransformer2 import TransformerManager
tm = TransformerManager()
fail= []
for c in string.printable:
    for k in string.printable:
        try:
            tm.check_complete(c+k)
        except:
            fail.append(c+k)
fail
```

I'm tempted to catch any exception and return 'invalid' / SyntaxError to at least no crash user session.... and refine

Resolving patch:
diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py
--- a/IPython/core/inputtransformer2.py
+++ b/IPython/core/inputtransformer2.py
@@ -418,6 +418,8 @@ def transform(self, lines):
         lines_after = lines[self.q_line + 1:]
 
         m = _help_end_re.search(content)
+        if not m:
+            raise SyntaxError(content)
         assert m is not None, content
         target = m.group(1)
         esc = m.group(3)
@@ -460,6 +462,8 @@ def make_tokens_by_line(lines):
     except tokenize.TokenError:
         # Input ended in a multiline string or expression. That's OK for us.
         pass
+    if not tokens_by_line[-1]:
+        tokens_by_line.pop()
     
     return tokens_by_line
 
@@ -522,9 +526,13 @@ def do_one_token_transform(self, lines):
         if not candidates:
             # Nothing to transform
             return False, lines
-
-        transformer = min(candidates, key=TokenTransformBase.sortby)
-        return True, transformer.transform(lines)
+        ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
+        for transformer in ordered_transformers:
+            try:
+                return True, transformer.transform(lines)
+            except SyntaxError:
+                pass
+        return False, lines
 
     def do_token_transforms(self, lines):
         for _ in range(TRANSFORM_LOOP_LIMIT):
@@ -591,10 +599,13 @@ def check_complete(self, cell: str):
             return 'invalid', None
 
         tokens_by_line = make_tokens_by_line(lines)
+        if not tokens_by_line:
+            return 'incomplete', find_last_indent(lines)
         if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
             # We're in a multiline string or expression
             return 'incomplete', find_last_indent(lines)
-
+        if len(tokens_by_line) == 1:
+            return 'incomplete', find_last_indent(lines)
         # Find the last token on the previous line that's not NEWLINE or COMMENT
         toks_last_line = tokens_by_line[-2]
         ix = len(toks_last_line) - 1


Test patch:
diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py
--- a/IPython/core/tests/test_inputtransformer2.py
+++ b/IPython/core/tests/test_inputtransformer2.py
@@ -5,6 +5,7 @@
 transformations.
 """
 import nose.tools as nt
+import string
 
 from IPython.core import inputtransformer2 as ipt2
 from IPython.core.inputtransformer2 import make_tokens_by_line
@@ -100,6 +101,16 @@
 [r"get_ipython().set_next_input('(a,\nb) = zip');get_ipython().run_line_magic('pinfo', 'zip')" + "\n"]
 )
 
+def check_make_token_by_line_never_ends_empty():
+    """
+    Check that not sequence of single or double characters ends up leading to en empty list of tokens
+    """
+    from string import printable
+    for c in printable:
+        nt.assert_not_equal(make_tokens_by_line(c)[-1], [])
+        for k in printable:
+            nt.assert_not_equal(make_tokens_by_line(c+k)[-1], [])
+
 def check_find(transformer, case, match=True):
     sample, expected_start, _  = case
     tbl = make_tokens_by_line(sample)
@@ -190,6 +201,17 @@ def test_check_complete():
     nt.assert_equal(cc("for a in range(5):"), ('incomplete', 4))
     nt.assert_equal(cc("raise = 2"), ('invalid', None))
     nt.assert_equal(cc("a = [1,\n2,"), ('incomplete', 0))
+    nt.assert_equal(cc(")"), ('incomplete', 0))
+    nt.assert_equal(cc("\\\r\n"), ('incomplete', 0))
     nt.assert_equal(cc("a = '''\n   hi"), ('incomplete', 3))
     nt.assert_equal(cc("def a():\n x=1\n global x"), ('invalid', None))
     nt.assert_equal(cc("a \\ "), ('invalid', None))  # Nothing allowed after backslash
+
+    # no need to loop on all the letters/numbers.
+    short = '12abAB'+string.printable[62:]
+    for c in short:
+        # test does not raise:
+        cc(c)
+        for k in short:
+            cc(c+k)
+