Repository: ipython/ipython

Issue:
Wrong Tooltip with shift-tab
@andram opened jupyter/notebook#520 at 2015-10-01 10:09:29 UTC

If a notebook cell starts with a multiline string, then shift-tab gets confused and shows the wrong tooltips for following lines. For example consider a cell containing the following 5 lines:

```
"""
Docstring
"""
max()
min()
```

now position the cursor inside the brackets of max(). Expected behaviour is that the docstring for `max` is shown. Actual behaviour is that the docstring for min is shown. (see attached screenshot for current github version of jupyter notebook)

![notebook_tooltip_problem](https://cloud.githubusercontent.com/assets/3459178/10218173/b2415676-682c-11e5-8551-708c1dc1ab53.png)



Discussion:

Resolving patch:
diff --git a/IPython/utils/tokenutil.py b/IPython/utils/tokenutil.py
--- a/IPython/utils/tokenutil.py
+++ b/IPython/utils/tokenutil.py
@@ -72,15 +72,24 @@ def token_at_cursor(cell, cursor_pos=0):
     cell = cast_unicode_py2(cell)
     names = []
     tokens = []
-    offset = 0
     call_names = []
+    
+    offsets = {1: 0} # lines start at 1
     for tup in generate_tokens(StringIO(cell).readline):
         
         tok = Token(*tup)
         
         # token, text, start, end, line = tup
-        start_col = tok.start[1]
-        end_col = tok.end[1]
+        start_line, start_col = tok.start
+        end_line, end_col = tok.end
+        if end_line + 1 not in offsets:
+            # keep track of offsets for each line
+            lines = tok.line.splitlines(True)
+            for lineno, line in zip(range(start_line + 1, end_line + 2), lines):
+                if lineno not in offsets:
+                    offsets[lineno] = offsets[lineno-1] + len(line)
+        
+        offset = offsets[start_line]
         # allow '|foo' to find 'foo' at the beginning of a line
         boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
         if offset + start_col >= boundary:
@@ -103,14 +112,12 @@ def token_at_cursor(cell, cursor_pos=0):
             elif tok.text == ')' and call_names:
                 call_names.pop(-1)
         
-        if offset + end_col > cursor_pos:
+        tokens.append(tok)
+        
+        if offsets[end_line] + end_col > cursor_pos:
             # we found the cursor, stop reading
             break
         
-        tokens.append(tok)
-        if tok.token in (tokenize2.NEWLINE, tokenize2.NL):
-            offset += len(tok.line)
-    
     if call_names:
         return call_names[-1]
     elif names:


Test patch:
diff --git a/IPython/utils/tests/test_tokenutil.py b/IPython/utils/tests/test_tokenutil.py
--- a/IPython/utils/tests/test_tokenutil.py
+++ b/IPython/utils/tests/test_tokenutil.py
@@ -53,6 +53,25 @@ def test_multiline():
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
 
+def test_multiline_token():
+    cell = '\n'.join([
+        '"""\n\nxxxxxxxxxx\n\n"""',
+        '5, """',
+        'docstring',
+        'multiline token',
+        '""", [',
+        '2, 3, "complicated"]',
+        'b = hello("string", there)'
+    ])
+    expected = 'hello'
+    start = cell.index(expected) + 1
+    for i in range(start, start + len(expected)):
+        expect_token(expected, cell, i)
+    expected = 'hello'
+    start = cell.index(expected) + 1
+    for i in range(start, start + len(expected)):
+        expect_token(expected, cell, i)
+
 def test_nested_call():
     cell = "foo(bar(a=5), b=10)"
     expected = 'foo'