Repository: gristlabs/asttokens

Issue:
IndexError on f((x)[:, 0])
I've had this for a while but only now found the time to boil down to a minimal reproducing case. :X

Reproducer:

```py
import asttokens
asttokens.ASTTokens('f((x)[:, 0])', parse=True)
```

Running this file gives:

```pytb
Traceback (most recent call last):
  File "/tmp/reproducer.py", line 2, in <module>
    asttokens.ASTTokens('f((x)[:, 0])', parse=True)
  File "asttokens/asttokens.py", line 65, in __init__
    self.mark_tokens(self._tree)
  File "asttokens/asttokens.py", line 76, in mark_tokens
    MarkTokens(self).visit_tree(root_node)
  File "asttokens/mark_tokens.py", line 47, in visit_tree
    util.visit_tree(node, self._visit_before_children, self._visit_after_children)
  File "asttokens/util.py", line 192, in visit_tree
    ret = postvisit(current, par_value, value)
  File "asttokens/mark_tokens.py", line 87, in _visit_after_children
    first, last = self._expand_to_matching_pairs(first, last, node)
  File "asttokens/mark_tokens.py", line 142, in _expand_to_matching_pairs
    last = self._code.next_token(last_token)
  File "asttokens/asttokens.py", line 141, in next_token
    while is_non_coding_token(self._tokens[i].type):
IndexError: list index out of range
```

Discussion:

Patch:
diff --git a/asttokens/mark_tokens.py b/asttokens/mark_tokens.py
index 6ab95e4..82acb55 100644
--- a/asttokens/mark_tokens.py
+++ b/asttokens/mark_tokens.py
@@ -103,22 +103,6 @@ class MarkTokens(object):
       newline = self._code.find_token(start_token, token.ENDMARKER)
     return self._code.prev_token(newline)
 
-  def _iter_non_child_tokens(self, first_token, last_token, node):
-    """
-    Generates all tokens in [first_token, last_token] range that do not belong to any children of
-    node. E.g. `foo(bar)` has children `foo` and `bar`, but we would yield the `(`.
-    """
-    tok = first_token
-    for n in self._iter_children(node):
-      for t in self._code.token_range(tok, self._code.prev_token(n.first_token)):
-        yield t
-      if n.last_token.index >= last_token.index:
-        return
-      tok = self._code.next_token(n.last_token)
-
-    for t in self._code.token_range(tok, last_token):
-      yield t
-
   def _expand_to_matching_pairs(self, first_token, last_token, node):
     """
     Scan tokens in [first_token, last_token] range that are between node's children, and for any
@@ -128,7 +112,7 @@ class MarkTokens(object):
     # child nodes). If we find any closing ones, we match them to the opens.
     to_match_right = []
     to_match_left = []
-    for tok in self._iter_non_child_tokens(first_token, last_token, node):
+    for tok in self._code.token_range(first_token, last_token):
       tok_info = tok[:2]
       if to_match_right and tok_info == to_match_right[-1]:
         to_match_right.pop()


Test patch:
diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py
index 58ab323..20dc84d 100644
--- a/tests/test_mark_tokens.py
+++ b/tests/test_mark_tokens.py
@@ -520,3 +520,8 @@ bar = ('x y z'   # comment2
                      {"Call:(obj.attribute.get_callback() or default_callback)()"})
     self.assertIn('BoolOp:obj.attribute.get_callback() or default_callback', m.view_nodes_at(8, 5))
     m.verify_all_nodes(self)
+
+  def test_complex_slice_and_parens(self):
+    source = 'f((x)[:, 0])'
+    m = self.create_mark_checker(source)
+    m.verify_all_nodes(self)