Repository: erikrose/parsimonious

Issue:
TokenMatcher exception
_TokenMatcher.\_uncached\_match_ has different behavior than _Literal.\_uncached\_match_ regarding overread.
_Literal_'s one returns None on overread thanks to its use of _startswith_.
_TokenMatcher_'s one generates an `IndexError: list index out of range`.

To reproduce the exception the example below uses a _OneOrMore_ expr that calls _TokenMatcher.\_uncached\_match_ with pos >= len(token_list):
``` py
s = [Token('token1'), Token('token2')]
grammar = TokenGrammar("""
            foo = token1 "token2"+
            token1 = "token1"
            """)
grammar.parse(s)
```

Simple fix is to check pos < len(token_list):
``` py
    def _uncached_match(self, token_list, pos, cache, error):
        if pos < len(token_list) and token_list[pos].type == self.literal:
            return Node(self, token_list, pos, pos + 1)
```

Comments:

Patch:
diff --git a/README.rst b/README.rst
index a6c8e07..8955b3d 100644
--- a/README.rst
+++ b/README.rst
@@ -282,6 +282,15 @@ Syntax Reference
 
 ``(things)``            Parentheses are used for grouping, like in every other
                         language.
+
+``thing{n}``            Exactly ``n`` repetitions of ``thing``.
+
+``thing{n,m}``          Between ``n`` and ``m`` repititions (inclusive.)
+
+``thing{,m}``           At most ``m`` repetitions of ``thing``.
+
+``thing{n,}``           At least ``n`` repetitions of ``thing``.
+
 ====================    ========================================================
 
 .. _flags: https://docs.python.org/3/howto/regex.html#compilation
@@ -434,7 +443,11 @@ Version History
 ===============
 
 (Next release)
+  * Add support for range ``{min,max}`` repetition expressions (righthandabacus)
+  * Fix bug in ``*`` and ``+`` for token grammars (lucaswiman)
   * Add support for grammars on bytestrings (lucaswiman)
+  * Fix LazyReference resolution bug #134 (righthandabacus)
+
   .. warning::
 
       This release makes backward-incompatible changes:
diff --git a/parsimonious/exceptions.py b/parsimonious/exceptions.py
index b51c336..29cdef9 100644
--- a/parsimonious/exceptions.py
+++ b/parsimonious/exceptions.py
@@ -30,14 +30,17 @@ class ParseError(StrAndRepr, Exception):
         match."""
         # This is a method rather than a property in case we ever wanted to
         # pass in which line endings we want to use.
-        return self.text.count('\n', 0, self.pos) + 1
+        if isinstance(self.text, list):  # TokenGrammar
+            return None
+        else:
+            return self.text.count('\n', 0, self.pos) + 1
 
     def column(self):
         """Return the 1-based column where the expression ceased to match."""
         # We choose 1-based because that's what Python does with SyntaxErrors.
         try:
             return self.pos - self.text.rindex('\n', 0, self.pos)
-        except ValueError:
+        except (ValueError, AttributeError):
             return self.pos + 1
 
 
diff --git a/parsimonious/expressions.py b/parsimonious/expressions.py
index 0679527..bf10656 100644
--- a/parsimonious/expressions.py
+++ b/parsimonious/expressions.py
@@ -375,105 +375,71 @@ class Lookahead(Compound):
     """An expression which consumes nothing, even if its contained expression
     succeeds"""
 
-    # TODO: Merge this and Not for better cache hit ratios and less code.
-    # Downside: pretty-printed grammars might be spelled differently than what
-    # went in. That doesn't bother me.
+    __slots__ = ['negativity']
 
-    def _uncached_match(self, text, pos, cache, error):
-        node = self.members[0].match_core(text, pos, cache, error)
-        if node is not None:
-            return Node(self, text, pos, pos)
-
-    def _as_rhs(self):
-        return u'&%s' % self._unicode_members()[0]
+    def __init__(self, member, *, negative=False, **kwargs):
+        super(Lookahead, self).__init__(member, **kwargs)
+        self.negativity = bool(negative)
 
-
-class Not(Compound):
-    """An expression that succeeds only if the expression within it doesn't
-
-    In any case, it never consumes any characters; it's a negative lookahead.
-
-    """
     def _uncached_match(self, text, pos, cache, error):
-        # FWIW, the implementation in Parsing Techniques in Figure 15.29 does
-        # not bother to cache NOTs directly.
         node = self.members[0].match_core(text, pos, cache, error)
-        if node is None:
+        if (node is None) == self.negativity: # negative lookahead == match only if not found
             return Node(self, text, pos, pos)
 
     def _as_rhs(self):
-        # TODO: Make sure this parenthesizes the member properly if it's an OR
-        # or AND.
-        return u'!%s' % self._unicode_members()[0]
+        return u'%s%s' % ('!' if self.negativity else '&', self._unicode_members()[0])
 
+def Not(term):
+    return Lookahead(term, negative=True)
 
 # Quantifiers. None of these is strictly necessary, but they're darn handy.
 
-class Optional(Compound):
-    """An expression that succeeds whether or not the contained one does
-
-    If the contained expression succeeds, it goes ahead and consumes what it
-    consumes. Otherwise, it consumes nothing.
-
-    """
-    def _uncached_match(self, text, pos, cache, error):
-        node = self.members[0].match_core(text, pos, cache, error)
-        return (Node(self, text, pos, pos) if node is None else
-                Node(self, text, pos, node.end, children=[node]))
-
-    def _as_rhs(self):
-        return u'%s?' % self._unicode_members()[0]
+class Quantifier(Compound):
+    """An expression wrapper like the */+/?/{n,m} quantifier in regexes."""
 
+    __slots__ = ['min', 'max']
 
-# TODO: Merge with OneOrMore.
-class ZeroOrMore(Compound):
-    """An expression wrapper like the * quantifier in regexes."""
-
-    def _uncached_match(self, text, pos, cache, error):
-        new_pos = pos
-        children = []
-        while True:
-            node = self.members[0].match_core(text, new_pos, cache, error)
-            if node is None or not (node.end - node.start):
-                # Node was None or 0 length. 0 would otherwise loop infinitely.
-                return Node(self, text, pos, new_pos, children)
-            children.append(node)
-            new_pos += node.end - node.start
-
-    def _as_rhs(self):
-        return u'%s*' % self._unicode_members()[0]
-
-
-class OneOrMore(Compound):
-    """An expression wrapper like the + quantifier in regexes.
-
-    You can also pass in an alternate minimum to make this behave like "2 or
-    more", "3 or more", etc.
-
-    """
-    __slots__ = ['min']
-
-    # TODO: Add max. It should probably succeed if there are more than the max
-    # --just not consume them.
-
-    def __init__(self, member, name='', min=1):
-        super(OneOrMore, self).__init__(member, name=name)
+    def __init__(self, member, *, min=0, max=float('inf'), name='', **kwargs):
+        super(Quantifier, self).__init__(member, name=name, **kwargs)
         self.min = min
+        self.max = max
 
     def _uncached_match(self, text, pos, cache, error):
         new_pos = pos
         children = []
-        while True:
+        size = len(text)
+        while new_pos < size and len(children) < self.max:
             node = self.members[0].match_core(text, new_pos, cache, error)
             if node is None:
-                break
+                break # no more matches
             children.append(node)
             length = node.end - node.start
-            if length == 0:  # Don't loop infinitely.
+            if len(children) >= self.min and length == 0:  # Don't loop infinitely
                 break
             new_pos += length
         if len(children) >= self.min:
             return Node(self, text, pos, new_pos, children)
 
     def _as_rhs(self):
-        return u'%s+' % self._unicode_members()[0]
+        if self.min == 0 and self.max == 1:
+            qualifier = '?'
+        elif self.min == 0 and self.max == float('inf'):
+            qualifier = '*'
+        elif self.min == 1 and self.max == float('inf'):
+            qualifier = '+'
+        elif self.max == float('inf'):
+            qualifier = '{%d,}' % self.min
+        elif self.min == 0:
+            qualifier = '{,%d}' % self.max
+        else:
+            qualifier = '{%d,%d}' % (self.min, self.max)
+        return '%s%s' % (self._unicode_members()[0], qualifier)
+
+def ZeroOrMore(member, name=''):
+    return Quantifier(member, name=name, min=0, max=float('inf'))
+
+def OneOrMore(member, name='', min=1):
+    return Quantifier(member, name=name, min=min, max=float('inf'))
+
+def Optional(member, name=''):
+    return Quantifier(member, name=name, min=0, max=1)
diff --git a/parsimonious/grammar.py b/parsimonious/grammar.py
index 617f4d1..ff4739f 100644
--- a/parsimonious/grammar.py
+++ b/parsimonious/grammar.py
@@ -10,7 +10,7 @@ from textwrap import dedent
 
 from parsimonious.exceptions import BadGrammar, UndefinedLabel
 from parsimonious.expressions import (Literal, Regex, Sequence, OneOf,
-    Lookahead, Optional, ZeroOrMore, OneOrMore, Not, TokenMatcher,
+    Lookahead, Quantifier, Optional, ZeroOrMore, OneOrMore, Not, TokenMatcher,
     expression, is_callable)
 from parsimonious.nodes import NodeVisitor
 from parsimonious.utils import evaluate_string
@@ -241,7 +241,7 @@ rule_syntax = (r'''
     atom = reference / literal / regex / parenthesized
     regex = "~" spaceless_literal ~"[ilmsuxa]*"i _
     parenthesized = "(" _ expression ")" _
-    quantifier = ~"[*+?]" _
+    quantifier = ~r"[*+?]|\{\d*,\d+\}|\{\d+,\d*\}|\{\d+\}" _
     reference = label !equals
 
     # A subsequent equal sign is the only thing that distinguishes a label
@@ -305,7 +305,17 @@ class RuleVisitor(NodeVisitor):
 
     def visit_quantified(self, node, quantified):
         atom, quantifier = quantified
-        return self.quantifier_classes[quantifier.text](atom)
+        try:
+            return self.quantifier_classes[quantifier.text](atom)
+        except KeyError:
+            # This should pass: assert re.full_match("\{(\d*)(,(\d*))?\}", quantifier)
+            quantifier = quantifier.text[1:-1].split(",")
+            if len(quantifier) == 1:
+                min_match = max_match = int(quantifier[0])
+            else:
+                min_match = int(quantifier[0]) if quantifier[0] else 0
+                max_match = int(quantifier[1]) if quantifier[1] else float('inf')
+            return Quantifier(atom, min=min_match, max=max_match)
 
     def visit_lookahead_term(self, node, lookahead_term):
         ampersand, term, _ = lookahead_term
@@ -405,7 +415,7 @@ class RuleVisitor(NodeVisitor):
         """
         return visited_children or node  # should semantically be a tuple
 
-    def _resolve_refs(self, rule_map, expr, done):
+    def _resolve_refs(self, rule_map, expr, resolved):
         """Return an expression with all its lazy references recursively
         resolved.
 
@@ -415,24 +425,42 @@ class RuleVisitor(NodeVisitor):
         :arg done: The set of Expressions that have already been or are
             currently being resolved, to ward off redundant work and prevent
             infinite recursion for circular refs
-
         """
-        if isinstance(expr, LazyReference):
+        if expr in resolved:
+            newref = resolved[expr]
+        elif isinstance(expr, LazyReference):
             label = str(expr)
             try:
                 reffed_expr = rule_map[label]
             except KeyError:
                 raise UndefinedLabel(expr)
-            return self._resolve_refs(rule_map, reffed_expr, done)
+            newref = resolved[expr] = self._resolve_refs(rule_map, reffed_expr, resolved)
         else:
-            if getattr(expr, 'members', ()) and expr not in done:
+            if getattr(expr, 'members', ()) and expr not in resolved:
                 # Prevents infinite recursion for circular refs. At worst, one
                 # of `expr.members` can refer back to `expr`, but it can't go
                 # any farther.
-                done.add(expr)
-                expr.members = tuple(self._resolve_refs(rule_map, member, done)
+                resolved[expr] = expr
+                expr.members = tuple(self._resolve_refs(rule_map, member, resolved)
                                      for member in expr.members)
-            return expr
+            newref = expr
+        return newref
+
+    def _find_unresolved(self, rules):
+        """Recursively find all LazyReference objects and return them as a set"""
+        seen = set()
+        to_resolve = set()
+        def _find_referenced_rules(rulelist):
+            for expr in rulelist:
+                if expr in seen:
+                    continue
+                seen.add(expr)
+                if isinstance(expr, LazyReference):
+                    to_resolve.add(expr)
+                elif getattr(expr, 'members', None):
+                    _find_referenced_rules(expr.members)
+        _find_referenced_rules(rules)
+        return to_resolve
 
     def visit_rules(self, node, rules_list):
         """Collate all the rules into a map. Return (map, default rule).
@@ -458,9 +486,18 @@ class RuleVisitor(NodeVisitor):
         rule_map.update(self.custom_rules)
 
         # Resolve references. This tolerates forward references.
-        done = set()
-        rule_map = OrderedDict((expr.name, self._resolve_refs(rule_map, expr, done))
-                               for expr in rule_map.values())
+        # We use a job pool `to_resolve` to remember all rules to resolve. It is
+        # initialized with all existing rules in `rule_map` and all
+        # LazyReference rules found later will be added as well.
+        to_resolve = set(rule_map.values())
+        resolved = {}
+        while to_resolve:
+            expr = to_resolve.pop()
+            newexpr = self._resolve_refs(rule_map, expr, resolved)
+            if getattr(expr, 'name', None):
+                rule_map[expr.name] = newexpr
+            if getattr(newexpr, 'members', ()):
+                to_resolve.update(self._find_unresolved(newexpr.members))
 
         # isinstance() is a temporary hack around the fact that * rules don't
         # always get transformed into lists by NodeVisitor. We should fix that;


Test patch:
diff --git a/parsimonious/tests/test_expressions.py b/parsimonious/tests/test_expressions.py
index 01b7e5d..dbf8af9 100644
--- a/parsimonious/tests/test_expressions.py
+++ b/parsimonious/tests/test_expressions.py
@@ -3,7 +3,7 @@ from unittest import TestCase
 
 from parsimonious.exceptions import ParseError, IncompleteParseError
 from parsimonious.expressions import (Literal, Regex, Sequence, OneOf, Not,
-                                      Optional, ZeroOrMore, OneOrMore, Expression)
+                                      Quantifier, Optional, ZeroOrMore, OneOrMore, Expression)
 from parsimonious.grammar import Grammar, rule_grammar
 from parsimonious.nodes import Node
 
@@ -24,7 +24,7 @@ class LengthTests(TestCase):
 
         """
         node_length = None if node is None else node.end - node.start
-        self.assertTrue(node_length == length)
+        assert node_length == length
 
     def test_regex(self):
         self.len_eq(Literal('hello').match('ehello', 1), 5)  # simple
@@ -50,6 +50,8 @@ class LengthTests(TestCase):
     def test_optional(self):
         self.len_eq(Sequence(Optional(Literal('a')), Literal('b')).match('b'), 1)  # contained expr fails
         self.len_eq(Sequence(Optional(Literal('a')), Literal('b')).match('ab'), 2)  # contained expr succeeds
+        self.len_eq(Optional(Literal('a')).match('aa'), 1)
+        self.len_eq(Optional(Literal('a')).match('bb'), 0)
 
     def test_zero_or_more(self):
         self.len_eq(ZeroOrMore(Literal('b')).match(''), 0)  # zero
@@ -64,7 +66,10 @@ class LengthTests(TestCase):
         self.len_eq(OneOrMore(Literal('b')).match('b'), 1)  # one
         self.len_eq(OneOrMore(Literal('b')).match('bbb'), 3)  # more
         self.len_eq(OneOrMore(Literal('b'), min=3).match('bbb'), 3)  # with custom min; success
+        self.len_eq(Quantifier(Literal('b'), min=3, max=5).match('bbbb'), 4)  # with custom min and max; success
+        self.len_eq(Quantifier(Literal('b'), min=3, max=5).match('bbbbbb'), 5)  # with custom min and max; success
         self.assertRaises(ParseError, OneOrMore(Literal('b'), min=3).match, 'bb')  # with custom min; failure
+        self.assertRaises(ParseError, Quantifier(Literal('b'), min=3, max=5).match, 'bb')  # with custom min and max; failure
         self.len_eq(OneOrMore(Regex('^')).match('bb'), 0)  # attempt infinite loop
 
 
@@ -267,6 +272,20 @@ class RepresentationTests(TestCase):
         self.assertEqual(str(Grammar('foo = "bar" ("baz" "eggs")* "spam"')),
                          u"foo = 'bar' ('baz' 'eggs')* 'spam'")
 
+        # Quantifiers
+        self.assertEqual(str(Grammar('foo = "bar" ("baz" "eggs"){2,4} "spam"')),
+                         "foo = 'bar' ('baz' 'eggs'){2,4} 'spam'")
+        self.assertEqual(str(Grammar('foo = "bar" ("baz" "eggs"){2,} "spam"')),
+                         "foo = 'bar' ('baz' 'eggs'){2,} 'spam'")
+        self.assertEqual(str(Grammar('foo = "bar" ("baz" "eggs"){1,} "spam"')),
+                         "foo = 'bar' ('baz' 'eggs')+ 'spam'")
+        self.assertEqual(str(Grammar('foo = "bar" ("baz" "eggs"){,4} "spam"')),
+                         "foo = 'bar' ('baz' 'eggs'){,4} 'spam'")
+        self.assertEqual(str(Grammar('foo = "bar" ("baz" "eggs"){0,1} "spam"')),
+                         "foo = 'bar' ('baz' 'eggs')? 'spam'")
+        self.assertEqual(str(Grammar('foo = "bar" ("baz" "eggs"){0,} "spam"')),
+                         "foo = 'bar' ('baz' 'eggs')* 'spam'")
+
         # OneOf
         self.assertEqual(str(Grammar('foo = "bar" ("baz" / "eggs") "spam"')),
                          u"foo = 'bar' ('baz' / 'eggs') 'spam'")
@@ -305,7 +324,7 @@ class SlotsTests(TestCase):
         But it does.
 
         """
-        class Smoo(Optional):
+        class Smoo(Quantifier):
             __slots__ = ['smoo']
 
             def __init__(self):
diff --git a/parsimonious/tests/test_grammar.py b/parsimonious/tests/test_grammar.py
index c9f0aa3..e982d61 100644
--- a/parsimonious/tests/test_grammar.py
+++ b/parsimonious/tests/test_grammar.py
@@ -3,8 +3,8 @@
 from sys import version_info
 from unittest import TestCase
 
-import sys
 import pytest
+import sys
 
 from parsimonious.exceptions import BadGrammar, UndefinedLabel, ParseError, VisitationError
 from parsimonious.expressions import Literal, Lookahead, Regex, Sequence, TokenMatcher, is_callable
@@ -286,6 +286,31 @@ class GrammarTests(TestCase):
             """)
         grammar.parse('(34)')
 
+    def test_resolve_refs_completeness(self):
+        """Smoke-test another circumstance where lazy references don't get resolved."""
+        grammar = Grammar(r"""
+            block = "{" _ item* "}" _
+
+            # An item is an element of a block.
+            item = number / word / block / paren
+
+            # Parens are for delimiting subexpressions.
+            paren = "(" _ item* ")" _
+
+            # Words are barewords, unquoted things, other than literals, that can live
+            # in lists. We may renege on some of these chars later, especially ".". We
+            # may add Unicode.
+            word = spaceless_word _
+            spaceless_word = ~r"[-a-z`~!@#$%^&*_+=|\\;<>,.?][-a-z0-9`~!@#$%^&*_+=|\\;<>,.?]*"i
+
+            number = ~r"[0-9]+" _ # There are decimals and strings and other stuff back on the "parsing" branch, once you get this working.
+
+            _ = meaninglessness*
+            meaninglessness = whitespace
+            whitespace = ~r"\s+"
+            """)
+        grammar.parse('{log (add 3 to 5)}')
+
     def test_infinite_loop(self):
         """Smoke-test a grammar that was causing infinite loops while building.
 
@@ -462,6 +487,43 @@ class GrammarTests(TestCase):
             list(grammar.keys()),
             ['r%s' % i for i in range(100)])
 
+    def test_repetitions(self):
+        grammar = Grammar(r'''
+            left_missing = "a"{,5}
+            right_missing = "a"{5,}
+            exact = "a"{5}
+            range = "a"{2,5}
+            optional = "a"?
+            plus = "a"+
+            star = "a"*
+        ''')
+        should_parse = [
+            ("left_missing", ["a" * i for i in range(6)]),
+            ("right_missing", ["a" * i for i in range(5, 8)]),
+            ("exact", ["a" * 5]),
+            ("range", ["a" * i for i in range(2, 6)]),
+            ("optional", ["", "a"]),
+            ("plus", ["a", "aa"]),
+            ("star", ["", "a", "aa"]),
+        ]
+        for rule, examples in should_parse:
+            for example in examples:
+                assert grammar[rule].parse(example)
+
+        should_not_parse = [
+            ("left_missing", ["a" * 6]),
+            ("right_missing", ["a" * i for i in range(5)]),
+            ("exact", ["a" * i for i in list(range(5)) + list(range(6, 10))]),
+            ("range", ["a" * i for i in list(range(2)) + list(range(6, 10))]),
+            ("optional", ["aa"]),
+            ("plus", [""]),
+            ("star", ["b"]),
+        ]
+        for rule, examples in should_not_parse:
+            for example in examples:
+                with pytest.raises(ParseError):
+                    grammar[rule].parse(example)
+
 
 class TokenGrammarTests(TestCase):
     """Tests for the TokenGrammar class and associated machinery"""
@@ -483,17 +545,36 @@ class TokenGrammarTests(TestCase):
         grammar = TokenGrammar("""
             foo = "token1" "token2"
             """)
-        self.assertRaises(ParseError,
-                      grammar.parse,
-                      [Token('tokenBOO'), Token('token2')])
+        with pytest.raises(ParseError) as e:
+            grammar.parse([Token('tokenBOO'), Token('token2')])
+        assert "Rule 'foo' didn't match at" in str(e.value)
 
     def test_token_repr(self):
         t = Token(u'💣')
         self.assertTrue(isinstance(t.__repr__(), str))
-        if sys.version_info < (3, 0):
-            self.assertEqual(u'<Token "💣">'.encode('utf-8'), t.__repr__())
-        else:
-            self.assertEqual(u'<Token "💣">', t.__repr__())
+        self.assertEqual(u'<Token "💣">', t.__repr__())
+
+    def test_token_star_plus_expressions(self):
+        a = Token("a")
+        b = Token("b")
+        grammar = TokenGrammar("""
+            foo = "a"*
+            bar = "a"+
+        """)
+        assert grammar["foo"].parse([]) is not None
+        assert grammar["foo"].parse([a]) is not None
+        assert grammar["foo"].parse([a, a]) is not None
+
+        with pytest.raises(ParseError):
+            grammar["foo"].parse([a, b])
+        with pytest.raises(ParseError):
+            grammar["foo"].parse([b])
+
+        assert grammar["bar"].parse([a]) is not None
+        with pytest.raises(ParseError):
+            grammar["bar"].parse([a, b])
+        with pytest.raises(ParseError):
+            grammar["bar"].parse([b])
 
 
 def test_precedence_of_string_modifiers():