Repository: erikrose/parsimonious

Issue:
incorrect precedence for modifiers of string literals
The precedence of literal modifiers like `r"string"` is wrong. `r` gets parsed as a rule reference, and the string literal is interpreted as-is. 

I discovered this while trying to add support for binary grammars.  I hope to fix this as part of that project, but wanted to note it in an issue in case I don't get around to it.

## How to reproduce:
```
>>> g = Grammar("""
    default = r"\b"
    r = "something"
""")
>>> g.parse("something\b")
s = 'something\x08'
Node(<Sequence default = r '\x08'>, s, 0, 10, children=[Node(<Literal r = 'something'>, s, 0, 9), Node(<Literal '\x08'>, s, 9, 10)])
>>> g.parse(r"\b")
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 113, in parse
    return self.default_rule.parse(text, pos=pos)
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/expressions.py", line 130, in parse
    node = self.match(text, pos=pos)
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/expressions.py", line 147, in match
    raise error
parsimonious.exceptions.ParseError: Rule 'default' didn't match at '\b' (line 1, column 1).
```

If the `r` rule is omitted:
```
>>> from parsimonious.grammar import Grammar
>>> Grammar("""
    default = r"\b"
""")
Traceback (most recent call last):
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 413, in _resolve_refs
    reffed_expr = rule_map[label]
KeyError: 'r'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 67, in __init__
    exprs, first = self._expressions_from_rules(rules, decorated_custom_rules)
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 104, in _expressions_from_rules
    return RuleVisitor(custom_rules).visit(tree)
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/nodes.py", line 213, in visit
    return method(node, [self.visit(n) for n in node])
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 452, in visit_rules
    rule_map = OrderedDict((expr.name, self._resolve_refs(rule_map, expr, done))
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 452, in <genexpr>
    rule_map = OrderedDict((expr.name, self._resolve_refs(rule_map, expr, done))
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 423, in _resolve_refs
    expr.members = tuple(self._resolve_refs(rule_map, member, done)
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 423, in <genexpr>
    expr.members = tuple(self._resolve_refs(rule_map, member, done)
  File "/Users/lucaswiman/opensource/parsimonious/parsimonious/grammar.py", line 415, in _resolve_refs
    raise UndefinedLabel(expr)
parsimonious.exceptions.UndefinedLabel: The label "r" was never defined.
```

Discussion:

Patch:
diff --git a/README.rst b/README.rst
index b27ecfd..c0ba2ae 100644
--- a/README.rst
+++ b/README.rst
@@ -236,6 +236,14 @@ Syntax Reference
                         conventions for "raw" and Unicode strings help support
                         fiddly characters.
 
+``b"some literal"``     A bytes literal. Using bytes literals and regular
+                        expressions allows your grammar to parse binary files.
+						Note that all literals and regular expressions must be
+						of the same type within a grammar. In grammars that
+						process bytestrings, you should make the grammar string
+						an ``r"""string"""`` so that byte literals like ``\xff``
+						work correctly.
+
 [space]                 Sequences are made out of space- or tab-delimited
                         things. ``a b c`` matches spots where those 3
                         terms appear in that order.
@@ -269,6 +277,9 @@ Syntax Reference
                         them out of simpler primitives. Parsimonious uses the
                         regex_ library instead of the built-in re module.
 
+``~br"regex"``          A bytes regex; required if your grammar parses
+                        bytestrings.
+
 ``(things)``            Parentheses are used for grouping, like in every other
                         language.
 ====================    ========================================================
@@ -421,6 +432,11 @@ Niceties
 
 Version History
 ===============
+(Next release)
+  * Add support for grammars on bytestrings (lucaswiman)
+  * Fix precedence of string literal modifiers u/r/b.
+    This will break grammars with no spaces between a
+	reference and a string literal. (lucaswiman)
 
 0.9.0
   * Add support for Python 3.7, 3.8, 3.9, 3.10 (righthandabacus, Lonnen)
diff --git a/parsimonious/grammar.py b/parsimonious/grammar.py
index c1e324a..617f4d1 100644
--- a/parsimonious/grammar.py
+++ b/parsimonious/grammar.py
@@ -6,6 +6,7 @@ by hand.
 
 """
 from collections import OrderedDict
+from textwrap import dedent
 
 from parsimonious.exceptions import BadGrammar, UndefinedLabel
 from parsimonious.expressions import (Literal, Regex, Sequence, OneOf,
@@ -226,8 +227,8 @@ rule_syntax = (r'''
     literal = spaceless_literal _
 
     # So you can't spell a regex like `~"..." ilm`:
-    spaceless_literal = ~"u?r?\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*\""is /
-                        ~"u?r?'[^'\\\\]*(?:\\\\.[^'\\\\]*)*'"is
+    spaceless_literal = ~"u?r?b?\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*\""is /
+                        ~"u?r?b?'[^'\\\\]*(?:\\\\.[^'\\\\]*)*'"is
 
     expression = ored / sequence / term
     or_term = "/" _ term
@@ -246,7 +247,7 @@ rule_syntax = (r'''
     # A subsequent equal sign is the only thing that distinguishes a label
     # (which begins a new rule) from a reference (which is just a pointer to a
     # rule defined somewhere else):
-    label = ~"[a-zA-Z_][a-zA-Z_0-9]*" _
+    label = ~"[a-zA-Z_][a-zA-Z_0-9]*(?![\"'])" _
 
     # _ = ~r"\s*(?:#[^\r\n]*)?\s*"
     _ = meaninglessness*
@@ -286,6 +287,7 @@ class RuleVisitor(NodeVisitor):
 
         """
         self.custom_rules = custom_rules or {}
+        self._last_literal_node_and_type = None
 
     def visit_parenthesized(self, node, parenthesized):
         """Treat a parenthesized subexpression as just its contents.
@@ -368,7 +370,19 @@ class RuleVisitor(NodeVisitor):
 
     def visit_spaceless_literal(self, spaceless_literal, visited_children):
         """Turn a string literal into a ``Literal`` that recognizes it."""
-        return Literal(evaluate_string(spaceless_literal.text))
+        literal_value = evaluate_string(spaceless_literal.text)
+        if self._last_literal_node_and_type:
+            last_node, last_type = self._last_literal_node_and_type
+            if last_type != type(literal_value):
+                raise BadGrammar(dedent(f"""\
+                    Found {last_node.text} ({last_type}) and {spaceless_literal.text} ({type(literal_value)}) string literals.
+                    All strings in a single grammar must be of the same type.
+                """)
+                )
+
+        self._last_literal_node_and_type = spaceless_literal, type(literal_value)
+
+        return Literal(literal_value)
 
     def visit_literal(self, node, literal):
         """Pick just the literal out of a literal-and-junk combo."""
diff --git a/parsimonious/utils.py b/parsimonious/utils.py
index 3dc27cc..7c7e863 100644
--- a/parsimonious/utils.py
+++ b/parsimonious/utils.py
@@ -12,9 +12,11 @@ class StrAndRepr(object):
 
 def evaluate_string(string):
     """Piggyback on Python's string support so we can have backslash escaping
-    and niceties like \n, \t, etc. string.decode('string_escape') would have
-    been a lower-level possibility.
+    and niceties like \n, \t, etc.
 
+    This also supports:
+    1. b"strings", allowing grammars to parse bytestrings, in addition to str.
+    2. r"strings" to simplify regexes.
     """
     return ast.literal_eval(string)
 


Test patch:
diff --git a/parsimonious/tests/test_grammar.py b/parsimonious/tests/test_grammar.py
index f495f81..c9f0aa3 100644
--- a/parsimonious/tests/test_grammar.py
+++ b/parsimonious/tests/test_grammar.py
@@ -4,8 +4,9 @@ from sys import version_info
 from unittest import TestCase
 
 import sys
+import pytest
 
-from parsimonious.exceptions import UndefinedLabel, ParseError
+from parsimonious.exceptions import BadGrammar, UndefinedLabel, ParseError, VisitationError
 from parsimonious.expressions import Literal, Lookahead, Regex, Sequence, TokenMatcher, is_callable
 from parsimonious.grammar import rule_grammar, RuleVisitor, Grammar, TokenGrammar, LazyReference
 from parsimonious.nodes import Node
@@ -493,3 +494,59 @@ class TokenGrammarTests(TestCase):
             self.assertEqual(u'<Token "💣">'.encode('utf-8'), t.__repr__())
         else:
             self.assertEqual(u'<Token "💣">', t.__repr__())
+
+
+def test_precedence_of_string_modifiers():
+    # r"strings", etc. should be parsed as a single literal, not r followed
+    # by a string literal.
+    g = Grammar(r"""
+        escaped_bell = r"\b"
+        r = "irrelevant"
+    """)
+    assert isinstance(g["escaped_bell"], Literal)
+    assert g["escaped_bell"].literal == "\\b"
+    with pytest.raises(ParseError):
+        g.parse("irrelevant\b")
+
+    g2 = Grammar(r"""
+        escaped_bell = r"\b"
+    """)
+    assert g2.parse("\\b")
+
+
+def test_binary_grammar():
+    g = Grammar(r"""
+        file = header body terminator
+        header = b"\xFF" length b"~"
+        length = ~rb"\d+"
+        body = ~b"[^\xFF]*"
+        terminator = b"\xFF"
+    """)
+    length = 22
+    assert g.parse(b"\xff22~" + (b"a" * 22) + b"\xff") is not None
+
+
+def test_inconsistent_string_types_in_grammar():
+    with pytest.raises(VisitationError) as e:
+        Grammar(r"""
+            foo = b"foo"
+            bar = "bar"
+        """)
+    assert e.value.original_class is BadGrammar
+    with pytest.raises(VisitationError) as e:
+        Grammar(r"""
+            foo = ~b"foo"
+            bar = "bar"
+        """)
+    assert e.value.original_class is BadGrammar
+
+    # The following should parse without errors because they use the same
+    # string types:
+    Grammar(r"""
+        foo = b"foo"
+        bar = b"bar"
+    """)
+    Grammar(r"""
+        foo = "foo"
+        bar = "bar"
+    """)