Repository: maxfischer2781/bootpeg

Issue:
Provide rule commitment
Top-down PEG parsers commonly provide a "no backup" operator ([PEP 617 ``~``](https://www.python.org/dev/peps/pep-0617/#id9)/[pyparsing ``-``](https://pyparsing-docs.readthedocs.io/en/latest/HowToUsePyparsing.html#operators)) that causes a failure if a follow-up rule does not match. A common example are matching parentheses: an opening parenthesis requires a closing parenthesis:
```
# PEP 617
group: "(" - expr - ")"
```
This "commit to rule" operator simplifies error detection/reporting.

Since Pika matches spuriously, it cannot directly commit to a rule and raise an error. Instead, reporting must be delayed until the outer rule has been matched.

Comments:

Patch:
diff --git a/bootpeg/grammars/bpeg.bpeg b/bootpeg/grammars/bpeg.bpeg
index db1d421..9996a58 100644
--- a/bootpeg/grammars/bpeg.bpeg
+++ b/bootpeg/grammars/bpeg.bpeg
@@ -11,9 +11,9 @@ end_line:
 literal:
     | ('"' :: '"') | ("'" :: "'") { Literal(.*[1:-1]) }
 range:
-    | first=literal spaces "-" spaces last=literal { Range(.first.value, .last.value) }
+    | first=literal spaces "-" ~ spaces last=literal { Range(.first.value, .last.value) }
 delimited:
-    | start=expr spaces "::" spaces stop=expr { Delimited(.start, .stop) }
+    | start=expr spaces "::" ~ spaces stop=expr { Delimited(.start, .stop) }
 
 terminal:
     | term=(range | nothing | literal | anything) { .term }
@@ -24,26 +24,28 @@ identifier:
     | ( ( "a" - "z" ) | ( "A" - "Z" ) | "_" )+ { .* }
 
 optional:
-    | "[" spaces expr=expr spaces "]" { either(.expr, Nothing()) }
+    | "[" ~ spaces expr=expr spaces "]" { either(.expr, Nothing()) }
 group:
-    | "(" spaces expr=expr spaces ")" { .expr }
+    | "(" ~ spaces expr=expr spaces ")" { .expr }
 choice:
-    | try=expr spaces "|" spaces else=expr { either(.try, .else) }
+    | try=expr spaces "|" ~ spaces else=expr { either(.try, .else) }
 sequence:
     | head=expr spaces tail=expr { chain(.head, .tail) }
 repeat:
     | expr=expr spaces "+" { Repeat(.expr) }
     | expr=expr spaces "*" { either(Repeat(.expr), Nothing()) }
 capture:
-    | name=identifier spaces "=" spaces expr=(reference | group) { Capture(.name, .expr) }
+    | name=identifier spaces "=" ~ spaces expr=(reference | group) { Capture(.name, .expr) }
 reference:
     | name=identifier { Reference(.name) }
 reject:
-    | "!" spaces expr=expr { Not(.expr) }
+    | "!" ~ spaces expr=expr { Not(.expr) }
 require:
-    | "&" spaces expr=expr { And(.expr) }
+    | "&" ~ spaces expr=expr { And(.expr) }
+commit:
+    | "~" ~ spaces expr=expr { require(.expr) }
 expr:
-    | expr=(choice | sequence | repeat | delimited | capture | reference | group | optional | reject | require | terminal) { .expr }
+    | expr=(choice | require | sequence | repeat | delimited | capture | reference | group | optional | reject | commit | terminal) { .expr }
 
 rule:
     | "|" spaces expr=expr spaces action=("{" :: "}") { Rule(.expr, Action(.action[1:-1])) }
@@ -54,7 +56,7 @@ comment:
 blank:
     | spaces end_line { Discard() }
 define:
-    | name=identifier ":" spaces end_line rules=rules { (.name, .rules) }
+    | name=identifier ":" ~ spaces end_line rules=rules { (.name, .rules) }
 rules:
     | " " spaces try=rule spaces end_line else=rules { either(.try, .else) }
     | " " spaces rule=rule spaces end_line { .rule }
diff --git a/bootpeg/pika/act.py b/bootpeg/pika/act.py
index 54c94c8..620c80f 100644
--- a/bootpeg/pika/act.py
+++ b/bootpeg/pika/act.py
@@ -1,7 +1,7 @@
 """
 Pika bottom-up Peg parser extension to transform parsed source
 """
-from typing import TypeVar, Any, Dict, Tuple, Mapping
+from typing import TypeVar, Any, Dict, Tuple, Mapping, Union, NamedTuple
 import re
 
 from .peg import Clause, D, Match, MemoTable, MemoKey, Literal, nested_str, Reference
@@ -49,7 +49,7 @@ class Capture(Clause[D]):
 
     __slots__ = ("name", "sub_clauses", "_hash")
 
-    def __init__(self, name, sub_clause: Clause[D]):
+    def __init__(self, name: str, sub_clause: Clause[D]):
         self.name = name
         self.sub_clauses = (sub_clause,)
         self._hash = None
@@ -192,6 +192,105 @@ class Action:
         return f"{self.__class__.__name__}({self.literal!r})"
 
 
+class Commit(Clause[D]):
+    """
+    The commitment to a clause, requiring a match of the sub_clause
+
+    This clause always matches, even if the sub_clause does not.
+    As a result, it "cuts" away alternatives of *not* matching the sub_clause.
+
+    Since Pika matches spuriously and can accumulate several failures,
+    a ``Cut`` does not raise an error during parsing.
+    Any :py:meth:`~.failed` match should be reported after parsing.
+    """
+
+    __slots__ = ("sub_clauses", "_hash")
+
+    def __init__(self, sub_clause: Clause[D]):
+        self.sub_clauses = (sub_clause,)
+        self._hash = None
+
+    @property
+    def maybe_zero(self):
+        return True
+
+    # The difference between a successful and failed match
+    # is that we do/don't have a parent match.
+    # A match length may be 0 in either case, if the parent
+    # matched 0 length or not at all.
+    def match(self, source: D, at: int, memo: MemoTable):
+        # While `Commit` handles cases in which the child is not matched,
+        # Pika never tries to match `Commit` without the child matching.
+        # An "empty" match is created by the memo as needed.
+        parent_match = memo[MemoKey(at, self.sub_clauses[0])]
+        return Match(parent_match.length, (parent_match,), at, self)
+
+    @classmethod
+    def failed(cls, match: Match) -> bool:
+        """Check whether the given ``match`` only succeeded due to the ``Cut``."""
+        assert isinstance(match.clause, cls), f"a {cls.__name__} match is required"
+        return not match.sub_matches and not match.clause.sub_clauses[0].maybe_zero
+
+    def __eq__(self, other):
+        return isinstance(other, Commit) and self.sub_clauses == other.sub_clauses
+
+    @cache_hash
+    def __hash__(self):
+        return hash(self.sub_clauses)
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self.sub_clauses[0]!r})"
+
+    def __str__(self):
+        return f"~{self.sub_clauses[0]}"
+
+
+class CommitFailure(NamedTuple):
+    """Information on a single failed :py:class:`~.Commit` match"""
+
+    position: int
+    commit: Commit
+
+
+class NamedFailure(NamedTuple):
+    """Information on all failures of a :py:class:`~.Reference` match"""
+
+    name: str
+    failures: Tuple[CommitFailure]
+
+
+def leaves(failure):
+    if isinstance(failure, CommitFailure):
+        yield failure
+    elif isinstance(failure, NamedFailure):
+        for child in failure.failures:
+            yield from leaves(child)
+    else:
+        raise NotImplementedError
+
+
+class CapturedParseFailure(Exception):
+    """Parsing captured match failures"""
+
+    def __init__(self, *failures: Union[NamedFailure, CommitFailure]):
+        self.failures = failures
+        leave_failures = sorted(
+            {leave for failure in failures for leave in leaves(failure)},
+            key=lambda failure: failure.position,
+        )
+        leave_reports = (
+            f"{str(failure.commit.sub_clauses[0])!r}@{failure.position}"
+            for failure in leave_failures
+        )
+        super().__init__(f'failed expected parse of {", ".join(leave_reports)}')
+
+    @property
+    def positions(self):
+        return {
+            leave.position for failure in self.failures for leave in leaves(failure)
+        }
+
+
 class TransformFailure(Exception):
     def __init__(self, clause, matches, captures, exc: Exception):
         super().__init__(f"failed to transform {clause}: {exc}")
@@ -202,25 +301,48 @@ class TransformFailure(Exception):
 
 
 def transform(head: Match, memo: MemoTable, namespace: Mapping[str, Any]):
-    return postorder_transform(head, memo.source, namespace)
+    matches, captures, failures = postorder_transform(head, memo.source, namespace)
+    if not failures:
+        return matches, captures
+    else:
+        raise CapturedParseFailure(*failures)
 
 
 # TODO: Use trampoline/coroutines for infinite depth
 def postorder_transform(
     match: Match, source: D, namespace: Mapping[str, Any]
-) -> Tuple[Any, Dict[str, Any]]:
-    matches, captures = (), {}
+) -> Tuple[Tuple, Dict[str, Any], Tuple]:
+    matches: Tuple = ()
+    captures: Dict[str, Any] = {}
+    failures: Tuple = ()
     for sub_match in match.sub_matches:
-        sub_matches, sub_captures = postorder_transform(sub_match, source, namespace)
+        sub_matches, sub_captures, sub_failures = postorder_transform(
+            sub_match, source, namespace
+        )
         matches += sub_matches
         captures.update(sub_captures)
+        failures += sub_failures
     position, clause = match.position, match.clause
-    if isinstance(clause, Capture):
+    if isinstance(clause, Commit):
+        if Commit.failed(match):
+            failures = (*failures, CommitFailure(position, clause))
+    elif isinstance(clause, Reference):
+        new_failures = tuple(
+            failure for failure in failures if not isinstance(failure, NamedFailure)
+        )
+        if new_failures:
+            failures = (
+                NamedFailure(clause.target, new_failures),
+                *(failure for failure in failures if isinstance(failure, NamedFailure)),
+            )
+    if failures:
+        return (), {}, failures
+    elif isinstance(clause, Capture):
         assert len(matches) <= 1, "Captured rule must provide no more than one value"
         captures[Action.mangle + clause.name] = (
             matches[0] if matches else source[position : position + match.length]
         )
-        return (), captures
+        return (), captures, failures
     elif isinstance(clause, Rule):
         matches = matches if matches else source[position : position + match.length]
         try:
@@ -229,5 +351,5 @@ def postorder_transform(
             raise
         except Exception as exc:
             raise TransformFailure(clause, matches, captures, exc)
-        return (result,) if not isinstance(result, Discard) else (), {}
-    return matches, captures
+        return (result,) if not isinstance(result, Discard) else (), {}, failures
+    return matches, captures, failures
diff --git a/bootpeg/pika/boot.bpeg b/bootpeg/pika/boot.bpeg
index 1696a92..4396b7f 100644
--- a/bootpeg/pika/boot.bpeg
+++ b/bootpeg/pika/boot.bpeg
@@ -38,8 +38,10 @@ reject:
     | "!" spaces expr=expr { Not(.expr) }
 require:
     | "&" spaces expr=expr { And(.expr) }
+commit:
+    | "~" spaces expr=expr { require(.expr) }
 expr:
-    | expr=(choice | sequence | repeat | delimited | capture | reference | group | optional | reject | require | range | literal | anything | nothing) { .expr }
+    | expr=(choice | commit | sequence | repeat | delimited | capture | reference | group | optional | reject | require | range | literal | anything | nothing) { .expr }
 
 rule:
     | "|" spaces expr=expr spaces "{" body=(((!"}") .)+) "}" { Rule(.expr, Action(.body)) }
diff --git a/bootpeg/pika/boot.py b/bootpeg/pika/boot.py
index 2a09b3b..defc556 100644
--- a/bootpeg/pika/boot.py
+++ b/bootpeg/pika/boot.py
@@ -22,9 +22,11 @@ from .front import (
     Rule,
     Action,
     Discard,
+    Commit,
     transform,
     chain,
     either,
+    require,
     Range,
     Delimited,
     unescape,
@@ -126,6 +128,8 @@ namespace = {
         Rule,
         Action,
         Discard,
+        Commit,
+        require,
         Range,
         Delimited,
         unescape,
@@ -326,20 +330,21 @@ def bootpeg() -> Parser:
 
 if __name__ == "__main__":
     parser = bootpeg()
+    print("Generation:", -1, "(min)")
+    display(parser)
     for iteration in range(2):
         with open(boot_path) as boot_peg:
-            print("Generation:", iteration)
-            display(parser)
             parser = range_parse(
                 boot_peg.read(),
                 parser,
             )
+            print("Generation:", iteration, "(boot)")
+            display(parser)
     for iteration in range(2, 4):
         with open(full_path) as base_peg:
-            print("Generation:", iteration)
-            display(min_parser)
             parser = range_parse(
                 base_peg.read(),
                 parser,
             )
-    display(parser)
+            print("Generation:", iteration, "(bpeg)")
+            display(parser)
diff --git a/bootpeg/pika/front.py b/bootpeg/pika/front.py
index b6edefc..e2de01e 100644
--- a/bootpeg/pika/front.py
+++ b/bootpeg/pika/front.py
@@ -19,7 +19,16 @@ from .peg import (
     Terminal,
     D,
 )
-from .act import Debug, Capture, Rule, transform, Action, Discard
+from .act import (
+    Debug,
+    Capture,
+    Rule,
+    transform,
+    Action,
+    Discard,
+    Commit,
+    CapturedParseFailure,
+)
 from ..utility import cache_hash
 
 __all__ = [
@@ -40,6 +49,8 @@ __all__ = [
     "Rule",
     "Action",
     "Discard",
+    "Commit",
+    "CapturedParseFailure",
     "transform",
     # helpers
     "chain",
@@ -50,7 +61,7 @@ __all__ = [
 ]
 
 
-def chain(left, right) -> Sequence:
+def chain(left: Clause[D], right: Clause[D]) -> Sequence[D]:
     """Chain two clauses efficiently"""
     if isinstance(left, Sequence):
         if isinstance(right, Sequence):
@@ -61,7 +72,7 @@ def chain(left, right) -> Sequence:
     return Sequence(left, right)
 
 
-def either(left, right) -> Choice:
+def either(left: Clause[D], right: Clause[D]) -> Choice[D]:
     """Choose between two clauses efficiently"""
     if isinstance(left, Choice):
         if isinstance(right, Choice):
@@ -72,6 +83,19 @@ def either(left, right) -> Choice:
     return Choice(left, right)
 
 
+def require(target: Clause[D]) -> Clause[D]:
+    """Commit to clauses efficiently"""
+    # It is tempting to try and remove `maybe_zero` clauses here.
+    # However, we cannot actually do that: There is no full grammar here, just clauses
+    # – but at least Reference needs the grammar to derive its `maybe_zero`.
+    if isinstance(target, Commit):
+        return target
+    elif isinstance(target, Sequence):
+        return Sequence(*(require(clause) for clause in target.sub_clauses))
+    else:
+        return Commit(target)
+
+
 class Range(Terminal[D]):
     """
     A terminal matching anything between two bounds inclusively
diff --git a/docs/source/grammar_bpeg.rst b/docs/source/grammar_bpeg.rst
index 02eea0f..383a02b 100644
--- a/docs/source/grammar_bpeg.rst
+++ b/docs/source/grammar_bpeg.rst
@@ -77,6 +77,20 @@ any: ``e*``
 
     Match ``e`` zero or several times. Equivalent to ``[ e+ ]``.
 
+commit: ``~ e``
+
+    Match ``e`` or fail. Always succeeds, may be zero width.
+
+    Failure to match ``e`` records the failure but proceeds "as if" ``e`` matched.
+    Useful for accurate failure reports.
+    ::
+
+        # fail on empty and mismatched parentheses
+        '(' ~ expr ')' | expr
+
+    Binds tighter than sequences and less tight than choices:
+    ``~e1 e2 | e3`` is equivalent to ``(~e1 ~e2) | e3``.
+
 capture: ``name=e``
 
     Capture the result of matching ``e`` with a given ``name`` for use in a rule action.


Test patch:
diff --git a/tests/test_grammars/test_bpeg.py b/tests/test_grammars/test_bpeg.py
index ab99117..d2a97bf 100644
--- a/tests/test_grammars/test_bpeg.py
+++ b/tests/test_grammars/test_bpeg.py
@@ -14,6 +14,7 @@ from bootpeg.pika.front import (
     Reference,
     Range,
     Delimited,
+    CapturedParseFailure,
 )
 
 
@@ -43,3 +44,16 @@ def test_roundtrip(clause):
         "parse_test"
     ]
     assert parsed_rule.sub_clauses[0] == clause
+
+
+commit_failures = (("top:\n | ~\n", {9}),)
+
+
+@pytest.mark.parametrize("source, positions", commit_failures)
+def test_commit(source, positions):
+    try:
+        bpeg.parse(source)
+    except CapturedParseFailure as cpf:
+        assert cpf.positions == positions
+    else:
+        assert not positions, "Expected parse failures, found none"