Repository: sqlfluff/sqlfluff

Issue:
[EXPERIMENT]: Rethink Matching routines
This is another experiment, and also a biggie. It's a rethink of matching as part of #5124.

This will need some tidying to get it into a state that it's reviewable, but given the scale of it - I think I shouldn't take it much further without getting some of it merged.

It's mostly additions for now, so I now need to strip out the things that we can get rid of as a result. Opening PR for testing and in particular for coverage.


Discussion:
This is far too big, I'll divide it up into pieces before merging. I think other things need to come first.

Patch:
diff --git a/src/sqlfluff/core/parser/grammar/base.py b/src/sqlfluff/core/parser/grammar/base.py
--- a/src/sqlfluff/core/parser/grammar/base.py
+++ b/src/sqlfluff/core/parser/grammar/base.py
@@ -1,7 +1,6 @@
 """Base grammar, Ref, Anything and Nothing."""
 
 import copy
-from dataclasses import dataclass
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -14,21 +13,16 @@
     Tuple,
     TypeVar,
     Union,
-    cast,
 )
 from uuid import UUID, uuid4
 
-from sqlfluff.core.errors import SQLParseError
 from sqlfluff.core.parser.context import ParseContext
 from sqlfluff.core.parser.helpers import trim_non_code_segments
-from sqlfluff.core.parser.match_logging import (
-    LateBoundJoinSegmentsCurtailed,
-    parse_match_logging,
-)
+from sqlfluff.core.parser.match_logging import parse_match_logging
 from sqlfluff.core.parser.match_result import MatchResult
 from sqlfluff.core.parser.match_wrapper import match_wrapper
 from sqlfluff.core.parser.matchable import Matchable
-from sqlfluff.core.parser.segments import BaseSegment, BracketedSegment, allow_ephemeral
+from sqlfluff.core.parser.segments import BaseSegment, allow_ephemeral
 from sqlfluff.core.parser.types import MatchableType, SimpleHintType
 from sqlfluff.core.string_helpers import curtail_string
 
@@ -36,49 +30,6 @@
     from sqlfluff.core.dialects.base import Dialect
 
 
-def first_trimmed_raw(seg: BaseSegment) -> str:
-    """Trim whitespace off a whole element raw.
-
-    Used as a helper function in BaseGrammar._look_ahead_match.
-
-    For existing compound segments, we should assume that within
-    that segment, things are internally consistent, that means
-    rather than enumerating all the individual segments of a longer
-    one we just dump out the whole segment, but splitting off the
-    first element separated by whitespace. This is a) faster and
-    also b) prevents some really horrible bugs with bracket matching.
-    See https://github.com/sqlfluff/sqlfluff/issues/433
-
-    This fetches the _whole_ raw of a potentially compound segment
-    to match against, trimming off any whitespace. This is the
-    most efficient way to get at the first element of a potentially
-    longer segment.
-    """
-    s = seg.raw_upper.split(maxsplit=1)
-    return s[0] if s else ""
-
-
-@dataclass
-class BracketInfo:
-    """BracketInfo tuple for keeping track of brackets during matching.
-
-    This is used in BaseGrammar._bracket_sensitive_look_ahead_match but
-    defined here for type checking.
-    """
-
-    bracket: BaseSegment
-    segments: Tuple[BaseSegment, ...]
-
-    def to_segment(self, end_bracket: Tuple[BaseSegment, ...]) -> BracketedSegment:
-        """Turn the contained segments into a bracketed segment."""
-        assert len(end_bracket) == 1
-        return BracketedSegment(
-            segments=self.segments,
-            start_bracket=(self.bracket,),
-            end_bracket=cast(Tuple[BaseSegment], end_bracket),
-        )
-
-
 def cached_method_for_parse_context(
     func: Callable[[Any, ParseContext, Optional[Tuple[str]]], SimpleHintType]
 ) -> Callable[..., SimpleHintType]:
@@ -520,355 +471,6 @@ def _longest_trimmed_match(
         # If no match at all, return nothing
         return MatchResult.from_unmatched(segments), None
 
-    @classmethod
-    def _look_ahead_match(
-        cls,
-        segments: Tuple[BaseSegment, ...],
-        matchers: List[MatchableType],
-        parse_context: ParseContext,
-    ) -> Tuple[Tuple[BaseSegment, ...], MatchResult, Optional[MatchableType]]:
-        """Look ahead for matches beyond the first element of the segments list.
-
-        This function also contains the performance improved hash-matching approach to
-        searching for matches, which should significantly improve performance.
-
-        Prioritise the first match, and if multiple match at the same point the longest.
-        If two matches of the same length match at the same time, then it's the first in
-        the iterable of matchers.
-
-        Returns:
-            `tuple` of (unmatched_segments, match_object, matcher).
-
-        """
-        parse_match_logging(
-            cls.__name__,
-            "_look_ahead_match",
-            "IN",
-            parse_context=parse_context,
-            v_level=4,
-            ls=len(segments),
-            seg=LateBoundJoinSegmentsCurtailed(segments),
-        )
-
-        # Have we been passed an empty tuple?
-        if not segments:  # pragma: no cover TODO?
-            return ((), MatchResult.from_empty(), None)
-
-        # Here we enable a performance optimisation. Most of the time in this cycle
-        # happens in loops looking for simple matchers which we should
-        # be able to find a shortcut for.
-
-        parse_match_logging(
-            cls.__name__,
-            "_look_ahead_match",
-            "SI",
-            parse_context=parse_context,
-            v_level=4,
-        )
-
-        best_simple_match = None
-        simple_match = None
-        for idx, seg in enumerate(segments):
-            trimmed_seg = first_trimmed_raw(seg)
-            for matcher in matchers:
-                simple_match = None
-                simple = matcher.simple(parse_context=parse_context)
-                if not simple:  # pragma: no cover
-                    # NOTE: For all bundled dialects, this clause is true, but until
-                    # the RegexMatcher is completely deprecated (and therefore that
-                    # `.simple()` must provide a result), it is still _possible_
-                    # to end up here.
-                    raise NotImplementedError(
-                        "All matchers passed to `._look_ahead_match()` are "
-                        "assumed to have a functioning `.simple()` option. "
-                        "In a future release it will be compulsory for _all_ "
-                        "matchables to implement `.simple()`. Please report "
-                        "this as a bug on GitHub along with your current query "
-                        f"and dialect.\nProblematic matcher: {matcher}"
-                    )
-                simple_raws, simple_types = simple
-
-                assert simple_raws or simple_types
-                if simple_raws:
-                    if trimmed_seg in simple_raws:
-                        simple_match = matcher
-
-                if simple_types and not simple_match:
-                    intersection = simple_types.intersection(seg.class_types)
-                    if intersection:
-                        simple_match = matcher
-
-                # If we couldn't achieve a simple match, move on to the next option.
-                if not simple_match:
-                    continue
-
-                # If there is, check the full version matches. If it doesn't
-                # then discount it and move on.
-                match = simple_match.match(segments[idx:], parse_context)
-                if not match:
-                    continue
-
-                best_simple_match = (
-                    segments[:idx],
-                    match,
-                    simple_match,
-                )
-                # Stop looking through matchers
-                break
-
-            # If we have a valid match, stop looking through segments
-            if best_simple_match:
-                break
-
-        # There are no other matchers, we can just shortcut now. Either with
-        # no match, or the best one we found (if we found one).
-        parse_match_logging(
-            cls.__name__,
-            "_look_ahead_match",
-            "SC",
-            parse_context=parse_context,
-            v_level=4,
-            bsm=None
-            if not best_simple_match
-            else (
-                len(best_simple_match[0]),
-                len(best_simple_match[1]),
-                best_simple_match[2],
-            ),
-        )
-
-        if best_simple_match:
-            return best_simple_match
-        else:
-            return ((), MatchResult.from_unmatched(segments), None)
-
-    @classmethod
-    def _bracket_sensitive_look_ahead_match(
-        cls,
-        segments: Tuple[BaseSegment, ...],
-        matchers: List[MatchableType],
-        parse_context: ParseContext,
-        start_bracket: Optional[MatchableType] = None,
-        end_bracket: Optional[MatchableType] = None,
-        bracket_pairs_set: str = "bracket_pairs",
-    ) -> Tuple[Tuple[BaseSegment, ...], MatchResult, Optional[MatchableType]]:
-        """Same as `_look_ahead_match` but with bracket counting.
-
-        NB: Given we depend on `_look_ahead_match` we can also utilise
-        the same performance optimisations which are implemented there.
-
-        bracket_pairs_set: Allows specific segments to override the available
-            bracket pairs. See the definition of "angle_bracket_pairs" in the
-            BigQuery dialect for additional context on why this exists.
-
-        Returns:
-            `tuple` of (unmatched_segments, match_object, matcher).
-
-        """
-        # Have we been passed an empty tuple?
-        if not segments:
-            return ((), MatchResult.from_unmatched(segments), None)
-
-        # Get hold of the bracket matchers from the dialect, and append them
-        # to the list of matchers. We get them from the relevant set on the
-        # dialect. We use zip twice to "unzip" them. We ignore the first
-        # argument because that's just the name.
-        _, start_bracket_refs, end_bracket_refs, persists = zip(
-            *parse_context.dialect.bracket_sets(bracket_pairs_set)
-        )
-        # These are matchables, probably StringParsers.
-        start_brackets = [
-            parse_context.dialect.ref(seg_ref) for seg_ref in start_bracket_refs
-        ]
-        end_brackets = [
-            parse_context.dialect.ref(seg_ref) for seg_ref in end_bracket_refs
-        ]
-        # Add any bracket-like things passed as arguments
-        if start_bracket:
-            start_brackets += [start_bracket]
-        if end_bracket:
-            end_brackets += [end_bracket]
-        bracket_matchers = start_brackets + end_brackets
-
-        # Make some buffers
-        seg_buff: Tuple[BaseSegment, ...] = segments
-        pre_seg_buff: Tuple[BaseSegment, ...] = ()
-        bracket_stack: List[BracketInfo] = []
-
-        # Iterate
-        while True:
-            # Do we have anything left to match on?
-            if seg_buff:
-                # Yes we have buffer left to work with.
-                # Are we already in a bracket stack?
-                if bracket_stack:
-                    # Yes, we're just looking for the closing bracket, or
-                    # another opening bracket.
-                    pre, match, matcher = cls._look_ahead_match(
-                        seg_buff,
-                        bracket_matchers,
-                        parse_context=parse_context,
-                    )
-
-                    if match:
-                        # NB: We can only consider this as a nested bracket if the start
-                        # and end tokens are not the same. If a matcher is both a start
-                        # and end token we cannot deepen the bracket stack. In general,
-                        # quoted strings are a typical example where the start and end
-                        # tokens are the same. Currently, though, quoted strings are
-                        # handled elsewhere in the parser, and there are no cases where
-                        # *this* code has to handle identical start and end brackets.
-                        # For now, consider this a small, speculative investment in a
-                        # possible future requirement.
-                        if matcher in start_brackets and matcher not in end_brackets:
-                            # Add any segments leading up to this to the previous
-                            # bracket.
-                            bracket_stack[-1].segments += pre
-                            # Add a bracket to the stack and add the matches from the
-                            # segment.
-                            bracket_stack.append(
-                                BracketInfo(
-                                    bracket=match.matched_segments[0],
-                                    segments=match.matched_segments,
-                                )
-                            )
-                            seg_buff = match.unmatched_segments
-                            continue
-                        elif matcher in end_brackets:
-                            # Found an end bracket. Does its type match that of
-                            # the innermost start bracket? E.g. ")" matches "(",
-                            # "]" matches "[".
-                            # For the start bracket we don't have the matcher
-                            # but we can work out the type, so we use that for
-                            # the lookup.
-                            start_index = [
-                                bracket.type for bracket in start_brackets
-                            ].index(bracket_stack[-1].bracket.get_type())
-                            # For the end index, we can just look for the matcher
-                            end_index = end_brackets.index(matcher)
-                            bracket_types_match = start_index == end_index
-                            if bracket_types_match:
-                                # Yes, the types match. So we've found a
-                                # matching end bracket. Pop the stack, construct
-                                # a bracketed segment and carry
-                                # on.
-
-                                # Complete the bracketed info
-                                bracket_stack[-1].segments += (
-                                    pre + match.matched_segments
-                                )
-                                # Construct a bracketed segment (as a tuple) if allowed.
-                                persist_bracket = persists[end_brackets.index(matcher)]
-                                if persist_bracket:
-                                    new_segments: Tuple[BaseSegment, ...] = (
-                                        bracket_stack[-1].to_segment(
-                                            end_bracket=match.matched_segments
-                                        ),
-                                    )
-                                else:
-                                    new_segments = bracket_stack[-1].segments
-                                # Remove the bracket set from the stack
-                                bracket_stack.pop()
-                                # If we're still in a bracket, add the new segments to
-                                # that bracket, otherwise add them to the buffer
-                                if bracket_stack:
-                                    bracket_stack[-1].segments += new_segments
-                                else:
-                                    pre_seg_buff += new_segments
-                                seg_buff = match.unmatched_segments
-                                continue
-                            else:
-                                # The types don't match. Error.
-                                raise SQLParseError(
-                                    f"Found unexpected end bracket!, "
-                                    f"was expecting "
-                                    f"{end_brackets[start_index]}, "
-                                    f"but got {matcher}",
-                                    segment=match.matched_segments[0],
-                                )
-
-                        else:  # pragma: no cover
-                            raise RuntimeError("I don't know how we get here?!")
-                    else:  # pragma: no cover
-                        # No match, we're in a bracket stack. Error.
-                        raise SQLParseError(
-                            "Couldn't find closing bracket for opening bracket.",
-                            segment=bracket_stack[-1].bracket,
-                        )
-                else:
-                    # No, we're open to more opening brackets or the thing(s)
-                    # that we're otherwise looking for.
-                    pre, match, matcher = cls._look_ahead_match(
-                        seg_buff,
-                        matchers + bracket_matchers,
-                        parse_context=parse_context,
-                    )
-
-                    if match:
-                        if matcher in matchers:
-                            # It's one of the things we were looking for!
-                            # Return.
-                            return (pre_seg_buff + pre, match, matcher)
-                        elif matcher in start_brackets:
-                            # We've found the start of a bracket segment.
-                            # NB: It might not *actually* be the bracket itself,
-                            # but could be some non-code element preceding it.
-                            # That's actually ok.
-
-                            # Add the bracket to the stack.
-                            bracket_stack.append(
-                                BracketInfo(
-                                    bracket=match.matched_segments[0],
-                                    segments=match.matched_segments,
-                                )
-                            )
-                            # The matched element has already been added to the bracket.
-                            # Add anything before it to the pre segment buffer.
-                            # Reset the working buffer.
-                            pre_seg_buff += pre
-                            seg_buff = match.unmatched_segments
-                            continue
-                        elif matcher in end_brackets:
-                            # We've found an unexpected end bracket! This is likely
-                            # because we're matching a section which should have ended.
-                            # If we had a match, it would have matched by now, so this
-                            # means no match.
-                            parse_match_logging(
-                                cls.__name__,
-                                "_bracket_sensitive_look_ahead_match",
-                                "UEXB",
-                                parse_context=parse_context,
-                                v_level=3,
-                                got=matcher,
-                            )
-                            # From here we'll drop out to the happy unmatched exit.
-                        else:  # pragma: no cover
-                            # This shouldn't happen!?
-                            raise NotImplementedError(
-                                "This shouldn't happen. Panic in "
-                                "_bracket_sensitive_look_ahead_match."
-                            )
-                    # Not in a bracket stack, but no match.
-                    # From here we'll drop out to the happy unmatched exit.
-            else:
-                # No we're at the end:
-                # Now check have we closed all our brackets?
-                if bracket_stack:  # pragma: no cover
-                    # No we haven't.
-                    raise SQLParseError(
-                        "Couldn't find closing bracket for opened brackets: "
-                        f"`{bracket_stack}`.",
-                        segment=bracket_stack[-1].bracket,
-                    )
-
-            # This is the happy unmatched path. This occurs when:
-            # - We reached the end with no open brackets.
-            # - No match while outside a bracket stack.
-            # - We found an unexpected end bracket before matching something
-            # interesting. We return with the mutated segments so we can reuse any
-            # bracket matching.
-            return ((), MatchResult.from_unmatched(pre_seg_buff + seg_buff), None)
-
     def __str__(self) -> str:  # pragma: no cover TODO?
         return repr(self)
 
@@ -1052,7 +654,7 @@ def _get_elem(self, dialect: "Dialect") -> MatchableType:
 
     def __repr__(self) -> str:
         return "<Ref: {}{}>".format(
-            str(self._ref), " [opt]" if self.is_optional() else ""
+            repr(self._ref), " [opt]" if self.is_optional() else ""
         )
 
     @match_wrapper(v_level=4)  # Log less for Ref
diff --git a/src/sqlfluff/core/parser/grammar/greedy.py b/src/sqlfluff/core/parser/grammar/greedy.py
--- a/src/sqlfluff/core/parser/grammar/greedy.py
+++ b/src/sqlfluff/core/parser/grammar/greedy.py
@@ -8,7 +8,7 @@
     BaseSegment,
     cached_method_for_parse_context,
 )
-from sqlfluff.core.parser.helpers import trim_non_code_segments
+from sqlfluff.core.parser.match_algorithms import greedy_match
 from sqlfluff.core.parser.match_result import MatchResult
 from sqlfluff.core.parser.match_wrapper import match_wrapper
 from sqlfluff.core.parser.segments import allow_ephemeral
@@ -50,112 +50,13 @@ def match(
         self, segments: Tuple[BaseSegment, ...], parse_context: ParseContext
     ) -> MatchResult:
         """Matching for GreedyUntil works just how you'd expect."""
-        return self.greedy_match(
+        return greedy_match(
             segments,
             parse_context,
             matchers=self._elements,
             include_terminator=False,
         )
 
-    @classmethod
-    def greedy_match(
-        cls,
-        segments: Tuple[BaseSegment, ...],
-        parse_context: ParseContext,
-        matchers: Sequence[MatchableType],
-        include_terminator: bool = False,
-    ) -> MatchResult:
-        """Matching for GreedyUntil works just how you'd expect."""
-        seg_buff = segments
-        seg_bank: Tuple[BaseSegment, ...] = ()  # Empty tuple
-
-        while True:
-            with parse_context.deeper_match(name="GreedyUntil") as ctx:
-                pre, mat, matcher = cls._bracket_sensitive_look_ahead_match(
-                    seg_buff, list(matchers), parse_context=ctx
-                )
-
-            if not mat:
-                # No terminator match? Return everything
-                return MatchResult.from_matched(segments)
-
-            # NOTE: For some terminators we only count them if they're preceded
-            # by whitespace, and others we don't. In principle, we aim that for
-            # _keywords_ we require whitespace, and for symbols we don't.
-            # We do this by looking at the `simple` method of the returned
-            # matcher, and if it's entirely alphabetical (as defined by
-            # str.isalpha()) then we infer that it's a keyword, and therefore
-            # _does_ require whitespace before it.
-            assert matcher, f"Match without matcher: {mat}"
-            _simple = matcher.simple(parse_context)
-            assert _simple, f"Terminators require a simple method: {matcher}"
-            _strings, _types = _simple
-            # NOTE: Typed matchers aren't common here, but we assume that they
-            # _don't_ require preceding whitespace.
-            # Do we need to enforce whitespace preceding?
-            if all(_s.isalpha() for _s in _strings) and not _types:
-                # Does the match include some whitespace already?
-                # Work forward
-                idx = 0
-                while True:
-                    elem = mat.matched_segments[idx]
-                    if elem.is_meta:  # pragma: no cover TODO?
-                        idx += 1
-                        continue
-                    elif elem.is_type(
-                        "whitespace", "newline"
-                    ):  # pragma: no cover TODO?
-                        allowable_match = True
-                        break
-                    else:
-                        # No whitespace before. Not allowed.
-                        allowable_match = False
-                        break
-
-                # If we're not ok yet, work backward to the preceding sections.
-                if not allowable_match:
-                    idx = -1
-                    while True:
-                        if len(pre) < abs(idx):  # pragma: no cover TODO?
-                            # If we're at the start, it's ok
-                            allowable_match = True
-                            break
-                        if pre[idx].is_meta:  # pragma: no cover TODO?
-                            idx -= 1
-                            continue
-                        elif pre[idx].is_type("whitespace", "newline"):
-                            allowable_match = True
-                            break
-                        else:
-                            # No whitespace before. Not allowed.
-                            allowable_match = False
-                            break
-
-                # If this match isn't preceded by whitespace and that is
-                # a requirement, then we can't use it. Carry on...
-                if not allowable_match:
-                    # Update our buffers and continue onward
-                    seg_bank = seg_bank + pre + mat.matched_segments
-                    seg_buff = mat.unmatched_segments
-                    # Loop around, don't return yet
-                    continue
-
-            # Return everything up to the match unless it's a gap matcher.
-            if include_terminator:
-                return MatchResult(
-                    seg_bank + pre + mat.matched_segments,
-                    mat.unmatched_segments,
-                )
-
-            # We can't claim any non-code segments, so we trim them off the end.
-            leading_nc, pre_seg_mid, trailing_nc = trim_non_code_segments(
-                seg_bank + pre
-            )
-            return MatchResult(
-                leading_nc + pre_seg_mid,
-                trailing_nc + mat.all_segments(),
-            )
-
 
 T = TypeVar("T", bound="StartsWith")
 
@@ -232,7 +133,7 @@ def match(
         # with, then we can still used the unmatched parts on the end.
         # We still need to deal with any non-code segments at the start.
         assert self.terminators
-        greedy_match = self.greedy_match(
+        greedy_matched = greedy_match(
             match.unmatched_segments,
             parse_context,
             # We match up to the terminators for this segment, but _also_
@@ -243,12 +144,12 @@ def match(
 
         # NB: If all we matched in the greedy match was non-code then we can't
         # claim it.
-        if not any(seg.is_code for seg in greedy_match.matched_segments):
+        if not any(seg.is_code for seg in greedy_matched.matched_segments):
             # So just return the original match.
             return match
 
         # Otherwise Combine the results.
         return MatchResult(
-            match.matched_segments + greedy_match.matched_segments,
-            greedy_match.unmatched_segments,
+            match.matched_segments + greedy_matched.matched_segments,
+            greedy_matched.unmatched_segments,
         )
diff --git a/src/sqlfluff/core/parser/grammar/sequence.py b/src/sqlfluff/core/parser/grammar/sequence.py
--- a/src/sqlfluff/core/parser/grammar/sequence.py
+++ b/src/sqlfluff/core/parser/grammar/sequence.py
@@ -15,6 +15,7 @@
 )
 from sqlfluff.core.parser.grammar.conditional import Conditional
 from sqlfluff.core.parser.helpers import check_still_complete, trim_non_code_segments
+from sqlfluff.core.parser.match_algorithms import bracket_sensitive_look_ahead_match
 from sqlfluff.core.parser.match_result import MatchResult
 from sqlfluff.core.parser.match_wrapper import match_wrapper
 from sqlfluff.core.parser.matchable import Matchable
@@ -386,7 +387,7 @@ def match(
             with parse_context.deeper_match(
                 name="Bracketed-End", clear_terminators=True
             ) as ctx:
-                content_segs, end_match, _ = self._bracket_sensitive_look_ahead_match(
+                content_segs, end_match, _ = bracket_sensitive_look_ahead_match(
                     segments=seg_buff,
                     matchers=[end_bracket],
                     parse_context=ctx,
diff --git a/src/sqlfluff/core/parser/match_algorithms.py b/src/sqlfluff/core/parser/match_algorithms.py
new file mode 100644
--- /dev/null
+++ b/src/sqlfluff/core/parser/match_algorithms.py
@@ -0,0 +1,452 @@
+"""Matching algorithms.
+
+These are mostly extracted from the body of either BaseSegment
+or BaseGrammar to un-bloat those classes.
+"""
+
+from dataclasses import dataclass
+from typing import List, Optional, Sequence, Tuple, cast
+
+from sqlfluff.core.errors import SQLParseError
+from sqlfluff.core.parser.context import ParseContext
+from sqlfluff.core.parser.helpers import trim_non_code_segments
+from sqlfluff.core.parser.match_result import MatchResult
+from sqlfluff.core.parser.segments import BaseSegment, BracketedSegment
+from sqlfluff.core.parser.types import MatchableType
+
+
+def first_trimmed_raw(seg: BaseSegment) -> str:
+    """Trim whitespace off a whole element raw.
+
+    Used as a helper function in BaseGrammar._look_ahead_match.
+
+    For existing compound segments, we should assume that within
+    that segment, things are internally consistent, that means
+    rather than enumerating all the individual segments of a longer
+    one we just dump out the whole segment, but splitting off the
+    first element separated by whitespace. This is a) faster and
+    also b) prevents some really horrible bugs with bracket matching.
+    See https://github.com/sqlfluff/sqlfluff/issues/433
+
+    This fetches the _whole_ raw of a potentially compound segment
+    to match against, trimming off any whitespace. This is the
+    most efficient way to get at the first element of a potentially
+    longer segment.
+    """
+    s = seg.raw_upper.split(maxsplit=1)
+    return s[0] if s else ""
+
+
+@dataclass
+class BracketInfo:
+    """BracketInfo tuple for keeping track of brackets during matching.
+
+    This is used in BaseGrammar._bracket_sensitive_look_ahead_match but
+    defined here for type checking.
+    """
+
+    bracket: BaseSegment
+    segments: Tuple[BaseSegment, ...]
+
+    def to_segment(self, end_bracket: Tuple[BaseSegment, ...]) -> BracketedSegment:
+        """Turn the contained segments into a bracketed segment."""
+        assert len(end_bracket) == 1
+        return BracketedSegment(
+            segments=self.segments,
+            start_bracket=(self.bracket,),
+            end_bracket=cast(Tuple[BaseSegment], end_bracket),
+        )
+
+
+def look_ahead_match(
+    segments: Tuple[BaseSegment, ...],
+    matchers: List[MatchableType],
+    parse_context: ParseContext,
+) -> Tuple[Tuple[BaseSegment, ...], MatchResult, Optional[MatchableType]]:
+    """Look ahead for matches beyond the first element of the segments list.
+
+    This function also contains the performance improved hash-matching approach to
+    searching for matches, which should significantly improve performance.
+
+    Prioritise the first match, and if multiple match at the same point the longest.
+    If two matches of the same length match at the same time, then it's the first in
+    the iterable of matchers.
+
+    Returns:
+        `tuple` of (unmatched_segments, match_object, matcher).
+
+    """
+    # Have we been passed an empty tuple?
+    if not segments:  # pragma: no cover TODO?
+        return ((), MatchResult.from_empty(), None)
+
+    # Here we enable a performance optimisation. Most of the time in this cycle
+    # happens in loops looking for simple matchers which we should
+    # be able to find a shortcut for.
+    best_simple_match = None
+    simple_match = None
+    for idx, seg in enumerate(segments):
+        trimmed_seg = first_trimmed_raw(seg)
+        for matcher in matchers:
+            simple_match = None
+            simple = matcher.simple(parse_context=parse_context)
+            if not simple:  # pragma: no cover
+                # NOTE: For all bundled dialects, this clause is true, but until
+                # the RegexMatcher is completely deprecated (and therefore that
+                # `.simple()` must provide a result), it is still _possible_
+                # to end up here.
+                raise NotImplementedError(
+                    "All matchers passed to `.look_ahead_match()` are "
+                    "assumed to have a functioning `.simple()` option. "
+                    "In a future release it will be compulsory for _all_ "
+                    "matchables to implement `.simple()`. Please report "
+                    "this as a bug on GitHub along with your current query "
+                    f"and dialect.\nProblematic matcher: {matcher}"
+                )
+            simple_raws, simple_types = simple
+
+            assert simple_raws or simple_types
+            if simple_raws:
+                if trimmed_seg in simple_raws:
+                    simple_match = matcher
+
+            if simple_types and not simple_match:
+                intersection = simple_types.intersection(seg.class_types)
+                if intersection:
+                    simple_match = matcher
+
+            # If we couldn't achieve a simple match, move on to the next option.
+            if not simple_match:
+                continue
+
+            # If there is, check the full version matches. If it doesn't
+            # then discount it and move on.
+            match = simple_match.match(segments[idx:], parse_context)
+            if not match:
+                continue
+
+            best_simple_match = (
+                segments[:idx],
+                match,
+                simple_match,
+            )
+            # Stop looking through matchers
+            break
+
+        # If we have a valid match, stop looking through segments
+        if best_simple_match:
+            break
+
+    if best_simple_match:
+        return best_simple_match
+    else:
+        return ((), MatchResult.from_unmatched(segments), None)
+
+
+def bracket_sensitive_look_ahead_match(
+    segments: Tuple[BaseSegment, ...],
+    matchers: List[MatchableType],
+    parse_context: ParseContext,
+    start_bracket: Optional[MatchableType] = None,
+    end_bracket: Optional[MatchableType] = None,
+    bracket_pairs_set: str = "bracket_pairs",
+) -> Tuple[Tuple[BaseSegment, ...], MatchResult, Optional[MatchableType]]:
+    """Same as `look_ahead_match` but with bracket counting.
+
+    NB: Given we depend on `look_ahead_match` we can also utilise
+    the same performance optimisations which are implemented there.
+
+    bracket_pairs_set: Allows specific segments to override the available
+        bracket pairs. See the definition of "angle_bracket_pairs" in the
+        BigQuery dialect for additional context on why this exists.
+
+    Returns:
+        `tuple` of (unmatched_segments, match_object, matcher).
+
+    """
+    # Have we been passed an empty tuple?
+    if not segments:
+        return ((), MatchResult.from_unmatched(segments), None)
+
+    # Get hold of the bracket matchers from the dialect, and append them
+    # to the list of matchers. We get them from the relevant set on the
+    # dialect. We use zip twice to "unzip" them. We ignore the first
+    # argument because that's just the name.
+    _, start_bracket_refs, end_bracket_refs, persists = zip(
+        *parse_context.dialect.bracket_sets(bracket_pairs_set)
+    )
+    # These are matchables, probably StringParsers.
+    start_brackets = [
+        parse_context.dialect.ref(seg_ref) for seg_ref in start_bracket_refs
+    ]
+    end_brackets = [parse_context.dialect.ref(seg_ref) for seg_ref in end_bracket_refs]
+    # Add any bracket-like things passed as arguments
+    if start_bracket:
+        start_brackets += [start_bracket]
+    if end_bracket:
+        end_brackets += [end_bracket]
+    bracket_matchers = start_brackets + end_brackets
+
+    # Make some buffers
+    seg_buff: Tuple[BaseSegment, ...] = segments
+    pre_seg_buff: Tuple[BaseSegment, ...] = ()
+    bracket_stack: List[BracketInfo] = []
+
+    # Iterate
+    while True:
+        # Do we have anything left to match on?
+        if seg_buff:
+            # Yes we have buffer left to work with.
+            # Are we already in a bracket stack?
+            if bracket_stack:
+                # Yes, we're just looking for the closing bracket, or
+                # another opening bracket.
+                pre, match, matcher = look_ahead_match(
+                    seg_buff,
+                    bracket_matchers,
+                    parse_context=parse_context,
+                )
+
+                if match:
+                    # NB: We can only consider this as a nested bracket if the start
+                    # and end tokens are not the same. If a matcher is both a start
+                    # and end token we cannot deepen the bracket stack. In general,
+                    # quoted strings are a typical example where the start and end
+                    # tokens are the same. Currently, though, quoted strings are
+                    # handled elsewhere in the parser, and there are no cases where
+                    # *this* code has to handle identical start and end brackets.
+                    # For now, consider this a small, speculative investment in a
+                    # possible future requirement.
+                    if matcher in start_brackets and matcher not in end_brackets:
+                        # Add any segments leading up to this to the previous
+                        # bracket.
+                        bracket_stack[-1].segments += pre
+                        # Add a bracket to the stack and add the matches from the
+                        # segment.
+                        bracket_stack.append(
+                            BracketInfo(
+                                bracket=match.matched_segments[0],
+                                segments=match.matched_segments,
+                            )
+                        )
+                        seg_buff = match.unmatched_segments
+                        continue
+                    elif matcher in end_brackets:
+                        # Found an end bracket. Does its type match that of
+                        # the innermost start bracket? E.g. ")" matches "(",
+                        # "]" matches "[".
+                        # For the start bracket we don't have the matcher
+                        # but we can work out the type, so we use that for
+                        # the lookup.
+                        start_index = [
+                            bracket.type for bracket in start_brackets
+                        ].index(bracket_stack[-1].bracket.get_type())
+                        # For the end index, we can just look for the matcher
+                        end_index = end_brackets.index(matcher)
+                        bracket_types_match = start_index == end_index
+                        if bracket_types_match:
+                            # Yes, the types match. So we've found a
+                            # matching end bracket. Pop the stack, construct
+                            # a bracketed segment and carry
+                            # on.
+
+                            # Complete the bracketed info
+                            bracket_stack[-1].segments += pre + match.matched_segments
+                            # Construct a bracketed segment (as a tuple) if allowed.
+                            persist_bracket = persists[end_brackets.index(matcher)]
+                            if persist_bracket:
+                                new_segments: Tuple[BaseSegment, ...] = (
+                                    bracket_stack[-1].to_segment(
+                                        end_bracket=match.matched_segments
+                                    ),
+                                )
+                            else:
+                                new_segments = bracket_stack[-1].segments
+                            # Remove the bracket set from the stack
+                            bracket_stack.pop()
+                            # If we're still in a bracket, add the new segments to
+                            # that bracket, otherwise add them to the buffer
+                            if bracket_stack:
+                                bracket_stack[-1].segments += new_segments
+                            else:
+                                pre_seg_buff += new_segments
+                            seg_buff = match.unmatched_segments
+                            continue
+                        else:
+                            # The types don't match. Error.
+                            raise SQLParseError(
+                                f"Found unexpected end bracket!, "
+                                f"was expecting "
+                                f"{end_brackets[start_index]}, "
+                                f"but got {matcher}",
+                                segment=match.matched_segments[0],
+                            )
+
+                    else:  # pragma: no cover
+                        raise RuntimeError("I don't know how we get here?!")
+                else:  # pragma: no cover
+                    # No match, we're in a bracket stack. Error.
+                    raise SQLParseError(
+                        "Couldn't find closing bracket for opening bracket.",
+                        segment=bracket_stack[-1].bracket,
+                    )
+            else:
+                # No, we're open to more opening brackets or the thing(s)
+                # that we're otherwise looking for.
+                pre, match, matcher = look_ahead_match(
+                    seg_buff,
+                    matchers + bracket_matchers,
+                    parse_context=parse_context,
+                )
+
+                if match:
+                    if matcher in matchers:
+                        # It's one of the things we were looking for!
+                        # Return.
+                        return (pre_seg_buff + pre, match, matcher)
+                    elif matcher in start_brackets:
+                        # We've found the start of a bracket segment.
+                        # NB: It might not *actually* be the bracket itself,
+                        # but could be some non-code element preceding it.
+                        # That's actually ok.
+
+                        # Add the bracket to the stack.
+                        bracket_stack.append(
+                            BracketInfo(
+                                bracket=match.matched_segments[0],
+                                segments=match.matched_segments,
+                            )
+                        )
+                        # The matched element has already been added to the bracket.
+                        # Add anything before it to the pre segment buffer.
+                        # Reset the working buffer.
+                        pre_seg_buff += pre
+                        seg_buff = match.unmatched_segments
+                        continue
+                    elif matcher in end_brackets:
+                        # We've found an unexpected end bracket! This is likely
+                        # because we're matching a section which should have ended.
+                        # If we had a match, it would have matched by now, so this
+                        # means no match.
+                        pass
+                        # From here we'll drop out to the happy unmatched exit.
+                    else:  # pragma: no cover
+                        # This shouldn't happen!?
+                        raise NotImplementedError(
+                            "This shouldn't happen. Panic in "
+                            "_bracket_sensitive_look_ahead_match."
+                        )
+                # Not in a bracket stack, but no match.
+                # From here we'll drop out to the happy unmatched exit.
+        else:
+            # No we're at the end:
+            # Now check have we closed all our brackets?
+            if bracket_stack:  # pragma: no cover
+                # No we haven't.
+                raise SQLParseError(
+                    "Couldn't find closing bracket for opened brackets: "
+                    f"`{bracket_stack}`.",
+                    segment=bracket_stack[-1].bracket,
+                )
+
+        # This is the happy unmatched path. This occurs when:
+        # - We reached the end with no open brackets.
+        # - No match while outside a bracket stack.
+        # - We found an unexpected end bracket before matching something
+        # interesting. We return with the mutated segments so we can reuse any
+        # bracket matching.
+        return ((), MatchResult.from_unmatched(pre_seg_buff + seg_buff), None)
+
+
+def greedy_match(
+    segments: Tuple[BaseSegment, ...],
+    parse_context: ParseContext,
+    matchers: Sequence[MatchableType],
+    include_terminator: bool = False,
+) -> MatchResult:
+    """Looks ahead to claim everything up to some future terminators."""
+    seg_buff = segments
+    seg_bank: Tuple[BaseSegment, ...] = ()  # Empty tuple
+
+    while True:
+        with parse_context.deeper_match(name="GreedyUntil") as ctx:
+            pre, mat, matcher = bracket_sensitive_look_ahead_match(
+                seg_buff, list(matchers), parse_context=ctx
+            )
+
+        if not mat:
+            # No terminator match? Return everything
+            return MatchResult.from_matched(segments)
+
+        # NOTE: For some terminators we only count them if they're preceded
+        # by whitespace, and others we don't. In principle, we aim that for
+        # _keywords_ we require whitespace, and for symbols we don't.
+        # We do this by looking at the `simple` method of the returned
+        # matcher, and if it's entirely alphabetical (as defined by
+        # str.isalpha()) then we infer that it's a keyword, and therefore
+        # _does_ require whitespace before it.
+        assert matcher, f"Match without matcher: {mat}"
+        _simple = matcher.simple(parse_context)
+        assert _simple, f"Terminators require a simple method: {matcher}"
+        _strings, _types = _simple
+        # NOTE: Typed matchers aren't common here, but we assume that they
+        # _don't_ require preceding whitespace.
+        # Do we need to enforce whitespace preceding?
+        if all(_s.isalpha() for _s in _strings) and not _types:
+            # Does the match include some whitespace already?
+            # Work forward
+            idx = 0
+            while True:
+                elem = mat.matched_segments[idx]
+                if elem.is_meta:  # pragma: no cover TODO?
+                    idx += 1
+                    continue
+                elif elem.is_type("whitespace", "newline"):  # pragma: no cover TODO?
+                    allowable_match = True
+                    break
+                else:
+                    # No whitespace before. Not allowed.
+                    allowable_match = False
+                    break
+
+            # If we're not ok yet, work backward to the preceding sections.
+            if not allowable_match:
+                idx = -1
+                while True:
+                    if len(pre) < abs(idx):  # pragma: no cover TODO?
+                        # If we're at the start, it's ok
+                        allowable_match = True
+                        break
+                    if pre[idx].is_meta:  # pragma: no cover TODO?
+                        idx -= 1
+                        continue
+                    elif pre[idx].is_type("whitespace", "newline"):
+                        allowable_match = True
+                        break
+                    else:
+                        # No whitespace before. Not allowed.
+                        allowable_match = False
+                        break
+
+            # If this match isn't preceded by whitespace and that is
+            # a requirement, then we can't use it. Carry on...
+            if not allowable_match:
+                # Update our buffers and continue onward
+                seg_bank = seg_bank + pre + mat.matched_segments
+                seg_buff = mat.unmatched_segments
+                # Loop around, don't return yet
+                continue
+
+        # Return everything up to the match unless it's a gap matcher.
+        if include_terminator:
+            return MatchResult(
+                seg_bank + pre + mat.matched_segments,
+                mat.unmatched_segments,
+            )
+
+        # We can't claim any non-code segments, so we trim them off the end.
+        leading_nc, pre_seg_mid, trailing_nc = trim_non_code_segments(seg_bank + pre)
+        return MatchResult(
+            leading_nc + pre_seg_mid,
+            trailing_nc + mat.all_segments(),
+        )
diff --git a/src/sqlfluff/core/parser/match_logging.py b/src/sqlfluff/core/parser/match_logging.py
--- a/src/sqlfluff/core/parser/match_logging.py
+++ b/src/sqlfluff/core/parser/match_logging.py
@@ -1,12 +1,9 @@
 """Classes to help with match logging."""
 
 import logging
-from typing import TYPE_CHECKING, Any, Tuple
-
-from sqlfluff.core.parser.helpers import join_segments_raw_curtailed
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:  # pragma: no cover
-    from sqlfluff.core.parser import BaseSegment
     from sqlfluff.core.parser.context import ParseContext
 
 
@@ -100,17 +97,3 @@ def parse_match_logging(
     ParseMatchLogObject(
         parse_context, grammar, func, msg, v_level=v_level, **kwargs
     ).log()
-
-
-class LateBoundJoinSegmentsCurtailed:
-    """Object to delay `join_segments_raw_curtailed` until later.
-
-    This allows us to defer the string manipulation involved
-    until actually required by the logger.
-    """
-
-    def __init__(self, segments: Tuple["BaseSegment", ...]) -> None:
-        self.segments = segments
-
-    def __str__(self) -> str:
-        return repr(join_segments_raw_curtailed(self.segments))


Test patch:
diff --git a/test/core/parser/grammar/grammar_base_test.py b/test/core/parser/grammar/grammar_base_test.py
--- a/test/core/parser/grammar/grammar_base_test.py
+++ b/test/core/parser/grammar/grammar_base_test.py
@@ -3,18 +3,15 @@
 NOTE: All of these tests depend somewhat on the KeywordSegment working as planned.
 """
 
-import pytest
 import logging
 
-from sqlfluff.core.parser import (
-    KeywordSegment,
-    StringParser,
-)
+import pytest
+
+from sqlfluff.core.parser import KeywordSegment, StringParser
 from sqlfluff.core.parser.context import ParseContext
-from sqlfluff.core.parser.segments import EphemeralSegment
-from sqlfluff.core.parser.grammar.base import BaseGrammar
 from sqlfluff.core.parser.grammar import OneOf, Sequence
-from sqlfluff.core.errors import SQLParseError
+from sqlfluff.core.parser.grammar.base import BaseGrammar
+from sqlfluff.core.parser.segments import EphemeralSegment
 
 # NB: All of these tests depend somewhat on the KeywordSegment working as planned
 
@@ -91,61 +88,6 @@ def test__parser__grammar__base__longest_trimmed_match__adv(seg_list, caplog):
     assert len(match) == 3
 
 
-@pytest.mark.parametrize(
-    "seg_list_slice,matcher_keywords,result_slice,winning_matcher,pre_match_slice",
-    [
-        # Basic version, we should find bar first
-        (slice(None, None), ["bar", "foo"], slice(None, 1), "bar", None),
-        # Look ahead for foo
-        (slice(None, None), ["foo"], slice(2, 3), "foo", slice(None, 2)),
-    ],
-)
-def test__parser__grammar__base__look_ahead_match(
-    seg_list_slice,
-    matcher_keywords,
-    result_slice,
-    winning_matcher,
-    pre_match_slice,
-    seg_list,
-):
-    """Test the _look_ahead_match method of the BaseGrammar."""
-    # Make the matcher keywords
-    matchers = [StringParser(keyword, KeywordSegment) for keyword in matcher_keywords]
-    # Fetch the matching keyword from above by index
-    winning_matcher = matchers[matcher_keywords.index(winning_matcher)]
-
-    ctx = ParseContext(dialect=None)
-    m = BaseGrammar._look_ahead_match(
-        seg_list[seg_list_slice],
-        matchers,
-        ctx,
-    )
-
-    # Check structure of the response.
-    assert isinstance(m, tuple)
-    assert len(m) == 3
-    # Unpack
-    result_pre_match, result_match, result_matcher = m
-
-    # Check the right matcher won
-    assert result_matcher == winning_matcher
-
-    # Make check tuple for the pre-match section
-    if pre_match_slice:
-        pre_match_slice = seg_list[pre_match_slice]
-    else:
-        pre_match_slice = ()
-    assert result_pre_match == pre_match_slice
-
-    # Make the check tuple
-    expected_result = make_result_tuple(
-        result_slice=result_slice,
-        matcher_keywords=matcher_keywords,
-        seg_list=seg_list,
-    )
-    assert result_match.matched_segments == expected_result
-
-
 def test__parser__grammar__base__ephemeral_segment(seg_list):
     """Test the ephemeral features on BaseGrammar.
 
@@ -165,110 +107,3 @@ def test__parser__grammar__base__ephemeral_segment(seg_list):
     chkpoint = m.matched_segments[0]
     # Check it's got the same content.
     assert chkpoint.segments == seg_list
-
-
-def test__parser__grammar__base__bracket_sensitive_look_ahead_match(
-    bracket_seg_list, fresh_ansi_dialect
-):
-    """Test the _bracket_sensitive_look_ahead_match method of the BaseGrammar."""
-    bs = StringParser("bar", KeywordSegment)
-    fs = StringParser("foo", KeywordSegment)
-    # We need a dialect here to do bracket matching
-    ctx = ParseContext(dialect=fresh_ansi_dialect)
-    # Basic version, we should find bar first
-    pre_section, match, matcher = BaseGrammar._bracket_sensitive_look_ahead_match(
-        bracket_seg_list, [fs, bs], ctx
-    )
-    assert pre_section == ()
-    assert matcher == bs
-    # NB the middle element is a match object
-    assert match.matched_segments == (
-        KeywordSegment("bar", bracket_seg_list[0].pos_marker),
-    )
-
-    # Look ahead for foo, we should find the one AFTER the brackets, not the
-    # on IN the brackets.
-    pre_section, match, matcher = BaseGrammar._bracket_sensitive_look_ahead_match(
-        bracket_seg_list, [fs], ctx
-    )
-    # NB: The bracket segments will have been mutated, so we can't directly compare.
-    # Make sure we've got a bracketed section in there.
-    assert len(pre_section) == 5
-    assert pre_section[2].is_type("bracketed")
-    assert len(pre_section[2].segments) == 4
-    assert matcher == fs
-    # We shouldn't match the whitespace with the keyword
-    assert match.matched_segments == (
-        KeywordSegment("foo", bracket_seg_list[8].pos_marker),
-    )
-    # Check that the unmatched segments are nothing.
-    assert not match.unmatched_segments
-
-
-def test__parser__grammar__base__bracket_fail_with_open_paren_close_square_mismatch(
-    generate_test_segments, fresh_ansi_dialect
-):
-    """Test _bracket_sensitive_look_ahead_match failure case.
-
-    Should fail when the type of a close bracket doesn't match the type of the
-    corresponding open bracket, but both are "definite" brackets.
-    """
-    fs = StringParser("foo", KeywordSegment)
-    # We need a dialect here to do bracket matching
-    ctx = ParseContext(dialect=fresh_ansi_dialect)
-    # Basic version, we should find bar first
-    with pytest.raises(SQLParseError) as sql_parse_error:
-        BaseGrammar._bracket_sensitive_look_ahead_match(
-            generate_test_segments(
-                [
-                    "select",
-                    " ",
-                    "*",
-                    " ",
-                    "from",
-                    "(",
-                    "foo",
-                    "]",  # Bracket types don't match (parens vs square)
-                ]
-            ),
-            [fs],
-            ctx,
-        )
-    assert sql_parse_error.match("Found unexpected end bracket")
-
-
-def test__parser__grammar__base__bracket_fail_with_unexpected_end_bracket(
-    generate_test_segments, fresh_ansi_dialect
-):
-    """Test _bracket_sensitive_look_ahead_match edge case.
-
-    Should fail gracefully and stop matching if we find a trailing unmatched.
-    """
-    fs = StringParser("foo", KeywordSegment)
-    # We need a dialect here to do bracket matching
-    ctx = ParseContext(dialect=fresh_ansi_dialect)
-    _, match, _ = BaseGrammar._bracket_sensitive_look_ahead_match(
-        generate_test_segments(
-            [
-                "bar",
-                "(",  # This bracket pair should be mutated
-                ")",
-                " ",
-                ")",  # This is the unmatched bracket
-                " ",
-                "foo",
-            ]
-        ),
-        [fs],
-        ctx,
-    )
-    # Check we don't match (even though there's a foo at the end)
-    assert not match
-    # Check the first bracket pair have been mutated.
-    segs = match.unmatched_segments
-    assert segs[1].is_type("bracketed")
-    assert segs[1].raw == "()"
-    assert len(segs[1].segments) == 2
-    # Check the trailing foo hasn't been mutated
-    assert segs[5].raw == "foo"
-    assert not isinstance(segs[5], KeywordSegment)
diff --git a/test/core/parser/grammar/grammar_ref_test.py b/test/core/parser/grammar/grammar_ref_test.py
--- a/test/core/parser/grammar/grammar_ref_test.py
+++ b/test/core/parser/grammar/grammar_ref_test.py
@@ -23,6 +23,12 @@ def test__parser__grammar__ref_eq():
     assert r1 not in check_list
 
 
+def test__parser__grammar__ref_repr():
+    """Test the __repr__ method of Ref."""
+    assert repr(Ref("foo")) == "<Ref: 'foo'>"
+    assert repr(Ref("bar", optional=True)) == "<Ref: 'bar' [opt]>"
+
+
 def test__parser__grammar_ref_exclude(generate_test_segments, fresh_ansi_dialect):
     """Test the Ref grammar exclude option."""
     ni = Ref("NakedIdentifierSegment", exclude=Ref.keyword("ABS"))
diff --git a/test/core/parser/match_algorithms_test.py b/test/core/parser/match_algorithms_test.py
new file mode 100644
--- /dev/null
+++ b/test/core/parser/match_algorithms_test.py
@@ -0,0 +1,192 @@
+"""Tests for the BaseGrammar and it's methods.
+
+NOTE: All of these tests depend somewhat on the KeywordSegment working as planned.
+"""
+
+import pytest
+
+from sqlfluff.core.errors import SQLParseError
+from sqlfluff.core.parser import KeywordSegment, StringParser
+from sqlfluff.core.parser.context import ParseContext
+from sqlfluff.core.parser.match_algorithms import (
+    bracket_sensitive_look_ahead_match,
+    look_ahead_match,
+)
+
+# NB: All of these tests depend somewhat on the KeywordSegment working as planned
+
+
+def make_result_tuple(result_slice, matcher_keywords, seg_list):
+    """Make a comparison tuple for test matching."""
+    # No result slice means no match.
+    if not result_slice:
+        return ()
+
+    return tuple(
+        KeywordSegment(elem.raw, pos_marker=elem.pos_marker)
+        if elem.raw in matcher_keywords
+        else elem
+        for elem in seg_list[result_slice]
+    )
+
+
+@pytest.mark.parametrize(
+    "seg_list_slice,matcher_keywords,result_slice,winning_matcher,pre_match_slice",
+    [
+        # Basic version, we should find bar first
+        (slice(None, None), ["bar", "foo"], slice(None, 1), "bar", None),
+        # Look ahead for foo
+        (slice(None, None), ["foo"], slice(2, 3), "foo", slice(None, 2)),
+    ],
+)
+def test__parser__algorithms__look_ahead_match(
+    seg_list_slice,
+    matcher_keywords,
+    result_slice,
+    winning_matcher,
+    pre_match_slice,
+    seg_list,
+):
+    """Test the look_ahead_match method of the BaseGrammar."""
+    # Make the matcher keywords
+    matchers = [StringParser(keyword, KeywordSegment) for keyword in matcher_keywords]
+    # Fetch the matching keyword from above by index
+    winning_matcher = matchers[matcher_keywords.index(winning_matcher)]
+
+    ctx = ParseContext(dialect=None)
+    m = look_ahead_match(
+        seg_list[seg_list_slice],
+        matchers,
+        ctx,
+    )
+
+    # Check structure of the response.
+    assert isinstance(m, tuple)
+    assert len(m) == 3
+    # Unpack
+    result_pre_match, result_match, result_matcher = m
+
+    # Check the right matcher won
+    assert result_matcher == winning_matcher
+
+    # Make check tuple for the pre-match section
+    if pre_match_slice:
+        pre_match_slice = seg_list[pre_match_slice]
+    else:
+        pre_match_slice = ()
+    assert result_pre_match == pre_match_slice
+
+    # Make the check tuple
+    expected_result = make_result_tuple(
+        result_slice=result_slice,
+        matcher_keywords=matcher_keywords,
+        seg_list=seg_list,
+    )
+    assert result_match.matched_segments == expected_result
+
+
+def test__parser__algorithms__bracket_sensitive_look_ahead_match(
+    bracket_seg_list, fresh_ansi_dialect
+):
+    """Test the bracket_sensitive_look_ahead_match method of the BaseGrammar."""
+    bs = StringParser("bar", KeywordSegment)
+    fs = StringParser("foo", KeywordSegment)
+    # We need a dialect here to do bracket matching
+    ctx = ParseContext(dialect=fresh_ansi_dialect)
+    # Basic version, we should find bar first
+    pre_section, match, matcher = bracket_sensitive_look_ahead_match(
+        bracket_seg_list, [fs, bs], ctx
+    )
+    assert pre_section == ()
+    assert matcher == bs
+    # NB the middle element is a match object
+    assert match.matched_segments == (
+        KeywordSegment("bar", bracket_seg_list[0].pos_marker),
+    )
+
+    # Look ahead for foo, we should find the one AFTER the brackets, not the
+    # on IN the brackets.
+    pre_section, match, matcher = bracket_sensitive_look_ahead_match(
+        bracket_seg_list, [fs], ctx
+    )
+    # NB: The bracket segments will have been mutated, so we can't directly compare.
+    # Make sure we've got a bracketed section in there.
+    assert len(pre_section) == 5
+    assert pre_section[2].is_type("bracketed")
+    assert len(pre_section[2].segments) == 4
+    assert matcher == fs
+    # We shouldn't match the whitespace with the keyword
+    assert match.matched_segments == (
+        KeywordSegment("foo", bracket_seg_list[8].pos_marker),
+    )
+    # Check that the unmatched segments are nothing.
+    assert not match.unmatched_segments
+
+
+def test__parser__algorithms__bracket_fail_with_open_paren_close_square_mismatch(
+    generate_test_segments, fresh_ansi_dialect
+):
+    """Test bracket_sensitive_look_ahead_match failure case.
+
+    Should fail when the type of a close bracket doesn't match the type of the
+    corresponding open bracket, but both are "definite" brackets.
+    """
+    fs = StringParser("foo", KeywordSegment)
+    # We need a dialect here to do bracket matching
+    ctx = ParseContext(dialect=fresh_ansi_dialect)
+    # Basic version, we should find bar first
+    with pytest.raises(SQLParseError) as sql_parse_error:
+        bracket_sensitive_look_ahead_match(
+            generate_test_segments(
+                [
+                    "select",
+                    " ",
+                    "*",
+                    " ",
+                    "from",
+                    "(",
+                    "foo",
+                    "]",  # Bracket types don't match (parens vs square)
+                ]
+            ),
+            [fs],
+            ctx,
+        )
+    assert sql_parse_error.match("Found unexpected end bracket")
+
+
+def test__parser__algorithms__bracket_fail_with_unexpected_end_bracket(
+    generate_test_segments, fresh_ansi_dialect
+):
+    """Test bracket_sensitive_look_ahead_match edge case.
+
+    Should fail gracefully and stop matching if we find a trailing unmatched.
+    """
+    fs = StringParser("foo", KeywordSegment)
+    # We need a dialect here to do bracket matching
+    ctx = ParseContext(dialect=fresh_ansi_dialect)
+    _, match, _ = bracket_sensitive_look_ahead_match(
+        generate_test_segments(
+            [
+                "bar",
+                "(",  # This bracket pair should be mutated
+                ")",
+                " ",
+                ")",  # This is the unmatched bracket
+                " ",
+                "foo",
+            ]
+        ),
+        [fs],
+        ctx,
+    )
+    # Check we don't match (even though there's a foo at the end)
+    assert not match
+    # Check the first bracket pair have been mutated.
+    segs = match.unmatched_segments
+    assert segs[1].is_type("bracketed")
+    assert segs[1].raw == "()"
+    assert len(segs[1].segments) == 2
+    # Check the trailing foo hasn't been mutated
+    assert segs[5].raw == "foo"
+    assert not isinstance(segs[5], KeywordSegment)