Repository: sqlfluff/sqlfluff

Issue:
`AnySetOf` grammar
<!--Note: This is for general enhancements to the project. Please use the Bug report template instead to raise parsing/linting/syntax issues for existing supported dialects-->
I know this has been talked about before in PRs so making an issue to formally track.

In many grammars there's a common situation where we have to denote several options that can be specified in any order but they cannot be specified more than once.

Our general approach to this in the project has been denote this using `AnyNumberOf` as this allows for the different orderings:
```python
AnyNumberOf(
    <option_1_grammar>,
    <option_2_grammar>,
    ...
)
```
However, the issue with this is that it places no limit on how many times each option can be specified.

This means that sqlfluff allows certain invalid statements to parse e.g.
```sql
CREATE TABLE ktw_account_binding (
    ktw_id VARCHAR(32) NOT NULL REFERENCES ref_table(bla)
    ON DELETE RESTRICT ON DELETE CASCADE ON DELETE CASCADE ON DELETE CASCADE
);
```
https://github.com/sqlfluff/sqlfluff/pull/2315#issuecomment-1013847846

We've accepted this limitation for the time being as it's more important to get the statements parsing for linting/formatting purposes rather than exactly reflecting the grammar (we'd expect a general degree of common sense when it comes to repeating these options).

That being said it would be nice to address this to refine our grammar and reduce dealing with contributor confusion.

`AnySetOf` would essentially allow all of it's grammar arguments to be parsed in any order a maximum of 1 time each. Hopefully we can inherit from `AnyNumberOf` to simplify this.


Discussion:

Patch:
diff --git a/src/sqlfluff/core/parser/__init__.py b/src/sqlfluff/core/parser/__init__.py
--- a/src/sqlfluff/core/parser/__init__.py
+++ b/src/sqlfluff/core/parser/__init__.py
@@ -23,6 +23,7 @@
     Delimited,
     Bracketed,
     AnyNumberOf,
+    AnySetOf,
     Ref,
     Anything,
     Nothing,
@@ -56,6 +57,7 @@
     "Delimited",
     "Bracketed",
     "AnyNumberOf",
+    "AnySetOf",
     "Ref",
     "Anything",
     "Nothing",
diff --git a/src/sqlfluff/core/parser/grammar/__init__.py b/src/sqlfluff/core/parser/grammar/__init__.py
--- a/src/sqlfluff/core/parser/grammar/__init__.py
+++ b/src/sqlfluff/core/parser/grammar/__init__.py
@@ -1,7 +1,12 @@
 """Definitions of grammars."""
 
 from sqlfluff.core.parser.grammar.base import Ref, Anything, Nothing
-from sqlfluff.core.parser.grammar.anyof import AnyNumberOf, OneOf, OptionallyBracketed
+from sqlfluff.core.parser.grammar.anyof import (
+    AnyNumberOf,
+    AnySetOf,
+    OneOf,
+    OptionallyBracketed,
+)
 from sqlfluff.core.parser.grammar.delimited import Delimited
 from sqlfluff.core.parser.grammar.greedy import GreedyUntil, StartsWith
 from sqlfluff.core.parser.grammar.sequence import Sequence, Bracketed
@@ -12,6 +17,7 @@
     "Anything",
     "Nothing",
     "AnyNumberOf",
+    "AnySetOf",
     "OneOf",
     "OptionallyBracketed",
     "Delimited",
diff --git a/src/sqlfluff/core/parser/grammar/anyof.py b/src/sqlfluff/core/parser/grammar/anyof.py
--- a/src/sqlfluff/core/parser/grammar/anyof.py
+++ b/src/sqlfluff/core/parser/grammar/anyof.py
@@ -22,6 +22,7 @@ class AnyNumberOf(BaseGrammar):
     def __init__(self, *args, **kwargs):
         self.max_times = kwargs.pop("max_times", None)
         self.min_times = kwargs.pop("min_times", 0)
+        self.max_times_per_element = kwargs.pop("max_times_per_element", None)
         # Any patterns to _prevent_ a match.
         self.exclude = kwargs.pop("exclude", None)
         super().__init__(*args, **kwargs)
@@ -121,7 +122,7 @@ def _prune_options(
 
     def _match_once(
         self, segments: Tuple[BaseSegment, ...], parse_context: ParseContext
-    ) -> MatchResult:
+    ) -> Tuple[MatchResult, Optional["MatchableType"]]:
         """Match the forward segments against the available elements once.
 
         This serves as the main body of OneOf, but also a building block
@@ -141,14 +142,14 @@ def _match_once(
             return MatchResult.from_unmatched(segments)
 
         with parse_context.deeper_match() as ctx:
-            match, _ = self._longest_trimmed_match(
+            match, matched_option = self._longest_trimmed_match(
                 segments,
                 available_options,
                 parse_context=ctx,
                 trim_noncode=False,
             )
 
-        return match
+        return match, matched_option
 
     @match_wrapper()
     @allow_ephemeral
@@ -171,6 +172,13 @@ def match(
         matched_segments: MatchResult = MatchResult.from_empty()
         unmatched_segments: Tuple[BaseSegment, ...] = segments
         n_matches = 0
+
+        # Keep track of the number of times each option has been matched.
+        available_options, _ = self._prune_options(
+            segments, parse_context=parse_context
+        )
+        available_option_counter = {str(o): 0 for o in available_options}
+
         while True:
             if self.max_times and n_matches >= self.max_times:
                 # We've matched as many times as we can
@@ -197,7 +205,23 @@ def match(
             else:
                 pre_seg = ()  # empty tuple
 
-            match = self._match_once(unmatched_segments, parse_context=parse_context)
+            match, matched_option = self._match_once(
+                unmatched_segments, parse_context=parse_context
+            )
+
+            # Increment counter for matched option.
+            if matched_option and (str(matched_option) in available_option_counter):
+                available_option_counter[str(matched_option)] += 1
+                # Check if we have matched an option too many times.
+                if (
+                    self.max_times_per_element
+                    and available_option_counter[str(matched_option)]
+                    > self.max_times_per_element
+                ):
+                    return MatchResult(
+                        matched_segments.matched_segments, unmatched_segments
+                    )
+
             if match:
                 matched_segments += pre_seg + match.matched_segments
                 unmatched_segments = match.unmatched_segments
@@ -240,3 +264,10 @@ def __init__(self, *args, **kwargs):
             args[0] if len(args) == 1 else Sequence(*args),
             **kwargs,
         )
+
+
+class AnySetOf(AnyNumberOf):
+    """Match any number of the elements but each element can only be matched once."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, max_times_per_element=1, **kwargs)
diff --git a/src/sqlfluff/dialects/dialect_ansi.py b/src/sqlfluff/dialects/dialect_ansi.py
--- a/src/sqlfluff/dialects/dialect_ansi.py
+++ b/src/sqlfluff/dialects/dialect_ansi.py
@@ -19,6 +19,7 @@
 from sqlfluff.core.dialects.common import AliasInfo, ColumnAliasInfo
 from sqlfluff.core.parser import (
     AnyNumberOf,
+    AnySetOf,
     Anything,
     BaseFileSegment,
     BaseSegment,
@@ -515,7 +516,7 @@
             ),
             optional=True,
         ),
-        AnyNumberOf(
+        AnySetOf(
             # ON DELETE clause, e.g. ON DELETE NO ACTION
             Sequence(
                 "ON",


Test patch:
diff --git a/test/core/parser/grammar_test.py b/test/core/parser/grammar_test.py
--- a/test/core/parser/grammar_test.py
+++ b/test/core/parser/grammar_test.py
@@ -12,6 +12,7 @@
     Indent,
 )
 from sqlfluff.core.parser.context import RootParseContext
+from sqlfluff.core.parser.grammar.anyof import AnySetOf
 from sqlfluff.core.parser.segments import EphemeralSegment, BaseSegment
 from sqlfluff.core.parser.grammar.base import BaseGrammar
 from sqlfluff.core.parser.grammar.noncode import NonCodeMatcher
@@ -678,3 +679,22 @@ def test__parser__grammar_noncode(seg_list, fresh_ansi_dialect):
         m = NonCodeMatcher().match(seg_list[1:], parse_context=ctx)
     # We should match one and only one segment
     assert len(m) == 1
+
+
+def test__parser__grammar_anysetof(generate_test_segments):
+    """Test the AnySetOf grammar."""
+    token_list = ["bar", "  \t ", "foo", "  \t ", "bar"]
+    seg_list = generate_test_segments(token_list)
+
+    bs = StringParser("bar", KeywordSegment)
+    fs = StringParser("foo", KeywordSegment)
+    g = AnySetOf(fs, bs)
+    with RootParseContext(dialect=None) as ctx:
+        # Check directly
+        assert g.match(seg_list, parse_context=ctx).matched_segments == (
+            KeywordSegment("bar", seg_list[0].pos_marker),
+            WhitespaceSegment("  \t ", seg_list[1].pos_marker),
+            KeywordSegment("foo", seg_list[2].pos_marker),
+        )
+        # Check with a bit of whitespace
+        assert not g.match(seg_list[1:], parse_context=ctx)