Repository: sqlfluff/sqlfluff

Issue:
L022 fails on cte with leading comma
Using the latest version from `master`

.sqlfluff
```
[sqlfluff:rules]
max_line_length = 115

[sqlfluff:indentation]
indented_joins = True

[sqlfluff:rules:L019]
comma_style = leading
```

test.sql
```sql
with cte_1 as (
    select
        *
    from
        table_1
)

, cte_2 as (
    select
        *
    from
        table_2
)

select * from table_3
```

```
$ sqlfluff lint test.sql
== [test.sql] FAIL
L:   8 | P:   3 | L022 | Blank line expected but not found after CTE definition.
```

If I remove the leading comma config, and make the comma trailing, then it passes


Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 20390e176..459b14070 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,10 +16,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Created new Github Organisation (https://github.com/sqlfluff) and
   migrated from https://github.com/alanmcruickshank/sqlfluff to
   https://github.com/sqlfluff/sqlfluff.
-- Changed the handling of `*` and `a.b.*` expressions to have their
-  own expressions. Any dependencies on this structure downstream
-  will be broken. This also fixes the linting of both kinds of expressions
-  with regard to L013 and L025.
 
 ## [0.3.6] - 2020-09-24
 
diff --git a/src/sqlfluff/default_config.cfg b/src/sqlfluff/default_config.cfg
index 7276e0b6f..989110f7e 100644
--- a/src/sqlfluff/default_config.cfg
+++ b/src/sqlfluff/default_config.cfg
@@ -26,10 +26,8 @@ dbt_var = {% macro var(variable) %}item{% endmacro %}
 tab_space_size = 4
 max_line_length = 80
 indent_unit = space
+comma_style = trailing
 
 # Some rules have their own specific config.
 [sqlfluff:rules:L010]
 capitalisation_policy = consistent
-
-[sqlfluff:rules:L019]
-comma_style = trailing
diff --git a/src/sqlfluff/dialects/dialect_ansi.py b/src/sqlfluff/dialects/dialect_ansi.py
index 4db5fa8dd..86cf0d268 100644
--- a/src/sqlfluff/dialects/dialect_ansi.py
+++ b/src/sqlfluff/dialects/dialect_ansi.py
@@ -93,7 +93,7 @@ ansi_dialect.add(
     EndSquareBracketSegment=KeywordSegment.make(']', name='end_square_bracket', type='end_square_bracket'),
     CommaSegment=KeywordSegment.make(',', name='comma', type='comma'),
     DotSegment=KeywordSegment.make('.', name='dot', type='dot'),
-    StarSegment=KeywordSegment.make('*', name='star', type='star'),
+    StarSegment=KeywordSegment.make('*', name='star'),
     TildeSegment=KeywordSegment.make('~', name='tilde'),
     CastOperatorSegment=KeywordSegment.make('::', name='casting_operator', type='casting_operator'),
     PlusSegment=KeywordSegment.make('+', name='plus', type='binary_operator'),
@@ -246,13 +246,6 @@ class ObjectReferenceSegment(BaseSegment):
         code_only=False
     )
 
-    @staticmethod
-    def _iter_reference_parts(elem):
-        """Extract the elements of a reference and yield."""
-        # trim on quotes and split out any dots.
-        for part in elem.raw_trimmed().split('.'):
-            yield part, elem
-
     def iter_raw_references(self):
         """Generate a list of reference strings and elements.
 
@@ -262,29 +255,14 @@ class ObjectReferenceSegment(BaseSegment):
         """
         # Extract the references from those identifiers (because some may be quoted)
         for elem in self.recursive_crawl('identifier'):
-            yield from self._iter_reference_parts(elem)
+            # trim on quotes and split out any dots.
+            for part in elem.raw_trimmed().split('.'):
+                yield part, elem
 
     def is_qualified(self):
         """Return if there is more than one element to the reference."""
         return len(list(self.iter_raw_references())) > 1
 
-    def qualification(self):
-        """Return the qualification type of this reference."""
-        return 'qualified' if self.is_qualified() else 'unqualified'
-
-    def extract_reference(self, level):
-        """Extract a reference of a given level.
-
-        e.g. level 1 = the object.
-        level 2 = the table
-        level 3 = the schema
-        etc...
-        """
-        refs = list(self.iter_raw_references())
-        if len(refs) >= level:
-            return refs[-level]
-        return None
-
 
 @ansi_dialect.segment()
 class ArrayAccessorSegment(BaseSegment):
@@ -548,15 +526,9 @@ class TableExpressionSegment(BaseSegment):
         return None
 
 
-@ansi_dialect.segment()
-class WildcardIdentifierSegment(ObjectReferenceSegment):
-    """Any identifier of the form a.b.*.
-
-    This inherits iter_raw_references from the
-    ObjectReferenceSegment.
-    """
-    type = "wildcard_identifier"
-    match_grammar = Sequence(
+ansi_dialect.add(
+    # This is a hook point to allow subclassing for other dialects
+    WildcardSelectTargetElementGrammar=Sequence(
         # *, blah.*, blah.blah.*, etc.
         AnyNumberOf(
             Sequence(
@@ -566,33 +538,8 @@ class WildcardIdentifierSegment(ObjectReferenceSegment):
             )
         ),
         Ref('StarSegment'), code_only=False
-    )
-
-    def iter_raw_references(self):
-        """Generate a list of reference strings and elements.
-
-        Each element is a tuple of (str, segment). If some are
-        split, then a segment may appear twice, but the substring
-        will only appear once.
-        """
-        # Extract the references from those identifiers (because some may be quoted)
-        for elem in self.recursive_crawl(('identifier', 'star')):
-            yield from self._iter_reference_parts(elem)
-
-
-@ansi_dialect.segment()
-class WildcardExpressionSegment(BaseSegment):
-    """A star (*) expression for a SELECT clause.
-
-    This is seperate from the identifier to allow for
-    some dialects which extend this logic to allow
-    REPLACE, EXCEPT or similar clauses e.g. BigQuery.
-    """
-    type = "wildcard_expression"
-    match_grammar = Sequence(
-        # *, blah.*, blah.blah.*, etc.
-        Ref('WildcardIdentifierSegment')
-    )
+    ),
+)
 
 
 @ansi_dialect.segment()
@@ -602,7 +549,7 @@ class SelectTargetElementSegment(BaseSegment):
     # Important to split elements before parsing, otherwise debugging is really hard.
     match_grammar = OneOf(
         # *, blah.*, blah.blah.*, etc.
-        Ref('WildcardExpressionSegment'),
+        Ref('WildcardSelectTargetElementGrammar'),
         GreedyUntil(
             'FROM', 'LIMIT',
             Ref('CommaSegment'),
@@ -613,7 +560,7 @@ class SelectTargetElementSegment(BaseSegment):
 
     parse_grammar = OneOf(
         # *, blah.*, blah.blah.*, etc.
-        Ref('WildcardExpressionSegment'),
+        Ref('WildcardSelectTargetElementGrammar'),
         Sequence(
             OneOf(
                 Ref('LiteralGrammar'),
@@ -645,7 +592,7 @@ class SelectClauseSegment(BaseSegment):
     match_grammar = StartsWith(
         Sequence(
             'SELECT',
-            Ref('WildcardExpressionSegment', optional=True)
+            Ref('WildcardSelectTargetElementGrammar', optional=True)
         ),
         terminator=OneOf('FROM', 'LIMIT', Ref('SetOperatorSegment')),
         enforce_whitespace_preceeding_terminator=True
diff --git a/src/sqlfluff/dialects/dialect_bigquery.py b/src/sqlfluff/dialects/dialect_bigquery.py
index 4538d370c..efe28499e 100644
--- a/src/sqlfluff/dialects/dialect_bigquery.py
+++ b/src/sqlfluff/dialects/dialect_bigquery.py
@@ -6,7 +6,7 @@ and
 https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#string_and_bytes_literals
 """
 
-from ..parser import (BaseSegment, NamedSegment, OneOf, Ref, Sequence, Bracketed, Delimited)
+from ..parser import (BaseSegment, NamedSegment, OneOf, Ref, Sequence, Bracketed, Delimited, AnyNumberOf)
 
 from .dialect_ansi import ansi_dialect
 
@@ -80,22 +80,25 @@ bigquery_dialect.replace(
             Ref('SingleIdentifierGrammar'),
             optional=True
         )
-    )
-)
-
-
-@bigquery_dialect.segment(replace=True)
-class WildcardExpressionSegment(BaseSegment):
-    """An extension of the star expression for Bigquery."""
-    type = 'wildcard_expression'
-    match_grammar = Sequence(
+    ),
+    WildcardSelectTargetElementGrammar=Sequence(
         # *, blah.*, blah.blah.*, etc.
-        Ref('WildcardIdentifierSegment'),
+        Sequence(
+            AnyNumberOf(
+                Sequence(
+                    Ref('SingleIdentifierGrammar'),
+                    Ref('DotSegment'),
+                    code_only=True
+                )
+            ),
+            Ref('StarSegment'), code_only=False
+        ),
         # Optional EXCEPT or REPLACE clause
         # https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#select_replace
         Ref('ExceptClauseSegment', optional=True),
         Ref('ReplaceClauseSegment', optional=True)
-    )
+    ),
+)
 
 
 @bigquery_dialect.segment()
diff --git a/src/sqlfluff/parser/segments_base.py b/src/sqlfluff/parser/segments_base.py
index 128d35f5d..a1c3f702e 100644
--- a/src/sqlfluff/parser/segments_base.py
+++ b/src/sqlfluff/parser/segments_base.py
@@ -949,16 +949,9 @@ class BaseSegment:
         return buff
 
     def recursive_crawl(self, seg_type):
-        """Recursively crawl for segments of a given type.
-
-        Args:
-            seg_type: :obj:`str` or :obj:`tuple` of :obj:`str`: which specifies
-                the type of elements to look for.
-        """
+        """Recursively crawl for segments of a given type."""
         # Check this segment
-        if isinstance(seg_type, str) and self.type == seg_type:
-            yield self
-        elif isinstance(seg_type, tuple) and self.type in seg_type:
+        if self.type == seg_type:
             yield self
         # Recurse
         for seg in self.segments:
diff --git a/src/sqlfluff/rules/std.py b/src/sqlfluff/rules/std.py
index b71af1c83..d2f6c29ce 100644
--- a/src/sqlfluff/rules/std.py
+++ b/src/sqlfluff/rules/std.py
@@ -1311,7 +1311,7 @@ class Rule_L013(BaseCrawler):
         if segment.type == 'select_target_element':
             if not any(e.type == 'alias_expression' for e in segment.segments):
                 types = {e.type for e in segment.segments if e.name != 'star'}
-                unallowed_types = types - {'whitespace', 'newline', 'object_reference', 'wildcard_expression'}
+                unallowed_types = types - {'whitespace', 'newline', 'object_reference'}
                 if len(unallowed_types) > 0:
                     # No fixes, because we don't know what the alias should be,
                     # the user should document it themselves.
@@ -2164,8 +2164,6 @@ class Rule_L020(BaseCrawler):
             # potential others.
             sc = segment.get_child('select_clause')
             reference_buffer = list(sc.recursive_crawl('object_reference'))
-            # Add any wildcard references
-            reference_buffer += list(sc.recursive_crawl('wildcard_identifier'))
             for potential_clause in ('where_clause', 'groupby_clause', 'having_clause', 'orderby_clause'):
                 clause = segment.get_child(potential_clause)
                 if clause:
@@ -2293,6 +2291,13 @@ class Rule_L022(BaseCrawler):
 
     """
 
+    def __init__(self, comma_style='trailing', **kwargs):
+        """Initialise, extracting the comma_style from the config."""
+        if comma_style not in ['trailing', 'leading']:
+            raise ValueError("Unexpected `comma_style`: {0!r}".format(comma_style))
+        self.comma_style = comma_style
+        super(Rule_L022, self).__init__(**kwargs)
+
     def _eval(self, segment, **kwargs):
         """Blank line expected but not found after CTE definition."""
         error_buffer = []
@@ -2301,8 +2306,9 @@ class Rule_L022(BaseCrawler):
             blank_line_found = False
             blank_line_started = False
             fix_point = None
+            cte_end_segments = ('end_bracket', 'comma') if self.comma_style == "trailing" else ('end_bracket')
             for seg in segment.segments:
-                if seg.type in ('end_bracket', 'comma'):
+                if seg.type in cte_end_segments:
                     expecting_blank_line = True
                     blank_line_found = False
                     blank_line_started = False
@@ -2465,6 +2471,22 @@ class Rule_L025(Rule_L020):
 
     """
 
+    @staticmethod
+    def _extract_type_tbl_reference(reference):
+        """Extract the type of reference and the referenced alias.
+
+        Returns:
+            :obj:`tuple` of (alias_type, alias)
+
+        """
+        this_ref_type = 'qualified' if reference.is_qualified() else 'unqualified'
+        ref_elems = list(reference.iter_raw_references())
+        if len(ref_elems) >= 2:
+            tbl_ref = ref_elems[-2]
+        else:
+            tbl_ref = None
+        return this_ref_type, tbl_ref
+
     def _lint_references_and_aliases(self, aliases, references, col_aliases, using_cols, parent_select):
         """Check all aliased references against tables referenced in the query."""
         # A buffer to keep any violations.
@@ -2472,7 +2494,7 @@ class Rule_L025(Rule_L020):
         # Check all the references that we have, keep track of which aliases we refer to.
         tbl_refs = set()
         for r in references:
-            tbl_ref = r.extract_reference(level=2)
+            _, tbl_ref = self._extract_type_tbl_reference(r)
             if tbl_ref:
                 tbl_refs.add(tbl_ref[0])
 
@@ -2517,7 +2539,7 @@ class Rule_L026(Rule_L025):
 
         # Check all the references that we have, do they reference present aliases?
         for r in references:
-            tbl_ref = r.extract_reference(level=2)
+            _, tbl_ref = self._extract_type_tbl_reference(r)
             # Check whether the string in the list of strings
             if tbl_ref and tbl_ref[0] not in [a[0] for a in aliases]:
                 # Last check, this *might* be a correlated subquery reference.
@@ -2570,7 +2592,7 @@ class Rule_L027(Rule_L025):
         violation_buff = []
         # Check all the references that we have.
         for r in references:
-            this_ref_type = r.qualification()
+            this_ref_type, _ = self._extract_type_tbl_reference(r)
             if (
                 this_ref_type == 'unqualified'
                 and r.raw not in col_aliases
@@ -2642,10 +2664,7 @@ class Rule_L028(Rule_L025):
         # Check all the references that we have.
         seen_ref_types = set()
         for r in references:
-            # We skip any unqualified wildcard references (i.e. *). They shouldn't count.
-            if not r.is_qualified() and r.type == 'wildcard_reference':
-                continue
-            this_ref_type = r.qualification()
+            this_ref_type, _ = self._extract_type_tbl_reference(r)
             if self.single_table_references == 'consistent':
                 if seen_ref_types and this_ref_type not in seen_ref_types:
                     violation_buff.append(


Test patch:
diff --git a/test/fixtures/parser/ansi/select_simple_b.yml b/test/fixtures/parser/ansi/select_simple_b.yml
index 1342270ef..3335e1644 100644
--- a/test/fixtures/parser/ansi/select_simple_b.yml
+++ b/test/fixtures/parser/ansi/select_simple_b.yml
@@ -5,9 +5,7 @@ file:
       select_clause:
         keyword: select
         select_target_element:
-          wildcard_expression:
-            wildcard_identifier:
-              star: '*'
+          keyword: '*'
       from_clause:
         keyword: from
         table_expression:
diff --git a/test/fixtures/parser/bigquery/select_replace_2.yml b/test/fixtures/parser/bigquery/select_replace_2.yml
index dc800f7cc..e61f60345 100644
--- a/test/fixtures/parser/bigquery/select_replace_2.yml
+++ b/test/fixtures/parser/bigquery/select_replace_2.yml
@@ -4,26 +4,24 @@ file:
       select_clause:
         keyword: SELECT
         select_target_element:
-          wildcard_expression:
-            wildcard_identifier:
-              star: '*'
-            select_replace_clause:
-              keyword: REPLACE
-              start_bracket: (
-              select_target_element:
-                function:
-                  function_name: CAST
-                  start_bracket: (
-                  expression:
-                    literal: '1'
-                  keyword: AS
-                  data_type:
-                    data_type_identifier: BOOLEAN
-                  end_bracket: )
-                alias_expression:
-                  keyword: AS
-                  identifier: foo
-              end_bracket: )
+          keyword: '*'
+          select_replace_clause:
+            keyword: REPLACE
+            start_bracket: (
+            select_target_element:
+              function:
+                function_name: CAST
+                start_bracket: (
+                expression:
+                  literal: '1'
+                keyword: AS
+                data_type:
+                  data_type_identifier: BOOLEAN
+                end_bracket: )
+              alias_expression:
+                keyword: AS
+                identifier: foo
+            end_bracket: )
       from_clause:
         keyword: FROM
         table_expression:
diff --git a/test/fixtures/parser/snowflake/snowflake_pivot.yml b/test/fixtures/parser/snowflake/snowflake_pivot.yml
index b448fae6a..95bc87585 100644
--- a/test/fixtures/parser/snowflake/snowflake_pivot.yml
+++ b/test/fixtures/parser/snowflake/snowflake_pivot.yml
@@ -4,9 +4,7 @@ file:
       select_clause:
         keyword: SELECT
         select_target_element:
-          wildcard_expression:
-            wildcard_identifier:
-              star: '*'
+          keyword: '*'
       from_clause:
         keyword: FROM
         table_expression:
diff --git a/test/fixtures/templater/jinja_g_macros/jinja.yml b/test/fixtures/templater/jinja_g_macros/jinja.yml
index 2eca31495..8cf65cac9 100644
--- a/test/fixtures/templater/jinja_g_macros/jinja.yml
+++ b/test/fixtures/templater/jinja_g_macros/jinja.yml
@@ -50,9 +50,7 @@ file:
             select_clause:
               keyword: select
               select_target_element:
-                wildcard_expression:
-                  wildcard_identifier:
-                    star: '*'
+                keyword: '*'
             from_clause:
               keyword: from
               table_expression:
diff --git a/test/rules_std_test.py b/test/rules_std_test.py
index 5f50f1b47..918e0733b 100644
--- a/test/rules_std_test.py
+++ b/test/rules_std_test.py
@@ -2,6 +2,8 @@
 
 import pytest
 
+import sqlfluff.rules.std as std_rules
+
 from sqlfluff.rules.std import std_rule_set
 from sqlfluff.linter import Linter
 from sqlfluff.config import FluffConfig
@@ -24,10 +26,7 @@ def assert_rule_fail_in_sql(code, sql, configs=None):
     print("Parsed:\n {0}".format(parsed.stringify()))
     lerrs, _, _, _ = r.crawl(parsed, dialect=cfg.get('dialect_obj'), fix=True)
     print("Errors Found: {0}".format(lerrs))
-    if not any(v.rule.code == code for v in lerrs):
-        pytest.fail(
-            "No {0} failures found in query which should fail.".format(code),
-            pytrace=False)
+    assert any(v.rule.code == code for v in lerrs)
     fixed = parsed  # use this as our buffer (yes it's a bit of misnomer right here)
     while True:
         # We get the errors again, but this time skip the assertion
@@ -49,6 +48,7 @@ def assert_rule_fail_in_sql(code, sql, configs=None):
             if fixes == l_fixes:
                 raise RuntimeError(
                     "Fixes aren't being applied: {0!r}".format(fixes))
+
     return fixed.raw
 
 
@@ -61,10 +61,7 @@ def assert_rule_pass_in_sql(code, sql, configs=None):
     print("Parsed:\n {0}".format(parsed.stringify()))
     lerrs, _, _, _ = r.crawl(parsed, dialect=cfg.get('dialect_obj'), fix=True)
     print("Errors Found: {0}".format(lerrs))
-    if any(v.rule.code == code for v in lerrs):
-        pytest.fail(
-            "Found {0} failures in query which should pass.".format(code),
-            pytrace=False)
+    assert not any(v.rule.code == code for v in lerrs)
 
 
 # ############## STD RULES TESTS
@@ -134,6 +131,15 @@ def assert_rule_pass_in_sql(code, sql, configs=None):
      {'rules': {'L019': {'comma_style': 'trailing'}}}),
     ('L019', 'pass', 'SELECT\n    a\n    , b\n    FROM c', None,
      {'rules': {'L019': {'comma_style': 'leading'}}}),
+    # Leading commas in with statement
+    ('L019', 'fail', ('WITH cte_1 as (\n    SELECT *\n    FROM table_1\n)\n\n'
+                      ', cte_2 as (\n    SELECT *\n    FROM table_2\n)\n\n'
+                      'SELECT * FROM table_3'), None,
+        {'rules': {'L019': {'comma_style': 'trailing'}}}),
+    ('L019', 'pass', ('WITH cte_1 as (\n    SELECT *\n    FROM table_1\n)\n\n'
+                      ', cte_2 as (\n    SELECT *\n    FROM table_2\n)\n\n'
+                      'SELECT * FROM table_3'), None,
+        {'rules': {'L019': {'comma_style': 'leading'}}}),
     # Trailing commas
     ('L019', 'fail', 'SELECT\n    a,\n    b\n    FROM c', None,
      {'rules': {'L019': {'comma_style': 'leading'}}}),
@@ -198,10 +204,7 @@ def assert_rule_pass_in_sql(code, sql, configs=None):
     ('L026', 'pass', 'SELECT * FROM db.sc.tbl2\nWHERE a NOT IN (SELECT tbl2.a FROM db.sc.tbl1)\n', None, None),  # Correlated subquery.
     # Make sure comments are aligned properly
     ('L003', 'pass', 'SELECT *\nFROM\n    t1\n-- Comment\nJOIN t2 USING (user_id)', None, None),
-    ('L003', 'fail', 'SELECT *\nFROM\n    t1\n    -- Comment\nJOIN t2 USING (user_id)', 'SELECT *\nFROM\n    t1\n-- Comment\nJOIN t2 USING (user_id)', None),
-    # L013 & L025 Fixes with https://github.com/sqlfluff/sqlfluff/issues/449
-    ('L013', 'pass', 'select ps.*, pandgs.blah from ps join pandgs using(moo)', None, None),
-    ('L025', 'pass', 'select ps.*, pandgs.blah from ps join pandgs using(moo)', None, None)
+    ('L003', 'fail', 'SELECT *\nFROM\n    t1\n    -- Comment\nJOIN t2 USING (user_id)', 'SELECT *\nFROM\n    t1\n-- Comment\nJOIN t2 USING (user_id)', None)
 ])
 def test__rules__std_string(rule, pass_fail, qry, fixed, configs):
     """Test that a rule passes/fails on a given string.
@@ -281,3 +284,15 @@ def test__rules__std_L003_process_raw_stack(generate_test_segments):
     print(res)
     assert sorted(res.keys()) == [1, 2]
     assert res[2]['indent_size'] == 5
+
+
+@pytest.mark.parametrize("rule,rule_init", [
+    ("L010", {"capitalisation_policy": "blah"}),
+    ("L019", {"comma_style": "blah"}),
+    ("L022", {"comma_style": "blah"})
+])
+def test_improper_configs_are_rejected(rule, rule_init):
+    """Ensure that unsupported configs raise a ValueError."""
+    rule_class = getattr(std_rules, "Rule_{}".format(rule))
+    with pytest.raises(ValueError):
+        rule_class(**rule_init)