Repository: sqlfluff/sqlfluff
Issue: In rule tests, fail if a rule that is not "@document_fix_compatible" returns "fixes"

Some of the rules (for example, [L002](https://github.com/sqlfluff/sqlfluff/blob/master/src/sqlfluff/core/rules/std/L002.py#L51L58)) support fixes but do not use "@document_fix_compatible". I think we want **all** fixable rules to be noted as such in the documentation.

We can enforce this by having the tests (probably `test/core/rules/rule_test_cases_test.py::test__rule_test_case`) report failure if a rule test case ever returns fixes but that rule does not specify `@document_fix_compatible`. 

Changed files:
src/sqlfluff/core/rules/doc_decorators.py, src/sqlfluff/core/rules/std/L024.py, src/sqlfluff/core/rules/std/L030.py, src/sqlfluff/core/rules/std/L014.py, test/core/rules/rule_test_cases_test.py, src/sqlfluff/core/rules/std/L002.py, test/core/rules/std_test.py

diff --git a/src/sqlfluff/core/rules/doc_decorators.py b/src/sqlfluff/core/rules/doc_decorators.py
index b40958ac..7c54571f 100644
--- a/src/sqlfluff/core/rules/doc_decorators.py
+++ b/src/sqlfluff/core/rules/doc_decorators.py
@@ -4,12 +4,20 @@ from sqlfluff.core.rules.config_info import STANDARD_CONFIG_INFO_DICT
 from sqlfluff.core.rules.base import rules_logger  # noqa
 
 
+FIX_COMPATIBLE = "``sqlfluff fix`` compatible."
+
+
 def document_fix_compatible(cls):
     """Mark the rule as fixable in the documentation."""
-    cls.__doc__ = cls.__doc__.replace("\\n", "\\n\\n``sqlfluff fix`` compatible.\\n", 1)
+    cls.__doc__ = cls.__doc__.replace("\\n", f"\\n\\n{FIX_COMPATIBLE}\\n", 1)
     return cls
 
 
+def is_fix_compatible(cls) -> bool:
+    """Return whether the rule is documented as fixable."""
+    return FIX_COMPATIBLE in cls.__doc__
+
+
 def document_configuration(cls, ruleset="std"):
     """Add a 'Configuration' section to a Rule docstring.
 
diff --git a/src/sqlfluff/core/rules/std/L002.py b/src/sqlfluff/core/rules/std/L002.py
index 1a10d3ac..6c04937c 100644
--- a/src/sqlfluff/core/rules/std/L002.py
+++ b/src/sqlfluff/core/rules/std/L002.py
@@ -1,10 +1,14 @@
 """Implementation of Rule L002."""
 
 from sqlfluff.core.rules.base import BaseCrawler, LintResult, LintFix
-from sqlfluff.core.rules.doc_decorators import document_configuration
+from sqlfluff.core.rules.doc_decorators import (
+    document_configuration,
+    document_fix_compatible,
+)
 
 
 @document_configuration
+@document_fix_compatible
 class Rule_L002(BaseCrawler):
     """Mixed Tabs and Spaces in single whitespace.
 
diff --git a/src/sqlfluff/core/rules/std/L014.py b/src/sqlfluff/core/rules/std/L014.py
index cd99c12b..b57fd842 100644
--- a/src/sqlfluff/core/rules/std/L014.py
+++ b/src/sqlfluff/core/rules/std/L014.py
@@ -2,11 +2,15 @@
 
 from typing import Tuple, List
 
-from sqlfluff.core.rules.doc_decorators import document_configuration
+from sqlfluff.core.rules.doc_decorators import (
+    document_configuration,
+    document_fix_compatible,
+)
 from sqlfluff.core.rules.std.L010 import Rule_L010
 
 
 @document_configuration
+@document_fix_compatible
 class Rule_L014(Rule_L010):
     """Inconsistent capitalisation of unquoted identifiers.
 
diff --git a/src/sqlfluff/core/rules/std/L024.py b/src/sqlfluff/core/rules/std/L024.py
index da48276b..575a271e 100644
--- a/src/sqlfluff/core/rules/std/L024.py
+++ b/src/sqlfluff/core/rules/std/L024.py
@@ -1,9 +1,11 @@
 """Implementation of Rule L024."""
 
 
+from sqlfluff.core.rules.doc_decorators import document_fix_compatible
 from sqlfluff.core.rules.std.L023 import Rule_L023
 
 
+@document_fix_compatible
 class Rule_L024(Rule_L023):
     """Single whitespace expected after USING in JOIN clause.
 
diff --git a/src/sqlfluff/core/rules/std/L030.py b/src/sqlfluff/core/rules/std/L030.py
index 3bc2ad78..577309e2 100644
--- a/src/sqlfluff/core/rules/std/L030.py
+++ b/src/sqlfluff/core/rules/std/L030.py
@@ -2,11 +2,15 @@
 
 from typing import List, Tuple
 
-from sqlfluff.core.rules.doc_decorators import document_configuration
+from sqlfluff.core.rules.doc_decorators import (
+    document_configuration,
+    document_fix_compatible,
+)
 from sqlfluff.core.rules.std.L010 import Rule_L010
 
 
 @document_configuration
+@document_fix_compatible
 class Rule_L030(Rule_L010):
     """Inconsistent capitalisation of function names.
 
diff --git a/test/core/rules/rule_test_cases_test.py b/test/core/rules/rule_test_cases_test.py
index 3278a4fd..514ed9ae 100644
--- a/test/core/rules/rule_test_cases_test.py
+++ b/test/core/rules/rule_test_cases_test.py
@@ -5,7 +5,9 @@ from glob import glob
 import pytest
 import oyaml as yaml
 
-from .std_test import rules__test_helper, RuleTestCase
+from sqlfluff.core.config import FluffConfig
+from sqlfluff.core.rules.doc_decorators import is_fix_compatible
+from .std_test import get_rule_from_set, rules__test_helper, RuleTestCase
 
 ids = []
 test_cases = []
@@ -28,4 +30,10 @@ for path in sorted(glob(test_cases_path)):
 @pytest.mark.parametrize("test_case", test_cases, ids=ids)
 def test__rule_test_case(test_case):
     """Run the tests."""
-    rules__test_helper(test_case)
+    res = rules__test_helper(test_case)
+    if res is not None and res != test_case.fail_str:
+        cfg = FluffConfig(configs=test_case.configs)
+        rule = get_rule_from_set(test_case.rule, config=cfg)
+        assert is_fix_compatible(
+            rule
+        ), f'Rule {test_case.rule} returned fixes but does not specify "@document_fix_compatible".'
diff --git a/test/core/rules/std_test.py b/test/core/rules/std_test.py
index d854de2a..45529e88 100644
--- a/test/core/rules/std_test.py
+++ b/test/core/rules/std_test.py
@@ -107,6 +107,7 @@ def rules__test_helper(test_case):
         # If a `fixed` value is provided then check it matches
         if test_case.fix_str:
             assert res == test_case.fix_str
+        return res
 
 
 class Rule_T042(BaseCrawler):