Repository: sqlfluff/sqlfluff

Issue:
Whitespace token is_whitespace is False
I expect segment.is_whitespace of a Whitespace token is True, however, it is set to False.

## Expected Behaviour
segment.is_whitespace return True

## Observed Behaviour
segment.is_whitespace return False
## Steps to Reproduce

## Version
Include the output of `sqlfluff --version` along with your Python version

## Configuration
```
Include your SQLFluff configuration here
```



Discussion:
To triage this issue, I searched the SQLFluff code to find all uses of `is_whitespace`. This is the only one I found:
```
src/sqlfluff/core/parser/segments/base.py:72:    is_whitespace = False
```

@alanmcruickshank: What's the purpose of `is_whitespace`?

I see that long ago (2019), there was a class `WhitespaceSegment` (also a `NewlineSegment`). Now it's not a class -- instead, it'd defined in `src/sqlfluff/core/rules/base.py`.
Once #866 is merged I'll pick up the rest of this which relates to some of the lexer objects.

Patch:
diff --git a/src/sqlfluff/core/dialects/dialect_ansi.py b/src/sqlfluff/core/dialects/dialect_ansi.py
--- a/src/sqlfluff/core/dialects/dialect_ansi.py
+++ b/src/sqlfluff/core/dialects/dialect_ansi.py
@@ -50,7 +50,7 @@
 ansi_dialect.set_lexer_struct(
     [
         # name, type, pattern, kwargs
-        ("whitespace", "regex", r"[\t ]+", dict(type="whitespace")),
+        ("whitespace", "regex", r"[\t ]+", dict(type="whitespace", is_whitespace=True)),
         (
             "inline_comment",
             "regex",
@@ -64,9 +64,14 @@
             dict(
                 is_comment=True,
                 type="comment",
-                subdivide=dict(type="newline", name="newline", regex=r"\r\n|\n"),
+                subdivide=dict(
+                    type="newline", name="newline", regex=r"\r\n|\n", is_whitespace=True
+                ),
                 trim_post_subdivide=dict(
-                    type="whitespace", name="whitespace", regex=r"[\t ]+"
+                    type="whitespace",
+                    name="whitespace",
+                    regex=r"[\t ]+",
+                    is_whitespace=True,
                 ),
             ),
         ),
@@ -83,7 +88,7 @@
         ("not_equal", "regex", r"!=|<>", dict(is_code=True)),
         ("greater_than_or_equal", "regex", r">=", dict(is_code=True)),
         ("less_than_or_equal", "regex", r"<=", dict(is_code=True)),
-        ("newline", "regex", r"\r\n|\n", dict(type="newline")),
+        ("newline", "regex", r"\r\n|\n", dict(type="newline", is_whitespace=True)),
         ("casting_operator", "regex", r"::", dict(is_code=True)),
         ("concat_operator", "regex", r"\|\|", dict(is_code=True)),
         ("equals", "singleton", "=", dict(is_code=True)),
diff --git a/src/sqlfluff/core/parser/lexer.py b/src/sqlfluff/core/parser/lexer.py
--- a/src/sqlfluff/core/parser/lexer.py
+++ b/src/sqlfluff/core/parser/lexer.py
@@ -74,12 +74,10 @@ def _trim(self, matched, start_pos):
         idx = 0
 
         if self.trim_post_subdivide:
-            trimmer = re.compile(self.trim_post_subdivide["regex"], re.DOTALL)
-            TrimClass = RawSegment.make(
-                self.trim_post_subdivide["regex"],
-                name=self.trim_post_subdivide["name"],
-                type=self.trim_post_subdivide["type"],
-            )
+            class_kwargs = self.trim_post_subdivide.copy()
+            pattern = class_kwargs.pop("regex")
+            trimmer = re.compile(pattern, re.DOTALL)
+            TrimClass = RawSegment.make(pattern, **class_kwargs)
 
             for trim_mat in trimmer.finditer(matched):
                 trim_span = trim_mat.span()
@@ -132,12 +130,10 @@ def _subdivide(self, matched, start_pos):
             seg_buff = ()
             str_buff = matched
             pos_buff = start_pos
-            divider = re.compile(self.subdivide["regex"], re.DOTALL)
-            DividerClass = RawSegment.make(
-                self.subdivide["regex"],
-                name=self.subdivide["name"],
-                type=self.subdivide["type"],
-            )
+            class_kwargs = self.subdivide.copy()
+            pattern = class_kwargs.pop("regex")
+            divider = re.compile(pattern, re.DOTALL)
+            DividerClass = RawSegment.make(pattern, **class_kwargs)
 
             while True:
                 # Iterate through subdividing as appropriate


Test patch:
diff --git a/test/core/dialects/ansi_test.py b/test/core/dialects/ansi_test.py
--- a/test/core/dialects/ansi_test.py
+++ b/test/core/dialects/ansi_test.py
@@ -162,3 +162,14 @@ def test__dialect__ansi_specific_segment_not_parse(raw, err_locations, caplog):
     assert len(parsed.violations) > 0
     locs = [(v.line_no(), v.line_pos()) for v in parsed.violations]
     assert locs == err_locations
+
+
+def test__dialect__ansi_is_whitespace():
+    """Test proper tagging with is_whitespace."""
+    lnt = Linter()
+    with open("test/fixtures/parser/ansi/select_in_multiline_comment.sql") as f:
+        parsed = lnt.parse_string(f.read())
+    # Check all the segments that *should* be whitespace, ARE
+    for raw_seg in parsed.tree.iter_raw_seg():
+        if raw_seg.type in ("whitespace", "newline"):
+            assert raw_seg.is_whitespace