Repository: tobymao/sqlglot

Issue:
\r\n sometimes causes a parsing error
Repro

```python
from sqlglot import parse_one
parse_one('select a\r\nfrom b')
``` 

It appears that when the fix went in for line numbers, that the advance function skips the \n, but doesn't advance start, so the text becomes \nfrom which doesn't match.

Discussion:

Patch:
diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py
index ef431119..ef3dc237 100644
--- a/sqlglot/tokens.py
+++ b/sqlglot/tokens.py
@@ -587,6 +587,7 @@ class Tokenizer(metaclass=_Tokenizer):
             # Ensures we don't count an extra line if we get a \r\n line break sequence
             if self._char == "\r" and self._peek == "\n":
                 i = 2
+                self._start += 1
 
             self._col = 1
             self._line += 1
diff --git a/sqlglotrs/src/tokenizer.rs b/sqlglotrs/src/tokenizer.rs
index eb800f09..f386ce63 100644
--- a/sqlglotrs/src/tokenizer.rs
+++ b/sqlglotrs/src/tokenizer.rs
@@ -140,6 +140,7 @@ impl<'a> TokenizerState<'a> {
             // Ensures we don't count an extra line if we get a \r\n line break sequence.
             if self.current_char == '\r' && self.peek_char == '\n' {
                 i = 2;
+                self.start += 1;
             }
 
             self.column = 1;


Test patch:
diff --git a/tests/test_tokens.py b/tests/test_tokens.py
index b97f54a6..970c1ac2 100644
--- a/tests/test_tokens.py
+++ b/tests/test_tokens.py
@@ -71,6 +71,20 @@ x"""
         self.assertEqual(tokens[2].line, 2)
         self.assertEqual(tokens[3].line, 3)
 
+    def test_crlf(self):
+        tokens = Tokenizer().tokenize("SELECT a\r\nFROM b")
+        tokens = [(token.token_type, token.text) for token in tokens]
+
+        self.assertEqual(
+            tokens,
+            [
+                (TokenType.SELECT, "SELECT"),
+                (TokenType.VAR, "a"),
+                (TokenType.FROM, "FROM"),
+                (TokenType.VAR, "b"),
+            ],
+        )
+
     def test_command(self):
         tokens = Tokenizer().tokenize("SHOW;")
         self.assertEqual(tokens[0].token_type, TokenType.SHOW)
diff --git a/tests/test_transpile.py b/tests/test_transpile.py
index b732b459..fb8f8313 100644
--- a/tests/test_transpile.py
+++ b/tests/test_transpile.py
@@ -89,6 +89,7 @@ class TestTranspile(unittest.TestCase):
         self.validate("SELECT MIN(3)>=MIN(2)", "SELECT MIN(3) >= MIN(2)")
         self.validate("SELECT 1>0", "SELECT 1 > 0")
         self.validate("SELECT 3>=3", "SELECT 3 >= 3")
+        self.validate("SELECT a\r\nFROM b", "SELECT a FROM b")
 
     def test_comments(self):
         self.validate(