Repository: RustPython/RustPython (language: rust)

Issue:
Tokenizer is missing `NL` token
The CPython tokenizer has both a `NEWLINE` and `NL` token.

Given this code:

```python
x = (
  "a"
  "b"
)

x = "a" \
  "b"
```

RustPython currently outputs the same tokenization for these two snippets (modulo the left and right parens -- notice that in both cases, we have `String` followed by `String`):

```
Name {
    name: "x",
}
Equal
Lpar
String {
    value: "a",
    kind: Normal,
}
String {
    value: "b",
    kind: Normal,
}
Rpar
Newline
Name {
    name: "x",
}
Equal
String {
    value: "a",
    kind: Normal,
}
String {
    value: "b",
    kind: Normal,
}
Newline
```

Whereas in CPython, there are `NL` tokens in the first case that aren't present for continuations:

```
❯ python -m tokenize test.py
0,0-0,0:            ENCODING       'utf-8'
1,0-1,1:            NAME           'x'
1,2-1,3:            OP             '='
1,4-1,5:            OP             '('
1,5-1,6:            NL             '\n'
2,2-2,5:            STRING         '"a"'
2,5-2,6:            NL             '\n'
3,2-3,5:            STRING         '"b"'
3,5-3,6:            NL             '\n'
4,0-4,1:            OP             ')'
4,1-4,2:            NEWLINE        '\n'
5,0-5,1:            NL             '\n'
6,0-6,1:            NAME           'x'
6,2-6,3:            OP             '='
6,4-6,7:            STRING         '"a"'
7,2-7,5:            STRING         '"b"'
7,5-7,6:            NEWLINE        '\n'
8,0-8,0:            ENDMARKER      ''
```


Discussion / hints:
I remember quickly looking into this but not replying in the other issue. `NL` seems to be a token only used for the python implementation of the tokenizer (i.e `tokenize.py`), [this comment](https://github.com/python/cpython/blob/79c10b7da84f52999dc483fc62c8e758ad3eff23/Grammar/Tokens#L65) indicates this.

So, running the same tokenizer from RustPython does include it:

```python
cargo run --release -- -m tokenize tok.py
0,0-0,0:            ENCODING       'utf-8'        
1,0-1,1:            NAME           'x'            
1,2-1,3:            OP             '='            
1,4-1,5:            OP             '('            
1,5-1,6:            NL             '\n'           
2,2-2,5:            STRING         '"a"'          
2,5-2,6:            NL             '\n'           
3,2-3,5:            STRING         '"b"'          
3,5-3,6:            NL             '\n'           
4,0-4,1:            OP             ')'            
4,1-4,2:            NEWLINE        '\n'           
5,0-5,1:            NL             '\n'           
6,0-6,1:            NAME           'x'            
6,2-6,3:            OP             '='            
6,4-6,7:            STRING         '"a"'          
7,2-7,5:            STRING         '"b"'          
7,5-7,6:            NEWLINE        '\n'           
8,0-8,0:            ENDMARKER      '' 
```

`NL` is used for physical lines while `NEWLINE` for logical ones, the C tokenizer appears to just join the physical lines to make a logical one while the Python version doesn't, not sure why that is.
I am working on porting a tokenizer-level linter flake8-commas to Ruff, and it relies on NL.

I reckon NL should be emitted in an `else` case here:

https://github.com/RustPython/RustPython/blob/main/compiler/parser/src/lexer.rs#L1073-L1083

CPython tokenize.py code for comparison:

https://github.com/python/cpython/blob/e8097d49f64e47f0ad7568e7181e6e2c466b6964/Lib/tokenize.py#L540-L544

https://github.com/python/cpython/blob/e8097d49f64e47f0ad7568e7181e6e2c466b6964/Lib/tokenize.py#L507-L508

I'll try to see if it works when I get a chance.

BTW, I think the `NL`/`NEWLINE` names are pretty bad, so maybe in RustPython should call `NL` something like `NonLogicalNewline`?

Fix patch:
diff --git a/compiler/parser/src/lexer.rs b/compiler/parser/src/lexer.rs
index cb38503b3e..1c124f6272 100644
--- a/compiler/parser/src/lexer.rs
+++ b/compiler/parser/src/lexer.rs
@@ -1075,10 +1075,13 @@ where
                 self.next_char();
                 let tok_end = self.get_pos();
 
-                // Depending on the nesting level, we emit newline or not:
+                // Depending on the nesting level, we emit a logical or
+                // non-logical newline:
                 if self.nesting == 0 {
                     self.at_begin_of_line = true;
                     self.emit((tok_start, Tok::Newline, tok_end));
+                } else {
+                    self.emit((tok_start, Tok::NonLogicalNewline, tok_end));
                 }
             }
             ' ' | '\t' | '\x0C' => {
@@ -1464,7 +1467,16 @@ mod tests {
         $(
             #[test]
             fn $name() {
-                let source = format!("x = [{}    1,2{}]{}", $eol, $eol, $eol);
+                let source = r"x = [
+
+    1,2
+,(3,
+4,
+), {
+5,
+6,\
+7}]
+".replace("\n", $eol);
                 let tokens = lex_source(&source);
                 assert_eq!(
                     tokens,
@@ -1474,9 +1486,32 @@ mod tests {
                         },
                         Tok::Equal,
                         Tok::Lsqb,
+                        Tok::NonLogicalNewline,
+                        Tok::NonLogicalNewline,
                         Tok::Int { value: BigInt::from(1) },
                         Tok::Comma,
                         Tok::Int { value: BigInt::from(2) },
+                        Tok::NonLogicalNewline,
+                        Tok::Comma,
+                        Tok::Lpar,
+                        Tok::Int { value: BigInt::from(3) },
+                        Tok::Comma,
+                        Tok::NonLogicalNewline,
+                        Tok::Int { value: BigInt::from(4) },
+                        Tok::Comma,
+                        Tok::NonLogicalNewline,
+                        Tok::Rpar,
+                        Tok::Comma,
+                        Tok::Lbrace,
+                        Tok::NonLogicalNewline,
+                        Tok::Int { value: BigInt::from(5) },
+                        Tok::Comma,
+                        Tok::NonLogicalNewline,
+                        Tok::Int { value: BigInt::from(6) },
+                        Tok::Comma,
+                        // Continuation here - no NonLogicalNewline.
+                        Tok::Int { value: BigInt::from(7) },
+                        Tok::Rbrace,
                         Tok::Rsqb,
                         Tok::Newline,
                     ]
@@ -1492,6 +1527,50 @@ mod tests {
         test_newline_in_brackets_unix_eol: UNIX_EOL,
     }
 
+    #[test]
+    fn test_non_logical_newline_in_string_continuation() {
+        let source = r"(
+    'a'
+    'b'
+
+    'c' \
+    'd'
+)";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Lpar,
+                Tok::NonLogicalNewline,
+                stok("a"),
+                Tok::NonLogicalNewline,
+                stok("b"),
+                Tok::NonLogicalNewline,
+                Tok::NonLogicalNewline,
+                stok("c"),
+                stok("d"),
+                Tok::NonLogicalNewline,
+                Tok::Rpar,
+                Tok::Newline,
+            ]
+        );
+    }
+
+    #[test]
+    fn test_logical_newline_line_comment() {
+        let source = "#Hello\n#World";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Comment("#Hello".to_owned()),
+                // tokenize.py does put an NL here...
+                Tok::Comment("#World".to_owned()),
+                // ... and here, but doesn't seem very useful.
+            ]
+        );
+    }
+
     #[test]
     fn test_operators() {
         let source = "//////=/ /";
diff --git a/compiler/parser/src/parser.rs b/compiler/parser/src/parser.rs
index 4277d3002e..4d9b52bb5e 100644
--- a/compiler/parser/src/parser.rs
+++ b/compiler/parser/src/parser.rs
@@ -96,7 +96,7 @@ pub fn parse_located(
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. }));
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
 
     python::TopParser::new()
         .parse(tokenizer)
diff --git a/compiler/parser/src/token.rs b/compiler/parser/src/token.rs
index 72aa89d683..b51b2f4776 100644
--- a/compiler/parser/src/token.rs
+++ b/compiler/parser/src/token.rs
@@ -25,6 +25,7 @@ pub enum Tok {
         triple_quoted: bool,
     },
     Newline,
+    NonLogicalNewline,
     Indent,
     Dedent,
     StartModule,
@@ -136,6 +137,7 @@ impl fmt::Display for Tok {
                 write!(f, "{kind}{quotes}{value}{quotes}")
             }
             Newline => f.write_str("Newline"),
+            NonLogicalNewline => f.write_str("NonLogicalNewline"),
             Indent => f.write_str("Indent"),
             Dedent => f.write_str("Dedent"),
             StartModule => f.write_str("StartProgram"),


Test patch: