Repository: asottile/tokenize-rt

Issue:
Consider emitting an `ESCAPED_NL` token
For example:

```python
x = y.\
    foo(
        bar,
    )
```

The current tokenization is:

```
1:0 NAME 'x'
?:? UNIMPORTANT_WS ' '
1:2 OP '='
?:? UNIMPORTANT_WS ' '
1:4 NAME 'y'
1:5 OP '.'
?:? UNIMPORTANT_WS '\\\n    '
2:4 NAME 'foo'
2:7 OP '('
2:8 NL '\n'
?:? UNIMPORTANT_WS '        '
3:8 NAME 'bar'
3:11 OP ','
3:12 NL '\n'
?:? UNIMPORTANT_WS '    '
4:4 OP ')'
4:5 NEWLINE '\n'
5:0 ENDMARKER ''
```

It would be cool to split the UNIMPORTANT_WS token which contains the escaped newline

Comments:

Patch:
diff --git a/README.md b/README.md
index 3e9e23c..6fdf84d 100644
--- a/README.md
+++ b/README.md
@@ -5,8 +5,9 @@ tokenize-rt
 ===========
 
 The stdlib `tokenize` module does not properly roundtrip.  This wrapper
-around the stdlib provides an additional token `UNIMPORTANT_WS`, and a `Token`
-data type.  Use `src_to_tokens` and `tokens_to_src` to roundtrip.
+around the stdlib provides two additional tokens `ESCAPED_NL` and
+`UNIMPORTANT_WS`, and a `Token` data type.  Use `src_to_tokens` and
+`tokens_to_src` to roundtrip.
 
 This library is useful if you're writing a refactoring tool based on the
 python tokenization.
@@ -21,6 +22,8 @@ python tokenization.
 
 ### `tokenize_rt.tokens_to_src(Sequence[Token]) -> text`
 
+### `tokenize_rt.ECSAPED_NL`
+
 ### `tokenize_rt.UNIMPORTANT_WS`
 
 ### `tokenize_rt.Token(name, src, line=None, utf8_byte_offset=None)`
@@ -28,9 +31,9 @@ python tokenization.
 Construct a token
 
 - `name`: one of the token names listed in `token.tok_name` or
-  `UNIMPORTANT_WS`
+  `ESCAPED_NL` or `UNIMPORTANT_WS`
 - `src`: token's source as text
 - `line`: the line number that this token appears on.  This will be `None` for
-   `UNIMPORTANT_WS` tokens.
+   `ESCAPED_NL` and `UNIMPORTANT_WS` tokens.
 - `utf8_byte_offset`: the utf8 byte offset that this token appears on in the
-  line.  This will be `None` for `UNIMPORTANT_WS` tokens.
+  line.  This will be `None` for `ESCAPED_NL` and `UNIMPORTANT_WS` tokens.
diff --git a/tokenize_rt.py b/tokenize_rt.py
index bc5ca7d..4513200 100644
--- a/tokenize_rt.py
+++ b/tokenize_rt.py
@@ -7,6 +7,7 @@ import io
 import tokenize
 
 
+ESCAPED_NL = 'ESCAPED_NL'
 UNIMPORTANT_WS = 'UNIMPORTANT_WS'
 Token = collections.namedtuple(
     'Token', ('name', 'src', 'line', 'utf8_byte_offset'),
@@ -32,8 +33,16 @@ def src_to_tokens(src):
                 newtok += lines[lineno]
             if scol > 0:
                 newtok += lines[sline][:scol]
+
+            # a multiline unimportant whitespace may contain escaped newlines
+            while '\\\n' in newtok:
+                ws, nl, newtok = newtok.partition('\\\n')
+                if ws:
+                    tokens.append(Token(UNIMPORTANT_WS, ws))
+                tokens.append(Token(ESCAPED_NL, nl))
             if newtok:
                 tokens.append(Token(UNIMPORTANT_WS, newtok))
+
         elif scol > last_col:
             tokens.append(Token(UNIMPORTANT_WS, line[last_col:scol]))
 


Test patch:
diff --git a/tests/tokenize_rt_test.py b/tests/tokenize_rt_test.py
index d01f0fb..6977e5f 100644
--- a/tests/tokenize_rt_test.py
+++ b/tests/tokenize_rt_test.py
@@ -5,6 +5,7 @@ import io
 
 import pytest
 
+from tokenize_rt import ESCAPED_NL
 from tokenize_rt import main
 from tokenize_rt import src_to_tokens
 from tokenize_rt import Token
@@ -26,6 +27,41 @@ def test_src_to_tokens_simple():
     ]
 
 
+def test_src_to_tokens_escaped_nl():
+    src = (
+        'x = \\\n'
+        '    5'
+    )
+    ret = src_to_tokens(src)
+    assert ret == [
+        Token('NAME', 'x', line=1, utf8_byte_offset=0),
+        Token(UNIMPORTANT_WS, ' ', line=None, utf8_byte_offset=None),
+        Token('OP', '=', line=1, utf8_byte_offset=2),
+        Token(UNIMPORTANT_WS, ' ', line=None, utf8_byte_offset=None),
+        Token(ESCAPED_NL, '\\\n', line=None, utf8_byte_offset=None),
+        Token(UNIMPORTANT_WS, '    ', line=None, utf8_byte_offset=None),
+        Token('NUMBER', '5', line=2, utf8_byte_offset=4),
+        Token('ENDMARKER', '', line=3, utf8_byte_offset=0),
+    ]
+
+
+def test_src_to_tokens_escaped_nl_no_left_ws():
+    src = (
+        'x =\\\n'
+        '    5'
+    )
+    ret = src_to_tokens(src)
+    assert ret == [
+        Token('NAME', 'x', line=1, utf8_byte_offset=0),
+        Token(UNIMPORTANT_WS, ' ', line=None, utf8_byte_offset=None),
+        Token('OP', '=', line=1, utf8_byte_offset=2),
+        Token(ESCAPED_NL, '\\\n', line=None, utf8_byte_offset=None),
+        Token(UNIMPORTANT_WS, '    ', line=None, utf8_byte_offset=None),
+        Token('NUMBER', '5', line=2, utf8_byte_offset=4),
+        Token('ENDMARKER', '', line=3, utf8_byte_offset=0),
+    ]
+
+
 @pytest.mark.parametrize(
     'filename',
     (