Repository: asottile/pyupgrade

Issue:
consider fixing invalid escape sequences
```
$ python2 -W once -c 'print("\d test")'
\d test
$ python3 -W once -c 'print("\d test")'
<string>:1: DeprecationWarning: invalid escape sequence \d
\d test
$ python3 -W once -c 'print(r"\d test")'
\d test
```


Comments:

Patch:
diff --git a/README.md b/README.md
index 521b142..ec37d48 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,21 @@ u"foo"      # 'foo'
 u'''foo'''  # '''foo'''
 ```
 
+### Invalid escape sequences
+
+```python
+# strings with only invalid sequences become raw strings
+'\d'    # r'\d'
+# strings with mixed valid / invalid sequences get escaped
+'\n\d'  # '\n\\d'
+# `ur` is not a valid string prefix in python3
+u'\d'   # u'\\d'
+
+# note: pyupgrade is timid in one case (that's usually a mistake)
+# in python2.x `'\u2603'` is the same as `'\\u2603'` without `unicode_literals`
+# but in python3.x, that's our friend ☃
+```
+
 ### Long literals
 
 Availability:
diff --git a/pyupgrade.py b/pyupgrade.py
index 8385410..d6a196c 100644
--- a/pyupgrade.py
+++ b/pyupgrade.py
@@ -8,6 +8,7 @@ import copy
 import io
 import re
 import string
+import warnings
 
 from tokenize_rt import ESCAPED_NL
 from tokenize_rt import Offset
@@ -60,7 +61,10 @@ def _ast_to_offset(node):
 
 
 def ast_parse(contents_text):
-    return ast.parse(contents_text.encode('UTF-8'))
+    # intentionally ignore warnings, we might be fixing warning-ridden syntax
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore')
+        return ast.parse(contents_text.encode('UTF-8'))
 
 
 def inty(s):
@@ -431,6 +435,65 @@ def _fix_unicode_literals(contents_text, py3_plus):
     return tokens_to_src(tokens)
 
 
+# https://docs.python.org/3/reference/lexical_analysis.html
+ESCAPE_STARTS = frozenset((
+    '\n', '\\', "'", '"', 'a', 'b', 'f', 'n', 'r', 't', 'v',
+    '0', '1', '2', '3', '4', '5', '6', '7',  # octal escapes
+    'x',  # hex escapes
+    # only valid in non-bytestrings
+    'N', 'u', 'U',
+))
+ESCAPE_STARTS_BYTES = ESCAPE_STARTS - frozenset(('N', 'u', 'U'))
+ESCAPE_RE = re.compile(r'\\.')
+
+
+def _fix_escape_sequences(contents_text):
+    last_name = None
+    tokens = src_to_tokens(contents_text)
+    for i, token in enumerate(tokens):
+        if token.name == 'NAME':
+            last_name = token
+            continue
+        elif token.name != 'STRING':
+            last_name = None
+            continue
+
+        match = STRING_PREFIXES_RE.match(token.src)
+        prefix = match.group(1)
+        rest = match.group(2)
+
+        if last_name is not None:  # pragma: no cover (py2 bug)
+            actual_prefix = (last_name.src + prefix).lower()
+        else:  # pragma: no cover (py3 only)
+            actual_prefix = prefix.lower()
+
+        if 'r' in actual_prefix or '\\' not in rest:
+            continue
+
+        if 'b' in actual_prefix:
+            valid_escapes = ESCAPE_STARTS_BYTES
+        else:
+            valid_escapes = ESCAPE_STARTS
+
+        escape_sequences = {m[1] for m in ESCAPE_RE.findall(rest)}
+        has_valid_escapes = escape_sequences & valid_escapes
+        has_invalid_escapes = escape_sequences - valid_escapes
+
+        def cb(match):
+            matched = match.group()
+            if matched[1] in valid_escapes:
+                return matched
+            else:
+                return r'\{}'.format(matched)
+
+        if has_invalid_escapes and (has_valid_escapes or 'u' in actual_prefix):
+            tokens[i] = token._replace(src=prefix + ESCAPE_RE.sub(cb, rest))
+        elif has_invalid_escapes and not has_valid_escapes:
+            tokens[i] = token._replace(src=prefix + 'r' + rest)
+
+    return tokens_to_src(tokens)
+
+
 def _fix_long_literals(contents_text):
     tokens = src_to_tokens(contents_text)
     for i, token in enumerate(tokens):
@@ -1248,6 +1311,7 @@ def fix_file(filename, args):
     contents_text = _fix_sets(contents_text)
     contents_text = _fix_format_literals(contents_text)
     contents_text = _fix_unicode_literals(contents_text, args.py3_plus)
+    contents_text = _fix_escape_sequences(contents_text)
     contents_text = _fix_long_literals(contents_text)
     contents_text = _fix_octal_literals(contents_text)
     if not args.keep_percent_format:


Test patch:
diff --git a/tests/pyupgrade_test.py b/tests/pyupgrade_test.py
index d468953..82acd34 100644
--- a/tests/pyupgrade_test.py
+++ b/tests/pyupgrade_test.py
@@ -8,6 +8,7 @@ import sys
 import pytest
 
 from pyupgrade import _fix_dictcomps
+from pyupgrade import _fix_escape_sequences
 from pyupgrade import _fix_format_literals
 from pyupgrade import _fix_fstrings
 from pyupgrade import _fix_long_literals
@@ -327,6 +328,48 @@ def test_unicode_literals(s, py3_plus, expected):
     assert ret == expected
 
 
+@pytest.mark.parametrize(
+    's',
+    (
+        '""',
+        r'r"\d"', r"r'\d'", r'r"""\d"""', r"r'''\d'''",
+        # python2 has a bug where `rb'foo'` is tokenized as NAME + STRING
+        r'rb"\d"',
+        # make sure we don't replace an already valid string
+        r'"\\d"',
+        # in python2 `'\u2603'` is literally \\u2603, but transforming based
+        # on that would be incorrect in python3.
+        # intentionally timid here to avoid breaking working python3 code
+        '"\\u2603"',
+        # don't touch already valid escapes
+        r'"\r\n"',
+    ),
+)
+def test_fix_escape_sequences_noop(s):
+    assert _fix_escape_sequences(s) == s
+
+
+@pytest.mark.parametrize(
+    ('s', 'expected'),
+    (
+        # no valid escape sequences, make a raw literal
+        (r'"\d"', r'r"\d"'),
+        # when there are valid escape sequences, need to use backslashes
+        (r'"\n\d"', r'"\n\\d"'),
+        # `ur` is not a valid string prefix in python3.x
+        (r'u"\d"', r'u"\\d"'),
+        # `rb` is not a valid string prefix in python2.x
+        (r'b"\d"', r'br"\d"'),
+        # 8 and 9 aren't valid octal digits
+        (r'"\8"', r'r"\8"'), (r'"\9"', r'r"\9"'),
+        # explicit byte strings should not honor string-specific escapes
+        ('b"\\u2603"', 'br"\\u2603"'),
+    ),
+)
+def test_fix_escape_sequences(s, expected):
+    assert _fix_escape_sequences(s) == expected
+
+
 @pytest.mark.xfail(sys.version_info >= (3,), reason='python2 "feature"')
 @pytest.mark.parametrize(
     ('s', 'expected'),