Repository: asottile/pyupgrade

Issue:
TokenError
Thank you very much for writing this tool!!

I want to run pyupgrade over a project that probably still contains code that raises a syntax error when imported in a python3 interpreter.

``pyupgrade --py36-plus`` outputs is the following in a python3.7 conda env:

```
filename1.py
filename2.py
Traceback (most recent call last):
  File "/home/thomas/miniconda/envs/py37/bin/pyupgrade", line 10, in <module>
    sys.exit(main())
  File "/home/thomas/miniconda/envs/py37/lib/python3.7/site-packages/pyupgrade.py", line 1428, in main
    ret |= fix_file(filename, args)
  File "/home/thomas/miniconda/envs/py37/lib/python3.7/site-packages/pyupgrade.py", line 1397, in fix_file
    contents_text = _fix_format_literals(contents_text)
  File "/home/thomas/miniconda/envs/py37/lib/python3.7/site-packages/pyupgrade.py", line 107, in _fix_format_literals
    tokens = src_to_tokens(contents_text)
  File "/home/thomas/miniconda/envs/py37/lib/python3.7/site-packages/tokenize_rt.py", line 44, in src_to_tokens
    ) in tokenize.generate_tokens(tokenize_target.readline):
  File "/home/thomas/miniconda/envs/py37/lib/python3.7/tokenize.py", line 579, in _tokenize
    raise TokenError("EOF in multi-line statement", (lnum, 0))
tokenize.TokenError: ('EOF in multi-line statement', (350, 0))
```

Unfortunately this exception is not very helpful, since it doesn't really tell me which file lead to this exception. Note that filename2.py is not the  culprit, because it doesn't have a line with number 350.

IMO pyupgrade should avoid showning tracebacks by defaults, but instead output filename+linum of the faulty files.

Tested it with

```
(py37) > $ conda list pyupgrade                                                                                 [±master ●●]
# packages in environment at /home/thomas/miniconda/envs/py37:
#
# Name                    Version                   Build  Channel
pyupgrade                 1.17.0                     py_0    conda-forge
```

BTW pyupgrade doesn't have a `--version` cli flag.

Discussion:

Patch:
diff --git a/pyupgrade.py b/pyupgrade.py
index 2b251f4..31a603e 100644
--- a/pyupgrade.py
+++ b/pyupgrade.py
@@ -104,7 +104,10 @@ def _rewrite_string_literal(literal):
 
 
 def _fix_format_literals(contents_text):
-    tokens = src_to_tokens(contents_text)
+    try:
+        tokens = src_to_tokens(contents_text)
+    except tokenize.TokenError:
+        return contents_text
 
     to_replace = []
     string_start = None
@@ -413,7 +416,10 @@ def _fix_py2_compatible(contents_text):
     )):
         return contents_text
 
-    tokens = src_to_tokens(contents_text)
+    try:
+        tokens = src_to_tokens(contents_text)
+    except tokenize.TokenError:  # pragma: no cover (bpo-2180)
+        return contents_text
     for i, token in reversed_enumerate(tokens):
         if token.offset in visitor.dicts:
             _process_dict_comp(tokens, i, visitor.dicts[token.offset])
@@ -859,7 +865,10 @@ def _fix_percent_format(contents_text):
     if not visitor.found:
         return contents_text
 
-    tokens = src_to_tokens(contents_text)
+    try:
+        tokens = src_to_tokens(contents_text)
+    except tokenize.TokenError:  # pragma: no cover (bpo-2180)
+        return contents_text
 
     for i, token in reversed_enumerate(tokens):
         node = visitor.found.get(token.offset)
@@ -1195,6 +1204,11 @@ def _fix_py3_plus(contents_text):
     )):
         return contents_text
 
+    try:
+        tokens = src_to_tokens(contents_text)
+    except tokenize.TokenError:  # pragma: no cover (bpo-2180)
+        return contents_text
+
     def _replace(i, mapping, node):
         new_token = Token('CODE', _get_tmpl(mapping, node))
         if isinstance(node, ast.Name):
@@ -1205,7 +1219,6 @@ def _fix_py3_plus(contents_text):
                 j += 1
             tokens[i:j + 1] = [new_token]
 
-    tokens = src_to_tokens(contents_text)
     for i, token in reversed_enumerate(tokens):
         if not token.src:
             continue
@@ -1357,7 +1370,10 @@ def _fix_fstrings(contents_text):
     if not visitor.found:
         return contents_text
 
-    tokens = src_to_tokens(contents_text)
+    try:
+        tokens = src_to_tokens(contents_text)
+    except tokenize.TokenError:  # pragma: no cover (bpo-2180)
+        return contents_text
     for i, token in reversed_enumerate(tokens):
         node = visitor.found.get(token.offset)
         if node is None:


Test patch:
diff --git a/tests/pyupgrade_test.py b/tests/pyupgrade_test.py
index 8de1903..102e7b9 100644
--- a/tests/pyupgrade_test.py
+++ b/tests/pyupgrade_test.py
@@ -1524,3 +1524,18 @@ def test_py36_plus_fstrings(tmpdir):
     assert f.read() == '"{} {}".format(hello, world)'
     assert main((f.strpath, '--py36-plus')) == 1
     assert f.read() == 'f"{hello} {world}"'
+
+
+def test_noop_token_error(tmpdir):
+    f = tmpdir.join('f.py')
+    f.write(
+        # force some rewrites (ast is ok https://bugs.python.org/issue2180)
+        'set(())\n'
+        '"%s" % (1,)\n'
+        'six.b("foo")\n'
+        '"{}".format(a)\n'
+        # token error
+        'x = \\\n'
+        '5\\\n'
+    )
+    assert main((f.strpath, '--py36-plus')) == 0