Repository: ambv/retype

Issue:
lib2to3_parse assumes that the ParseError will always refer to an existing line
There seems to be a case where `ParseError` will report the line number after the last line number, causing an `IndexError` in retype:

Example file (core.py):
```
def get_message():
    return '123'
```

Example stub (types/core.pyi):
```
def get_message() -> str: ...
```

```
$>retype --traceback core.py
error: core.py: list index out of range
Traceback (most recent call last):
  File "retype.py", line 110, in retype_path
    retype_file(src, pyi_dir, targets, quiet=quiet, hg=hg)
  File "retype.py", line 132, in retype_file
    src_node = lib2to3_parse(src_txt)
  File "retype.py", line 161, in lib2to3_parse
    faulty_line = src_txt.splitlines()[lineno - 1]
IndexError: list index out of range
```

I haven't gone digging yet to see why the `Driver` is failing to parse this, but it seems that this should be fixed as well.

**Tested using:** 
- `Python 3.6.1 (v3.6.1:69c0db5, Mar 21 2017, 17:54:52) [MSC v.1900 32 bit (Intel)] on win32`
- `Python 3.6.1 (default, May 11 2017, 22:14:44) [GCC 4.9.2] on linux`


Discussion:

Patch:
diff --git a/retype.py b/retype.py
index eee4b8e..d59e1b0 100644
--- a/retype.py
+++ b/retype.py
@@ -12,6 +12,7 @@ from pathlib import Path
 import re
 import sys
 import threading
+import tokenize
 import traceback
 
 import click
@@ -138,9 +139,9 @@ def retype_file(src, pyi_dir, targets, *, quiet=False, hg=False):
 
     Type comments in sources are normalized to type annotations.
     """
-    with open(src) as src_file:
-        src_txt = src_file.read()
-    src_node = lib2to3_parse(src_txt)
+    with tokenize.open(src) as src_buffer:
+        src_encoding = src_buffer.encoding
+        src_node = lib2to3_parse(src_buffer.read())
     try:
         with open((pyi_dir / src.name).with_suffix('.pyi')) as pyi_file:
             pyi_txt = pyi_file.read()
@@ -156,7 +157,7 @@ def retype_file(src, pyi_dir, targets, *, quiet=False, hg=False):
         reapply_all(pyi_ast.body, src_node)
     fix_remaining_type_comments(src_node)
     targets.mkdir(parents=True, exist_ok=True)
-    with open(targets / src.name, 'w') as target_file:
+    with open(targets / src.name, 'w', encoding=src_encoding) as target_file:
         target_file.write(lib2to3_unparse(src_node, hg=hg))
     return targets / src.name
 
@@ -169,7 +170,11 @@ def lib2to3_parse(src_txt):
         result = drv.parse_string(src_txt, True)
     except ParseError as pe:
         lineno, column = pe.context[1]
-        faulty_line = src_txt.splitlines()[lineno - 1]
+        lines = src_txt.splitlines()
+        if src_txt[-1] != '\n':
+            faulty_line = "The source is missing a trailing newline."
+        else:
+            faulty_line = lines[lineno - 1]
         raise ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") from None
 
     if isinstance(result, Leaf):


Test patch:
diff --git a/tests/test_retype.py b/tests/test_retype.py
index 5142782..dfb4d99 100644
--- a/tests/test_retype.py
+++ b/tests/test_retype.py
@@ -2134,6 +2134,17 @@ class PrintStmtTestCase(RetypeTestCase):
             str(exception),
         )
 
+class ParseErrorTestCase(RetypeTestCase):
+    def test_missing_trailing_newline_crash(self) -> None:
+        pyi_txt = "def f() -> None: ...\n"
+        src_txt = """
+        def f():
+            pass"""
+        exception = self.assertReapplyRaises(pyi_txt, src_txt, ValueError)
+        self.assertEqual(
+            'Cannot parse: 4:0: The source is missing a trailing newline.',
+            str(exception),
+        )
 
 class PostProcessTestCase(RetypeTestCase):
     def test_straddling_variable_comments(self) -> None: