Repository: python/mypy

Issue:
Archaic control character 0x1f causes crash in linecount report
I have this file (foo.py):
```py
def foo():
    # ()
    return ""
```
The comment contains a weird control character hex 1f. Running mypy like this:
```
mypy --xml-report .xml-report foo.py --tb
```
causes this crash:
```
foo.py: error: INTERNAL ERROR -- please report a bug at https://github.com/python/mypy/issues version: 0.650+dev.94c54aa755f2d7826fed5e0e2276053ba49da8e6
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/__main__.py", line 11, in <module>
    main(None)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/main.py", line 92, in main
    res = build.build(sources, options, None, flush_errors, fscache)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 155, in build
    result = _build(sources, options, alt_lib_path, flush_errors, fscache)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 204, in _build
    graph = dispatch(sources, manager)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 2172, in dispatch
    process_graph(graph, manager)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 2469, in process_graph
    process_stale_scc(graph, scc, manager)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 2605, in process_stale_scc
    graph[id].finish_passes()
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 1767, in finish_passes
    manager.report_file(self.tree, self.type_map(), self.options)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 1519, in wrap_context
    yield
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 1767, in finish_passes
    manager.report_file(self.tree, self.type_map(), self.options)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 643, in report_file
    self.reports.file(file, type_map, options)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/report.py", line 83, in file
    reporter.on_file(tree, type_map, options)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/report.py", line 450, in on_file
    any_info=self._get_any_info_for_line(visitor, lineno))
  File "src/lxml/etree.pyx", line 3112, in lxml.etree.SubElement
  File "src/lxml/apihelpers.pxi", line 199, in lxml.etree._makeSubElement
  File "src/lxml/apihelpers.pxi", line 194, in lxml.etree._makeSubElement
  File "src/lxml/apihelpers.pxi", line 297, in lxml.etree._initNodeAttributes
  File "src/lxml/apihelpers.pxi", line 311, in lxml.etree._addAttributeToNode
  File "src/lxml/apihelpers.pxi", line 1439, in lxml.etree._utf8
ValueError: All strings must be XML compatible: Unicode or ASCII, no NULL bytes or control characters
foo.py: : note: use --pdb to drop into pdb
Generated XML report: /Users/guido/src/server/.xml-report/index.xml
```


Discussion:
Also, if the directory `.xml-report`  doesn't already exist, I get additional tracebacks (instead of the final `Generated XML report: ...` line):
```
Traceback (most recent call last):
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 1519, in wrap_context
    yield
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 1767, in finish_passes
    manager.report_file(self.tree, self.type_map(), self.options)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 643, in report_file
    self.reports.file(file, type_map, options)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/report.py", line 83, in file
    reporter.on_file(tree, type_map, options)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/report.py", line 450, in on_file
    any_info=self._get_any_info_for_line(visitor, lineno))
  File "src/lxml/etree.pyx", line 3112, in lxml.etree.SubElement
  File "src/lxml/apihelpers.pxi", line 199, in lxml.etree._makeSubElement
  File "src/lxml/apihelpers.pxi", line 194, in lxml.etree._makeSubElement
  File "src/lxml/apihelpers.pxi", line 297, in lxml.etree._initNodeAttributes
  File "src/lxml/apihelpers.pxi", line 311, in lxml.etree._addAttributeToNode
  File "src/lxml/apihelpers.pxi", line 1439, in lxml.etree._utf8
ValueError: All strings must be XML compatible: Unicode or ASCII, no NULL bytes or control characters

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 204, in _build
    graph = dispatch(sources, manager)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 2172, in dispatch
    process_graph(graph, manager)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 2469, in process_graph
    process_stale_scc(graph, scc, manager)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 2605, in process_stale_scc
    graph[id].finish_passes()
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 1767, in finish_passes
    manager.report_file(self.tree, self.type_map(), self.options)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 1523, in wrap_context
    report_internal_error(err, self.path, 0, self.manager.errors, self.options)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/errors.py", line 615, in report_internal_error
    raise SystemExit(1)
SystemExit: 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/__main__.py", line 11, in <module>
    main(None)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/main.py", line 92, in main
    res = build.build(sources, options, None, flush_errors, fscache)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 155, in build
    result = _build(sources, options, alt_lib_path, flush_errors, fscache)
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/build.py", line 214, in _build
    reports.finish()
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/report.py", line 87, in finish
    reporter.on_finish()
  File "/Users/guido/src/server/.mypy/venv/lib/python3.6/site-packages/mypy/report.py", line 666, in on_finish
    last_xml.write(out_path, encoding='utf-8')
  File "src/lxml/etree.pyx", line 2039, in lxml.etree._ElementTree.write
  File "src/lxml/serializer.pxi", line 721, in lxml.etree._tofilelike
  File "src/lxml/serializer.pxi", line 780, in lxml.etree._create_output_buffer
  File "src/lxml/serializer.pxi", line 770, in lxml.etree._create_output_buffer
FileNotFoundError: [Errno 2] No such file or directory
```
This seems to be unrelated to the control character, it's just due to the situation where `on_finish()` is called without `on_file()` ever being called -- there's no `ensure_dir_exists()` call on `on_finish()`.

Resolving patch:
diff --git a/mypy/report.py b/mypy/report.py
--- a/mypy/report.py
+++ b/mypy/report.py
@@ -90,6 +90,8 @@ def finish(self) -> None:
 class AbstractReporter(metaclass=ABCMeta):
     def __init__(self, reports: Reports, output_dir: str) -> None:
         self.output_dir = output_dir
+        if output_dir != '<memory>':
+            stats.ensure_dir_exists(output_dir)
 
     @abstractmethod
     def on_file(self, tree: MypyFile, type_map: Dict[Expression, Type], options: Options) -> None:
@@ -124,8 +126,6 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
         super().__init__(reports, output_dir)
         self.counts = {}  # type: Dict[str, Tuple[int, int, int, int]]
 
-        stats.ensure_dir_exists(output_dir)
-
     def on_file(self,
                 tree: MypyFile,
                 type_map: Dict[Expression, Type],
@@ -170,7 +170,6 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
         super().__init__(reports, output_dir)
         self.counts = {}  # type: Dict[str, Tuple[int, int]]
         self.any_types_counter = {}  # type: Dict[str, typing.Counter[int]]
-        stats.ensure_dir_exists(output_dir)
 
     def on_file(self,
                 tree: MypyFile,
@@ -361,8 +360,6 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
         super().__init__(reports, output_dir)
         self.lines_covered = {}  # type: Dict[str, List[int]]
 
-        stats.ensure_dir_exists(output_dir)
-
     def on_file(self,
                 tree: MypyFile,
                 type_map: Dict[Expression, Type],
@@ -418,6 +415,10 @@ def __init__(self, reports: Reports, output_dir: str) -> None:
         self.last_xml = None  # type: Optional[Any]
         self.files = []  # type: List[FileInfo]
 
+    # XML doesn't like control characters, but they are sometimes
+    # legal in source code (e.g. comments, string literals).
+    control_fixer = str.maketrans(''.join(chr(i) for i in range(32)), '?' * 32)
+
     def on_file(self,
                 tree: MypyFile,
                 type_map: Dict[Expression, Type],
@@ -446,7 +447,7 @@ def on_file(self,
                 etree.SubElement(root, 'line',
                                  number=str(lineno),
                                  precision=stats.precision_names[status],
-                                 content=line_text.rstrip('\n'),
+                                 content=line_text.rstrip('\n').translate(self.control_fixer),
                                  any_info=self._get_any_info_for_line(visitor, lineno))
         # Assumes a layout similar to what XmlReporter uses.
         xslt_path = os.path.relpath('mypy-html.xslt', path)
@@ -737,7 +738,6 @@ def on_finish(self) -> None:
         last_xml = self.memory_xml.last_xml
         assert last_xml is not None
         out_path = os.path.join(self.output_dir, 'index.txt')
-        stats.ensure_dir_exists(os.path.dirname(out_path))
         transformed_txt = bytes(self.xslt_txt(last_xml))
         with open(out_path, 'wb') as out_file:
             out_file.write(transformed_txt)


Test patch:
diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py
--- a/mypy/test/testcmdline.py
+++ b/mypy/test/testcmdline.py
@@ -52,24 +52,31 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
     env['PYTHONPATH'] = PREFIX
     process = subprocess.Popen(fixed + args,
                                stdout=subprocess.PIPE,
-                               stderr=subprocess.STDOUT,
+                               stderr=subprocess.PIPE,
                                cwd=test_temp_dir,
                                env=env)
-    outb = process.stdout.read()
+    outb, errb = process.communicate()
+    result = process.returncode
     # Split output into lines.
     out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
+    err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()]
 
     if "PYCHARM_HOSTED" in os.environ:
-        pos = next((p for p, i in enumerate(out) if i.startswith('pydev debugger: ')), None)
-        if pos is not None:
-            del out[pos]  # the attaching debugger message itself
-            del out[pos]  # plus the extra new line added
+        for pos, line in enumerate(err):
+            if line.startswith('pydev debugger: '):
+                # Delete the attaching debugger message itself, plus the extra newline added.
+                del err[pos:pos + 2]
+                break
 
-    result = process.wait()
     # Remove temp file.
     os.remove(program_path)
     # Compare actual output to expected.
     if testcase.output_files:
+        # Ignore stdout, but we insist on empty stderr and zero status.
+        if err or result:
+            raise AssertionError(
+                'Expected zero status and empty stderr, got %d and\n%s' %
+                (result, '\n'.join(err + out)))
         for path, expected_content in testcase.output_files:
             if not os.path.exists(path):
                 raise AssertionError(
@@ -85,7 +92,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
                                        'Output file {} did not match its expected output'.format(
                                            path))
     else:
-        out = normalize_error_messages(out)
+        out = normalize_error_messages(err + out)
         obvious_result = 1 if out else 0
         if obvious_result != result:
             out.append('== Return code: {}'.format(result))
diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test
--- a/test-data/unit/reports.test
+++ b/test-data/unit/reports.test
@@ -77,7 +77,7 @@ def untyped_function():
 # cmd: mypy --html-report report n.py
 [file n.py]
 class A(object):
-    pass
+    pass  # hex 1f here: ()
 
 [file report/mypy-html.css]
 [file report/index.html]
@@ -96,7 +96,7 @@ class A(object):
 <span id="L2" class="lineno"><a class="lineno" href="#L2">2</a></span>
 </pre></td>
 <td class="table-code"><pre><span class="line-empty" title="No Anys on this line!">class A(object):</span>
-<span class="line-empty" title="No Anys on this line!">    pass</span>
+<span class="line-empty" title="No Anys on this line!">    pass  # hex 1f here: (?)</span>
 </pre></td>
 </tr></tbody>
 </table>