Repository: HypothesisWorks/hypothesis

Issue:
Enhance scripts/fix_doctests.py now that Sphinx 1.7 has been released
[`scripts/fix_doctests.py`](https://github.com/HypothesisWorks/hypothesis-python/blob/master/scripts/fix_doctests.py) is a nifty tool that replaces the expected output of failing doctests with whatever we actually got.  

However, it currently includes some workarounds for buggy reporting in Sphinx, which I fixed in sphinx-doc/sphinx#4584 and which has now been released (🎉).

- Line numbers may be `?` for unknown, instead of an incorrect number
- Line numbers are no longer off by one
- File names are accurately reported, at least on Python 3, so we shouldn't assume all docstrings are in `strategies.py`
- The script should check that it's running Sphinx 1.7 and Sphinx is not on Python 2


Discussion:

Resolving patch:
diff --git a/scripts/fix_doctests.py b/scripts/fix_doctests.py
--- a/scripts/fix_doctests.py
+++ b/scripts/fix_doctests.py
@@ -22,25 +22,43 @@
 import os
 import re
 import sys
+from textwrap import dedent
 from subprocess import PIPE, run
 from collections import defaultdict
+from distutils.version import StrictVersion
 
 import hypothesistooling as tools
 
 
+def dedent_lines(lines, force_newline=None):
+    """Remove common leading whitespace from a list of strings."""
+    if not lines:
+        return []
+    joiner = '' if lines[0].endswith('\n') else '\n'
+    lines = dedent(joiner.join(lines)).split('\n')
+    if force_newline is False:
+        return lines
+    if force_newline is True:
+        return [l + '\n' for l in lines]
+    return [l + (bool(joiner) * '\n') for l in lines]
+
+
 class FailingExample(object):
 
     def __init__(self, chunk):
         """Turn a chunk of text into an object representing the test."""
-        location, *lines = [l + '\n' for l in chunk.split('\n') if l.strip()]
+        # Determine and save the location of the actual doctest
+        location, *lines = [l for l in chunk.split('\n') if l.strip()]
         self.location = location.strip()
-        pattern = 'File "(.+?)", line (\d+?), in .+'
+        pattern = r'File "(.+?)", line (\d+|\?+), in .+'
         file, line = re.match(pattern, self.location).groups()
         self.file = os.path.join('docs', file)
-        self.line = int(line) + 1
-        got = lines.index('Got:\n')
-        self.expected_lines = lines[lines.index('Expected:\n') + 1:got]
-        self.got_lines = lines[got + 1:]
+        self.line = None if '?' in line else int(line)
+        # Select the expected and returned output of the test
+        got = lines.index('Got:')
+        self.expected_lines = \
+            dedent_lines(lines[lines.index('Expected:') + 1:got])
+        self.got_lines = dedent_lines(lines[got + 1:])
         self.checked_ok = None
         self.adjust()
 
@@ -49,33 +67,38 @@ def indices(self):
         return slice(self.line, self.line + len(self.expected_lines))
 
     def adjust(self):
-        with open(self.file) as f:
-            lines = f.readlines()
-        # The raw line number is the first line of *input*, so adjust to
-        # first line of output by skipping lines which start with a prompt
-        while self.line < len(lines):
-            if lines[self.line].strip()[:4] not in ('>>> ', '... '):
-                break
-            self.line += 1
-        # Sadly the filename and line number for doctests in docstrings is
-        # wrong - see https://github.com/sphinx-doc/sphinx/issues/4223
-        # Luckily, we can just cheat because they're all in one file for now!
-        # (good luck if this changes without an upstream fix...)
-        if lines[self.indices] != self.expected_lines:
+        if self.line is None and self.file.endswith('.rst'):
+            # Sphinx reports an unknown line number when the doctest is
+            # included from a docstring, so docutils must have misreported the
+            # file location.  We thus force it to the most likley candidate:
             self.file = 'src/hypothesis/strategies.py'
-            with open(self.file) as f:
-                lines = f.readlines()
+        with open(self.file) as f:
+            lines = f.read().split('\n')
+        if self.line is not None:
+            # The raw line number is the first line of *input*, so adjust to
+            # first line of output by skipping lines which start with a prompt
+            while self.line < len(lines):
+                if lines[self.line].lstrip()[:4] not in ('>>> ', '... '):
+                    break
+                self.line += 1
+        else:
+            # If the location within a file wasn't reported, we have to go
+            # looking for it.
+            stripped = [l.lstrip() for l in lines]
             self.line = 0
-            while self.expected_lines[0] in lines:
-                self.line = lines[self.line:].index(self.expected_lines[0])
-                if lines[self.indices] == self.expected_lines:
+            while self.expected_lines[0] in stripped[self.line:]:
+                self.line = stripped[self.line:].index(self.expected_lines[0])
+                candidate = dedent_lines(lines[self.indices], False)
+                if candidate == self.expected_lines:
                     break
+                self.line += 1
         # Finally, set the flag for location quality
-        self.checked_ok = lines[self.indices] == self.expected_lines
+        self.checked_ok = \
+            dedent_lines(lines[self.indices], False) == self.expected_lines
 
     def __repr__(self):
         return '{}\nExpected: {!r:.60}\nGot:      {!r:.60}'.format(
-            self.location, self.expected, self.got)
+            self.location, self.expected_lines, self.got_lines)
 
 
 def get_doctest_output():
@@ -96,8 +119,19 @@ def get_doctest_output():
             for fname, examples in tests.items()}
 
 
+def indent_like(lines, like):
+    """Indent ``lines`` to the same level as ``like``."""
+    prefix = len(like[0].rstrip()) - len(dedent_lines(like)[0].rstrip())
+    return [prefix * ' ' + l for l in dedent_lines(lines, force_newline=True)]
+
+
 def main():
     os.chdir(tools.ROOT)
+    version = run(['sphinx-build', '--version'], stdout=PIPE,
+                  encoding='utf-8').stdout.lstrip('sphinx-build ')
+    if StrictVersion(version) < '1.7':
+        print('This script requires Sphinx 1.7 or later; got %s.\n' % version)
+        sys.exit(2)
     failing = get_doctest_output()
     if not failing:
         print('All doctests are OK')
@@ -110,7 +144,8 @@ def main():
         with open(fname) as f:
             lines = f.readlines()
         for ex in examples:
-            lines[ex.indices] = ex.got_lines
+            # Note: can't indent earlier, as we don't know file indentation
+            lines[ex.indices] = indent_like(ex.got_lines, lines[ex.indices])
         with open(fname, 'w') as f:
             f.writelines(lines)
 


Test patch: