Repository: HypothesisWorks/hypothesis

Issue:
--hypothesis-show-statistics doesn't work with pytest-xdist
If you use the flag while running with xdist then you'll get no statistics. This is presumably because the tests are running in a different process than the collector, so the data isn't getting communicated back.

To reproduce e.g. run `pytest --hypothesis-show-statistics tests/cover/test_testdecorators.py`, which will use the default Hypothesis configuration and run with two child runners and show no statistics.


Discussion:
Related to pytest-dev/pytest-xdist#79.  Our current code, which doesn't work with xdist, is:

https://github.com/HypothesisWorks/hypothesis/blob/fe60c8aae7fd8478ef08acfd9f8f852ec004c1e7/hypothesis-python/src/hypothesis/extra/pytestplugin.py#L107-L113

It looks like we'll need to write some special-case logic to pass the report string back to the master instance when running under `xdist`, and probably also to receive the data.  See e.g. [the hook docs](https://docs.pytest.org/en/documentation-restructure/how-to/writing_plugins.html#_pytest.hookspec.pytest_runtest_makereport) and [xdist/workermanage.py](https://github.com/pytest-dev/pytest-xdist/blob/master/xdist/workermanage.py) as starting points.

-----------

Update: the reporting quoted above works just fine; it was relying on the global `gathered_statistics` dict that broke our reporting under `xdist`.  PR incoming.

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
--- a/hypothesis-python/src/hypothesis/extra/pytestplugin.py
+++ b/hypothesis-python/src/hypothesis/extra/pytestplugin.py
@@ -17,6 +17,8 @@
 
 from __future__ import division, print_function, absolute_import
 
+from distutils.version import LooseVersion
+
 import pytest
 
 from hypothesis import core, settings
@@ -103,7 +105,9 @@ def pytest_runtest_call(item):
         store = StoringReporter(item.config)
 
         def note_statistics(stats):
-            gathered_statistics[item.nodeid] = stats
+            lines = [item.nodeid + ':', ''] + stats.get_description() + ['']
+            gathered_statistics[item.nodeid] = lines
+            item.hypothesis_statistics = lines
 
         with collector.with_value(note_statistics):
             with with_reporter(store):
@@ -120,42 +124,42 @@ def pytest_runtest_makereport(item, call):
             'Hypothesis',
             '\n'.join(item.hypothesis_report_information)
         ))
+    if hasattr(item, 'hypothesis_statistics') and report.when == 'teardown':
+        # Running on pytest < 3.5 where user_properties doesn't exist, fall
+        # back on the global gathered_statistics (which breaks under xdist)
+        if hasattr(report, 'user_properties'):  # pragma: no branch
+            val = ('hypothesis-stats', item.hypothesis_statistics)
+            # Workaround for https://github.com/pytest-dev/pytest/issues/4034
+            if isinstance(report.user_properties, tuple):
+                report.user_properties += (val,)
+            else:
+                report.user_properties.append(val)
 
 
 def pytest_terminal_summary(terminalreporter):
     if not terminalreporter.config.getoption(PRINT_STATISTICS_OPTION):
         return
     terminalreporter.section('Hypothesis Statistics')
-    for name, statistics in gathered_statistics.items():
-        terminalreporter.write_line(name + ':')
-        terminalreporter.write_line('')
 
-        if not statistics.has_runs:
-            terminalreporter.write_line('  - Test was never run')
-            continue
+    if LooseVersion(pytest.__version__) < '3.5':  # pragma: no cover
+        if not gathered_statistics:
+            terminalreporter.write_line(
+                'Reporting Hypothesis statistics with pytest-xdist enabled '
+                'requires pytest >= 3.5'
+            )
+        for lines in gathered_statistics.values():
+            for li in lines:
+                terminalreporter.write_line(li)
+        return
 
-        terminalreporter.write_line((
-            '  - %d passing examples, %d failing examples,'
-            ' %d invalid examples') % (
-            statistics.passing_examples, statistics.failing_examples,
-            statistics.invalid_examples,
-        ))
-        terminalreporter.write_line(
-            '  - Typical runtimes: %s' % (statistics.runtimes,)
-        )
-        terminalreporter.write_line(
-            '  - Fraction of time spent in data generation: %s' % (
-                statistics.draw_time_percentage,))
-        terminalreporter.write_line(
-            '  - Stopped because %s' % (statistics.exit_reason,)
-        )
-        if statistics.events:
-            terminalreporter.write_line('  - Events:')
-            for event in statistics.events:
-                terminalreporter.write_line(
-                    '    * %s' % (event,)
-                )
-        terminalreporter.write_line('')
+    # terminalreporter.stats is a dict, where the empty string appears to
+    # always be the key for a list of _pytest.reports.TestReport objects
+    # (where we stored the statistics data in pytest_runtest_makereport above)
+    for test_report in terminalreporter.stats.get('', []):
+        for name, lines in test_report.user_properties:
+            if name == 'hypothesis-stats' and test_report.when == 'teardown':
+                for li in lines:
+                    terminalreporter.write_line(li)
 
 
 def pytest_collection_modifyitems(items):
diff --git a/hypothesis-python/src/hypothesis/statistics.py b/hypothesis-python/src/hypothesis/statistics.py
--- a/hypothesis-python/src/hypothesis/statistics.py
+++ b/hypothesis-python/src/hypothesis/statistics.py
@@ -103,6 +103,25 @@ def __init__(self, engine):
             self.draw_time_percentage = '~ %d%%' % (
                 round(draw_time_percentage),)
 
+    def get_description(self):
+        """Return a list of lines describing the statistics, to be printed."""
+        if not self.has_runs:
+            return ['  - Test was never run']
+        lines = [
+            '  - %d passing examples, %d failing examples, %d invalid examples'
+            % (self.passing_examples, self.failing_examples,
+               self.invalid_examples),
+            '  - Typical runtimes: %s' % (self.runtimes,),
+            '  - Fraction of time spent in data generation: %s' % (
+                self.draw_time_percentage,
+            ),
+            '  - Stopped because %s' % (self.exit_reason,)
+        ]
+        if self.events:
+            lines.append('  - Events:')
+            lines += ['    * %s' % (event,) for event in self.events]
+        return lines
+
 
 def note_engine_for_statistics(engine):
     callback = collector.value


Test patch:
diff --git a/hypothesis-python/tests/pytest/test_statistics.py b/hypothesis-python/tests/pytest/test_statistics.py
--- a/hypothesis-python/tests/pytest/test_statistics.py
+++ b/hypothesis-python/tests/pytest/test_statistics.py
@@ -17,6 +17,10 @@
 
 from __future__ import division, print_function, absolute_import
 
+from distutils.version import LooseVersion
+
+import pytest
+
 from hypothesis.extra.pytestplugin import PRINT_STATISTICS_OPTION
 
 pytest_plugins = 'pytester'
@@ -67,6 +71,19 @@ def test_prints_statistics_given_option(testdir):
     assert '< 10% of examples satisfied assumptions' in out
 
 
+@pytest.mark.skipif(LooseVersion(pytest.__version__) < '3.5', reason='too old')
+def test_prints_statistics_given_option_under_xdist(testdir):
+    script = testdir.makepyfile(TESTSUITE)
+    result = testdir.runpytest(script, PRINT_STATISTICS_OPTION, '-n', '2')
+    out = '\n'.join(result.stdout.lines)
+    assert 'Hypothesis Statistics' in out
+    assert 'timeout=0.2' in out
+    assert 'max_examples=100' in out
+    assert '< 10% of examples satisfied assumptions' in out
+    # Check that xdist doesn't have us report the same thing twice
+    assert out.count('Stopped because settings.timeout=0.2') == 1
+
+
 UNITTEST_TESTSUITE = """
 
 from hypothesis import given