Repository: HypothesisWorks/hypothesis

Issue:
Fix junitxml
:sparkles: _**This is an old work account. Please reference @brandonchinn178 for all future communication**_ :sparkles:
<!-- updated by mention_personal_account_in_comments.py -->

---

Pytest 5.4 changed the API such that `item.config._xml` is no longer set directly, but rather set in `item.config._store.get(xml_key)`.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/_hypothesis_pytestplugin.py b/hypothesis-python/src/_hypothesis_pytestplugin.py
--- a/hypothesis-python/src/_hypothesis_pytestplugin.py
+++ b/hypothesis-python/src/_hypothesis_pytestplugin.py
@@ -25,6 +25,11 @@
 
 import pytest
 
+try:
+    from _pytest.junitxml import xml_key
+except ImportError:
+    xml_key = "_xml"  # type: ignore
+
 LOAD_PROFILE_OPTION = "--hypothesis-profile"
 VERBOSITY_OPTION = "--hypothesis-verbosity"
 PRINT_STATISTICS_OPTION = "--hypothesis-show-statistics"
@@ -53,6 +58,8 @@
 check to assure Hypothesis that you understand what you are doing.
 """
 
+STATS_KEY = "_hypothesis_stats"
+
 
 class StoringReporter:
     def __init__(self, config):
@@ -255,9 +262,7 @@ def pytest_runtest_call(item):
 
             def note_statistics(stats):
                 stats["nodeid"] = item.nodeid
-                item.hypothesis_statistics = base64.b64encode(
-                    describe_statistics(stats).encode()
-                ).decode()
+                item.hypothesis_statistics = describe_statistics(stats)
 
             with collector.with_value(note_statistics):
                 with with_reporter(store):
@@ -266,6 +271,16 @@ def note_statistics(stats):
             if store.results:
                 item.hypothesis_report_information = list(store.results)
 
+    def _stash_get(config, key, default):
+        if hasattr(config, "stash"):
+            # pytest 7
+            return config.stash.get(key, default)
+        elif hasattr(config, "_store"):
+            # pytest 5.4
+            return config._store.get(key, default)
+        else:
+            return getattr(config, key, default)
+
     @pytest.hookimpl(hookwrapper=True)
     def pytest_runtest_makereport(item, call):
         report = (yield).get_result()
@@ -274,15 +289,28 @@ def pytest_runtest_makereport(item, call):
                 ("Hypothesis", "\n".join(item.hypothesis_report_information))
             )
         if hasattr(item, "hypothesis_statistics") and report.when == "teardown":
+            stats = item.hypothesis_statistics
+            stats_base64 = base64.b64encode(stats.encode()).decode()
+
             name = "hypothesis-statistics-" + item.nodeid
-            try:
-                item.config._xml.add_global_property(name, item.hypothesis_statistics)
-            except AttributeError:
-                # --junitxml not passed, or Pytest 4.5 (before add_global_property)
-                # We'll fail xunit2 xml schema checks, upgrade pytest if you care.
-                report.user_properties.append((name, item.hypothesis_statistics))
+
+            # Include hypothesis information to the junit XML report.
+            #
+            # Note that when `pytest-xdist` is enabled, `xml_key` is not present in the
+            # stash, so we don't add anything to the junit XML report in that scenario.
+            # https://github.com/pytest-dev/pytest/issues/7767#issuecomment-1082436256
+            xml = _stash_get(item.config, xml_key, None)
+            if xml:
+                xml.add_global_property(name, stats_base64)
+
+            # If there's a terminal report, include our summary stats for each test
+            terminalreporter = item.config.pluginmanager.getplugin("terminalreporter")
+            if terminalreporter is not None:
+                # ideally, we would store this on terminalreporter.config.stash, but
+                # pytest-xdist doesn't copy that back to the controller
+                report.__dict__[STATS_KEY] = stats
+
             # If there's an HTML report, include our summary stats for each test
-            stats = base64.b64decode(item.hypothesis_statistics.encode()).decode()
             pytest_html = item.config.pluginmanager.getplugin("html")
             if pytest_html is not None:  # pragma: no cover
                 report.extra = getattr(report, "extra", []) + [
@@ -290,30 +318,13 @@ def pytest_runtest_makereport(item, call):
                 ]
 
     def pytest_terminal_summary(terminalreporter):
-        if not terminalreporter.config.getoption(PRINT_STATISTICS_OPTION):
-            return
-        terminalreporter.section("Hypothesis Statistics")
-
-        def report(properties):
-            for name, value in properties:
-                if name.startswith("hypothesis-statistics-"):
-                    if hasattr(value, "uniobj"):
-                        # Under old versions of pytest, `value` was a `py.xml.raw`
-                        # rather than a string, so we get the (unicode) string off it.
-                        value = value.uniobj
-                    line = base64.b64decode(value.encode()).decode() + "\n\n"
-                    terminalreporter.write_line(line)
-
-        try:
-            global_properties = terminalreporter.config._xml.global_properties
-        except AttributeError:
-            # terminalreporter.stats is a dict, where the empty string appears to
-            # always be the key for a list of _pytest.reports.TestReport objects
-            for test_report in terminalreporter.stats.get("", []):
-                if test_report.when == "teardown":
-                    report(test_report.user_properties)
-        else:
-            report(global_properties)
+        if terminalreporter.config.getoption(PRINT_STATISTICS_OPTION):
+            terminalreporter.section("Hypothesis Statistics")
+            for reports in terminalreporter.stats.values():
+                for report in reports:
+                    stats = report.__dict__.get(STATS_KEY)
+                    if stats:
+                        terminalreporter.write_line(stats + "\n\n")
 
     def pytest_collection_modifyitems(items):
         if "hypothesis" not in sys.modules:
diff --git a/tooling/src/hypothesistooling/__main__.py b/tooling/src/hypothesistooling/__main__.py
--- a/tooling/src/hypothesistooling/__main__.py
+++ b/tooling/src/hypothesistooling/__main__.py
@@ -356,7 +356,7 @@ def documentation():
         )
 
 
-def run_tox(task, version):
+def run_tox(task, version, *args):
     python = install.python_executable(version)
 
     # Create a version of the name that tox will pick up for the correct
@@ -373,7 +373,7 @@ def run_tox(task, version):
     env["PATH"] = os.path.dirname(python) + ":" + env["PATH"]
     print(env["PATH"])
 
-    pip_tool("tox", "-e", task, env=env, cwd=hp.HYPOTHESIS_PYTHON)
+    pip_tool("tox", "-e", task, *args, env=env, cwd=hp.HYPOTHESIS_PYTHON)
 
 
 # See update_python_versions() above
@@ -449,12 +449,22 @@ def check_pypy38():
     run_tox("pypy3-full", PYPY38)
 
 
+@task()
+def tox(*args):
+    if len(args) < 2:
+        print("Usage: ./build.sh tox TOX_ENV PY_VERSION [tox args]")
+        sys.exit(1)
+    run_tox(args[0], args[1], *args[2:])
+
+
 def standard_tox_task(name):
     TASKS["check-" + name] = python_tests(lambda: run_tox(name, PYMAIN))
 
 
 standard_tox_task("nose")
 standard_tox_task("pytest46")
+standard_tox_task("pytest54")
+standard_tox_task("pytest62")
 
 for n in [22, 32, 40]:
     standard_tox_task(f"django{n}")


Test patch:
diff --git a/hypothesis-python/tests/pytest/test_junit.py b/hypothesis-python/tests/pytest/test_junit.py
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/pytest/test_junit.py
@@ -0,0 +1,73 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Copyright the Hypothesis Authors.
+# Individual contributors are listed in AUTHORS.rst and the git log.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+pytest_plugins = "pytester"
+
+
+TESTSUITE = """
+from hypothesis import given
+from hypothesis.strategies import integers
+
+@given(integers())
+def test_valid(x):
+    assert x == x
+
+@given(integers())
+def test_invalid(x):
+    assert x != x
+"""
+
+
+def _run_and_get_junit(testdir, *args):
+    script = testdir.makepyfile(TESTSUITE)
+    testdir.runpytest(script, "--junit-xml=out.xml", *args)
+    return ET.parse(Path(testdir.tmpdir) / "out.xml").getroot()
+
+
+def _findall_from_root(junit_xml, path):
+    if junit_xml.tag == "testsuites":
+        return junit_xml.findall(f"./testsuite/{path}")
+    # This case only exists for tests against Pytest before 5.1.0;
+    # see https://github.com/pytest-dev/pytest/commit/a43ba78d3bde
+    return junit_xml.findall(f"./{path}")
+
+
+def suite_properties_ok(junit_xml):
+    # Check whether <properties> is included in <testsuite>.  This is currently not
+    # the case when using pytest-xdist, which is a shame, but we can live with it.
+    testsuite_props = _findall_from_root(junit_xml, "properties")
+    return len(testsuite_props) == 1 and {
+        prop.get("name") for prop in testsuite_props[0].findall("property")
+    } == {
+        "hypothesis-statistics-test_outputs_valid_xunit2.py::test_valid",
+        "hypothesis-statistics-test_outputs_valid_xunit2.py::test_invalid",
+    }
+
+
+def test_outputs_valid_xunit2(testdir):
+    # The thing we really care about with pytest-xdist + junitxml is that we don't
+    # break xunit2 compatibility by putting <properties> inside <testcase>.
+    junit_xml = _run_and_get_junit(testdir)
+    testcase_props = _findall_from_root(junit_xml, "testcase/properties")
+    assert len(testcase_props) == 0
+    # Check whether <properties> is included in <testsuite>
+    assert suite_properties_ok(junit_xml)
+
+
+def test_outputs_valid_xunit2_with_xdist(testdir):
+    junit_xml = _run_and_get_junit(testdir, "-n2")
+    testcase_props = _findall_from_root(junit_xml, "testcase/properties")
+    assert len(testcase_props) == 0
+    # If <properties> is included in <testsuite>, this assertion will fail.
+    # That would be a GOOD THING, and we would remove the `not` to prevent regressions.
+    assert not suite_properties_ok(junit_xml)