Repository: HypothesisWorks/hypothesis

Issue:
Coverage ignores .coveragerc
Upgrading `hypothesis` beyond `3.29.0` broke our test setup: apparently, `hypothesis` now interferes with `coverage` and makes it ignore `.coveragerc`, causing lots of entries in the coverage report and violating `--fail-under=100`. (The entries now include missed paths in installed packages and in Python itself.)

(Also, the upgrade tripled test running time and caused deprecation warnings of the type "test took too long"; possibly related: #879.)

Versions:
- Python: `3.6.2`
- `hypothesis`: `3.30.0`
- `coverage`: `4.4.1`
- (`virtualenv`: `15.1.0`)

Setting `use_coverage` to `False` fixed these issues.


Discussion:
Whoops, sorry! Ignoring coveragerc for our own internal usage is intended behaviour, but it certainly shouldn't be propagating excluded paths back to the enclosing coverage. That's pretty bad. I'll try to get a fix out in the next couple of days.

Resolving patch:
diff --git a/src/hypothesis/core.py b/src/hypothesis/core.py
--- a/src/hypothesis/core.py
+++ b/src/hypothesis/core.py
@@ -538,22 +538,6 @@ def hypothesis_check_include(filename):  # pragma: no cover
     return filename.endswith('.py')
 
 
-def hypothesis_should_trace(original_filename, frame):  # pragma: no cover
-    disp = FileDisposition()
-    assert original_filename is not None
-    disp.original_filename = original_filename
-    disp.canonical_filename = encoded_filepath(
-        canonical_filename(original_filename))
-    disp.source_filename = disp.canonical_filename
-    disp.reason = ''
-    disp.file_tracer = None
-    disp.has_dynamic_filename = False
-    disp.trace = hypothesis_check_include(disp.canonical_filename)
-    if not disp.trace:
-        disp.reason = 'hypothesis internal reasons'
-    return disp
-
-
 def escalate_warning(msg, slug=None):  # pragma: no cover
     if slug is not None:
         msg = '%s (%s)' % (msg, slug)
@@ -593,6 +577,8 @@ def __init__(self, test_runner, search_strategy, test, settings, random):
         self.__was_flaky = False
         self.random = random
         self.__warned_deadline = False
+        self.__existing_collector = None
+
         if self.settings.deadline is None:
             self.test = test
         else:
@@ -625,6 +611,7 @@ def timed_test(*args, **kwargs):
             self.test = timed_test
 
         self.coverage_data = CoverageData()
+        self.files_to_propagate = set()
 
         if settings.use_coverage and not IN_COVERAGE_TESTS:  # pragma: no cover
             if Collector._collectors:
@@ -633,7 +620,7 @@ def timed_test(*args, **kwargs):
             self.collector = Collector(
                 branch=True,
                 timid=FORCE_PURE_TRACER,
-                should_trace=hypothesis_should_trace,
+                should_trace=self.should_trace,
                 check_include=hypothesis_check_include,
                 concurrency='thread',
                 warn=escalate_warning,
@@ -647,7 +634,28 @@ def timed_test(*args, **kwargs):
         else:
             self.collector = None
 
+    def should_trace(self, original_filename, frame):  # pragma: no cover
+        disp = FileDisposition()
+        assert original_filename is not None
+        disp.original_filename = original_filename
+        disp.canonical_filename = encoded_filepath(
+            canonical_filename(original_filename))
+        disp.source_filename = disp.canonical_filename
+        disp.reason = ''
+        disp.file_tracer = None
+        disp.has_dynamic_filename = False
+        disp.trace = hypothesis_check_include(disp.canonical_filename)
+        if not disp.trace:
+            disp.reason = 'hypothesis internal reasons'
+        elif self.__existing_collector is not None:
+            check = self.__existing_collector.should_trace(
+                original_filename, frame)
+            if check.trace:
+                self.files_to_propagate.add(check.canonical_filename)
+        return disp
+
     def hijack_collector(self, collector):  # pragma: no cover
+        self.__existing_collector = collector
         original_save_data = collector.save_data
 
         def save_data(covdata):
@@ -657,14 +665,14 @@ def save_data(covdata):
                     filename: {
                         arc: None
                         for arc in self.coverage_data.arcs(filename)}
-                    for filename in self.coverage_data.measured_files()
+                    for filename in self.files_to_propagate
                 })
             else:
                 covdata.add_lines({
                     filename: {
                         line: None
                         for line in self.coverage_data.lines(filename)}
-                    for filename in self.coverage_data.measured_files()
+                    for filename in self.files_to_propagate
                 })
             collector.save_data = original_save_data
         collector.save_data = save_data


Test patch:
diff --git a/tests/nocover/test_coverage.py b/tests/nocover/test_coverage.py
--- a/tests/nocover/test_coverage.py
+++ b/tests/nocover/test_coverage.py
@@ -81,7 +81,7 @@ def test(a, b, c):
         some_function_to_test(a, b, c)
 
     cov = Coverage(
-        config_file=False, data_file=str(tmpdir.join('.coveragerc')),
+        config_file=False, data_file=str(tmpdir.join('.coverage')),
         branch=branch, timid=timid,
     )
     cov._warn = escalate_warning
@@ -93,3 +93,28 @@ def test(a, b, c):
     lines = data.lines(__file__)
     for i in hrange(LINE_START + 1, LINE_END + 1):
         assert i in lines
+
+
+def rnd():
+    import random
+    return random.gauss(0, 1)
+
+
+@pytest.mark.parametrize('branch', [False, True])
+@pytest.mark.parametrize('timid', [False, True])
+def test_does_not_trace_files_outside_inclusion(tmpdir, branch, timid):
+    @given(st.booleans())
+    def test(a):
+        rnd()
+
+    cov = Coverage(
+        config_file=False, data_file=str(tmpdir.join('.coverage')),
+        branch=branch, timid=timid, include=[__file__],
+    )
+    cov._warn = escalate_warning
+    cov.start()
+    test()
+    cov.stop()
+
+    data = cov.get_data()
+    assert len(list(data.measured_files())) == 1