Repository: pantsbuild/pants

Issue:
Add support for collecting global test coverage.
Right now Pants doesn't report on untested files as warned about under 'Coverage will not report on unencountered files' here: https://www.pantsbuild.org/docs/python-test-goal#coverage

This can be problematic if you want a global view of test coverage.


Discussion:
I experimented this morning and its very quick to generate a dummy coverage data file for all sources with `coverage run --source=src/root empty.py`. Combining that with the coverage data files from test runs produces a report that misses no source files at all.
Very cool! I spent a lot of time trying to get this working and could not for the life of me get it figured out. 

This is great news!

Resolving patch:
diff --git a/src/python/pants/backend/python/goals/coverage_py.py b/src/python/pants/backend/python/goals/coverage_py.py
--- a/src/python/pants/backend/python/goals/coverage_py.py
+++ b/src/python/pants/backend/python/goals/coverage_py.py
@@ -36,12 +36,14 @@
 from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest
 from pants.engine.addresses import Address, Addresses
 from pants.engine.fs import (
+    EMPTY_DIGEST,
     AddPrefix,
     CreateDigest,
     Digest,
     DigestContents,
     FileContent,
     MergeDigests,
+    PathGlobs,
     Snapshot,
 )
 from pants.engine.process import ProcessResult
@@ -49,6 +51,7 @@
 from pants.engine.target import TransitiveTargets, TransitiveTargetsRequest
 from pants.engine.unions import UnionRule
 from pants.option.custom_types import file_option
+from pants.source.source_root import AllSourceRoots
 from pants.util.logging import LogLevel
 
 """
@@ -163,6 +166,16 @@ def register_options(cls, register):
                 f"non-standard location."
             ),
         )
+        register(
+            "--global-report",
+            type=bool,
+            default=False,
+            help=(
+                "If true, Pants will generate a global coverage report.\n\nThe global report will "
+                "include all Python source files in the workspace and not just those depended on "
+                "by the tests that were run."
+            ),
+        )
 
     @property
     def filter(self) -> Tuple[str, ...]:
@@ -195,6 +208,10 @@ def config_request(self) -> ConfigFilesRequest:
             },
         )
 
+    @property
+    def global_report(self) -> bool:
+        return cast(bool, self.options.global_report)
+
 
 @dataclass(frozen=True)
 class PytestCoverageData(CoverageData):
@@ -320,29 +337,103 @@ class MergedCoverageData:
 
 @rule(desc="Merge Pytest coverage data", level=LogLevel.DEBUG)
 async def merge_coverage_data(
-    data_collection: PytestCoverageDataCollection, coverage_setup: CoverageSetup
+    data_collection: PytestCoverageDataCollection,
+    coverage_setup: CoverageSetup,
+    coverage: CoverageSubsystem,
+    source_roots: AllSourceRoots,
 ) -> MergedCoverageData:
-    if len(data_collection) == 1:
+    if len(data_collection) == 1 and not coverage.global_report:
         return MergedCoverageData(data_collection[0].digest)
-    # We prefix each .coverage file with its corresponding address to avoid collisions.
-    coverage_digests = await MultiGet(
-        Get(Digest, AddPrefix(data.digest, prefix=data.address.path_safe_spec))
-        for data in data_collection
-    )
-    input_digest = await Get(Digest, MergeDigests(coverage_digests))
-    prefixes = sorted(f"{data.address.path_safe_spec}/.coverage" for data in data_collection)
+
+    coverage_digest_gets = []
+    coverage_data_file_paths = []
+    for data in data_collection:
+        # We prefix each .coverage file with its corresponding address to avoid collisions.
+        coverage_digest_gets.append(
+            Get(Digest, AddPrefix(data.digest, prefix=data.address.path_safe_spec))
+        )
+        coverage_data_file_paths.append(f"{data.address.path_safe_spec}/.coverage")
+
+    if coverage.global_report:
+        global_coverage_base_dir = PurePath("__global_coverage__")
+
+        global_coverage_config_path = global_coverage_base_dir / "pyproject.toml"
+        global_coverage_config_content = toml.dumps(
+            {
+                "tool": {
+                    "coverage": {
+                        "run": {
+                            "relative_files": True,
+                            "source": list(source_root.path for source_root in source_roots),
+                        }
+                    }
+                }
+            }
+        ).encode()
+
+        no_op_exe_py_path = global_coverage_base_dir / "no-op-exe.py"
+
+        all_sources_digest, no_op_exe_py_digest, global_coverage_config_digest = await MultiGet(
+            Get(
+                Digest,
+                PathGlobs(globs=[f"{source_root.path}/**/*.py" for source_root in source_roots]),
+            ),
+            Get(Digest, CreateDigest([FileContent(path=str(no_op_exe_py_path), content=b"")])),
+            Get(
+                Digest,
+                CreateDigest(
+                    [
+                        FileContent(
+                            path=str(global_coverage_config_path),
+                            content=global_coverage_config_content,
+                        ),
+                    ]
+                ),
+            ),
+        )
+        extra_sources_digest = await Get(
+            Digest, MergeDigests((all_sources_digest, no_op_exe_py_digest))
+        )
+        input_digest = await Get(
+            Digest, MergeDigests((extra_sources_digest, global_coverage_config_digest))
+        )
+        result = await Get(
+            ProcessResult,
+            VenvPexProcess(
+                coverage_setup.pex,
+                argv=("run", "--rcfile", str(global_coverage_config_path), str(no_op_exe_py_path)),
+                input_digest=input_digest,
+                output_files=(".coverage",),
+                description="Create base global Pytest coverage report.",
+                level=LogLevel.DEBUG,
+            ),
+        )
+        coverage_digests = await MultiGet(
+            Get(
+                Digest, AddPrefix(digest=result.output_digest, prefix=str(global_coverage_base_dir))
+            ),
+            *coverage_digest_gets,
+        )
+        coverage_data_file_paths.append(str(global_coverage_base_dir / ".coverage"))
+        input_digest = await Get(Digest, MergeDigests(coverage_digests))
+    else:
+        extra_sources_digest = EMPTY_DIGEST
+        input_digest = await Get(Digest, MergeDigests(await MultiGet(coverage_digest_gets)))
+
     result = await Get(
         ProcessResult,
         VenvPexProcess(
             coverage_setup.pex,
-            argv=("combine", *prefixes),
+            argv=("combine", *sorted(coverage_data_file_paths)),
             input_digest=input_digest,
             output_files=(".coverage",),
-            description=f"Merge {len(prefixes)} Pytest coverage reports.",
+            description=f"Merge {len(coverage_data_file_paths)} Pytest coverage reports.",
             level=LogLevel.DEBUG,
         ),
     )
-    return MergedCoverageData(result.output_digest)
+    return MergedCoverageData(
+        await Get(Digest, MergeDigests((result.output_digest, extra_sources_digest)))
+    )
 
 
 @rule(desc="Generate Pytest coverage reports", level=LogLevel.DEBUG)


Test patch:
diff --git a/src/python/pants/backend/python/goals/coverage_py_integration_test.py b/src/python/pants/backend/python/goals/coverage_py_integration_test.py
--- a/src/python/pants/backend/python/goals/coverage_py_integration_test.py
+++ b/src/python/pants/backend/python/goals/coverage_py_integration_test.py
@@ -57,6 +57,10 @@ def test_add():
         )
         """
     ),
+    "src/python/core/BUILD": "python_library()",
+    "src/python/core/__init__.py": "",
+    "src/python/core/untested.py": "CONSTANT = 42",
+    "foo/bar.py": "BAZ = True",
     # Test that a `tests/` source root accurately gets coverage data for the `src/`
     # root.
     "tests/python/project_test/__init__.py": "",
@@ -113,6 +117,7 @@ def run_coverage(tmpdir: str, *more_args: str) -> PantsResult:
         f"{tmpdir}/tests/python/project_test:multiply",
         f"{tmpdir}/tests/python/project_test:arithmetic",
         f"{tmpdir}/tests/python/project_test/no_src",
+        f"--source-root-patterns=['/{tmpdir}/src/python', '{tmpdir}/tests/python', '{tmpdir}/foo']",
         *more_args,
     ]
     result = run_pants(command)
@@ -148,6 +153,35 @@ def test_coverage() -> None:
     )
 
 
+def test_coverage_global() -> None:
+    with setup_tmpdir(SOURCES) as tmpdir:
+        result = run_coverage(tmpdir, "--coverage-py-global-report")
+    assert (
+        dedent(
+            f"""\
+            Name                                                          Stmts   Miss  Cover
+            ---------------------------------------------------------------------------------
+            {tmpdir}/foo/bar.py                                            1      1     0%
+            {tmpdir}/src/python/core/__init__.py                           0      0   100%
+            {tmpdir}/src/python/core/untested.py                           1      1     0%
+            {tmpdir}/src/python/project/__init__.py                        0      0   100%
+            {tmpdir}/src/python/project/lib.py                             6      0   100%
+            {tmpdir}/src/python/project/lib_test.py                        3      0   100%
+            {tmpdir}/src/python/project/random.py                          2      2     0%
+            {tmpdir}/tests/python/project_test/__init__.py                 0      0   100%
+            {tmpdir}/tests/python/project_test/no_src/BUILD.py             1      1     0%
+            {tmpdir}/tests/python/project_test/no_src/__init__.py          0      0   100%
+            {tmpdir}/tests/python/project_test/no_src/test_no_src.py       2      0   100%
+            {tmpdir}/tests/python/project_test/test_arithmetic.py          3      0   100%
+            {tmpdir}/tests/python/project_test/test_multiply.py            3      0   100%
+            ---------------------------------------------------------------------------------
+            TOTAL                                                            22      5    77%
+            """
+        )
+        in result.stderr
+    ), result.stderr
+
+
 def test_coverage_with_filter() -> None:
     with setup_tmpdir(SOURCES) as tmpdir:
         result = run_coverage(tmpdir, "--coverage-py-filter=['project.lib', 'project_test.no_src']")