Repository: pantsbuild/pants

Issue:
Include only declared dependencies at JVM compilation and runtime
Currently the entire content of a resolve is [available to the compiler](https://github.com/pantsbuild/pants/blob/61bb3bdd3ef555883ce3839f20e663b2a9cf9de1/src/python/pants/backend/java/compile/javac.py#L128-L134): instead, we should expose only the direct `jvm_artifact` dependencies of each target to it.

Likewise, [classpath materialization](https://github.com/pantsbuild/pants/blob/61bb3bdd3ef555883ce3839f20e663b2a9cf9de1/src/python/pants/backend/java/classpath.py#L65-L71) should include only the transitive artifact dependencies.


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/java/compile/javac.py b/src/python/pants/backend/java/compile/javac.py
--- a/src/python/pants/backend/java/compile/javac.py
+++ b/src/python/pants/backend/java/compile/javac.py
@@ -29,11 +29,15 @@
 from pants.jvm.compile import rules as jvm_compile_rules
 from pants.jvm.jdk_rules import JdkSetup
 from pants.jvm.resolve.coursier_fetch import (
+    Coordinate,
+    Coordinates,
     CoursierResolvedLockfile,
     CoursierResolveKey,
+    FilterDependenciesRequest,
     MaterializedClasspath,
     MaterializedClasspathRequest,
 )
+from pants.jvm.target_types import JvmArtifactFieldSet
 from pants.util.logging import LogLevel
 
 logger = logging.getLogger(__name__)
@@ -118,7 +122,19 @@ async def compile_java_source(
             exit_code=0,
         )
 
-    lockfile = await Get(CoursierResolvedLockfile, CoursierResolveKey, request.resolve)
+    filter_coords = Coordinates(
+        (
+            Coordinate.from_jvm_artifact_target(dep)
+            for item in CoarsenedTargets(request.component.dependencies).closure()
+            for dep in item.members
+            if JvmArtifactFieldSet.is_applicable(dep)
+        )
+    )
+
+    unfiltered_lockfile = await Get(CoursierResolvedLockfile, CoursierResolveKey, request.resolve)
+    lockfile = await Get(
+        CoursierResolvedLockfile, FilterDependenciesRequest(filter_coords, unfiltered_lockfile)
+    )
 
     dest_dir = "classfiles"
     (
diff --git a/src/python/pants/backend/scala/compile/scalac.py b/src/python/pants/backend/scala/compile/scalac.py
--- a/src/python/pants/backend/scala/compile/scalac.py
+++ b/src/python/pants/backend/scala/compile/scalac.py
@@ -31,12 +31,15 @@
 from pants.jvm.resolve.coursier_fetch import (
     ArtifactRequirements,
     Coordinate,
+    Coordinates,
     CoursierResolvedLockfile,
     CoursierResolveKey,
+    FilterDependenciesRequest,
     MaterializedClasspath,
     MaterializedClasspathRequest,
 )
 from pants.jvm.resolve.coursier_setup import Coursier
+from pants.jvm.target_types import JvmArtifactFieldSet
 from pants.util.logging import LogLevel
 
 logger = logging.getLogger(__name__)
@@ -99,7 +102,19 @@ async def compile_scala_source(
             exit_code=0,
         )
 
-    lockfile = await Get(CoursierResolvedLockfile, CoursierResolveKey, request.resolve)
+    filter_coords = Coordinates(
+        (
+            Coordinate.from_jvm_artifact_target(dep)
+            for item in CoarsenedTargets(request.component.dependencies).closure()
+            for dep in item.members
+            if JvmArtifactFieldSet.is_applicable(dep)
+        )
+    )
+
+    unfiltered_lockfile = await Get(CoursierResolvedLockfile, CoursierResolveKey, request.resolve)
+    lockfile = await Get(
+        CoursierResolvedLockfile, FilterDependenciesRequest(filter_coords, unfiltered_lockfile)
+    )
 
     transitive_dependency_classfiles_fallible = await MultiGet(
         Get(
diff --git a/src/python/pants/jvm/resolve/coursier_fetch.py b/src/python/pants/jvm/resolve/coursier_fetch.py
--- a/src/python/pants/jvm/resolve/coursier_fetch.py
+++ b/src/python/pants/jvm/resolve/coursier_fetch.py
@@ -28,10 +28,14 @@
 )
 from pants.engine.process import BashBinary, Process, ProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
-from pants.engine.target import Targets, TransitiveTargets, TransitiveTargetsRequest
+from pants.engine.target import Target, Targets, TransitiveTargets, TransitiveTargetsRequest
 from pants.jvm.resolve.coursier_setup import Coursier
 from pants.jvm.subsystems import JvmSubsystem
 from pants.jvm.target_types import (
+    JvmArtifactArtifactField,
+    JvmArtifactFieldSet,
+    JvmArtifactGroupField,
+    JvmArtifactVersionField,
     JvmCompatibleResolveNamesField,
     JvmLockfileSources,
     JvmRequirementsField,
@@ -91,6 +95,22 @@ def to_coord_str(self, versioned: bool = True) -> str:
             return f"{self.group}:{self.artifact}:{self.version}"
         return f"{self.group}:{self.artifact}"
 
+    @staticmethod
+    def from_jvm_artifact_target(target: Target) -> Coordinate:
+        if not JvmArtifactFieldSet.is_applicable(target):
+            raise CoursierError(
+                "`Coordinate.from_jvm_artifact_target()` only works on targets with "
+                "`JvmArtifactFieldSet` fields present."
+            )
+
+        group = target[JvmArtifactGroupField].value
+        artifact = target[JvmArtifactArtifactField].value
+        version = target[JvmArtifactVersionField].value
+
+        # These are all required, but mypy doesn't think so.
+        assert group is not None and artifact is not None and version is not None
+        return Coordinate(group=group, artifact=artifact, version=version)
+
 
 class Coordinates(DeduplicatedCollection[Coordinate]):
     """An ordered list of `Coordinate`s."""
@@ -316,6 +336,43 @@ async def coursier_resolve_lockfile(
     )
 
 
+@dataclass(frozen=True)
+class FilterDependenciesRequest:
+    direct_dependencies: Coordinates
+    lockfile: CoursierResolvedLockfile
+    transitive: bool = True
+
+
+@rule
+async def filter_for_dependencies(
+    request: FilterDependenciesRequest,
+) -> CoursierResolvedLockfile:
+    """Returns a filtered version of the input lockfile that only contains the direct dependencies
+    of a given target.
+
+    This should reduce the volume of files that are copied to various compilation tasks.
+    """
+
+    # Assumption: all coordinates in the resolved lockfile will be compatible with the
+    # direct_dependencies in some way, so we do not need to do version testing here.
+
+    entries = {(i.coord.group, i.coord.artifact): i for i in request.lockfile.entries}
+
+    dependencies = {(i.group, i.artifact) for i in request.direct_dependencies}
+
+    if request.transitive:
+        # Coursier already stores the transitive dependencies (`dependencies` vs `direct`
+        # dependencies), so we can just iterate through those to get the full transitive dep list.
+        dependencies |= {
+            (j.group, j.artifact)
+            for dependency in dependencies
+            for j in entries[dependency].dependencies
+        }
+
+    entries_out = (entries[dependency] for dependency in dependencies)
+    return CoursierResolvedLockfile(entries=tuple(entries_out))
+
+
 @dataclass(frozen=True)
 class FetchOneCoordRequest:
     coord: Coordinate


Test patch:
diff --git a/src/python/pants/backend/scala/compile/scalac_test.py b/src/python/pants/backend/scala/compile/scalac_test.py
--- a/src/python/pants/backend/scala/compile/scalac_test.py
+++ b/src/python/pants/backend/scala/compile/scalac_test.py
@@ -272,6 +272,7 @@ def test_compile_with_maven_deps(rule_runner: RuleRunner) -> None:
                 )
                 scala_sources(
                     name = 'main',
+                    dependencies = [":joda-time_joda-time"],
                 )
                 """
             ),
@@ -306,7 +307,7 @@ def main(args: Array[String]): Unit = {
 
 
 @maybe_skip_jdk_test
-def test_compile_with_missing_maven_dep_fails(rule_runner: RuleRunner) -> None:
+def test_compile_with_undeclared_jvm_artifact_target_fails(rule_runner: RuleRunner) -> None:
     rule_runner.write_files(
         {
             "BUILD": dedent(
@@ -345,3 +346,52 @@ def main(args: Array[String]): Unit = {
     fallible_result = rule_runner.request(FallibleCompiledClassfiles, [request])
     assert fallible_result.result == CompileResult.FAILED and fallible_result.stderr
     assert "error: object joda is not a member of package org" in fallible_result.stderr
+
+
+@maybe_skip_jdk_test
+def test_compile_with_undeclared_jvm_artifact_dependency_fails(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "BUILD": dedent(
+                """\
+                jvm_artifact(
+                    name = "joda-time_joda-time",
+                    group = "joda-time",
+                    artifact = "joda-time",
+                    version = "2.10.10",
+                )
+                scala_sources(
+                    name = 'main',
+                    dependencies = [],  # `joda-time` needs to be here for compile to succeed
+                )
+                """
+            ),
+            "coursier_resolve.lockfile": CoursierResolvedLockfile(entries=())
+            .to_json()
+            .decode("utf-8"),
+            "Example.scala": dedent(
+                """
+                package org.pantsbuild.example
+
+                import org.joda.time.DateTime
+
+                object Main {
+                    def main(args: Array[String]): Unit = {
+                        val dt = new DateTime()
+                        println(dt.getYear)
+                    }
+                }
+                """
+            ),
+        }
+    )
+
+    request = CompileScalaSourceRequest(
+        component=expect_single_expanded_coarsened_target(
+            rule_runner, Address(spec_path="", target_name="main")
+        ),
+        resolve=make_resolve(rule_runner),
+    )
+    fallible_result = rule_runner.request(FallibleCompiledClassfiles, [request])
+    assert fallible_result.result == CompileResult.FAILED and fallible_result.stderr
+    assert "error: object joda is not a member of package org" in fallible_result.stderr
diff --git a/src/python/pants/jvm/resolve/coursier_fetch_filter_test.py b/src/python/pants/jvm/resolve/coursier_fetch_filter_test.py
new file mode 100644
--- /dev/null
+++ b/src/python/pants/jvm/resolve/coursier_fetch_filter_test.py
@@ -0,0 +1,117 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from typing import DefaultDict, Sequence
+from unittest import mock
+
+import pytest
+
+from pants.core.util_rules.external_tool import rules as external_tool_rules
+from pants.jvm.resolve.coursier_fetch import (
+    Coordinate,
+    Coordinates,
+    CoursierLockfileEntry,
+    CoursierResolvedLockfile,
+    FilterDependenciesRequest,
+)
+from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
+from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules
+from pants.jvm.util_rules import rules as util_rules
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    rule_runner = RuleRunner(
+        rules=[
+            *coursier_fetch_rules(),
+            *coursier_setup_rules(),
+            *external_tool_rules(),
+            *util_rules(),
+            QueryRule(CoursierResolvedLockfile, (FilterDependenciesRequest,)),
+        ],
+    )
+    return rule_runner
+
+
+coord1 = Coordinate("test", "art1", "1.0.0")
+coord2 = Coordinate("test", "art2", "1.0.0")
+coord3 = Coordinate("test", "art3", "1.0.0")
+coord4 = Coordinate("test", "art4", "1.0.0")
+coord5 = Coordinate("test", "art5", "1.0.0")
+
+
+# No depedencies (coord1)
+# 1 direct dependency, more transitive dependencies (coord2)
+# 1 where direct dependencies provide no transitive dependencies (coord 4)
+# 1 where direct dependencies provide repeated dependencies (coord5)
+direct: dict[Coordinate, set[Coordinate]] = {
+    coord1: set(),
+    coord2: {
+        coord3,
+    },  # 1, 2, 3, 4, 5
+    coord3: {coord1, coord4, coord5},  # 1, 3, 4, 5
+    coord4: {
+        coord1,
+    },  # 1, 4
+    coord5: {coord1, coord4},  # 1, 4, 5
+}
+
+
+@pytest.fixture
+def lockfile() -> CoursierResolvedLockfile:
+
+    # Calculate transitive deps
+    transitive_ = {(i, k) for i, j in direct.items() for k in j}
+    while True:
+        old_len = len(transitive_)
+        transitive_ |= {(i, k) for i, j in transitive_ for k in direct[j]}
+        if old_len == len(transitive_):
+            break
+    transitive = DefaultDict(set)
+    for (i, j) in transitive_:
+        transitive[i].add(j)
+
+    entries = (
+        CoursierLockfileEntry(
+            coord=coord,
+            file_name=f"{coord.artifact}.jar",
+            direct_dependencies=Coordinates(direct[coord]),
+            dependencies=Coordinates(transitive[coord]),
+            file_digest=mock.Mock(),
+        )
+        for coord in direct
+    )
+
+    return CoursierResolvedLockfile(entries=tuple(entries))
+
+
+def test_no_deps(rule_runner: RuleRunner, lockfile: CoursierResolvedLockfile) -> None:
+    filtered = filter(rule_runner, [coord1], lockfile, False)
+    assert filtered == [coord1]
+
+
+def test_filter_non_transitive_ignores_deps(
+    rule_runner: RuleRunner, lockfile: CoursierResolvedLockfile
+) -> None:
+    filtered = filter(rule_runner, [coord2], lockfile, False)
+    assert filtered == [coord2]
+
+
+def test_filter_transitive_includes_transitive_deps(
+    rule_runner: RuleRunner, lockfile: CoursierResolvedLockfile
+) -> None:
+    filtered = filter(rule_runner, [coord2], lockfile, True)
+    assert set(filtered) == {coord1, coord2, coord3, coord4, coord5}
+    # Entries should only appear once.
+    assert len(filtered) == 5
+
+
+def filter(rule_runner, coordinates, lockfile, transitive) -> Sequence[Coordinate]:
+    filtered = rule_runner.request(
+        CoursierResolvedLockfile,
+        [FilterDependenciesRequest(Coordinates(coordinates), lockfile, transitive)],
+    )
+    return list(i.coord for i in filtered.entries)