Repository: pantsbuild/pants

Issue:
Add support for `files` in `scalatest`/`junit`
See https://www.pantsbuild.org/docs/resources#files for information on how these are provided in python, which should be 95% accurate for the JVM.


Discussion:

Resolving patch:
diff --git a/src/python/pants/jvm/non_jvm_dependencies.py b/src/python/pants/jvm/non_jvm_dependencies.py
new file mode 100644
--- /dev/null
+++ b/src/python/pants/jvm/non_jvm_dependencies.py
@@ -0,0 +1,62 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from dataclasses import dataclass
+
+from pants.core.target_types import (
+    FilesGeneratingSourcesField,
+    FileSourceField,
+    RelocatedFilesSourcesField,
+)
+from pants.engine.internals.native_engine import EMPTY_DIGEST
+from pants.engine.rules import collect_rules, rule
+from pants.engine.target import FieldSet
+from pants.engine.unions import UnionRule
+from pants.jvm.compile import (
+    ClasspathEntry,
+    ClasspathEntryRequest,
+    CompileResult,
+    FallibleClasspathEntry,
+)
+
+
+@dataclass(frozen=True)
+class FileFieldSet(FieldSet):
+    required_fields = (FileSourceField,)
+    sources: FileSourceField
+
+
+@dataclass(frozen=True)
+class FilesGeneratorFieldSet(FieldSet):
+    required_fields = (FilesGeneratingSourcesField,)
+    sources: FilesGeneratingSourcesField
+
+
+@dataclass(frozen=True)
+class RelocatedFilesFieldSet(FieldSet):
+    required_fields = (RelocatedFilesSourcesField,)
+    sources: RelocatedFilesSourcesField
+
+
+class NoopClasspathEntryRequest(ClasspathEntryRequest):
+    field_sets = (FileFieldSet, FilesGeneratorFieldSet, RelocatedFilesFieldSet)
+
+
+@rule(desc="Compile with javac")
+async def noop_classpath_entry(
+    request: NoopClasspathEntryRequest,
+) -> FallibleClasspathEntry:
+
+    return FallibleClasspathEntry(
+        f"Empty classpath for no-op classpath target {request.component}",
+        CompileResult.SUCCEEDED,
+        ClasspathEntry(EMPTY_DIGEST, [], []),
+        exit_code=0,
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(ClasspathEntryRequest, NoopClasspathEntryRequest),
+    ]


Test patch:
diff --git a/src/python/pants/backend/scala/test/scalatest.py b/src/python/pants/backend/scala/test/scalatest.py
--- a/src/python/pants/backend/scala/test/scalatest.py
+++ b/src/python/pants/backend/scala/test/scalatest.py
@@ -8,6 +8,8 @@
 from pants.backend.scala.target_types import ScalatestTestSourceField
 from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
 from pants.core.goals.test import TestDebugRequest, TestFieldSet, TestResult, TestSubsystem
+from pants.core.target_types import FileSourceField
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.engine.addresses import Addresses
 from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs, RemovePrefix, Snapshot
 from pants.engine.process import (
@@ -18,6 +20,7 @@
     ProcessCacheScope,
 )
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.target import Dependencies, DependenciesRequest, SourcesField, Targets
 from pants.engine.unions import UnionRule
 from pants.jvm.classpath import Classpath
 from pants.jvm.goals import lockfile
@@ -40,6 +43,7 @@ class ScalatestTestFieldSet(TestFieldSet):
 
     sources: ScalatestTestSourceField
     jdk_version: JvmJdkField
+    dependencies: Dependencies
 
 
 class ScalatestToolLockfileSentinel(GenerateToolLockfileSentinel):
@@ -65,17 +69,27 @@ async def setup_scalatest_for_target(
     scalatest: Scalatest,
     test_subsystem: TestSubsystem,
 ) -> TestSetup:
-    jdk = await Get(
-        JdkEnvironment, JdkRequest, JdkRequest.from_field(request.field_set.jdk_version)
+
+    jdk, dependencies = await MultiGet(
+        Get(JdkEnvironment, JdkRequest, JdkRequest.from_field(request.field_set.jdk_version)),
+        Get(Targets, DependenciesRequest(request.field_set.dependencies)),
     )
 
     lockfile_request = await Get(GenerateJvmLockfileFromTool, ScalatestToolLockfileSentinel())
-    classpath, scalatest_classpath = await MultiGet(
+    classpath, scalatest_classpath, files = await MultiGet(
         Get(Classpath, Addresses([request.field_set.address])),
         Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)),
+        Get(
+            SourceFiles,
+            SourceFilesRequest(
+                (dep.get(SourcesField) for dep in dependencies),
+                for_sources_types=(FileSourceField,),
+                enable_codegen=True,
+            ),
+        ),
     )
 
-    merged_classpath_digest = await Get(Digest, MergeDigests(classpath.digests()))
+    input_digest = await Get(Digest, MergeDigests((*classpath.digests(), files.snapshot.digest)))
 
     toolcp_relpath = "__toolcp"
     extra_immutable_input_digests = {
@@ -115,7 +129,7 @@ async def setup_scalatest_for_target(
             reports_dir,
             *scalatest.options.args,
         ],
-        input_digest=merged_classpath_digest,
+        input_digest=input_digest,
         extra_immutable_input_digests=extra_immutable_input_digests,
         output_directories=(reports_dir,),
         description=f"Run Scalatest runner for {request.field_set.address}",
diff --git a/src/python/pants/backend/scala/test/scalatest_test.py b/src/python/pants/backend/scala/test/scalatest_test.py
--- a/src/python/pants/backend/scala/test/scalatest_test.py
+++ b/src/python/pants/backend/scala/test/scalatest_test.py
@@ -20,11 +20,13 @@
 from pants.backend.scala.test.scalatest import rules as scalatest_rules
 from pants.build_graph.address import Address
 from pants.core.goals.test import TestResult
+from pants.core.target_types import FilesGeneratorTarget, FileTarget, RelocatedFiles
 from pants.core.util_rules import config_files, source_files
 from pants.engine.addresses import Addresses
 from pants.engine.target import CoarsenedTargets
 from pants.jvm import classpath
 from pants.jvm.jdk_rules import rules as jdk_util_rules
+from pants.jvm.non_jvm_dependencies import rules as non_jvm_dependencies_rules
 from pants.jvm.resolve.common import Coordinate
 from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
 from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules
@@ -46,6 +48,7 @@ def rule_runner() -> RuleRunner:
             *coursier_fetch_rules(),
             *coursier_setup_rules(),
             *jdk_util_rules(),
+            *non_jvm_dependencies_rules(),
             *scalac_rules(),
             *scalatest_rules(),
             *scala_target_types_rules(),
@@ -59,6 +62,9 @@ def rule_runner() -> RuleRunner:
         ],
         target_types=[
             JvmArtifactTarget,
+            FileTarget,
+            FilesGeneratorTarget,
+            RelocatedFiles,
             ScalaSourcesGeneratorTarget,
             ScalatestTestsGeneratorTarget,
         ],
@@ -117,6 +123,66 @@ class SimpleSpec extends AnyFunSpec {
     assert test_result.xml_results and test_result.xml_results.files
 
 
+@maybe_skip_jdk_test
+def test_file_deps_success(rule_runner: RuleRunner) -> None:
+    scalatest_coord = Coordinate(group="org.scalatest", artifact="scalatest_2.13", version="3.2.10")
+    rule_runner.write_files(
+        {
+            "3rdparty/jvm/default.lock": importlib.resources.read_text(
+                *Scalatest.default_lockfile_resource
+            ),
+            "BUILD": dedent(
+                f"""\
+                jvm_artifact(
+                  name = 'org.scalatest_scalatest',
+                  group = '{scalatest_coord.group}',
+                  artifact = '{scalatest_coord.artifact}',
+                  version = '{scalatest_coord.version}',
+                )
+
+                scalatest_tests(
+                    name='example-test',
+                    dependencies= [
+                        ':org.scalatest_scalatest',
+                        ':ducks',
+                    ],
+                )
+
+                file(
+                    name="ducks",
+                    source="ducks.txt",
+                )
+
+                """
+            ),
+            "SimpleSpec.scala": dedent(
+                """
+                package org.pantsbuild.example;
+
+                import org.scalatest.funspec.AnyFunSpec
+                import java.nio.file.Files
+                import java.nio.file.Path
+
+                class SimpleSpec extends AnyFunSpec {
+                  describe("Ducks") {
+                    it("should be ducks") {
+                      assert(Files.readString(Path.of("ducks.txt")) == "lol ducks")
+                    }
+                  }
+                }
+                """
+            ),
+            "ducks.txt": "lol ducks",
+        }
+    )
+
+    test_result = run_scalatest_test(rule_runner, "example-test", "SimpleSpec.scala")
+
+    assert test_result.exit_code == 0
+    assert "Tests: succeeded 1, failed 0, canceled 0, ignored 0, pending 0" in test_result.stdout
+    assert test_result.xml_results and test_result.xml_results.files
+
+
 def run_scalatest_test(
     rule_runner: RuleRunner, target_name: str, relative_file_path: str
 ) -> TestResult:
diff --git a/src/python/pants/jvm/test/junit.py b/src/python/pants/jvm/test/junit.py
--- a/src/python/pants/jvm/test/junit.py
+++ b/src/python/pants/jvm/test/junit.py
@@ -7,6 +7,8 @@
 from pants.backend.java.subsystems.junit import JUnit
 from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
 from pants.core.goals.test import TestDebugRequest, TestFieldSet, TestResult, TestSubsystem
+from pants.core.target_types import FileSourceField
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.engine.addresses import Addresses
 from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs, RemovePrefix, Snapshot
 from pants.engine.process import (
@@ -17,6 +19,7 @@
     ProcessCacheScope,
 )
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.target import Dependencies, DependenciesRequest, SourcesField, Targets
 from pants.engine.unions import UnionRule
 from pants.jvm.classpath import Classpath
 from pants.jvm.goals import lockfile
@@ -39,6 +42,7 @@ class JunitTestFieldSet(TestFieldSet):
 
     sources: JunitTestSourceField
     jdk_version: JvmJdkField
+    dependencies: Dependencies
 
 
 class JunitToolLockfileSentinel(GenerateToolLockfileSentinel):
@@ -65,17 +69,26 @@ async def setup_junit_for_target(
     test_subsystem: TestSubsystem,
 ) -> TestSetup:
 
-    jdk = await Get(
-        JdkEnvironment, JdkRequest, JdkRequest.from_field(request.field_set.jdk_version)
+    jdk, dependencies = await MultiGet(
+        Get(JdkEnvironment, JdkRequest, JdkRequest.from_field(request.field_set.jdk_version)),
+        Get(Targets, DependenciesRequest(request.field_set.dependencies)),
     )
 
     lockfile_request = await Get(GenerateJvmLockfileFromTool, JunitToolLockfileSentinel())
-    classpath, junit_classpath = await MultiGet(
+    classpath, junit_classpath, files = await MultiGet(
         Get(Classpath, Addresses([request.field_set.address])),
         Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)),
+        Get(
+            SourceFiles,
+            SourceFilesRequest(
+                (dep.get(SourcesField) for dep in dependencies),
+                for_sources_types=(FileSourceField,),
+                enable_codegen=True,
+            ),
+        ),
     )
 
-    merged_classpath_digest = await Get(Digest, MergeDigests(classpath.digests()))
+    input_digest = await Get(Digest, MergeDigests((*classpath.digests(), files.snapshot.digest)))
 
     toolcp_relpath = "__toolcp"
     extra_immutable_input_digests = {
@@ -112,7 +125,7 @@ async def setup_junit_for_target(
             reports_dir,
             *junit.options.args,
         ],
-        input_digest=merged_classpath_digest,
+        input_digest=input_digest,
         extra_immutable_input_digests=extra_immutable_input_digests,
         output_directories=(reports_dir,),
         description=f"Run JUnit 5 ConsoleLauncher against {request.field_set.address}",
diff --git a/src/python/pants/jvm/test/junit_test.py b/src/python/pants/jvm/test/junit_test.py
--- a/src/python/pants/jvm/test/junit_test.py
+++ b/src/python/pants/jvm/test/junit_test.py
@@ -5,6 +5,7 @@
 
 import re
 from textwrap import dedent
+from typing import Iterable
 
 import pytest
 
@@ -16,6 +17,7 @@
 from pants.backend.scala.target_types import rules as scala_target_types_rules
 from pants.build_graph.address import Address
 from pants.core.goals.test import TestResult
+from pants.core.target_types import FilesGeneratorTarget, FileTarget, RelocatedFiles
 from pants.core.util_rules import config_files, source_files
 from pants.core.util_rules.external_tool import rules as external_tool_rules
 from pants.engine.addresses import Addresses
@@ -23,6 +25,7 @@
 from pants.engine.target import CoarsenedTargets
 from pants.jvm import classpath
 from pants.jvm.jdk_rules import rules as java_util_rules
+from pants.jvm.non_jvm_dependencies import rules as non_jvm_dependencies_rules
 from pants.jvm.resolve.common import ArtifactRequirement, Coordinate, Coordinates
 from pants.jvm.resolve.coursier_fetch import CoursierLockfileEntry
 from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
@@ -56,10 +59,14 @@ def rule_runner() -> RuleRunner:
             *source_files.rules(),
             *target_types_rules(),
             *util_rules(),
+            *non_jvm_dependencies_rules(),
             QueryRule(CoarsenedTargets, (Addresses,)),
             QueryRule(TestResult, (JunitTestFieldSet,)),
         ],
         target_types=[
+            FileTarget,
+            FilesGeneratorTarget,
+            RelocatedFiles,
             JvmArtifactTarget,
             JavaSourcesGeneratorTarget,
             JunitTestsGeneratorTarget,
@@ -606,6 +613,101 @@ class SimpleTest {
     assert re.search(r"1 tests found", test_result.stdout) is not None
 
 
+def _write_file_dependencies(rule_runner: RuleRunner, junit_deps: Iterable[str], path_to_read: str):
+    junit_deps_str = ", ".join(f"'{i}'" for i in junit_deps)
+
+    rule_runner.write_files(
+        {
+            "3rdparty/jvm/default.lock": JUNIT4_RESOLVED_LOCKFILE.serialize(
+                [ArtifactRequirement(coordinate=JUNIT_COORD)]
+            ),
+            "BUILD": dedent(
+                f"""\
+                jvm_artifact(
+                  name = 'junit_junit',
+                  group = '{JUNIT_COORD.group}',
+                  artifact = '{JUNIT_COORD.artifact}',
+                  version = '{JUNIT_COORD.version}',
+                )
+                junit_tests(
+                    name='example-test',
+                    dependencies= [
+                        ':junit_junit',
+                        {junit_deps_str}
+                    ],
+                )
+                file(
+                    name="duck",
+                    source="ducks.txt",
+                )
+                files(
+                    name="ducks",
+                    sources=["*.txt"],
+                )
+                relocated_files(
+                    name="relocated_ducks",
+                    files_targets=[":duck"],
+                    src="",
+                    dest="ducks",
+                )
+                """
+            ),
+            "SimpleTest.java": dedent(
+                f"""
+                package org.pantsbuild.example;
+
+                import junit.framework.TestCase;
+                import java.nio.file.Files;
+                import java.nio.file.Path;
+
+                public class SimpleTest extends TestCase {{
+                   public void testHello() throws Exception {{
+                        assertEquals("lol ducks", Files.readString(Path.of("{path_to_read}")));
+                   }}
+                }}
+                """
+            ),
+            "ducks.txt": "lol ducks",
+        }
+    )
+
+
+@maybe_skip_jdk_test
+def test_vintage_file_dependency(rule_runner: RuleRunner) -> None:
+    _write_file_dependencies(rule_runner, [":duck"], "ducks.txt")
+    test_result = run_junit_test(rule_runner, "example-test", "SimpleTest.java")
+
+    assert test_result.exit_code == 0
+    assert re.search(r"Finished:\s+testHello", test_result.stdout) is not None
+    assert re.search(r"1 tests successful", test_result.stdout) is not None
+    assert re.search(r"1 tests found", test_result.stdout) is not None
+
+
+@maybe_skip_jdk_test
+def test_vintage_files_dependencies(rule_runner: RuleRunner) -> None:
+    _write_file_dependencies(rule_runner, [":ducks"], "ducks.txt")
+
+    test_result = run_junit_test(rule_runner, "example-test", "SimpleTest.java")
+
+    assert test_result.exit_code == 0
+    assert re.search(r"Finished:\s+testHello", test_result.stdout) is not None
+    assert re.search(r"1 tests successful", test_result.stdout) is not None
+    assert re.search(r"1 tests found", test_result.stdout) is not None
+
+
+@pytest.mark.skip  # TODO(14537) `relocated_files` doesn't presently work, un-skip when fixing that.
+@maybe_skip_jdk_test
+def test_vintage_relocated_files_dependency(rule_runner: RuleRunner) -> None:
+    _write_file_dependencies(rule_runner, [":relocated_ducks"], "ducks/ducks.txt")
+
+    test_result = run_junit_test(rule_runner, "example-test", "SimpleTest.java")
+
+    assert test_result.exit_code == 0
+    assert re.search(r"Finished:\s+testHello", test_result.stdout) is not None
+    assert re.search(r"1 tests successful", test_result.stdout) is not None
+    assert re.search(r"1 tests found", test_result.stdout) is not None
+
+
 def run_junit_test(
     rule_runner: RuleRunner, target_name: str, relative_file_path: str
 ) -> TestResult: