Repository: pantsbuild/pants

Issue:
Scala import extraction for inference
Inference for Scala will require (at a minimum) import extraction from Scala sources. In v1 this was accomplished with https://scalameta.org/, which still seems to be active.

https://scalameta.org/docs/trees/guide.html#parse-trees


Discussion:

Resolving patch:
diff --git a/src/python/pants/backend/java/subsystems/java_infer.py b/src/python/pants/backend/java/subsystems/java_infer.py
--- a/src/python/pants/backend/java/subsystems/java_infer.py
+++ b/src/python/pants/backend/java/subsystems/java_infer.py
@@ -34,6 +34,7 @@ def register_options(cls, register):
         _default_package_mapping_url = git_url(
             "src/python/pants/backend/java/dependency_inference/jvm_artifact_mappings.py"
         )
+        # TODO: Move to `coursier` or a generic `jvm` subsystem.
         register(
             "--third-party-import-mapping",
             type=dict,
diff --git a/src/python/pants/backend/scala/dependency_inference/rules.py b/src/python/pants/backend/scala/dependency_inference/rules.py
--- a/src/python/pants/backend/scala/dependency_inference/rules.py
+++ b/src/python/pants/backend/scala/dependency_inference/rules.py
@@ -21,6 +21,12 @@
     WrappedTarget,
 )
 from pants.engine.unions import UnionRule
+from pants.jvm.dependency_inference import artifact_mapper
+from pants.jvm.dependency_inference.artifact_mapper import (
+    AvailableThirdPartyArtifacts,
+    ThirdPartyPackageToArtifactMapping,
+    find_artifact_mapping,
+)
 from pants.jvm.dependency_inference.symbol_mapper import FirstPartySymbolMapping
 from pants.util.ordered_set import OrderedSet
 
@@ -36,6 +42,8 @@ async def infer_scala_dependencies_via_source_analysis(
     request: InferScalaSourceDependencies,
     scala_infer_subsystem: ScalaInferSubsystem,
     first_party_symbol_map: FirstPartySymbolMapping,
+    third_party_artifact_mapping: ThirdPartyPackageToArtifactMapping,
+    available_artifacts: AvailableThirdPartyArtifacts,
 ) -> InferredDependencies:
     if not scala_infer_subsystem.imports:
         return InferredDependencies([])
@@ -53,7 +61,11 @@ async def infer_scala_dependencies_via_source_analysis(
 
     dependencies: OrderedSet[Address] = OrderedSet()
     for symbol in symbols:
-        matches = first_party_symbol_map.symbols.addresses_for_symbol(symbol)
+        first_party_matches = first_party_symbol_map.symbols.addresses_for_symbol(symbol)
+        third_party_matches = find_artifact_mapping(
+            symbol, third_party_artifact_mapping, available_artifacts
+        )
+        matches = first_party_matches.union(third_party_matches)
         if not matches:
             continue
 
@@ -74,6 +86,7 @@ async def infer_scala_dependencies_via_source_analysis(
 def rules():
     return [
         *collect_rules(),
+        *artifact_mapper.rules(),
         *scala_parser.rules(),
         *symbol_mapper.rules(),
         UnionRule(InferDependenciesRequest, InferScalaSourceDependencies),


Test patch:
diff --git a/src/python/pants/backend/java/dependency_inference/rules_test.py b/src/python/pants/backend/java/dependency_inference/rules_test.py
--- a/src/python/pants/backend/java/dependency_inference/rules_test.py
+++ b/src/python/pants/backend/java/dependency_inference/rules_test.py
@@ -6,10 +6,6 @@
 import pytest
 
 from pants.backend.java.compile.javac import rules as javac_rules
-from pants.backend.java.dependency_inference.java_parser import rules as java_parser_rules
-from pants.backend.java.dependency_inference.java_parser_launcher import (
-    rules as java_parser_launcher_rules,
-)
 from pants.backend.java.dependency_inference.rules import InferJavaSourceDependencies
 from pants.backend.java.dependency_inference.rules import rules as dep_inference_rules
 from pants.backend.java.target_types import (
@@ -29,11 +25,6 @@
     InferredDependencies,
     Targets,
 )
-from pants.jvm.dependency_inference.artifact_mapper import (
-    FrozenTrieNode,
-    ThirdPartyPackageToArtifactMapping,
-    UnversionedCoordinate,
-)
 from pants.jvm.jdk_rules import rules as java_util_rules
 from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
 from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules
@@ -56,8 +47,6 @@ def rule_runner() -> RuleRunner:
             *coursier_setup_rules(),
             *dep_inference_rules(),
             *external_tool_rules(),
-            *java_parser_launcher_rules(),
-            *java_parser_rules(),
             *java_target_rules(),
             *java_util_rules(),
             *javac_rules(),
@@ -68,7 +57,6 @@ def rule_runner() -> RuleRunner:
             QueryRule(ExplicitlyProvidedDependencies, [DependenciesRequest]),
             QueryRule(InferredDependencies, [InferJavaSourceDependencies]),
             QueryRule(Targets, [UnparsedAddressInputs]),
-            QueryRule(ThirdPartyPackageToArtifactMapping, []),
         ],
         target_types=[JavaSourcesGeneratorTarget, JunitTestsGeneratorTarget, JvmArtifact],
     )
@@ -558,206 +546,3 @@ class C {}
     # B.java does NOT have a dependency on A.java, as it would if we just had subtargets without
     # inferred dependencies.
     assert rule_runner.request(Addresses, [DependenciesRequest(lib[Dependencies])]) == Addresses()
-
-
-@maybe_skip_jdk_test
-def test_third_party_mapping_parsing(rule_runner: RuleRunner) -> None:
-    rule_runner.set_options(
-        [
-            "--java-infer-third-party-import-mapping={'io.github.frenchtoast.savory.**': 'github-frenchtoast:savory'}"
-        ],
-        env_inherit=PYTHON_BOOTSTRAP_ENV,
-    )
-    mapping = rule_runner.request(ThirdPartyPackageToArtifactMapping, [])
-    root_node = mapping.mapping_root
-
-    # Handy trie traversal function to placate mypy
-    def traverse(*children) -> FrozenTrieNode:
-        node = root_node
-        for child in children:
-            new_node = node.find_child(child)
-            if not new_node:
-                coord = ".".join(children)
-                raise Exception(f"Could not find the package specifed by {coord}.")
-            node = new_node
-        return node
-
-    # Verify some provided by `JVM_ARTFACT_MAPPINGS`
-    assert set(traverse("org", "junit").coordinates) == {
-        UnversionedCoordinate(group="junit", artifact="junit")
-    }
-    assert set(traverse("org", "aopalliance").coordinates) == {
-        UnversionedCoordinate(group="aopalliance", artifact="aopalliance")
-    }
-
-    # Verify the one that we provided in the options is there too.
-    assert set(traverse("io", "github", "frenchtoast", "savory").coordinates) == {
-        UnversionedCoordinate(group="github-frenchtoast", artifact="savory")
-    }
-
-
-@maybe_skip_jdk_test
-def test_third_party_dep_inference(rule_runner: RuleRunner) -> None:
-    rule_runner.set_options(
-        ["--java-infer-third-party-import-mapping={'org.joda.time.**': 'joda-time:joda-time'}"],
-        env_inherit=PYTHON_BOOTSTRAP_ENV,
-    )
-    rule_runner.write_files(
-        {
-            "BUILD": dedent(
-                """\
-                jvm_artifact(
-                    name = "joda-time_joda-time",
-                    group = "joda-time",
-                    artifact = "joda-time",
-                    version = "2.10.10",
-                )
-
-                java_sources(name = 'lib')
-                """
-            ),
-            "PrintDate.java": dedent(
-                """\
-                package org.pantsbuild.example;
-
-                import org.joda.time.DateTime;
-
-                public class PrintDate {
-                    public static void main(String[] args) {
-                        DateTime dt = new DateTime();
-                        System.out.println(dt.toString());
-                    }
-                }
-                """
-            ),
-        }
-    )
-
-    lib = rule_runner.get_target(
-        Address("", target_name="lib", relative_file_path="PrintDate.java")
-    )
-    assert rule_runner.request(Addresses, [DependenciesRequest(lib[Dependencies])]) == Addresses(
-        [Address("", target_name="joda-time_joda-time")]
-    )
-
-
-@maybe_skip_jdk_test
-def test_third_party_dep_inference_fqtn(rule_runner: RuleRunner) -> None:
-    rule_runner.set_options(
-        ["--java-infer-third-party-import-mapping={'org.joda.time.**': 'joda-time:joda-time'}"],
-        env_inherit=PYTHON_BOOTSTRAP_ENV,
-    )
-    rule_runner.write_files(
-        {
-            "BUILD": dedent(
-                """\
-                jvm_artifact(
-                    name = "joda-time_joda-time",
-                    group = "joda-time",
-                    artifact = "joda-time",
-                    version = "2.10.10",
-                )
-
-                java_sources(name = 'lib')
-                """
-            ),
-            "PrintDate.java": dedent(
-                """\
-                package org.pantsbuild.example;
-
-                public class PrintDate {
-                    public static void main(String[] args) {
-                        org.joda.time.DateTime dt = new DateTime();
-                        System.out.println(dt.toString());
-                    }
-                }
-                """
-            ),
-        }
-    )
-
-    lib = rule_runner.get_target(
-        Address("", target_name="lib", relative_file_path="PrintDate.java")
-    )
-    assert rule_runner.request(Addresses, [DependenciesRequest(lib[Dependencies])]) == Addresses(
-        [Address("", target_name="joda-time_joda-time")]
-    )
-
-
-@maybe_skip_jdk_test
-def test_third_party_dep_inference_nonrecursive(rule_runner: RuleRunner) -> None:
-    rule_runner.set_options(
-        [
-            "--java-infer-third-party-import-mapping={'org.joda.time.**':'joda-time:joda-time', 'org.joda.time.DateTime':'joda-time:joda-time-2'}"
-        ],
-        env_inherit=PYTHON_BOOTSTRAP_ENV,
-    )
-    rule_runner.write_files(
-        {
-            "BUILD": dedent(
-                """\
-                jvm_artifact(
-                    name = "joda-time_joda-time",
-                    group = "joda-time",
-                    artifact = "joda-time",
-                    version = "2.10.10",
-                )
-
-                jvm_artifact(
-                    name = "joda-time_joda-time-2",
-                    group = "joda-time",
-                    artifact = "joda-time-2",  # doesn't really exist, but useful for this test
-                    version = "2.10.10",
-                )
-
-                java_sources(name = 'lib')
-                """
-            ),
-            "PrintDate.java": dedent(
-                """\
-                package org.pantsbuild.example;
-
-                import org.joda.time.DateTime;
-
-                public class PrintDate {
-                    public static void main(String[] args) {
-                        DateTime dt = new DateTime();
-                        System.out.println(dt.toString());
-                    }
-                }
-                """
-            ),
-            "PrintDate2.java": dedent(
-                """\
-                package org.pantsbuild.example;
-
-                import org.joda.time.LocalDateTime;
-
-                public class PrintDate {
-                    public static void main(String[] args) {
-                        DateTime dt = new LocalDateTime();
-                        System.out.println(dt.toString());
-                    }
-                }
-                """
-            ),
-        }
-    )
-
-    # First test whether the specific import mapping for org.joda.time.DateTime takes effect over the recursive
-    # mapping.
-    lib1 = rule_runner.get_target(
-        Address("", target_name="lib", relative_file_path="PrintDate.java")
-    )
-    assert rule_runner.request(Addresses, [DependenciesRequest(lib1[Dependencies])]) == Addresses(
-        [Address("", target_name="joda-time_joda-time-2")]
-    )
-
-    # Then try a file which should not match the specific import mapping and which will then match the
-    # recursive mapping.
-    lib2 = rule_runner.get_target(
-        Address("", target_name="lib", relative_file_path="PrintDate2.java")
-    )
-    assert rule_runner.request(Addresses, [DependenciesRequest(lib2[Dependencies])]) == Addresses(
-        [Address("", target_name="joda-time_joda-time")]
-    )
diff --git a/src/python/pants/jvm/dependency_inference/artifact_mapper_test.py b/src/python/pants/jvm/dependency_inference/artifact_mapper_test.py
new file mode 100644
--- /dev/null
+++ b/src/python/pants/jvm/dependency_inference/artifact_mapper_test.py
@@ -0,0 +1,258 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from textwrap import dedent
+
+import pytest
+
+from pants.backend.java.compile.javac import rules as javac_rules
+from pants.backend.java.dependency_inference.rules import rules as dep_inference_rules
+from pants.backend.java.target_types import JavaSourcesGeneratorTarget, JunitTestsGeneratorTarget
+from pants.backend.java.target_types import rules as java_target_rules
+from pants.core.util_rules import config_files, source_files
+from pants.core.util_rules.external_tool import rules as external_tool_rules
+from pants.engine.addresses import Address, Addresses
+from pants.engine.target import Dependencies, DependenciesRequest
+from pants.jvm.dependency_inference.artifact_mapper import (
+    FrozenTrieNode,
+    ThirdPartyPackageToArtifactMapping,
+    UnversionedCoordinate,
+)
+from pants.jvm.jdk_rules import rules as java_util_rules
+from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
+from pants.jvm.resolve.coursier_setup import rules as coursier_setup_rules
+from pants.jvm.target_types import JvmArtifact
+from pants.jvm.testutil import maybe_skip_jdk_test
+from pants.jvm.util_rules import rules as util_rules
+from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner
+
+NAMED_RESOLVE_OPTIONS = '--jvm-resolves={"test": "coursier_resolve.lockfile"}'
+DEFAULT_RESOLVE_OPTION = "--jvm-default-resolve=test"
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    rule_runner = RuleRunner(
+        rules=[
+            *config_files.rules(),
+            *coursier_fetch_rules(),
+            *coursier_setup_rules(),
+            *dep_inference_rules(),
+            *external_tool_rules(),
+            *java_target_rules(),
+            *java_util_rules(),
+            *javac_rules(),
+            *source_files.rules(),
+            *util_rules(),
+            QueryRule(Addresses, [DependenciesRequest]),
+            QueryRule(ThirdPartyPackageToArtifactMapping, []),
+        ],
+        target_types=[JavaSourcesGeneratorTarget, JunitTestsGeneratorTarget, JvmArtifact],
+    )
+    rule_runner.set_options(
+        args=[NAMED_RESOLVE_OPTIONS, DEFAULT_RESOLVE_OPTION], env_inherit=PYTHON_BOOTSTRAP_ENV
+    )
+    return rule_runner
+
+
+@maybe_skip_jdk_test
+def test_third_party_mapping_parsing(rule_runner: RuleRunner) -> None:
+    rule_runner.set_options(
+        [
+            "--java-infer-third-party-import-mapping={'io.github.frenchtoast.savory.**': 'github-frenchtoast:savory'}"
+        ],
+        env_inherit=PYTHON_BOOTSTRAP_ENV,
+    )
+    mapping = rule_runner.request(ThirdPartyPackageToArtifactMapping, [])
+    root_node = mapping.mapping_root
+
+    # Handy trie traversal function to placate mypy
+    def traverse(*children) -> FrozenTrieNode:
+        node = root_node
+        for child in children:
+            new_node = node.find_child(child)
+            if not new_node:
+                coord = ".".join(children)
+                raise Exception(f"Could not find the package specifed by {coord}.")
+            node = new_node
+        return node
+
+    # Verify some provided by `JVM_ARTFACT_MAPPINGS`
+    assert set(traverse("org", "junit").coordinates) == {
+        UnversionedCoordinate(group="junit", artifact="junit")
+    }
+    assert set(traverse("org", "aopalliance").coordinates) == {
+        UnversionedCoordinate(group="aopalliance", artifact="aopalliance")
+    }
+
+    # Verify the one that we provided in the options is there too.
+    assert set(traverse("io", "github", "frenchtoast", "savory").coordinates) == {
+        UnversionedCoordinate(group="github-frenchtoast", artifact="savory")
+    }
+
+
+@maybe_skip_jdk_test
+def test_third_party_dep_inference(rule_runner: RuleRunner) -> None:
+    rule_runner.set_options(
+        ["--java-infer-third-party-import-mapping={'org.joda.time.**': 'joda-time:joda-time'}"],
+        env_inherit=PYTHON_BOOTSTRAP_ENV,
+    )
+    rule_runner.write_files(
+        {
+            "BUILD": dedent(
+                """\
+                jvm_artifact(
+                    name = "joda-time_joda-time",
+                    group = "joda-time",
+                    artifact = "joda-time",
+                    version = "2.10.10",
+                )
+
+                java_sources(name = 'lib')
+                """
+            ),
+            "PrintDate.java": dedent(
+                """\
+                package org.pantsbuild.example;
+
+                import org.joda.time.DateTime;
+
+                public class PrintDate {
+                    public static void main(String[] args) {
+                        DateTime dt = new DateTime();
+                        System.out.println(dt.toString());
+                    }
+                }
+                """
+            ),
+        }
+    )
+
+    lib = rule_runner.get_target(
+        Address("", target_name="lib", relative_file_path="PrintDate.java")
+    )
+    assert rule_runner.request(Addresses, [DependenciesRequest(lib[Dependencies])]) == Addresses(
+        [Address("", target_name="joda-time_joda-time")]
+    )
+
+
+@maybe_skip_jdk_test
+def test_third_party_dep_inference_fqtn(rule_runner: RuleRunner) -> None:
+    rule_runner.set_options(
+        ["--java-infer-third-party-import-mapping={'org.joda.time.**': 'joda-time:joda-time'}"],
+        env_inherit=PYTHON_BOOTSTRAP_ENV,
+    )
+    rule_runner.write_files(
+        {
+            "BUILD": dedent(
+                """\
+                jvm_artifact(
+                    name = "joda-time_joda-time",
+                    group = "joda-time",
+                    artifact = "joda-time",
+                    version = "2.10.10",
+                )
+
+                java_sources(name = 'lib')
+                """
+            ),
+            "PrintDate.java": dedent(
+                """\
+                package org.pantsbuild.example;
+
+                public class PrintDate {
+                    public static void main(String[] args) {
+                        org.joda.time.DateTime dt = new DateTime();
+                        System.out.println(dt.toString());
+                    }
+                }
+                """
+            ),
+        }
+    )
+
+    lib = rule_runner.get_target(
+        Address("", target_name="lib", relative_file_path="PrintDate.java")
+    )
+    assert rule_runner.request(Addresses, [DependenciesRequest(lib[Dependencies])]) == Addresses(
+        [Address("", target_name="joda-time_joda-time")]
+    )
+
+
+@maybe_skip_jdk_test
+def test_third_party_dep_inference_nonrecursive(rule_runner: RuleRunner) -> None:
+    rule_runner.set_options(
+        [
+            "--java-infer-third-party-import-mapping={'org.joda.time.**':'joda-time:joda-time', 'org.joda.time.DateTime':'joda-time:joda-time-2'}"
+        ],
+        env_inherit=PYTHON_BOOTSTRAP_ENV,
+    )
+    rule_runner.write_files(
+        {
+            "BUILD": dedent(
+                """\
+                jvm_artifact(
+                    name = "joda-time_joda-time",
+                    group = "joda-time",
+                    artifact = "joda-time",
+                    version = "2.10.10",
+                )
+
+                jvm_artifact(
+                    name = "joda-time_joda-time-2",
+                    group = "joda-time",
+                    artifact = "joda-time-2",  # doesn't really exist, but useful for this test
+                    version = "2.10.10",
+                )
+
+                java_sources(name = 'lib')
+                """
+            ),
+            "PrintDate.java": dedent(
+                """\
+                package org.pantsbuild.example;
+
+                import org.joda.time.DateTime;
+
+                public class PrintDate {
+                    public static void main(String[] args) {
+                        DateTime dt = new DateTime();
+                        System.out.println(dt.toString());
+                    }
+                }
+                """
+            ),
+            "PrintDate2.java": dedent(
+                """\
+                package org.pantsbuild.example;
+
+                import org.joda.time.LocalDateTime;
+
+                public class PrintDate {
+                    public static void main(String[] args) {
+                        DateTime dt = new LocalDateTime();
+                        System.out.println(dt.toString());
+                    }
+                }
+                """
+            ),
+        }
+    )
+
+    # First test whether the specific import mapping for org.joda.time.DateTime takes effect over the recursive
+    # mapping.
+    lib1 = rule_runner.get_target(
+        Address("", target_name="lib", relative_file_path="PrintDate.java")
+    )
+    assert rule_runner.request(Addresses, [DependenciesRequest(lib1[Dependencies])]) == Addresses(
+        [Address("", target_name="joda-time_joda-time-2")]
+    )
+
+    # Then try a file which should not match the specific import mapping and which will then match the
+    # recursive mapping.
+    lib2 = rule_runner.get_target(
+        Address("", target_name="lib", relative_file_path="PrintDate2.java")
+    )
+    assert rule_runner.request(Addresses, [DependenciesRequest(lib2[Dependencies])]) == Addresses(
+        [Address("", target_name="joda-time_joda-time")]
+    )