Repository: pantsbuild/pants

Issue:
Allow to specify the dialect for ScalaParser
**Is your feature request related to a problem? Please describe.**
We are using Scala 2.13 with Source3, so some of scala3 syntax is used in our project, for example, `import p.*`, which should be a wildcard.

**Describe the solution you'd like**
I would like to specify the dialect for ScalaParser from the list here:
https://github.com/scalameta/scalameta/blob/main/scalameta/dialects/shared/src/main/scala/scala/meta/dialects/package.scala



Discussion:
Is there a dialect that is a superset of them all? The parser for dependency inference does not necessarily have to use the exact dialect matching the source code. It just needs to be able to "liberally" parse as much of the code as it can.
Probably scala3?
I think we can also create a dialect that chooses true for all options:
https://github.com/scalameta/scalameta/blob/main/scalameta/dialects/shared/src/main/scala/scala/meta/dialects/package.scala

That sounds like it should work. Would you like to make the change?
Yes. Which one do you prefer Scala3 or all true?
All true since it will be the superset (assuming no problems arise from that).
So I tried the superset dialect, and actually, our source code cannot be analyzed with Scala3 because we have a function called `export` which is a reserved keyword on Scala 3. So I'm worried the superset solution is not going to be that smooth.

I can try to add the dialect to the scala-infer subsystem.
My two cents is that for projects that have both Scala3 and Scala2.XX the ability to define the dialect per resolve might be useful. So in the scala-infer subsystem, the option can be `dialect_for_resolve`.

@tdyas if you the `dialect_for_resolve` solution is viable I can give that a try.



> I can try to add the dialect to the scala-infer subsystem. My two cents is that for projects that have both Scala3 and Scala2.XX the ability to define the dialect per resolve might be useful. So in the scala-infer subsystem, the option can be `dialect_for_resolve`.

Maybe Pants set the dialect from a combination of (1) the Scala version for the resolve and (2) a resolve-specific `bool` option that enables Scala 3 syntax for Scala 2.x versions? Such A "enable Scala 3 syntax" option would avoid duplicating the Scala versions already known from the `--scala-version-for-resolve` option and would be less confusing for a user. Thoughts?

Sounds good to me, does the "enable Scala 3 syntax" is global or per resolve?

Another option is to just assume Scala 3 syntax is enabled without any flag?
> Another option is to just assume Scala 3 syntax is enabled without any flag?

Good idea. Let's do that and then no need for the "enable Scala 3" flag.
@tdyas please checkout https://github.com/pantsbuild/pants/pull/15649

I was managed to support scala3 by checking for `-Xsource:3` in the scalac args.

Resolving patch:
diff --git a/src/python/pants/backend/scala/dependency_inference/scala_parser.py b/src/python/pants/backend/scala/dependency_inference/scala_parser.py
--- a/src/python/pants/backend/scala/dependency_inference/scala_parser.py
+++ b/src/python/pants/backend/scala/dependency_inference/scala_parser.py
@@ -9,8 +9,11 @@
 from dataclasses import dataclass
 from typing import Any, Iterator, Mapping
 
+from pants.backend.scala.subsystems.scala import ScalaSubsystem
+from pants.backend.scala.subsystems.scalac import Scalac
+from pants.build_graph.address import Address
 from pants.core.goals.generate_lockfiles import DEFAULT_TOOL_LOCKFILE, GenerateToolLockfileSentinel
-from pants.core.util_rules.source_files import SourceFiles
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.engine.fs import (
     AddPrefix,
     CreateDigest,
@@ -24,12 +27,15 @@
 from pants.engine.internals.selectors import Get, MultiGet
 from pants.engine.process import FallibleProcessResult, ProcessExecutionFailure, ProcessResult
 from pants.engine.rules import collect_rules, rule
+from pants.engine.target import WrappedTarget
 from pants.engine.unions import UnionRule
 from pants.jvm.compile import ClasspathEntry
 from pants.jvm.jdk_rules import InternalJdk, JvmProcess
 from pants.jvm.resolve.common import ArtifactRequirements, Coordinate
 from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest
 from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool
+from pants.jvm.subsystems import JvmSubsystem
+from pants.jvm.target_types import JvmResolveField
 from pants.option.global_options import ProcessCleanupOption
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
@@ -164,12 +170,40 @@ class ScalaParserCompiledClassfiles(ClasspathEntry):
     pass
 
 
+@dataclass(frozen=True)
+class AnalyzeScalaSourceRequest:
+    source_files: SourceFiles
+    scala_version: str
+    source3: bool
+
+
+@rule(level=LogLevel.DEBUG)
+async def create_analyze_scala_source_request(
+    scala_subsystem: ScalaSubsystem, jvm: JvmSubsystem, scalac: Scalac, request: SourceFilesRequest
+) -> AnalyzeScalaSourceRequest:
+    address = request.sources_fields[0].address
+
+    (wrapped_tgt, source_files) = await MultiGet(
+        Get(WrappedTarget, Address, address),
+        Get(SourceFiles, SourceFilesRequest, request),
+    )
+
+    tgt = wrapped_tgt.target
+    resolve = tgt[JvmResolveField].normalized_value(jvm)
+    scala_version = scala_subsystem.version_for_resolve(resolve)
+    source3 = "-Xsource:3" in scalac.args
+
+    return AnalyzeScalaSourceRequest(source_files, scala_version, source3)
+
+
 @rule(level=LogLevel.DEBUG)
 async def analyze_scala_source_dependencies(
     jdk: InternalJdk,
     processor_classfiles: ScalaParserCompiledClassfiles,
-    source_files: SourceFiles,
+    request: AnalyzeScalaSourceRequest,
 ) -> FallibleScalaSourceDependencyAnalysisResult:
+    source_files = request.source_files
+
     if len(source_files.files) > 1:
         raise ValueError(
             f"analyze_scala_source_dependencies expects sources with exactly 1 source file, but found {len(source_files.snapshot.files)}."
@@ -214,6 +248,8 @@ async def analyze_scala_source_dependencies(
                 "org.pantsbuild.backend.scala.dependency_inference.ScalaParser",
                 analysis_output_path,
                 source_path,
+                request.scala_version,
+                str(request.source3),
             ],
             input_digest=prefixed_source_files_digest,
             extra_immutable_input_digests=extra_immutable_input_digests,


Test patch:
diff --git a/src/python/pants/backend/scala/dependency_inference/scala_parser_test.py b/src/python/pants/backend/scala/dependency_inference/scala_parser_test.py
--- a/src/python/pants/backend/scala/dependency_inference/scala_parser_test.py
+++ b/src/python/pants/backend/scala/dependency_inference/scala_parser_test.py
@@ -7,13 +7,14 @@
 from pants.backend.scala import target_types
 from pants.backend.scala.dependency_inference import scala_parser
 from pants.backend.scala.dependency_inference.scala_parser import (
+    AnalyzeScalaSourceRequest,
     ScalaImport,
     ScalaSourceDependencyAnalysis,
 )
 from pants.backend.scala.target_types import ScalaSourceField, ScalaSourceTarget
 from pants.build_graph.address import Address
 from pants.core.util_rules import source_files
-from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.core.util_rules.source_files import SourceFilesRequest
 from pants.engine.target import SourcesField
 from pants.jvm import jdk_rules
 from pants.jvm import util_rules as jvm_util_rules
@@ -33,8 +34,8 @@ def rule_runner() -> RuleRunner:
             *jdk_rules.rules(),
             *target_types.rules(),
             *jvm_util_rules.rules(),
-            QueryRule(SourceFiles, (SourceFilesRequest,)),
-            QueryRule(ScalaSourceDependencyAnalysis, (SourceFiles,)),
+            QueryRule(AnalyzeScalaSourceRequest, (SourceFilesRequest,)),
+            QueryRule(ScalaSourceDependencyAnalysis, (AnalyzeScalaSourceRequest,)),
         ],
         target_types=[ScalaSourceTarget],
     )
@@ -52,8 +53,8 @@ def _analyze(rule_runner: RuleRunner, source: str) -> ScalaSourceDependencyAnaly
 
     target = rule_runner.get_target(address=Address("", target_name="source"))
 
-    source_files = rule_runner.request(
-        SourceFiles,
+    request = rule_runner.request(
+        AnalyzeScalaSourceRequest,
         [
             SourceFilesRequest(
                 (target.get(SourcesField),),
@@ -63,7 +64,7 @@ def _analyze(rule_runner: RuleRunner, source: str) -> ScalaSourceDependencyAnaly
         ],
     )
 
-    return rule_runner.request(ScalaSourceDependencyAnalysis, [source_files])
+    return rule_runner.request(ScalaSourceDependencyAnalysis, [request])
 
 
 def test_parser_simple(rule_runner: RuleRunner) -> None:
@@ -411,6 +412,28 @@ def test_package_object(rule_runner: RuleRunner) -> None:
     assert sorted(analysis.provided_symbols) == ["foo.bar", "foo.bar.Hello"]
 
 
+def test_source3(rule_runner: RuleRunner) -> None:
+    rule_runner.set_options(
+        args=[
+            "-ldebug",
+            "--scala-version-for-resolve={'jvm-default':'2.13.8'}",
+            "--scalac-args=['-Xsource:3']",
+        ],
+        env_inherit=PYTHON_BOOTSTRAP_ENV,
+    )
+    analysis = _analyze(
+        rule_runner,
+        textwrap.dedent(
+            """
+            package foo
+
+            import bar.*
+            """
+        ),
+    )
+    assert analysis.imports_by_scope.get("foo") == (ScalaImport("bar", None, True),)
+
+
 def test_extract_annotations(rule_runner: RuleRunner) -> None:
     analysis = _analyze(
         rule_runner,