Repository: pantsbuild/pants

Issue:
Failure to Prepare JDK Environment Behind Proxy
**Describe the bug**
Commands on Scala targets that require a JDK, such as `pants check ::` and `pants lint ::`, fail when the JVM index used by Coursier is unavailable.

**Pants version**
2.18.0

**OS**
Linux

**Additional info**
The following is a sample of the output from a server running behind a proxy, and therefore unable to reach the JVM index used by Coursier:
```
pants --changed-since=### lint
  86.22s	Ensure download of JDK --jvm=zulu:8.0.372.
[INFO] Long running tasks:
  79.06s	Run `scalafmt` on 73 files.
  79.06s	Run `scalafmt` on 18 files.
  79.06s	Run `scalafmt` on 5 files.
[INFO] Long running tasks:
  109.15s	Run `scalafmt` on 73 files.
  109.15s	Run `scalafmt` on 18 files.
  109.15s	Run `scalafmt` on 5 files.
[ERROR] 1 Exception encountered:

Engine traceback:
  in `lint` goal

ProcessExecutionFailure: Process 'Run `scalafmt` on 5 files.' failed with exit code 1.
stdout:

stderr:
Downloading https://github.com/coursier/jvm-index/raw/master/index.json
Downloaded https://github.com/coursier/jvm-index/raw/master/index.json
Exception in thread "main" java.lang.Exception: Error while getting https://github.com/coursier/jvm-index/raw/master/index.json: download error: Caught java.net.ConnectException (Connection timed out) while downloading https://github.com/coursier/jvm-index/raw/master/index.json
	at coursier.jvm.JvmIndex$.$anonfun$load$5(JvmIndex.scala:142)
	at coursier.jvm.JvmIndex$.$anonfun$load$5$adapted(JvmIndex.scala:140)
	at coursier.util.Task$.$anonfun$flatMap$extension$1(Task.scala:14)
	at coursier.util.Task$.$anonfun$flatMap$extension$1$adapted(Task.scala:14)
	at coursier.util.Task$.wrap(Task.scala:82)
	at coursier.util.Task$.$anonfun$flatMap$2(Task.scala:14)
	at scala.concurrent.Future.$anonfun$flatMap$1(Future.scala:307)
	at scala.concurrent.impl.Promise.$anonfun$transformWith$1(Promise.scala:41)
	at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
	at java.base@17.0.5/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base@17.0.5/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base@17.0.5/java.lang.Thread.run(Thread.java:833)
	at com.oracle.svm.core.thread.PlatformThreads.threadStartRoutine(PlatformThreads.java:775)
	at com.oracle.svm.core.posix.thread.PosixPlatformThreads.pthreadStartRoutine(PosixPlatformThreads.java:203)
```



Discussion:
I would like to try resolving this by adding the ability to pass a `jvm-index` argument to Coursier from Pants.  The Coursier argument is described here https://get-coursier.io/docs/cli-java#jvm-index.
That seems like a reasonable approach.

Resolving patch:
diff --git a/src/python/pants/jvm/jdk_rules.py b/src/python/pants/jvm/jdk_rules.py
--- a/src/python/pants/jvm/jdk_rules.py
+++ b/src/python/pants/jvm/jdk_rules.py
@@ -214,19 +214,24 @@ async def prepare_jdk_environment(
     if version is DefaultJdk.SYSTEM:
         coursier_jdk_option = "--system-jvm"
     else:
-        coursier_jdk_option = shlex.quote(f"--jvm={version}")
+        coursier_jdk_option = f"--jvm={version}"
+
+    if not coursier.jvm_index:
+        coursier_options = ["java-home", coursier_jdk_option]
+    else:
+        jvm_index_option = f"--jvm-index={coursier.jvm_index}"
+        coursier_options = ["java-home", jvm_index_option, coursier_jdk_option]
 
     # TODO(#16104) This argument re-writing code should use the native {chroot} support.
     # See also `run` for other argument re-writing code.
     def prefixed(arg: str) -> str:
+        quoted = shlex.quote(arg)
         if arg.startswith("__"):
-            return f"${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{arg}"
+            return f"${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{quoted}"
         else:
-            return arg
+            return quoted
 
-    optionally_prefixed_coursier_args = [
-        prefixed(arg) for arg in coursier.args(["java-home", coursier_jdk_option])
-    ]
+    optionally_prefixed_coursier_args = [prefixed(arg) for arg in coursier.args(coursier_options)]
     # NB: We `set +e` in the subshell to ensure that it exits as well.
     #  see https://unix.stackexchange.com/a/23099
     java_home_command = " ".join(("set +e;", *optionally_prefixed_coursier_args))
diff --git a/src/python/pants/jvm/resolve/coursier_setup.py b/src/python/pants/jvm/resolve/coursier_setup.py
--- a/src/python/pants/jvm/resolve/coursier_setup.py
+++ b/src/python/pants/jvm/resolve/coursier_setup.py
@@ -22,7 +22,7 @@
 from pants.engine.platform import Platform
 from pants.engine.process import Process
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
-from pants.option.option_types import StrListOption
+from pants.option.option_types import StrListOption, StrOption
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 from pants.util.memo import memoized_property
@@ -86,7 +86,6 @@
     """
 )
 
-
 # TODO: Coursier renders setrlimit error line on macOS.
 #   see https://github.com/pantsbuild/pants/issues/13942.
 POST_PROCESS_COURSIER_STDERR_SCRIPT = textwrap.dedent(  # noqa: PNT20
@@ -149,12 +148,26 @@ class CoursierSubsystem(TemplatedExternalTool):
             Maven style repositories to resolve artifacts from.
 
             Coursier will resolve these repositories in the order in which they are
-            specifed, and re-ordering repositories will cause artifacts to be
+            specified, and re-ordering repositories will cause artifacts to be
             re-downloaded. This can result in artifacts in lockfiles becoming invalid.
             """
         ),
     )
 
+    jvm_index = StrOption(
+        default="",
+        help=softwrap(
+            """
+            The JVM index to be used by Coursier.
+
+            Possible values are:
+              - cs: The default JVM index used and maintained by Coursier.
+              - cs-maven: Fetches a JVM index from the io.get-coursier:jvm-index Maven repository.
+              - <URL>: An arbitrary URL for a JVM index. Ex. https://url/of/your/index.json
+            """
+        ),
+    )
+
     def generate_exe(self, plat: Platform) -> str:
         tool_version = self.known_version(plat)
         url = (tool_version and tool_version.url_override) or self.generate_url(plat)
@@ -170,6 +183,7 @@ class Coursier:
     coursier: DownloadedExternalTool
     _digest: Digest
     repos: FrozenOrderedSet[str]
+    jvm_index: str
     _append_only_caches: FrozenDict[str, str]
 
     bin_dir: ClassVar[str] = "__coursier"
@@ -314,6 +328,7 @@ async def setup_coursier(
             ),
         ),
         repos=FrozenOrderedSet(coursier_subsystem.repos),
+        jvm_index=coursier_subsystem.jvm_index,
         _append_only_caches=python.APPEND_ONLY_CACHES,
     )
 


Test patch:
diff --git a/src/python/pants/jvm/jdk_rules_test.py b/src/python/pants/jvm/jdk_rules_test.py
--- a/src/python/pants/jvm/jdk_rules_test.py
+++ b/src/python/pants/jvm/jdk_rules_test.py
@@ -215,3 +215,34 @@ def test_pass_jvm_options_to_java_program(rule_runner: RuleRunner) -> None:
     assert "java.specification.version=11" in jvm_properties
     assert "pants.jvm.global=true" in jvm_properties
     assert "pants.jvm.extra=true" in jvm_properties
+
+
+@maybe_skip_jdk_test
+def test_jvm_not_found_when_empty_jvm_index(rule_runner: RuleRunner) -> None:
+    # Prepare empty JVM Index file
+    filename = "index.json"
+    file_content = textwrap.dedent(
+        """\
+        {}
+        """
+    )
+    rule_runner.write_files({filename: file_content})
+
+    jdk_release_version = "21"
+    jdk_binary = f"adoptium:1.{jdk_release_version}"
+
+    # Assert jdk_binary is found with default JVM Index
+    rule_runner.set_options([f"--jvm-tool-jdk={jdk_binary}"], env_inherit=PYTHON_BOOTSTRAP_ENV)
+    assert f'openjdk version "{jdk_release_version}' in run_javac_version(rule_runner)
+
+    # Assert jdk_binary is not found with empty JVM Index
+    rule_runner.set_options(
+        [
+            f"--coursier-jvm-index={rule_runner.build_root}/{filename}",
+            f"--jvm-tool-jdk={jdk_binary}",
+        ],
+        env_inherit=PYTHON_BOOTSTRAP_ENV,
+    )
+    expected_exception_msg = rf".*?JVM {jdk_binary} not found in index.*?"
+    with pytest.raises(ExecutionError, match=expected_exception_msg):
+        run_javac_version(rule_runner)