Repository: bazelbuild/bazel
Issue: repository_ctx.download doesn't use netrc
Labels: type: bug
P2
team-ExternalDeps

### Description of the problem / feature request:
It seems that `repository_ctx.download(url)` doesn't handle netrc auth (while `http_archive`/`http_file` can use it, also wget can fetch same url).

Rule in question is `jvm_maven_import_external` https://github.com/bazelbuild/bazel/blob/master/tools/build_defs/repo/jvm.bzl#L109 

### Bugs: what's the simplest, easiest way to reproduce this bug? Please provide a minimal example if possible.

Don't have repro yet, will try to add later

### What operating system are you running Bazel on?

NixOS

### What's the output of `bazel info release`?

release 4.1.0- (@non-git)

### If `bazel info release` returns "development version" or "(@non-git)", tell us how you built Bazel.

nixpkgs


Issue comments:
It doesn't out of the box, but it can be made to with a bit of boiler-plate:

```starlark
load(
    "@bazel_tools//tools/build_defs/repo:utils.bzl",
    "read_netrc",
    "use_netrc",
)
load("@bazel_skylib//lib:versions.bzl", "versions")

def _use_netrc(netrc, urls, patterns):
    if versions.is_at_most("3.0.0", versions.get()):
        return use_netrc(netrc, urls)
    return use_netrc(netrc, urls, patterns)

def get_auth(ctx, urls):
    """Given the list of URLs obtain the correct auth dict.
    Args:
        ctx: The repository context.
        urls: A list of URLs.
    Returns:
        A dictionary of workspace attributes to update.
    """

    # Mostly copied from https://github.com/bazelbuild/bazel/blob/3.7.2/tools/build_defs/repo/http.bzl
    if ctx.attr.netrc:
        netrc = read_netrc(ctx, ctx.attr.netrc)
        return _use_netrc(netrc, urls, ctx.attr.auth_patterns)

    if not ctx.os.name.startswith("windows"):
        if "HOME" in ctx.os.environ:
            netrcfile = "%s/.netrc" % (ctx.os.environ["HOME"],)
            if ctx.execute(["test", "-f", netrcfile]).return_code == 0:
                netrc = read_netrc(ctx, netrcfile)
                return _use_netrc(netrc, urls, ctx.attr.auth_patterns)
    elif "USERPROFILE" in ctx.os.environ and ctx.os.name.startswith("windows"):
        netrcfile = "%s/.netrc" % (ctx.os.environ["USERPROFILE"])
        if ctx.path(netrcfile).exists:
            netrc = read_netrc(ctx, netrcfile)
            return _use_netrc(netrc, urls, ctx.attr.auth_patterns)
        netrcfile = "%s/_netrc" % (ctx.os.environ["USERPROFILE"])
        if ctx.path(netrcfile).exists:
            netrc = read_netrc(ctx, netrcfile)
            return _use_netrc(netrc, urls, ctx.attr.auth_patterns)

    return {}
    
def _my_rule_implementation(ctx):
    auth = get_auth(ctx, all_urls)

    download_info = ctx.download_and_extract(
        ctx.attr.urls,
        ctx.attr.sha256,
        auth = auth,
    )
```

That said it isn't ideal for every rule implementation to have to do this.
In recent versions, it's even simpler:

```bzl
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "read_user_netrc", "use_netrc")

def _get_auth(ctx, urls):
    """Given the list of URLs obtain the correct auth dict."""
    if ctx.attr.netrc:
        netrc = read_netrc(ctx, ctx.attr.netrc)
    else:
        netrc = read_user_netrc(ctx)
    return use_netrc(netrc, urls, ctx.attr.auth_patterns)

def _my_rule_implementation(ctx):
    auth = _get_auth(ctx, all_urls)

    download_info = ctx.download_and_extract(
        ctx.attr.urls,
        ctx.attr.sha256,
        auth = auth,
    )

my_rule = rule(
  attrs = {
    "auth_patterns": attr.string_dict(),
    "netrc": attr.string(),
    "urls": attr.string_list(),
  },
  implementation = _my_rule_implementation,
)
```
> In recent versions, it's even simpler:
> 
> ```python
> load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "read_user_netrc", "use_netrc")
> 
> def _get_auth(ctx, urls):
>     """Given the list of URLs obtain the correct auth dict."""
>     if ctx.attr.netrc:
>         netrc = read_netrc(ctx, ctx.attr.netrc)
>     else:
>         netrc = read_user_netrc(ctx)
>     return use_netrc(netrc, urls, ctx.attr.auth_patterns)
> 
> def _my_rule_implementation(ctx):
>     auth = _get_auth(ctx, all_urls)
> 
>     download_info = ctx.download_and_extract(
>         ctx.attr.urls,
>         ctx.attr.sha256,
>         auth = auth,
>     )
> 
> my_rule = rule(
>   attrs = {
>     "auth_patterns": attr.string_dict(),
>     "netrc": attr.string(),
>     "urls": attr.string_list(),
>   },
>   implementation = _my_rule_implementation,
> )
> ```

I would have never found this in a billion years. Any link to documentation or your process on how you figured this out?
Everything in `@bazel_tools//`, including the built-in `http_archive`, lives in https://github.com/bazelbuild/bazel/tree/6.1.2/tools/.  You can just read the code there.  Documentation would be nice, though.
The always-comprehensive, always-accurate documentation: the Source.

:/
Note for anyone blocked on this: using the `NETRC` environment variable or the `--credential_helper` flag should work around the issue, until we implement a proper fix. See integration test at https://github.com/bazelbuild/bazel/blob/62799ed724391ef3cd792fae0d81910c06daf245/src/test/py/bazel/bzlmod/bzlmod_credentials_test.py#L135 for an example.

Pull request review comments:

Patch that resolved the issue:
diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
index ea3bea3221fa12..514a2dc2f37123 100644
--- a/MODULE.bazel.lock
+++ b/MODULE.bazel.lock
@@ -2159,7 +2159,7 @@
   "moduleExtensions": {
     "//:extensions.bzl%bazel_android_deps": {
       "general": {
-        "bzlTransitiveDigest": "PjK+f/kxkhda9tRFlKVdGfNszPoXs7CDXZUi+ZGWGYU=",
+        "bzlTransitiveDigest": "QhQuQNBvy9io+azl8gYOlszhd84XltqOUhc6WXnInFY=",
         "accumulatedFileDigests": {},
         "envVariables": {},
         "generatedRepoSpecs": {
@@ -2178,9 +2178,9 @@
     },
     "//:extensions.bzl%bazel_build_deps": {
       "general": {
-        "bzlTransitiveDigest": "PjK+f/kxkhda9tRFlKVdGfNszPoXs7CDXZUi+ZGWGYU=",
+        "bzlTransitiveDigest": "QhQuQNBvy9io+azl8gYOlszhd84XltqOUhc6WXnInFY=",
         "accumulatedFileDigests": {
-          "@@//src/test/tools/bzlmod:MODULE.bazel.lock": "10b96bd3c1eb194b0efe3a13fd06f2051abf36efb33414ad92048883ba471c7f",
+          "@@//src/test/tools/bzlmod:MODULE.bazel.lock": "cf51741dfa229ffdcb85071df1c72fbf76ee282235033e2e1a2f39b3aa716a54",
           "@@//:MODULE.bazel": "1ce286c2e04a814940dcb741cddee8772a3d8b97d22e12ee12e116dac57f1cd3"
         },
         "envVariables": {},
@@ -2429,7 +2429,7 @@
     },
     "//:extensions.bzl%bazel_test_deps": {
       "general": {
-        "bzlTransitiveDigest": "PjK+f/kxkhda9tRFlKVdGfNszPoXs7CDXZUi+ZGWGYU=",
+        "bzlTransitiveDigest": "QhQuQNBvy9io+azl8gYOlszhd84XltqOUhc6WXnInFY=",
         "accumulatedFileDigests": {},
         "envVariables": {},
         "generatedRepoSpecs": {
@@ -2479,7 +2479,7 @@
     },
     "//tools/android:android_extensions.bzl%remote_android_tools_extensions": {
       "general": {
-        "bzlTransitiveDigest": "iz3RFYDcsjupaT10sdSPAhA44WL3eDYkTEnYThllj1w=",
+        "bzlTransitiveDigest": "kyYmVfXVvQ/eqU6zuZ6/NCQC72VLGyxJvwX/5Kzw3E8=",
         "accumulatedFileDigests": {},
         "envVariables": {},
         "generatedRepoSpecs": {
@@ -2506,7 +2506,7 @@
     },
     "//tools/test:extensions.bzl%remote_coverage_tools_extension": {
       "general": {
-        "bzlTransitiveDigest": "cizrA62cv8WUgb0cCmx5B6PRijtr/I4TAWxg/4caNGU=",
+        "bzlTransitiveDigest": "dSavvzz4Z4n0JlBCZevqTBn9fRwlL29Y3/kjd1rvGAQ=",
         "accumulatedFileDigests": {},
         "envVariables": {},
         "generatedRepoSpecs": {
diff --git a/tools/build_defs/repo/http.bzl b/tools/build_defs/repo/http.bzl
index 81fc3f96dc77ab..de88e6e48e588a 100644
--- a/tools/build_defs/repo/http.bzl
+++ b/tools/build_defs/repo/http.bzl
@@ -45,11 +45,9 @@ load(
 )
 load(
     ":utils.bzl",
+    "get_auth",
     "patch",
-    "read_netrc",
-    "read_user_netrc",
     "update_attrs",
-    "use_netrc",
     "workspace_and_buildfile",
 )
 
@@ -119,16 +117,6 @@ Authorization: Bearer RANDOM-TOKEN
 </pre>
 """
 
-def _get_auth(ctx, urls):
-    """Given the list of URLs obtain the correct auth dict."""
-    if ctx.attr.netrc:
-        netrc = read_netrc(ctx, ctx.attr.netrc)
-    elif "NETRC" in ctx.os.environ:
-        netrc = read_netrc(ctx, ctx.os.environ["NETRC"])
-    else:
-        netrc = read_user_netrc(ctx)
-    return use_netrc(netrc, urls, ctx.attr.auth_patterns)
-
 def _update_integrity_attr(ctx, attrs, download_info):
     # We don't need to override the integrity attribute if sha256 is already specified.
     integrity_override = {} if ctx.attr.sha256 else {"integrity": download_info.integrity}
@@ -140,7 +128,7 @@ def _http_archive_impl(ctx):
         fail("Only one of build_file and build_file_content can be provided.")
 
     all_urls = _get_all_urls(ctx)
-    auth = _get_auth(ctx, all_urls)
+    auth = get_auth(ctx, all_urls)
 
     download_info = ctx.download_and_extract(
         all_urls,
@@ -182,7 +170,7 @@ def _http_file_impl(ctx):
     if download_path in forbidden_files or not str(download_path).startswith(str(repo_root)):
         fail("'%s' cannot be used as downloaded_file_path in http_file" % ctx.attr.downloaded_file_path)
     all_urls = _get_all_urls(ctx)
-    auth = _get_auth(ctx, all_urls)
+    auth = get_auth(ctx, all_urls)
     download_info = ctx.download(
         all_urls,
         "file/" + downloaded_file_path,
@@ -219,7 +207,7 @@ filegroup(
 def _http_jar_impl(ctx):
     """Implementation of the http_jar rule."""
     all_urls = _get_all_urls(ctx)
-    auth = _get_auth(ctx, all_urls)
+    auth = get_auth(ctx, all_urls)
     downloaded_file_name = ctx.attr.downloaded_file_name
     download_info = ctx.download(
         all_urls,
diff --git a/tools/build_defs/repo/jvm.bzl b/tools/build_defs/repo/jvm.bzl
index c7e87d2f4a97bd..a3ea79e17e5386 100644
--- a/tools/build_defs/repo/jvm.bzl
+++ b/tools/build_defs/repo/jvm.bzl
@@ -44,6 +44,10 @@ load(
     "DEFAULT_CANONICAL_ID_ENV",
     "get_default_canonical_id",
 )
+load(
+    ":utils.bzl",
+    "get_auth",
+)
 
 _HEADER = "# DO NOT EDIT: generated by jvm_import_external()"
 
@@ -124,6 +128,7 @@ def _jvm_import_external(repository_ctx):
         path,
         sha,
         canonical_id = repository_ctx.attr.canonical_id or get_default_canonical_id(repository_ctx, urls),
+        auth = get_auth(repository_ctx, urls),
     )
     if srcurls and _should_fetch_sources_in_current_env(repository_ctx):
         repository_ctx.download(
@@ -131,6 +136,7 @@ def _jvm_import_external(repository_ctx):
             srcpath,
             srcsha,
             canonical_id = repository_ctx.attr.canonical_id,
+            auth = get_auth(repository_ctx, srcurls),
         )
     repository_ctx.file("BUILD", "\n".join(lines))
     repository_ctx.file("%s/BUILD" % extension, "\n".join([
diff --git a/tools/build_defs/repo/utils.bzl b/tools/build_defs/repo/utils.bzl
index cda04c237ca7b5..dabb75332cbb1f 100644
--- a/tools/build_defs/repo/utils.bzl
+++ b/tools/build_defs/repo/utils.bzl
@@ -420,3 +420,27 @@ def read_user_netrc(ctx):
     if not ctx.path(netrcfile).exists:
         return {}
     return read_netrc(ctx, netrcfile)
+
+def get_auth(ctx, urls):
+    """Utility function to obtain the correct auth dict for a list of urls from .netrc file.
+
+    Support optional netrc and auth_patterns attributes if available.
+
+    Args:
+      ctx: The repository context of the repository rule calling this utility
+        function.
+      urls: the list of urls to read
+
+    Returns:
+      the auth dict which can be passed to repository_ctx.download
+    """
+    if hasattr(ctx.attr, "netrc") and ctx.attr.netrc:
+        netrc = read_netrc(ctx, ctx.attr.netrc)
+    elif "NETRC" in ctx.os.environ:
+        netrc = read_netrc(ctx, ctx.os.environ["NETRC"])
+    else:
+        netrc = read_user_netrc(ctx)
+    auth_patterns = {}
+    if hasattr(ctx.attr, "auth_patterns") and ctx.attr.auth_patterns:
+        auth_patterns = ctx.attr.auth_patterns
+    return use_netrc(netrc, urls, auth_patterns)


Test patch:
diff --git a/src/test/tools/bzlmod/MODULE.bazel.lock b/src/test/tools/bzlmod/MODULE.bazel.lock
index 80fcbbd46be616..709394de9f7ee9 100644
--- a/src/test/tools/bzlmod/MODULE.bazel.lock
+++ b/src/test/tools/bzlmod/MODULE.bazel.lock
@@ -646,7 +646,7 @@
     },
     "@@bazel_tools//tools/android:android_extensions.bzl%remote_android_tools_extensions": {
       "general": {
-        "bzlTransitiveDigest": "iz3RFYDcsjupaT10sdSPAhA44WL3eDYkTEnYThllj1w=",
+        "bzlTransitiveDigest": "kyYmVfXVvQ/eqU6zuZ6/NCQC72VLGyxJvwX/5Kzw3E8=",
         "accumulatedFileDigests": {},
         "envVariables": {},
         "generatedRepoSpecs": {
@@ -730,7 +730,7 @@
     },
     "@@bazel_tools//tools/test:extensions.bzl%remote_coverage_tools_extension": {
       "general": {
-        "bzlTransitiveDigest": "cizrA62cv8WUgb0cCmx5B6PRijtr/I4TAWxg/4caNGU=",
+        "bzlTransitiveDigest": "dSavvzz4Z4n0JlBCZevqTBn9fRwlL29Y3/kjd1rvGAQ=",
         "accumulatedFileDigests": {},
         "envVariables": {},
         "generatedRepoSpecs": {
@@ -750,7 +750,7 @@
     },
     "@@rules_java~7.1.0//java:extensions.bzl%toolchains": {
       "general": {
-        "bzlTransitiveDigest": "iUIRqCK7tkhvcDJCAfPPqSd06IHG0a8HQD0xeQyVAqw=",
+        "bzlTransitiveDigest": "8Iziusp18/KdkV+5SEUfQVW9SZzGjlEhlnyCj67JB60=",
         "accumulatedFileDigests": {},
         "envVariables": {},
         "generatedRepoSpecs": {
@@ -1290,7 +1290,7 @@
     },
     "@@rules_python~0.4.0//bzlmod:extensions.bzl%pip_install": {
       "general": {
-        "bzlTransitiveDigest": "rTru6D/C8vlaQDk4HOKyx4U/l6PCnj3Aq/gLraAqHgQ=",
+        "bzlTransitiveDigest": "HXTU3kpE37wW2sY8kraZBNNdfwttv4GVV9n6rlJf2NE=",
         "accumulatedFileDigests": {},
         "envVariables": {},
         "generatedRepoSpecs": {