Repository: moby/buildkit

Issue:
`$GITHUB_TOKEN` should be consumed for `ADD https://raw.githubusercontent.com/...` and `ADD https://github.com/...`
### Description

GitHub recently introduced the API rate limit to non-REST requests too

https://github.blog/changelog/2025-05-08-updated-rate-limits-for-unauthenticated-requests/

> These changes will apply to operations like cloning repositories over HTTPS, anonymously interacting with our REST APIs, and downloading files from raw.githubusercontent.com.

`source/http` and `source/git` should be able to fetch the `$GITHUB_TOKEN` secret from the client so as to relax the rate limit.

Probably the existing secret API can be reused.
However, if we want to let `buildx` to automatically expose the token, probably it needs a new API that can be only accessed from SourceOp, not from ExecOp, to reduce security concerns.
(So, probably `buildx` shouldn't expose the token by default?)

Discussion:
@tonistiigi: Doesn't this already work? The default secrets https://github.com/moby/buildkit/blob/v0.22.0-rc2/client/llb/source.go#L234-L235 should be added automatically in llb. Need to be either that secret value directly of with the host suffix https://github.com/moby/buildkit/blob/v0.22.0-rc2/source/git/source.go#L238-L249@AkihiroSuda: > Doesn't this already work? The default secrets https://github.com/moby/buildkit/blob/v0.22.0-rc2/client/llb/source.go#L234-L235 should be added automatically in llb. Need to be either that secret value directly of with the host suffix https://github.com/moby/buildkit/blob/v0.22.0-rc2/source/git/source.go#L238-L249

This still doesn't work for `ADD https://raw.githubusercontent.com`
@AkihiroSuda: BTW https://docs.docker.com/build/building/secrets/ should cover GitHub as well as GitLab
@tonistiigi: > This still doesn't work for ADD https://raw.githubusercontent.com

We have `AttrHTTPAuthHeaderSecret` now. The question is how to allow it to be set in Dockerfile. We could hardcode secret, smth like `HTTP_AUTH_HEADER_<HOST>`. Or invent some convention like `ADD https://user:[secret=FOO]@host/path`. The first one already matches what we are doing for Git, so I guess it is less controversial.@tonistiigi: > BTW https://docs.docker.com/build/building/secrets/ should cover GitHub as well as GitLab

What's the difference?@tonistiigi: Are we ok with the overhead of checking `HTTP_AUTH_HEADER_<HOST>`, `HTTP_AUTH_TOKEN_<HOST>` secret on session for every URL? Or do we need to add something in the Dockerfile? `ADD --auth-secret=... https://` ? As mentioned above, we already check secrets for Git, so I guess it would be ok for HTTP as well? If we do add a flag we could also make Git URLs use the same flag, although we still need to keep the default as well.

cc @marxarelli@AkihiroSuda: > > BTW https://docs.docker.com/build/building/secrets/ should cover GitHub as well as GitLab
> 
> What's the difference?

The name is the only difference I see.
It still has to be documented with a concrete example that uses `GITHUB_TOKEN`.
Probably a concrete GHA yaml should be listed too

> HTTP_AUTH_HEADER_<HOST>

SGTM in the BuildKit level, but buildx should provide a human-friendly wrapper for `GITHUB_TOKEN`.@marxarelli: > Are we ok with the overhead of checking `HTTP_AUTH_HEADER_<HOST>`, `HTTP_AUTH_TOKEN_<HOST>` secret on session for every URL? Or do we need to add something in the Dockerfile? `ADD --auth-secret=... https://` ? As mentioned above, we already check secrets for Git, so I guess it would be ok for HTTP as well? If we do add a flag we could also make Git URLs use the same flag, although we still need to keep the default as well.
> 
> cc [@marxarelli](https://github.com/marxarelli)

Thanks for looping me in!

I quite like the explicit `--auth-secret` flag; it seems more flexible and clear. That said, I think the conventional `HTTP_AUTH_HEADER_<HOST>` approach works too.

Solution patch:
diff --git a/source/http/source.go b/source/http/source.go
index 574ef95f3..06666be7a 100644
--- a/source/http/source.go
+++ b/source/http/source.go
@@ -34,6 +34,11 @@ import (
 	"github.com/pkg/errors"
 )
 
+const (
+	HTTPAuthHeaderSecretPrefix = "HTTP_AUTH_HEADER_"
+	HTTPAuthTokenSecretPrefix  = "HTTP_AUTH_TOKEN_"
+)
+
 // supportedUserHeaders defines supported user-defined header fields. Fields
 // not included here will be silently dropped.
 var supportedUserDefinedHeaders = map[string]bool{
@@ -506,18 +511,36 @@ func (hs *httpSourceHandler) newHTTPRequest(ctx context.Context, g session.Group
 		req.Header.Set(field.Name, field.Value)
 	}
 
+	type authSecret struct {
+		name  string
+		token bool
+	}
+
+	var secretNames []authSecret
 	if hs.src.AuthHeaderSecret != "" {
+		secretNames = append(secretNames, authSecret{name: hs.src.AuthHeaderSecret})
+	} else {
+		u, err := url.Parse(hs.src.URL)
+		if err == nil {
+			secretNames = append(secretNames, authSecret{name: HTTPAuthHeaderSecretPrefix + u.Hostname()})
+			secretNames = append(secretNames, authSecret{name: HTTPAuthTokenSecretPrefix + u.Hostname(), token: true})
+		}
+	}
+
+	for _, secret := range secretNames {
 		err := hs.sm.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error {
-			dt, err := secrets.GetSecret(ctx, caller, hs.src.AuthHeaderSecret)
+			dt, err := secrets.GetSecret(ctx, caller, secret.name)
 			if err != nil {
 				return err
 			}
-
-			req.Header.Set("Authorization", string(dt))
-
+			v := string(dt)
+			if secret.token {
+				v = "Bearer " + v
+			}
+			req.Header.Set("Authorization", v)
 			return nil
 		})
-		if err != nil {
+		if err != nil && hs.src.AuthHeaderSecret != "" {
 			return nil, errors.Wrapf(err, "failed to retrieve HTTP auth secret %s", hs.src.AuthHeaderSecret)
 		}
 	}



Test patch:
diff --git a/client/client_test.go b/client/client_test.go
index 80ec967b4..6c49b5e8c 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -112,6 +112,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testBuildHTTPSource,
 	testBuildHTTPSourceEtagScope,
 	testBuildHTTPSourceAuthHeaderSecret,
+	testBuildHTTPSourceHostTokenSecret,
 	testBuildHTTPSourceHeader,
 	testBuildPushAndValidate,
 	testBuildExportWithUncompressed,
@@ -3238,6 +3239,47 @@ func testBuildHTTPSourceAuthHeaderSecret(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, "Bearer foo", allReqs[0].Header.Get("Authorization"))
 }
 
+func testBuildHTTPSourceHostTokenSecret(t *testing.T, sb integration.Sandbox) {
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	modTime := time.Now().Add(-24 * time.Hour) // avoid false positive with current time
+
+	resp := httpserver.Response{
+		Etag:         identity.NewID(),
+		Content:      []byte("content1"),
+		LastModified: &modTime,
+	}
+
+	server := httpserver.NewTestServer(map[string]httpserver.Response{
+		"/foo": resp,
+	})
+	defer server.Close()
+
+	st := llb.HTTP(server.URL + "/foo")
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	_, err = c.Solve(
+		sb.Context(),
+		def,
+		SolveOpt{
+			Session: []session.Attachable{secretsprovider.FromMap(map[string][]byte{
+				"HTTP_AUTH_TOKEN_127.0.0.1": []byte("123456"),
+			})},
+		},
+		nil,
+	)
+	require.NoError(t, err)
+
+	allReqs := server.Stats("/foo").Requests
+	require.Equal(t, 1, len(allReqs))
+	require.Equal(t, http.MethodGet, allReqs[0].Method)
+	require.Equal(t, "Bearer 123456", allReqs[0].Header.Get("Authorization"))
+}
+
 func testBuildHTTPSourceHeader(t *testing.T, sb integration.Sandbox) {
 	c, err := New(sb.Context(), sb.Address())
 	require.NoError(t, err)