Repository: theupdateframework/go-tuf
Pull request #682 (2025-04-09 19:27:50)

Allow users to configure custom http.Client or http.RoundTripper in DefaultFetcher
Related to https://github.com/theupdateframework/go-tuf/issues/660

This updates the `DefaultFetcher` so users can optionally provide a custom http.Client or http.RoundTripper. Note that this removes the `timeout` parameter from the `DefaultFetcher#DownloadFile` method.

Once this is merged, I will follow up with another pull request allowing users to configure HTTP request retries in `DownloadFile`.


Patch:
diff --git a/metadata/config/config.go b/metadata/config/config.go
index cd8cc98..0c0ed36 100644
--- a/metadata/config/config.go
+++ b/metadata/config/config.go
@@ -18,6 +18,8 @@
 package config
 
 import (
+	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 
@@ -64,12 +66,12 @@ func New(remoteURL string, rootBytes []byte) (*UpdaterConfig, error) {
 		SnapshotMaxLength:  2000000, // bytes
 		TargetsMaxLength:   5000000, // bytes
 		// Updater configuration
-		Fetcher:               &fetcher.DefaultFetcher{}, // use the default built-in download fetcher
-		LocalTrustedRoot:      rootBytes,                 // trusted root.json
-		RemoteMetadataURL:     remoteURL,                 // URL of where the TUF metadata is
-		RemoteTargetsURL:      targetsURL,                // URL of where the target files should be downloaded from
-		DisableLocalCache:     false,                     // enable local caching of trusted metadata
-		PrefixTargetsWithHash: true,                      // use hash-prefixed target files with consistent snapshots
+		Fetcher:               fetcher.NewDefaultFetcher(), // use the default built-in download fetcher
+		LocalTrustedRoot:      rootBytes,                   // trusted root.json
+		RemoteMetadataURL:     remoteURL,                   // URL of where the TUF metadata is
+		RemoteTargetsURL:      targetsURL,                  // URL of where the target files should be downloaded from
+		DisableLocalCache:     false,                       // enable local caching of trusted metadata
+		PrefixTargetsWithHash: true,                        // use hash-prefixed target files with consistent snapshots
 		UnsafeLocalMode:       false,
 	}, nil
 }
@@ -87,3 +89,29 @@ func (cfg *UpdaterConfig) EnsurePathsExist() error {
 
 	return nil
 }
+
+func (cfg *UpdaterConfig) SetDefaultFetcherHTTPClient(client *http.Client) error {
+	// Check if the configured fetcher is the default fetcher
+	// since we are only configuring a timeout value for the default fetcher
+	df, ok := cfg.Fetcher.(*fetcher.DefaultFetcher)
+	if !ok {
+		return fmt.Errorf("fetcher is not type fetcher.DefaultFetcher")
+	}
+	df.SetHTTPClient(client)
+	cfg.Fetcher = df
+	return nil
+}
+
+func (cfg *UpdaterConfig) SetDefaultFetcherTransport(rt http.RoundTripper) error {
+	// Check if the configured fetcher is the default fetcher
+	// since we are only configuring a timeout value for the default fetcher
+	df, ok := cfg.Fetcher.(*fetcher.DefaultFetcher)
+	if !ok {
+		return fmt.Errorf("fetcher is not type fetcher.DefaultFetcher")
+	}
+	if err := df.SetTransport(rt); err != nil {
+		return err
+	}
+	cfg.Fetcher = df
+	return nil
+}
diff --git a/metadata/fetcher/fetcher.go b/metadata/fetcher/fetcher.go
index dfe2bc1..8955aa9 100644
--- a/metadata/fetcher/fetcher.go
+++ b/metadata/fetcher/fetcher.go
@@ -22,19 +22,28 @@ import (
 	"io"
 	"net/http"
 	"strconv"
-	"time"
 
 	"github.com/theupdateframework/go-tuf/v2/metadata"
 )
 
+// httpClient interface allows us to either provide a live http.Client
+// or a mock implementation for testing purposes
+type httpClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
 // Fetcher interface
 type Fetcher interface {
-	DownloadFile(urlPath string, maxLength int64, timeout time.Duration) ([]byte, error)
+	DownloadFile(urlPath string, maxLength int64) ([]byte, error)
 }
 
 // DefaultFetcher implements Fetcher
 type DefaultFetcher struct {
 	httpUserAgent string
+	client        httpClient
+	// for implementation of retry logic in a future pull request
+	// retryInterval time.Duration
+	// retryCount    int
 }
 
 func (d *DefaultFetcher) SetHTTPUserAgent(httpUserAgent string) {
@@ -43,8 +52,7 @@ func (d *DefaultFetcher) SetHTTPUserAgent(httpUserAgent string) {
 
 // DownloadFile downloads a file from urlPath, errors out if it failed,
 // its length is larger than maxLength or the timeout is reached.
-func (d *DefaultFetcher) DownloadFile(urlPath string, maxLength int64, timeout time.Duration) ([]byte, error) {
-	client := &http.Client{Timeout: timeout}
+func (d *DefaultFetcher) DownloadFile(urlPath string, maxLength int64) ([]byte, error) {
 	req, err := http.NewRequest("GET", urlPath, nil)
 	if err != nil {
 		return nil, err
@@ -54,7 +62,7 @@ func (d *DefaultFetcher) DownloadFile(urlPath string, maxLength int64, timeout t
 		req.Header.Set("User-Agent", d.httpUserAgent)
 	}
 	// Execute the request.
-	res, err := client.Do(req)
+	res, err := d.client.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -78,7 +86,7 @@ func (d *DefaultFetcher) DownloadFile(urlPath string, maxLength int64, timeout t
 	// Although the size has been checked above, use a LimitReader in case
 	// the reported size is inaccurate, or size is -1 which indicates an
 	// unknown length. We read maxLength + 1 in order to check if the read data
-	// surpased our set limit.
+	// surpassed our set limit.
 	data, err := io.ReadAll(io.LimitReader(res.Body, maxLength+1))
 	if err != nil {
 		return nil, err
@@ -91,3 +99,40 @@ func (d *DefaultFetcher) DownloadFile(urlPath string, maxLength int64, timeout t
 
 	return data, nil
 }
+
+func NewDefaultFetcher() *DefaultFetcher {
+	return &DefaultFetcher{
+		client: http.DefaultClient,
+	}
+}
+
+// NewFetcherWithHTTPClient creates a new DefaultFetcher with a custom httpClient
+func (f *DefaultFetcher) NewFetcherWithHTTPClient(hc httpClient) *DefaultFetcher {
+	return &DefaultFetcher{
+		client: hc,
+	}
+}
+
+// NewFetcherWithRoundTripper creates a new DefaultFetcher with a custom RoundTripper
+// The function will create a default http.Client and replace the transport with the provided RoundTripper implementation
+func (f *DefaultFetcher) NewFetcherWithRoundTripper(rt http.RoundTripper) *DefaultFetcher {
+	client := http.DefaultClient
+	client.Transport = rt
+	return &DefaultFetcher{
+		client: client,
+	}
+}
+
+func (f *DefaultFetcher) SetHTTPClient(hc httpClient) {
+	f.client = hc
+}
+
+func (f *DefaultFetcher) SetTransport(rt http.RoundTripper) error {
+	hc, ok := f.client.(*http.Client)
+	if !ok {
+		return fmt.Errorf("fetcher is not type fetcher.DefaultFetcher")
+	}
+	hc.Transport = rt
+	f.client = hc
+	return nil
+}
diff --git a/metadata/updater/updater.go b/metadata/updater/updater.go
index 70d42ca..108d35c 100644
--- a/metadata/updater/updater.go
+++ b/metadata/updater/updater.go
@@ -245,7 +245,7 @@ func (update *Updater) DownloadTarget(targetFile *metadata.TargetFiles, filePath
 		}
 	}
 	fullURL := fmt.Sprintf("%s%s", targetBaseURL, targetRemotePath)
-	data, err := update.cfg.Fetcher.DownloadFile(fullURL, targetFile.Length, time.Second*15)
+	data, err := update.cfg.Fetcher.DownloadFile(fullURL, targetFile.Length)
 	if err != nil {
 		return "", nil, err
 	}
@@ -648,7 +648,7 @@ func (update *Updater) downloadMetadata(roleName string, length int64, version s
 	} else {
 		urlPath = fmt.Sprintf("%s%s.%s.json", urlPath, version, url.PathEscape(roleName))
 	}
-	return update.cfg.Fetcher.DownloadFile(urlPath, length, time.Second*15)
+	return update.cfg.Fetcher.DownloadFile(urlPath, length)
 }
 
 // generateTargetFilePath generates path from TargetFiles


Test patch:
diff --git a/internal/testutils/simulator/repository_simulator.go b/internal/testutils/simulator/repository_simulator.go
index 43f48d9..7494fc0 100644
--- a/internal/testutils/simulator/repository_simulator.go
+++ b/internal/testutils/simulator/repository_simulator.go
@@ -303,7 +303,7 @@ func partition(s string, delimiter string) (string, string) {
 	return version, role
 }
 
-func (rs *RepositorySimulator) DownloadFile(urlPath string, maxLength int64, timeout time.Duration) ([]byte, error) {
+func (rs *RepositorySimulator) DownloadFile(urlPath string, maxLength int64) ([]byte, error) {
 	data, err := rs.fetch(urlPath)
 	if err != nil {
 		return data, err
diff --git a/metadata/fetcher/fetcher_test.go b/metadata/fetcher/fetcher_test.go
index 5bd8e6d..345335b 100644
--- a/metadata/fetcher/fetcher_test.go
+++ b/metadata/fetcher/fetcher_test.go
@@ -79,8 +79,9 @@ func TestDownLoadFile(t *testing.T) {
 			// this will only be printed if run in verbose mode or if test fails
 			t.Logf("Desc: %s", tt.desc)
 			// run the function under test
-			fetcher := DefaultFetcher{httpUserAgent: "Metadata_Unit_Test/1.0"}
-			data, err := fetcher.DownloadFile(tt.url, tt.maxLength, tt.timeout)
+			fetcher := NewDefaultFetcher()
+			fetcher.SetHTTPUserAgent("Metadata_Unit_Test/1.0")
+			data, err := fetcher.DownloadFile(tt.url, tt.maxLength)
 			// special case if we expect no error
 			if tt.wantErr == nil {
 				assert.NoErrorf(t, err, "expected no error but got %v", err)