Repository: gotenberg/gotenberg (go)

Issue:
`User-Agent`, `Accept-Encoding` and `Host` headers
I tried to generate a PDF using a Gotenberg container, but the resulting file was completely blank. I sent a POST request with [this URL](https://www.sec.gov/Archives/edgar/data/1067983/000119312524133049/d759664d8k.htm), but all I got was a white, empty page. You need to set the HTTP headers like [this](https://www.sec.gov/os/accessing-edgar-data). However, when I manually downloaded the HTML file and sent it to my Gotenberg container, the conversion worked like a charm. This made me think that maybe the `extraHttpHeaders` parameter isn't working as it should for `User-Agent`, `Accept-Encoding` and `Host`.

To get to the bottom of this, I was wondering if there's a way to take a closer look at the actual headers and body that a Gotenberg instance sends when it makes a GET request.

_Originally posted by @wjkoh in https://github.com/gotenberg/gotenberg/discussions/842#discussioncomment-9752869_

Pull request description:
feat(chromium): add user agent override
Closes #900.

New form field `userAgent`.

Patch:
diff --git a/go.mod b/go.mod
index 1bc9655..e6f6c89 100644
--- a/go.mod
+++ b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7
-	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/labstack/echo/v4 v4.12.0
 	github.com/labstack/gommon v0.4.2
@@ -28,13 +28,13 @@ require (
 	github.com/ulikunitz/xz v0.5.12 // indirect
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.23.0 // indirect
-	golang.org/x/image v0.16.0 // indirect
-	golang.org/x/net v0.25.0
+	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/image v0.17.0 // indirect
+	golang.org/x/net v0.26.0
 	golang.org/x/sync v0.7.0
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/term v0.20.0
-	golang.org/x/text v0.15.0
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/term v0.21.0
+	golang.org/x/text v0.16.0
 )
 
 require github.com/dlclark/regexp2 v1.11.0
@@ -65,6 +65,6 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
diff --git a/go.sum b/go.sum
index e207438..dc1ad69 100644
--- a/go.sum
+++ b/go.sum
@@ -61,8 +61,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
@@ -143,28 +143,28 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw=
-golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=
+golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
-google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
diff --git a/pkg/modules/chromium/browser.go b/pkg/modules/chromium/browser.go
index d7a3032..fd84915 100644
--- a/pkg/modules/chromium/browser.go
+++ b/pkg/modules/chromium/browser.go
@@ -227,6 +227,7 @@ func (b *chromiumBrowser) pdf(ctx context.Context, logger *zap.Logger, url, outp
 		clearCookiesActionFunc(logger, b.arguments.clearCookies),
 		disableJavaScriptActionFunc(logger, b.arguments.disableJavaScript),
 		setCookiesActionFunc(logger, options.Cookies),
+		userAgentOverride(logger, options.UserAgent),
 		extraHttpHeadersActionFunc(logger, options.ExtraHttpHeaders),
 		navigateActionFunc(logger, url, options.SkipNetworkIdleEvent),
 		hideDefaultWhiteBackgroundActionFunc(logger, options.OmitBackground, options.PrintBackground),
@@ -250,6 +251,7 @@ func (b *chromiumBrowser) screenshot(ctx context.Context, logger *zap.Logger, ur
 		clearCookiesActionFunc(logger, b.arguments.clearCookies),
 		disableJavaScriptActionFunc(logger, b.arguments.disableJavaScript),
 		setCookiesActionFunc(logger, options.Cookies),
+		userAgentOverride(logger, options.UserAgent),
 		extraHttpHeadersActionFunc(logger, options.ExtraHttpHeaders),
 		navigateActionFunc(logger, url, options.SkipNetworkIdleEvent),
 		hideDefaultWhiteBackgroundActionFunc(logger, options.OmitBackground, true),
diff --git a/pkg/modules/chromium/chromium.go b/pkg/modules/chromium/chromium.go
index cbf9618..374dd3b 100644
--- a/pkg/modules/chromium/chromium.go
+++ b/pkg/modules/chromium/chromium.go
@@ -108,8 +108,12 @@ type Options struct {
 	// Optional
 	Cookies []Cookie
 
-	// ExtraHttpHeaders are the HTTP headers to send by Chromium while loading
-	// the HTML document.
+	// UserAgent overrides the default 'User-Agent' HTTP header.
+	// Optional.
+	UserAgent string
+
+	// ExtraHttpHeaders are extra HTTP headers to send by Chromium while
+	// loading he HTML document.
 	// Optional.
 	ExtraHttpHeaders map[string]string
 
@@ -134,6 +138,7 @@ func DefaultOptions() Options {
 		WaitWindowStatus:        "",
 		WaitForExpression:       "",
 		Cookies:                 nil,
+		UserAgent:               "",
 		ExtraHttpHeaders:        nil,
 		EmulatedMediaType:       "",
 		OmitBackground:          false,
diff --git a/pkg/modules/chromium/routes.go b/pkg/modules/chromium/routes.go
index a043654..3ee8d64 100644
--- a/pkg/modules/chromium/routes.go
+++ b/pkg/modules/chromium/routes.go
@@ -35,6 +35,7 @@ func FormDataChromiumOptions(ctx *api.Context) (*api.FormData, Options) {
 		waitWindowStatus        string
 		waitForExpression       string
 		cookies                 []Cookie
+		userAgent               string
 		extraHttpHeaders        map[string]string
 		emulatedMediaType       string
 		omitBackground          bool
@@ -78,6 +79,7 @@ func FormDataChromiumOptions(ctx *api.Context) (*api.FormData, Options) {
 
 			return err
 		}).
+		String("userAgent", &userAgent, defaultOptions.UserAgent).
 		Custom("extraHttpHeaders", func(value string) error {
 			if value == "" {
 				extraHttpHeaders = defaultOptions.ExtraHttpHeaders
@@ -115,6 +117,7 @@ func FormDataChromiumOptions(ctx *api.Context) (*api.FormData, Options) {
 		WaitWindowStatus:        waitWindowStatus,
 		WaitForExpression:       waitForExpression,
 		Cookies:                 cookies,
+		UserAgent:               userAgent,
 		ExtraHttpHeaders:        extraHttpHeaders,
 		EmulatedMediaType:       emulatedMediaType,
 		OmitBackground:          omitBackground,
diff --git a/pkg/modules/chromium/tasks.go b/pkg/modules/chromium/tasks.go
index 2c593da..ac5d6e1 100644
--- a/pkg/modules/chromium/tasks.go
+++ b/pkg/modules/chromium/tasks.go
@@ -281,6 +281,23 @@ func setCookiesActionFunc(logger *zap.Logger, cookies []Cookie) chromedp.ActionF
 	}
 }
 
+func userAgentOverride(logger *zap.Logger, userAgent string) chromedp.ActionFunc {
+	return func(ctx context.Context) error {
+		if len(userAgent) == 0 {
+			logger.Debug("no user agent override")
+			return nil
+		}
+
+		logger.Debug(fmt.Sprintf("user agent override: %s", userAgent))
+		err := emulation.SetUserAgentOverride(userAgent).Do(ctx)
+		if err == nil {
+			return nil
+		}
+
+		return fmt.Errorf("set user agent override: %w", err)
+	}
+}
+
 func extraHttpHeadersActionFunc(logger *zap.Logger, extraHttpHeaders map[string]string) chromedp.ActionFunc {
 	return func(ctx context.Context) error {
 		if len(extraHttpHeaders) == 0 {


Test patch:
diff --git a/pkg/modules/chromium/browser_test.go b/pkg/modules/chromium/browser_test.go
index 4591f7b..84efcad 100644
--- a/pkg/modules/chromium/browser_test.go
+++ b/pkg/modules/chromium/browser_test.go
@@ -614,6 +614,41 @@ func TestChromiumBrowser_pdf(t *testing.T) {
 				"set cookie",
 			},
 		},
+		{
+			scenario: "user agent override",
+			browser: newChromiumBrowser(
+				browserArguments{
+					binPath:          os.Getenv("CHROMIUM_BIN_PATH"),
+					wsUrlReadTimeout: 5 * time.Second,
+					allowList:        regexp2.MustCompile("", 0),
+					denyList:         regexp2.MustCompile("", 0),
+				},
+			),
+			fs: func() *gotenberg.FileSystem {
+				fs := gotenberg.NewFileSystem()
+
+				err := os.MkdirAll(fs.WorkingDirPath(), 0o755)
+				if err != nil {
+					t.Fatalf(fmt.Sprintf("expected no error but got: %v", err))
+				}
+
+				err = os.WriteFile(fmt.Sprintf("%s/index.html", fs.WorkingDirPath()), []byte("<h1>User-Agent override</h1>"), 0o755)
+				if err != nil {
+					t.Fatalf("expected no error but got: %v", err)
+				}
+
+				return fs
+			}(),
+			options: PdfOptions{
+				Options: Options{UserAgent: "foo"},
+			},
+			noDeadline:  false,
+			start:       true,
+			expectError: false,
+			expectedLogEntries: []string{
+				fmt.Sprintf("user agent override: foo"),
+			},
+		},
 		{
 			scenario: "extra HTTP headers",
 			browser: newChromiumBrowser(
@@ -1990,6 +2025,7 @@ func TestChromiumBrowser_screenshot(t *testing.T) {
 				"cache not cleared",
 				"cookies not cleared",
 				"JavaScript not disabled",
+				"no user agent override",
 				"no extra HTTP headers",
 				"navigate to",
 				"default white background not hidden",
@@ -2037,6 +2073,7 @@ func TestChromiumBrowser_screenshot(t *testing.T) {
 				"cookies not cleared",
 				"JavaScript not disabled",
 				"no cookies to set",
+				"no user agent override",
 				"no extra HTTP headers",
 				"navigate to",
 				"default white background not hidden",
@@ -2084,6 +2121,7 @@ func TestChromiumBrowser_screenshot(t *testing.T) {
 				"cookies not cleared",
 				"JavaScript not disabled",
 				"no cookies to set",
+				"no user agent override",
 				"no extra HTTP headers",
 				"navigate to",
 				"default white background not hidden",