Repository: raviqqe/muffet (go)

Issue:
429 errors when using `--max-connections`
We have a bunch of GitHub issue links on a site, and even with `--max-connections=10 --buffer-size=8192 --color=always --rate-limit=2` we're running in a lot of 429 errors. Any suggestion on how to avoid this?

Pull request:
Fix host throttling
Close #172.


Patch:
diff --git a/command.go b/command.go
index 7dc9da8..ae4768b 100644
--- a/command.go
+++ b/command.go
@@ -45,21 +45,23 @@ func (c *command) runWithError(ss []string) (bool, error) {
 		return false, errors.New("verbose option not supported for JSON output")
 	}
 
-	client := newThrottledHttpClient(
-		c.httpClientFactory.Create(
-			httpClientOptions{
-				MaxConnectionsPerHost: args.MaxConnectionsPerHost,
-				BufferSize:            args.BufferSize,
-				MaxRedirections:       args.MaxRedirections,
-				Proxy:                 args.Proxy,
-				SkipTLSVerification:   args.SkipTLSVerification,
-				Timeout:               time.Duration(args.Timeout) * time.Second,
-				Headers:               args.Headers,
-			},
+	client := newRedirectHttpClient(
+		newThrottledHttpClient(
+			c.httpClientFactory.Create(
+				httpClientOptions{
+					MaxConnectionsPerHost: args.MaxConnectionsPerHost,
+					BufferSize:            args.BufferSize,
+					Proxy:                 args.Proxy,
+					SkipTLSVerification:   args.SkipTLSVerification,
+					Timeout:               time.Duration(args.Timeout) * time.Second,
+					Headers:               args.Headers,
+				},
+			),
+			args.RateLimit,
+			args.MaxConnections,
+			args.MaxConnectionsPerHost,
 		),
-		args.RateLimit,
-		args.MaxConnections,
-		args.MaxConnectionsPerHost,
+		args.MaxRedirections,
 	)
 
 	pp := newPageParser(newLinkFinder(args.ExcludedPatterns))
diff --git a/fasthttp_http_client.go b/fasthttp_http_client.go
index 5049bd2..4a74e04 100644
--- a/fasthttp_http_client.go
+++ b/fasthttp_http_client.go
@@ -1,8 +1,6 @@
 package main
 
 import (
-	"errors"
-	"fmt"
 	"net/url"
 	"strings"
 	"time"
@@ -11,14 +9,13 @@ import (
 )
 
 type fasthttpHttpClient struct {
-	client          *fasthttp.Client
-	maxRedirections int
-	timeout         time.Duration
-	headers         map[string]string
+	client  *fasthttp.Client
+	timeout time.Duration
+	headers map[string]string
 }
 
-func newFasthttpHttpClient(c *fasthttp.Client, maxRedirections int, timeout time.Duration, headers map[string]string) httpClient {
-	return &fasthttpHttpClient{c, maxRedirections, timeout, headers}
+func newFasthttpHttpClient(c *fasthttp.Client, timeout time.Duration, headers map[string]string) httpClient {
+	return &fasthttpHttpClient{c, timeout, headers}
 }
 
 func (c *fasthttpHttpClient) Get(u *url.URL) (httpResponse, error) {
@@ -30,42 +27,17 @@ func (c *fasthttpHttpClient) Get(u *url.URL) (httpResponse, error) {
 		req.Header.Add(k, v)
 	}
 
-	// Some HTTP servers require "Accept" headers to be set explicitly.
+	// Some HTTP servers require "Accept" headers set explicitly.
 	if !includeHeader(c.headers, "Accept") {
 		req.Header.Add("Accept", "*/*")
 	}
 
-	i := 0
-
-	for {
-		err := c.client.DoTimeout(&req, &res, c.timeout)
-		if err != nil && i > 0 {
-			return nil, fmt.Errorf("%w (following redirect %v)", err, req.URI())
-		} else if err != nil {
-			return nil, err
-		}
-
-		switch res.StatusCode() / 100 {
-		case 2:
-			return newFasthttpHttpResponse(req.URI(), &res), nil
-		case 3:
-			i++
-
-			if i > c.maxRedirections {
-				return nil, errors.New("too many redirections")
-			}
-
-			u := res.Header.Peek("Location")
-
-			if len(u) == 0 {
-				return nil, errors.New("location header not found")
-			}
-
-			req.URI().UpdateBytes(u)
-		default:
-			return nil, fmt.Errorf("%v", res.StatusCode())
-		}
+	err := c.client.DoTimeout(&req, &res, c.timeout)
+	if err != nil {
+		return nil, err
 	}
+
+	return newFasthttpHttpResponse(req.URI(), &res), nil
 }
 
 func includeHeader(hs map[string]string, h string) bool {
diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go
index 216031d..3e32957 100644
--- a/fasthttp_http_client_factory.go
+++ b/fasthttp_http_client_factory.go
@@ -34,7 +34,6 @@ func (*fasthttpHttpClientFactory) Create(o httpClientOptions) httpClient {
 			Dial:                     d,
 			NoDefaultUserAgentHeader: true,
 		},
-		o.MaxRedirections,
 		o.Timeout,
 		o.Headers,
 	)
diff --git a/http_client.go b/http_client.go
index e365ef7..d442f17 100644
--- a/http_client.go
+++ b/http_client.go
@@ -5,7 +5,7 @@ import (
 )
 
 type httpClient interface {
-	// Get sends a HTTP request with the GET method.
-	// Non-200 status codes always result in non-nil errors.
+	// Get sends an HTTP request with a GET method.
+	// It depends on implementation of each client what is considered as errors.
 	Get(url *url.URL) (httpResponse, error)
 }
diff --git a/http_client_factory.go b/http_client_factory.go
index 726a1e6..07125c5 100644
--- a/http_client_factory.go
+++ b/http_client_factory.go
@@ -4,8 +4,7 @@ import "time"
 
 type httpClientOptions struct {
 	MaxConnectionsPerHost,
-	BufferSize,
-	MaxRedirections int
+	BufferSize int
 	Proxy               string
 	SkipTLSVerification bool
 	Timeout             time.Duration
diff --git a/redirect_http_client.go b/redirect_http_client.go
new file mode 100644
index 0000000..b25433d
--- /dev/null
+++ b/redirect_http_client.go
@@ -0,0 +1,60 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+)
+
+type redirectHttpClient struct {
+	client          httpClient
+	maxRedirections int
+}
+
+func newRedirectHttpClient(c httpClient, maxRedirections int) httpClient {
+	return &redirectHttpClient{c, maxRedirections}
+}
+
+func (c *redirectHttpClient) Get(u *url.URL) (httpResponse, error) {
+	i := 0
+
+	for {
+		r, err := c.client.Get(u)
+		if err != nil {
+			return nil, c.formatError(err, i, u)
+		}
+
+		switch r.StatusCode() / 100 {
+		case 2:
+			return r, nil
+		case 3:
+			i++
+
+			if i > c.maxRedirections {
+				return nil, errors.New("too many redirections")
+			}
+
+			s := r.Header("Location")
+
+			if len(s) == 0 {
+				return nil, errors.New("location header not set")
+			}
+
+			u, err = u.Parse(s)
+
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, c.formatError(fmt.Errorf("%v", r.StatusCode()), i, u)
+		}
+	}
+}
+
+func (*redirectHttpClient) formatError(err error, redirections int, u *url.URL) error {
+	if redirections == 0 {
+		return err
+	}
+
+	return fmt.Errorf("%w (following redirect %v)", err, u.String())
+}


Test patch:
diff --git a/command_test.go b/command_test.go
index 6dfb901..74617dd 100644
--- a/command_test.go
+++ b/command_test.go
@@ -44,7 +44,7 @@ func TestCommandRun(t *testing.T) {
 				return nil, errors.New("")
 			}
 
-			return newFakeHttpResponse(200, s, "text/html", nil), nil
+			return newFakeHtmlResponse(s, ""), nil
 		},
 	).Run([]string{"http://foo.com"})
 
@@ -58,15 +58,13 @@ func TestCommandRunWithLinks(t *testing.T) {
 		func(u *url.URL) (*fakeHttpResponse, error) {
 			switch u.String() {
 			case "http://foo.com":
-				return newFakeHttpResponse(
-					200,
+				return newFakeHtmlResponse(
 					"http://foo.com",
-					"text/html",
-					[]byte(`<html><body><a href="/foo" /></body></html>`),
+					`<html><body><a href="/foo" /></body></html>`,
 				), nil
 			case "http://foo.com/foo":
 				visited = true
-				return newFakeHttpResponse(200, "http://foo.com", "text/html", nil), nil
+				return newFakeHtmlResponse("http://foo.com", ""), nil
 			}
 
 			return nil, errors.New("")
@@ -83,7 +81,7 @@ func TestCommandRunWithVerboseOption(t *testing.T) {
 	ok := newTestCommandWithStdout(
 		b,
 		func(u *url.URL) (*fakeHttpResponse, error) {
-			return newFakeHttpResponse(200, "http://foo.com", "text/html", nil), nil
+			return newFakeHtmlResponse("http://foo.com", ""), nil
 		},
 	).Run([]string{"-v", "http://foo.com"})
 
@@ -108,11 +106,9 @@ func TestCommandFailToRunWithInvalidLink(t *testing.T) {
 		b,
 		func(u *url.URL) (*fakeHttpResponse, error) {
 			if u.String() == "http://foo.com" {
-				return newFakeHttpResponse(
-					200,
+				return newFakeHtmlResponse(
 					"http://foo.com",
-					"text/html",
-					[]byte(`<html><body><a href="/foo" /></body></html>`),
+					`<html><body><a href="/foo" /></body></html>`,
 				), nil
 			}
 
@@ -130,7 +126,7 @@ func TestCommandFailToParseArguments(t *testing.T) {
 	c := newTestCommandWithStderr(
 		b,
 		func(u *url.URL) (*fakeHttpResponse, error) {
-			return newFakeHttpResponse(200, "", "text/html", nil), nil
+			return newFakeHtmlResponse("", ""), nil
 		},
 	)
 
@@ -160,7 +156,12 @@ func TestCommandFailToGetHTMLRootPage(t *testing.T) {
 	ok := newTestCommandWithStderr(
 		b,
 		func(u *url.URL) (*fakeHttpResponse, error) {
-			return newFakeHttpResponse(200, "", "image/png", nil), nil
+			return newFakeHttpResponse(
+				200,
+				"",
+				nil,
+				map[string]string{"content-type": "image/png"},
+			), nil
 		},
 	).Run([]string{"http://foo.com"})
 
@@ -192,7 +193,7 @@ func TestCommandShowHelp(t *testing.T) {
 	c := newTestCommandWithStdout(
 		b,
 		func(u *url.URL) (*fakeHttpResponse, error) {
-			return newFakeHttpResponse(200, "", "text/html", nil), nil
+			return newFakeHtmlResponse("", ""), nil
 		},
 	)
 
@@ -208,7 +209,7 @@ func TestCommandShowVersion(t *testing.T) {
 	c := newTestCommandWithStdout(
 		b,
 		func(u *url.URL) (*fakeHttpResponse, error) {
-			return newFakeHttpResponse(200, "", "text/html", nil), nil
+			return newFakeHtmlResponse("", ""), nil
 		},
 	)
 
@@ -227,11 +228,9 @@ func TestCommandFailToRunWithJSONOutput(t *testing.T) {
 		b,
 		func(u *url.URL) (*fakeHttpResponse, error) {
 			if u.String() == "http://foo.com" {
-				return newFakeHttpResponse(
-					200,
+				return newFakeHtmlResponse(
 					"http://foo.com",
-					"text/html",
-					[]byte(`<html><body><a href="/foo" /></body></html>`),
+					`<html><body><a href="/foo" /></body></html>`,
 				), nil
 			}
 
@@ -249,7 +248,7 @@ func TestCommandDoNotIncludeSuccessfulPageInJSONOutput(t *testing.T) {
 	ok := newTestCommandWithStdout(
 		b,
 		func(u *url.URL) (*fakeHttpResponse, error) {
-			return newFakeHttpResponse(200, "", "text/html", nil), nil
+			return newFakeHtmlResponse("", ""), nil
 		},
 	).Run([]string{"--json", "http://foo.com"})
 
diff --git a/fake_http_response_test.go b/fake_http_response_test.go
index cb7a8c9..1639f35 100644
--- a/fake_http_response_test.go
+++ b/fake_http_response_test.go
@@ -3,14 +3,29 @@ package main
 import "strings"
 
 type fakeHttpResponse struct {
-	statusCode  int
-	location    string
-	contentType string
-	body        []byte
+	statusCode int
+	location   string
+	body       []byte
+	headers    map[string]string
 }
 
-func newFakeHttpResponse(statusCode int, location string, contentType string, body []byte) *fakeHttpResponse {
-	return &fakeHttpResponse{statusCode, location, contentType, body}
+func newFakeHttpResponse(statusCode int, location string, body []byte, headers map[string]string) *fakeHttpResponse {
+	hs := make(map[string]string, len(headers))
+
+	for k, v := range headers {
+		hs[strings.ToLower(k)] = v
+	}
+
+	return &fakeHttpResponse{statusCode, location, body, hs}
+}
+
+func newFakeHtmlResponse(location string, body string) *fakeHttpResponse {
+	return newFakeHttpResponse(
+		200,
+		location,
+		[]byte(body),
+		map[string]string{"content-type": "text/html"},
+	)
 }
 
 func (r *fakeHttpResponse) URL() string {
@@ -22,8 +37,8 @@ func (r *fakeHttpResponse) StatusCode() int {
 }
 
 func (r *fakeHttpResponse) Header(name string) string {
-	if strings.ToLower(name) == "content-type" {
-		return r.contentType
+	if v, ok := r.headers[strings.ToLower(name)]; ok {
+		return v
 	}
 
 	return ""
diff --git a/fasthttp_http_client_factory_test.go b/fasthttp_http_client_factory_test.go
new file mode 100644
index 0000000..41f3696
--- /dev/null
+++ b/fasthttp_http_client_factory_test.go
@@ -0,0 +1,11 @@
+package main
+
+import "testing"
+
+func TestFasthttpHttpClientFactoryCreate(t *testing.T) {
+	newFasthttpHttpClientFactory().Create(httpClientOptions{})
+}
+
+func TestFasthttpHttpClientFactoryCreateWithProxy(t *testing.T) {
+	newFasthttpHttpClientFactory().Create(httpClientOptions{Proxy: "foo"})
+}
diff --git a/link_fetcher_test.go b/link_fetcher_test.go
index 9899a0a..3c2a822 100644
--- a/link_fetcher_test.go
+++ b/link_fetcher_test.go
@@ -30,12 +30,7 @@ func TestLinkFetcherFetch(t *testing.T) {
 					return nil, errors.New("")
 				}
 
-				return newFakeHttpResponse(
-					200,
-					"http://foo.com",
-					"text/html",
-					nil,
-				), nil
+				return newFakeHtmlResponse("http://foo.com", ""), nil
 			}),
 	)
 
@@ -59,12 +54,7 @@ func TestLinkFetcherFetchFromCache(t *testing.T) {
 
 				ok = false
 
-				return newFakeHttpResponse(
-					200,
-					s,
-					"text/html",
-					nil,
-				), nil
+				return newFakeHtmlResponse(s, ""), nil
 			}),
 	)
 
@@ -87,7 +77,7 @@ func TestLinkFetcherFetchCacheConcurrency(t *testing.T) {
 			func(u *url.URL) (*fakeHttpResponse, error) {
 				c++
 
-				return newFakeHttpResponse(200, "http://foo.com", "text/html", nil), nil
+				return newFakeHtmlResponse("http://foo.com", ""), nil
 			}),
 	)
 
@@ -119,7 +109,7 @@ func TestLinkFetcherFetchWithFragments(t *testing.T) {
 					return nil, errors.New("")
 				}
 
-				return newFakeHttpResponse(200, s, "text/html", []byte(`<p id="foo" />`)), nil
+				return newFakeHtmlResponse(s, `<p id="foo" />`), nil
 			},
 		),
 	)
@@ -144,7 +134,7 @@ func TestLinkFetcherFetchIgnoringFragments(t *testing.T) {
 					return nil, errors.New("")
 				}
 
-				return newFakeHttpResponse(200, s, "text/html", nil), nil
+				return newFakeHtmlResponse(s, ""), nil
 			},
 		),
 		linkFetcherOptions{IgnoreFragments: true},
@@ -168,7 +158,7 @@ func TestLinkFetcherFailToFetch(t *testing.T) {
 func TestLinkFetcherFailToParseURL(t *testing.T) {
 	f := newTestLinkFetcher(
 		newFakeHttpClient(func(*url.URL) (*fakeHttpResponse, error) {
-			return newFakeHttpResponse(200, "", "text/html", nil), nil
+			return newFakeHtmlResponse("", ""), nil
 		}))
 
 	_, _, err := f.Fetch(":")
diff --git a/page_checker_test.go b/page_checker_test.go
index 36ab260..2333d29 100644
--- a/page_checker_test.go
+++ b/page_checker_test.go
@@ -58,7 +58,7 @@ func TestPageCheckerCheckTwoPages(t *testing.T) {
 					return nil, errors.New("")
 				}
 
-				return newFakeHttpResponse(200, s, "text/html", nil), nil
+				return newFakeHtmlResponse(s, ""), nil
 			},
 		),
 	)
@@ -97,7 +97,7 @@ func TestPageCheckerDoNotCheckSamePageTwice(t *testing.T) {
 	c := newTestPageChecker(
 		newFakeHttpClient(
 			func(u *url.URL) (*fakeHttpResponse, error) {
-				return newFakeHttpResponse(200, "http://foo.com", "text/html", nil), nil
+				return newFakeHtmlResponse("http://foo.com", ""), nil
 			},
 		),
 	)
diff --git a/redirect_http_client_test.go b/redirect_http_client_test.go
new file mode 100644
index 0000000..db7fbaa
--- /dev/null
+++ b/redirect_http_client_test.go
@@ -0,0 +1,260 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const testUrl = "http://foo.com"
+
+func TestNewRedirectHttpClient(t *testing.T) {
+	newRedirectHttpClient(newFakeHttpClient(nil), 42)
+}
+
+func TestRedirectHttpClientGet(t *testing.T) {
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				if u.String() != testUrl {
+					return nil, errors.New("")
+				}
+
+				return newFakeHtmlResponse(testUrl, ""), nil
+			},
+		),
+		42,
+	).Get(u)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 200, r.StatusCode())
+}
+
+func TestRedirectHttpClientGetWithRedirect(t *testing.T) {
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	redirected := false
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				if u.String() != testUrl {
+					return nil, errors.New("")
+				} else if !redirected {
+					redirected = true
+
+					return newFakeHttpResponse(
+						300,
+						testUrl,
+						nil,
+						map[string]string{"location": testUrl},
+					), nil
+				}
+
+				return newFakeHtmlResponse("", ""), nil
+			},
+		),
+		42,
+	).Get(u)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 200, r.StatusCode())
+}
+
+func TestRedirectHttpClientGetWithRedirects(t *testing.T) {
+	const maxRedirections = 42
+
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	i := 0
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				if u.String() != testUrl {
+					return nil, errors.New("")
+				} else if i < maxRedirections {
+					i++
+
+					return newFakeHttpResponse(
+						300,
+						testUrl,
+						nil,
+						map[string]string{"location": testUrl},
+					), nil
+				}
+
+				return newFakeHtmlResponse("", ""), nil
+			},
+		),
+		maxRedirections,
+	).Get(u)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 200, r.StatusCode())
+	assert.Equal(t, maxRedirections, i)
+}
+
+func TestRedirectHttpClientGetWithRelativeRedirect(t *testing.T) {
+	const maxRedirections = 42
+
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	redirected := false
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				switch u.String() {
+				case "http://foo.com/foo":
+					return newFakeHtmlResponse("", ""), nil
+				case testUrl:
+					if !redirected {
+						redirected = true
+
+						return newFakeHttpResponse(
+							300,
+							testUrl,
+							nil,
+							map[string]string{"location": "/foo"},
+						), nil
+					}
+				}
+
+				return nil, errors.New("")
+			},
+		),
+		maxRedirections,
+	).Get(u)
+
+	assert.Nil(t, err)
+	assert.Equal(t, 200, r.StatusCode())
+	assert.Equal(t, redirected, true)
+}
+
+func TestRedirectHttpClientFailWithTooManyRedirects(t *testing.T) {
+	const maxRedirections = 42
+
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	i := 0
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				i++
+
+				return newFakeHttpResponse(
+					300,
+					testUrl,
+					nil,
+					map[string]string{"location": testUrl},
+				), nil
+			},
+		),
+		maxRedirections,
+	).Get(u)
+
+	assert.Nil(t, r)
+	assert.Equal(t, err.Error(), "too many redirections")
+	assert.Equal(t, maxRedirections+1, i)
+}
+
+func TestRedirectHttpClientFailWithUnsetLocationHeader(t *testing.T) {
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				return newFakeHttpResponse(300, testUrl, nil, nil), nil
+			},
+		),
+		42,
+	).Get(u)
+
+	assert.Nil(t, r)
+	assert.Equal(t, err.Error(), "location header not set")
+}
+
+func TestRedirectHttpClientFailWithInvalidLocationURL(t *testing.T) {
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				return newFakeHttpResponse(
+					300,
+					testUrl,
+					nil,
+					map[string]string{"location": ":"},
+				), nil
+			},
+		),
+		42,
+	).Get(u)
+
+	assert.Nil(t, r)
+	assert.Contains(t, err.Error(), "parse")
+}
+
+func TestRedirectHttpClientFailWithInvalidStatusCode(t *testing.T) {
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				return newFakeHttpResponse(404, testUrl, nil, nil), nil
+			},
+		),
+		42,
+	).Get(u)
+
+	assert.Nil(t, r)
+	assert.Equal(t, err.Error(), "404")
+}
+
+func TestRedirectHttpClientFailAfterRedirect(t *testing.T) {
+	u, err := url.Parse(testUrl)
+
+	assert.Nil(t, err)
+
+	redirected := false
+	r, err := newRedirectHttpClient(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				if !redirected {
+					redirected = true
+
+					return newFakeHttpResponse(
+						300,
+						testUrl,
+						nil,
+						map[string]string{"location": "/foo"},
+					), nil
+				}
+
+				return newFakeHttpResponse(404, "", nil, nil), nil
+			},
+		),
+		42,
+	).Get(u)
+
+	assert.Nil(t, r)
+	assert.Contains(t, err.Error(), "following redirect http://foo.com/foo")
+}
diff --git a/robots_txt_fetcher_test.go b/robots_txt_fetcher_test.go
index 8bc8044..b1522ba 100644
--- a/robots_txt_fetcher_test.go
+++ b/robots_txt_fetcher_test.go
@@ -24,11 +24,11 @@ func TestRobotsTxtFetcherFetchRobotsTxt(t *testing.T) {
 				return newFakeHttpResponse(
 					200,
 					s,
-					"text/plain",
 					[]byte(`
 						User-Agent: *
 						Disallow: /bar
 					`),
+					map[string]string{"content-type": "text/plain"},
 				), nil
 			})).Fetch(u)
 
diff --git a/sitemap_fetcher_test.go b/sitemap_fetcher_test.go
index 51b97cb..d5879d7 100644
--- a/sitemap_fetcher_test.go
+++ b/sitemap_fetcher_test.go
@@ -24,7 +24,6 @@ func TestSitemapFetcherFetchSitemap(t *testing.T) {
 				return newFakeHttpResponse(
 					200,
 					s,
-					"text/xml",
 					[]byte(`
 						<?xml version="1.0" encoding="UTF-8"?>
 						<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
@@ -33,6 +32,7 @@ func TestSitemapFetcherFetchSitemap(t *testing.T) {
 							</url>
 						</urlset>
 					`),
+					map[string]string{"content-type": "text/xml"},
 				), nil
 			})).Fetch(u)
 
@@ -64,7 +64,10 @@ func TestSitemapFetcherFailToParseSitemap(t *testing.T) {
 	_, err = newSitemapFetcher(
 		newFakeHttpClient(
 			func(u *url.URL) (*fakeHttpResponse, error) {
-				return newFakeHttpResponse(200, "", "text/xml", []byte(`<`)), nil
+				return newFakeHttpResponse(
+					200, "", []byte(`<`),
+					map[string]string{"content-type": "text/xml"},
+				), nil
 			},
 		),
 	).Fetch(u)