Repository: raviqqe/muffet (go)

Issue:
"too many redirections" with Twitter links
I've got pages with links to Twitter that have always been checked successfully with Muffet in the past. But starting about a week ago they fail consistently like this:

```
$ muffet --version
2.8.0

$ muffet --verbose https://twitter.com/twitter
failed to fetch root page: too many redirections
```

Suffice to say, the links work fine in a browser. I assume something changed on the Twitter side, maybe specifically to foil link checkers. However, before I go excluding all Twitter links from my automation I'm hoping someone who knows the Muffet code could give it a closer look and confirm if there's a way to make it succeed. FWIW, I've already tried some of my favorite workarounds I've used in similar situations in the past, such as setting headers to make my client look more "real":

```
$ muffet --verbose --header="User-Agent:Mozilla/5.0" --header="Accept-Encoding:zstd,br,gzip,deflate" https://twitter.com/twitter 
failed to fetch root page: too many redirections
```

Pull request:
Handle cookies
Fix #308.

Note that this logic is not perfect due to the current implementation of the standard `net/http/cookiejar` package which doesn't handle folded `set-cookie` headers properly. See also https://stackoverflow.com/questions/2880047/is-it-possible-to-set-more-than-one-cookie-with-a-single-set-cookie.

Patch:
diff --git a/arguments.go b/arguments.go
index a9dba4f..2fb8620 100644
--- a/arguments.go
+++ b/arguments.go
@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"errors"
+	"net/http"
 	"regexp"
 	"strings"
 
@@ -41,7 +42,7 @@ type arguments struct {
 	URL                 string
 	ExcludedPatterns    []*regexp.Regexp
 	IncludePatterns     []*regexp.Regexp
-	Headers             map[string]string
+	Header              http.Header
 }
 
 func getArguments(ss []string) (*arguments, error) {
@@ -70,7 +71,7 @@ func getArguments(ss []string) (*arguments, error) {
 		return nil, err
 	}
 
-	args.Headers, err = parseHeaders(args.RawHeaders)
+	args.Header, err = parseHeaders(args.RawHeaders)
 	if err != nil {
 		return nil, err
 	}
@@ -110,8 +111,8 @@ func compileRegexps(regexps []string) ([]*regexp.Regexp, error) {
 	return rs, nil
 }
 
-func parseHeaders(headers []string) (map[string]string, error) {
-	m := make(map[string]string, len(headers))
+func parseHeaders(headers []string) (http.Header, error) {
+	h := make(http.Header, len(headers))
 
 	for _, s := range headers {
 		i := strings.IndexRune(s, ':')
@@ -120,10 +121,10 @@ func parseHeaders(headers []string) (map[string]string, error) {
 			return nil, errors.New("invalid header format")
 		}
 
-		m[s[:i]] = strings.TrimSpace(s[i+1:])
+		h.Add(s[:i], strings.TrimSpace(s[i+1:]))
 	}
 
-	return m, nil
+	return h, nil
 }
 
 func reconcileDeprecatedArguments(args *arguments) {
diff --git a/command.go b/command.go
index 9dcee50..b78dbeb 100644
--- a/command.go
+++ b/command.go
@@ -54,7 +54,7 @@ func (c *command) runWithError(ss []string) (bool, error) {
 					Proxy:                 args.Proxy,
 					SkipTLSVerification:   args.SkipTLSVerification,
 					Timeout:               time.Duration(args.Timeout) * time.Second,
-					Headers:               args.Headers,
+					Header:                args.Header,
 				},
 			),
 			args.RateLimit,
diff --git a/fasthttp_http_client.go b/fasthttp_http_client.go
index 4a74e04..d7cd26b 100644
--- a/fasthttp_http_client.go
+++ b/fasthttp_http_client.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"net/http"
 	"net/url"
 	"strings"
 	"time"
@@ -11,24 +12,32 @@ import (
 type fasthttpHttpClient struct {
 	client  *fasthttp.Client
 	timeout time.Duration
-	headers map[string]string
+	header  http.Header
 }
 
-func newFasthttpHttpClient(c *fasthttp.Client, timeout time.Duration, headers map[string]string) httpClient {
-	return &fasthttpHttpClient{c, timeout, headers}
+func newFasthttpHttpClient(c *fasthttp.Client, timeout time.Duration, header http.Header) httpClient {
+	return &fasthttpHttpClient{c, timeout, header}
 }
 
-func (c *fasthttpHttpClient) Get(u *url.URL) (httpResponse, error) {
+func (c *fasthttpHttpClient) Get(u *url.URL, header http.Header) (httpResponse, error) {
 	req, res := fasthttp.Request{}, fasthttp.Response{}
 	req.SetRequestURI(u.String())
 	req.SetConnectionClose()
 
-	for k, v := range c.headers {
-		req.Header.Add(k, v)
+	for k, vs := range c.header {
+		for _, v := range vs {
+			req.Header.Add(k, v)
+		}
+	}
+
+	for k, vs := range header {
+		for _, v := range vs {
+			req.Header.Add(k, v)
+		}
 	}
 
 	// Some HTTP servers require "Accept" headers set explicitly.
-	if !includeHeader(c.headers, "Accept") {
+	if !includeHeader(c.header, "Accept") {
 		req.Header.Add("Accept", "*/*")
 	}
 
@@ -40,9 +49,9 @@ func (c *fasthttpHttpClient) Get(u *url.URL) (httpResponse, error) {
 	return newFasthttpHttpResponse(req.URI(), &res), nil
 }
 
-func includeHeader(hs map[string]string, h string) bool {
-	for k := range hs {
-		if strings.EqualFold(k, h) {
+func includeHeader(h http.Header, k string) bool {
+	for kk := range h {
+		if strings.EqualFold(kk, k) {
 			return true
 		}
 	}
diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go
index 5a67bd6..be89341 100644
--- a/fasthttp_http_client_factory.go
+++ b/fasthttp_http_client_factory.go
@@ -37,6 +37,6 @@ func (*fasthttpHttpClientFactory) Create(o httpClientOptions) httpClient {
 			MaxResponseBodySize:      o.MaxResponseBodySize,
 		},
 		o.Timeout,
-		o.Headers,
+		o.Header,
 	)
 }
diff --git a/http_client.go b/http_client.go
index d442f17..77a5216 100644
--- a/http_client.go
+++ b/http_client.go
@@ -1,11 +1,12 @@
 package main
 
 import (
+	"net/http"
 	"net/url"
 )
 
 type httpClient interface {
 	// Get sends an HTTP request with a GET method.
 	// It depends on implementation of each client what is considered as errors.
-	Get(url *url.URL) (httpResponse, error)
+	Get(url *url.URL, header http.Header) (httpResponse, error)
 }
diff --git a/http_client_factory.go b/http_client_factory.go
index 0c2e403..ec3c385 100644
--- a/http_client_factory.go
+++ b/http_client_factory.go
@@ -1,6 +1,9 @@
 package main
 
-import "time"
+import (
+	"net/http"
+	"time"
+)
 
 type httpClientOptions struct {
 	MaxConnectionsPerHost,
@@ -9,7 +12,7 @@ type httpClientOptions struct {
 	Proxy               string
 	SkipTLSVerification bool
 	Timeout             time.Duration
-	Headers             map[string]string
+	Header              http.Header
 }
 
 type httpClientFactory interface {
diff --git a/link_fetcher.go b/link_fetcher.go
index dafe9e3..ad1b27e 100644
--- a/link_fetcher.go
+++ b/link_fetcher.go
@@ -73,7 +73,7 @@ func (f *linkFetcher) sendRequest(s string) (int, *page, error) {
 		return 0, nil, err
 	}
 
-	r, err := f.client.Get(u)
+	r, err := f.client.Get(u, nil)
 
 	if err != nil {
 		return 0, nil, err
diff --git a/redirect_http_client.go b/redirect_http_client.go
index b25433d..5f49c79 100644
--- a/redirect_http_client.go
+++ b/redirect_http_client.go
@@ -3,6 +3,8 @@ package main
 import (
 	"errors"
 	"fmt"
+	"net/http"
+	"net/http/cookiejar"
 	"net/url"
 )
 
@@ -15,11 +17,25 @@ func newRedirectHttpClient(c httpClient, maxRedirections int) httpClient {
 	return &redirectHttpClient{c, maxRedirections}
 }
 
-func (c *redirectHttpClient) Get(u *url.URL) (httpResponse, error) {
+func (c *redirectHttpClient) Get(u *url.URL, header http.Header) (httpResponse, error) {
+	if header == nil {
+		header = http.Header{}
+	}
+
+	cj, err := cookiejar.New(nil)
+
+	if err != nil {
+		return nil, err
+	}
+
 	i := 0
 
 	for {
-		r, err := c.client.Get(u)
+		for _, c := range cj.Cookies(u) {
+			header.Add("cookie", c.String())
+		}
+
+		r, err := c.client.Get(u, header)
 		if err != nil {
 			return nil, c.formatError(err, i, u)
 		}
@@ -45,6 +61,8 @@ func (c *redirectHttpClient) Get(u *url.URL) (httpResponse, error) {
 			if err != nil {
 				return nil, err
 			}
+
+			cj.SetCookies(u, parseCookies(r.Header("set-cookie")))
 		default:
 			return nil, c.formatError(fmt.Errorf("%v", r.StatusCode()), i, u)
 		}
@@ -58,3 +76,9 @@ func (*redirectHttpClient) formatError(err error, redirections int, u *url.URL)
 
 	return fmt.Errorf("%w (following redirect %v)", err, u.String())
 }
+
+func parseCookies(s string) []*http.Cookie {
+	h := http.Header{}
+	h.Add("cookie", s)
+	return (&http.Request{Header: h}).Cookies()
+}
diff --git a/robots_txt_fetcher.go b/robots_txt_fetcher.go
index f770fe3..74df3c4 100644
--- a/robots_txt_fetcher.go
+++ b/robots_txt_fetcher.go
@@ -19,7 +19,7 @@ func (f *robotsTxtFetcher) Fetch(uu *url.URL) (*robotstxt.RobotsData, error) {
 	u := *uu
 	u.Path = "robots.txt"
 
-	r, err := f.client.Get(&u)
+	r, err := f.client.Get(&u, nil)
 	if err != nil {
 		return nil, f.formatError(err)
 	}
diff --git a/sitemap_fetcher.go b/sitemap_fetcher.go
index 8321e25..c038443 100644
--- a/sitemap_fetcher.go
+++ b/sitemap_fetcher.go
@@ -20,7 +20,7 @@ func (f *sitemapFetcher) Fetch(uu *url.URL) (map[string]struct{}, error) {
 	u := *uu
 	u.Path = "sitemap.xml"
 
-	r, err := f.client.Get(&u)
+	r, err := f.client.Get(&u, nil)
 	if err != nil {
 		return nil, f.formatGetError(err)
 	}
diff --git a/throttled_http_client.go b/throttled_http_client.go
index fecc323..a0c2cc3 100644
--- a/throttled_http_client.go
+++ b/throttled_http_client.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"net/http"
 	"net/url"
 )
 
@@ -18,7 +19,7 @@ func newThrottledHttpClient(c httpClient, requestPerSecond int, maxConnections,
 	}
 }
 
-func (c *throttledHttpClient) Get(u *url.URL) (httpResponse, error) {
+func (c *throttledHttpClient) Get(u *url.URL, header http.Header) (httpResponse, error) {
 	c.connections.Request()
 	defer c.connections.Release()
 
@@ -26,5 +27,5 @@ func (c *throttledHttpClient) Get(u *url.URL) (httpResponse, error) {
 	t.Request()
 	defer t.Release()
 
-	return c.client.Get(u)
+	return c.client.Get(u, header)
 }


Test patch:
diff --git a/arguments_test.go b/arguments_test.go
index c2a7f1a..edf120a 100644
--- a/arguments_test.go
+++ b/arguments_test.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"net/http"
 	"testing"
 
 	"github.com/bradleyjkemp/cupaloy"
@@ -68,28 +69,37 @@ func TestHelp(t *testing.T) {
 	cupaloy.SnapshotT(t, help())
 }
 
-func TestParseHeaders(t *testing.T) {
+func TestParseHeader(t *testing.T) {
 	for _, c := range []struct {
 		arguments []string
-		answer    map[string]string
+		header    http.Header
 	}{
 		{
 			nil,
-			map[string]string{},
+			http.Header{},
 		},
 		{
 			[]string{"MyHeader: foo"},
-			map[string]string{"MyHeader": "foo"},
+			(func() http.Header {
+				h := http.Header{}
+				h.Add("MyHeader", "foo")
+				return h
+			})(),
 		},
 		{
 			[]string{"MyHeader: foo", "YourHeader: bar"},
-			map[string]string{"MyHeader": "foo", "YourHeader": "bar"},
+			(func() http.Header {
+				h := http.Header{}
+				h.Add("MyHeader", "foo")
+				h.Add("YourHeader", "bar")
+				return h
+			})(),
 		},
 	} {
 		hs, err := parseHeaders(c.arguments)
 
 		assert.Nil(t, err)
-		assert.Equal(t, c.answer, hs)
+		assert.Equal(t, c.header, hs)
 	}
 }
 
diff --git a/fake_http_client_test.go b/fake_http_client_test.go
index ffde74d..f25855c 100644
--- a/fake_http_client_test.go
+++ b/fake_http_client_test.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"net/http"
 	"net/url"
 )
 
@@ -12,6 +13,6 @@ func newFakeHttpClient(h func(*url.URL) (*fakeHttpResponse, error)) *fakeHttpCli
 	return &fakeHttpClient{h}
 }
 
-func (c *fakeHttpClient) Get(u *url.URL) (httpResponse, error) {
+func (c *fakeHttpClient) Get(u *url.URL, _ http.Header) (httpResponse, error) {
 	return c.handler(u)
 }
diff --git a/redirect_http_client_test.go b/redirect_http_client_test.go
index db7fbaa..cdf5dfc 100644
--- a/redirect_http_client_test.go
+++ b/redirect_http_client_test.go
@@ -30,7 +30,7 @@ func TestRedirectHttpClientGet(t *testing.T) {
 			},
 		),
 		42,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, err)
 	assert.Equal(t, 200, r.StatusCode())
@@ -62,7 +62,7 @@ func TestRedirectHttpClientGetWithRedirect(t *testing.T) {
 			},
 		),
 		42,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, err)
 	assert.Equal(t, 200, r.StatusCode())
@@ -96,7 +96,7 @@ func TestRedirectHttpClientGetWithRedirects(t *testing.T) {
 			},
 		),
 		maxRedirections,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, err)
 	assert.Equal(t, 200, r.StatusCode())
@@ -134,7 +134,7 @@ func TestRedirectHttpClientGetWithRelativeRedirect(t *testing.T) {
 			},
 		),
 		maxRedirections,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, err)
 	assert.Equal(t, 200, r.StatusCode())
@@ -163,7 +163,7 @@ func TestRedirectHttpClientFailWithTooManyRedirects(t *testing.T) {
 			},
 		),
 		maxRedirections,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, r)
 	assert.Equal(t, err.Error(), "too many redirections")
@@ -182,7 +182,7 @@ func TestRedirectHttpClientFailWithUnsetLocationHeader(t *testing.T) {
 			},
 		),
 		42,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, r)
 	assert.Equal(t, err.Error(), "location header not set")
@@ -205,7 +205,7 @@ func TestRedirectHttpClientFailWithInvalidLocationURL(t *testing.T) {
 			},
 		),
 		42,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, r)
 	assert.Contains(t, err.Error(), "parse")
@@ -223,7 +223,7 @@ func TestRedirectHttpClientFailWithInvalidStatusCode(t *testing.T) {
 			},
 		),
 		42,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, r)
 	assert.Equal(t, err.Error(), "404")
@@ -253,7 +253,7 @@ func TestRedirectHttpClientFailAfterRedirect(t *testing.T) {
 			},
 		),
 		42,
-	).Get(u)
+	).Get(u, nil)
 
 	assert.Nil(t, r)
 	assert.Contains(t, err.Error(), "following redirect http://foo.com/foo")