Repository: raviqqe/muffet
Pull request #190 (2021-11-10 03:58:56)

Refactor

codecov[bot]: # [Codecov](https://codecov.io/gh/raviqqe/muffet/pull/190?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama) Report
> Merging [#190](https://codecov.io/gh/raviqqe/muffet/pull/190?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama) (2bc01e7) into [master](https://codecov.io/gh/raviqqe/muffet/commit/1f6910665a2957e30f6ef29e785cf2ed550e4f8f?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama) (1f69106) will **decrease** coverage by `0.18%`.
> The diff coverage is `62.50%`.

[![Impacted file tree graph](https://codecov.io/gh/raviqqe/muffet/pull/190/graphs/tree.svg?width=650&height=150&src=pr&token=qXJG42bGNw&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama)](https://codecov.io/gh/raviqqe/muffet/pull/190?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama)

```diff
@@            Coverage Diff             @@
##           master     #190      +/-   ##
==========================================
- Coverage   77.27%   77.08%   -0.19%     
==========================================
  Files          23       23              
  Lines         418      419       +1     
==========================================
  Hits          323      323              
- Misses         79       80       +1     
  Partials       16       16              
```


| [Impacted Files](https://codecov.io/gh/raviqqe/muffet/pull/190?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama) | Coverage Δ | |
|---|---|---|
| [fasthttp\_http\_client.go](https://codecov.io/gh/raviqqe/muffet/pull/190/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-ZmFzdGh0dHBfaHR0cF9jbGllbnQuZ28=) | `0.00% <0.00%> (ø)` | |
| [fasthttp\_http\_client\_factory.go](https://codecov.io/gh/raviqqe/muffet/pull/190/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-ZmFzdGh0dHBfaHR0cF9jbGllbnRfZmFjdG9yeS5nbw==) | `0.00% <0.00%> (ø)` | |
| [command.go](https://codecov.io/gh/raviqqe/muffet/pull/190/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-Y29tbWFuZC5nbw==) | `84.26% <100.00%> (ø)` | |
| [link\_fetcher.go](https://codecov.io/gh/raviqqe/muffet/pull/190/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-bGlua19mZXRjaGVyLmdv) | `77.55% <100.00%> (ø)` | |
| [robots\_txt\_fetcher.go](https://codecov.io/gh/raviqqe/muffet/pull/190/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-cm9ib3RzX3R4dF9mZXRjaGVyLmdv) | `81.81% <100.00%> (ø)` | |
| [sitemap\_fetcher.go](https://codecov.io/gh/raviqqe/muffet/pull/190/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-c2l0ZW1hcF9mZXRjaGVyLmdv) | `88.88% <100.00%> (ø)` | |
| [throttled\_http\_client.go](https://codecov.io/gh/raviqqe/muffet/pull/190/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-dGhyb3R0bGVkX2h0dHBfY2xpZW50Lmdv) | `75.00% <100.00%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/raviqqe/muffet/pull/190?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/raviqqe/muffet/pull/190?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama). Last update [1f69106...2bc01e7](https://codecov.io/gh/raviqqe/muffet/pull/190?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama).


Patch:
diff --git a/README.md b/README.md
index 68f561c..00c4eff 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,7 @@ website recursively.
 ## Features
 
 - Massive speed
+- High compatibility with web browsers
 - Colored outputs
 - Different tags support (`a`, `img`, `link`, `script`, etc)
 
diff --git a/command.go b/command.go
index 1063efe..623c4d4 100644
--- a/command.go
+++ b/command.go
@@ -54,6 +54,7 @@ func (c *command) runWithError(ss []string) (bool, error) {
 				Proxy:                 args.Proxy,
 				SkipTLSVerification:   args.SkipTLSVerification,
 				Timeout:               time.Duration(args.Timeout) * time.Second,
+				Headers:               args.Headers,
 			},
 		),
 		args.RateLimit,
@@ -66,7 +67,6 @@ func (c *command) runWithError(ss []string) (bool, error) {
 		client,
 		pp,
 		linkFetcherOptions{
-			args.Headers,
 			args.IgnoreFragments,
 		},
 	)
diff --git a/fasthttp_http_client.go b/fasthttp_http_client.go
index 06286f6..6fe6971 100644
--- a/fasthttp_http_client.go
+++ b/fasthttp_http_client.go
@@ -14,23 +14,24 @@ type fasthttpHTTPClient struct {
 	client          *fasthttp.Client
 	maxRedirections int
 	timeout         time.Duration
+	headers         map[string]string
 }
 
-func newFasthttpHTTPClient(c *fasthttp.Client, maxRedirections int, timeout time.Duration) httpClient {
-	return &fasthttpHTTPClient{c, maxRedirections, timeout}
+func newFasthttpHTTPClient(c *fasthttp.Client, maxRedirections int, timeout time.Duration, headers map[string]string) httpClient {
+	return &fasthttpHTTPClient{c, maxRedirections, timeout, headers}
 }
 
-func (c *fasthttpHTTPClient) Get(u *url.URL, headers map[string]string) (httpResponse, error) {
+func (c *fasthttpHTTPClient) Get(u *url.URL) (httpResponse, error) {
 	req, res := fasthttp.Request{}, fasthttp.Response{}
 	req.SetRequestURI(u.String())
 	req.SetConnectionClose()
 
-	for k, v := range headers {
+	for k, v := range c.headers {
 		req.Header.Add(k, v)
 	}
 
 	// Some HTTP servers require "Accept" headers to be set explicitly.
-	if !includeHeader(headers, "Accept") {
+	if !includeHeader(c.headers, "Accept") {
 		req.Header.Add("Accept", "*/*")
 	}
 
diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go
index 9038f34..9fb5567 100644
--- a/fasthttp_http_client_factory.go
+++ b/fasthttp_http_client_factory.go
@@ -36,5 +36,6 @@ func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient {
 		},
 		o.MaxRedirections,
 		o.Timeout,
+		o.Headers,
 	)
 }
diff --git a/http_client.go b/http_client.go
index 44e00d2..e365ef7 100644
--- a/http_client.go
+++ b/http_client.go
@@ -7,5 +7,5 @@ import (
 type httpClient interface {
 	// Get sends a HTTP request with the GET method.
 	// Non-200 status codes always result in non-nil errors.
-	Get(url *url.URL, headers map[string]string) (httpResponse, error)
+	Get(url *url.URL) (httpResponse, error)
 }
diff --git a/http_client_factory.go b/http_client_factory.go
index 0c65542..726a1e6 100644
--- a/http_client_factory.go
+++ b/http_client_factory.go
@@ -9,6 +9,7 @@ type httpClientOptions struct {
 	Proxy               string
 	SkipTLSVerification bool
 	Timeout             time.Duration
+	Headers             map[string]string
 }
 
 type httpClientFactory interface {
diff --git a/link_fetcher.go b/link_fetcher.go
index 7d16c14..59b6799 100644
--- a/link_fetcher.go
+++ b/link_fetcher.go
@@ -72,7 +72,7 @@ func (f *linkFetcher) sendRequest(s string) (int, *page, error) {
 		return 0, nil, err
 	}
 
-	r, err := f.client.Get(u, f.options.Headers)
+	r, err := f.client.Get(u)
 
 	if err != nil {
 		return 0, nil, err
diff --git a/link_fetcher_options.go b/link_fetcher_options.go
index e241b27..b3ece11 100644
--- a/link_fetcher_options.go
+++ b/link_fetcher_options.go
@@ -1,7 +1,5 @@
 package main
 
 type linkFetcherOptions struct {
-	// TODO Move to httpClient?
-	Headers         map[string]string
 	IgnoreFragments bool
 }
diff --git a/robots_txt_fetcher.go b/robots_txt_fetcher.go
index 74df3c4..f770fe3 100644
--- a/robots_txt_fetcher.go
+++ b/robots_txt_fetcher.go
@@ -19,7 +19,7 @@ func (f *robotsTxtFetcher) Fetch(uu *url.URL) (*robotstxt.RobotsData, error) {
 	u := *uu
 	u.Path = "robots.txt"
 
-	r, err := f.client.Get(&u, nil)
+	r, err := f.client.Get(&u)
 	if err != nil {
 		return nil, f.formatError(err)
 	}
diff --git a/sitemap_fetcher.go b/sitemap_fetcher.go
index c038443..8321e25 100644
--- a/sitemap_fetcher.go
+++ b/sitemap_fetcher.go
@@ -20,7 +20,7 @@ func (f *sitemapFetcher) Fetch(uu *url.URL) (map[string]struct{}, error) {
 	u := *uu
 	u.Path = "sitemap.xml"
 
-	r, err := f.client.Get(&u, nil)
+	r, err := f.client.Get(&u)
 	if err != nil {
 		return nil, f.formatGetError(err)
 	}
diff --git a/throttled_http_client.go b/throttled_http_client.go
index d5cc174..5495bdb 100644
--- a/throttled_http_client.go
+++ b/throttled_http_client.go
@@ -1,7 +1,10 @@
 package main
 
-import "net/url"
-import "go.uber.org/ratelimit"
+import (
+	"net/url"
+
+	"go.uber.org/ratelimit"
+)
 
 // TODO Throttle requests for each host.
 type throttledHTTPClient struct {
@@ -20,11 +23,11 @@ func newThrottledHTTPClient(c httpClient, rps int, maxConnections int) httpClien
 	return &throttledHTTPClient{c, l, newSemaphore(maxConnections)}
 }
 
-func (c *throttledHTTPClient) Get(u *url.URL, hs map[string]string) (httpResponse, error) {
+func (c *throttledHTTPClient) Get(u *url.URL) (httpResponse, error) {
 	c.semaphore.Request()
 	defer c.semaphore.Release()
 
 	c.limiter.Take()
 
-	return c.client.Get(u, hs)
+	return c.client.Get(u)
 }


Test patch:
diff --git a/fake_http_client_test.go b/fake_http_client_test.go
index 6db81e8..2e956bd 100644
--- a/fake_http_client_test.go
+++ b/fake_http_client_test.go
@@ -12,6 +12,6 @@ func newFakeHTTPClient(h func(*url.URL) (*fakeHTTPResponse, error)) *fakeHTTPCli
 	return &fakeHTTPClient{h}
 }
 
-func (c *fakeHTTPClient) Get(u *url.URL, headers map[string]string) (httpResponse, error) {
+func (c *fakeHTTPClient) Get(u *url.URL) (httpResponse, error) {
 	return c.handler(u)
 }