Repository: raviqqe/muffet
Pull request #138 (2020-12-17 03:22:25)

Unzip responses

codecov[bot]: # [Codecov](https://codecov.io/gh/raviqqe/muffet/pull/138?src=pr&el=h1) Report
> Merging [#138](https://codecov.io/gh/raviqqe/muffet/pull/138?src=pr&el=desc) (370376b) into [master](https://codecov.io/gh/raviqqe/muffet/commit/5dd06974e0190b60039f30814b91d287935743d3?el=desc) (5dd0697) will **decrease** coverage by `1.52%`.
> The diff coverage is `50.00%`.

[![Impacted file tree graph](https://codecov.io/gh/raviqqe/muffet/pull/138/graphs/tree.svg?width=650&height=150&src=pr&token=qXJG42bGNw)](https://codecov.io/gh/raviqqe/muffet/pull/138?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #138      +/-   ##
==========================================
- Coverage   79.32%   77.80%   -1.53%     
==========================================
  Files          23       23              
  Lines         387      401      +14     
==========================================
+ Hits          307      312       +5     
- Misses         68       74       +6     
- Partials       12       15       +3     
```


| [Impacted Files](https://codecov.io/gh/raviqqe/muffet/pull/138?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [fasthttp\_http\_response.go](https://codecov.io/gh/raviqqe/muffet/pull/138/diff?src=pr&el=tree#diff-ZmFzdGh0dHBfaHR0cF9yZXNwb25zZS5nbw==) | `0.00% <0.00%> (ø)` | |
| [link\_fetcher.go](https://codecov.io/gh/raviqqe/muffet/pull/138/diff?src=pr&el=tree#diff-bGlua19mZXRjaGVyLmdv) | `77.55% <50.00%> (-2.89%)` | :arrow_down: |
| [robots\_txt\_fetcher.go](https://codecov.io/gh/raviqqe/muffet/pull/138/diff?src=pr&el=tree#diff-cm9ib3RzX3R4dF9mZXRjaGVyLmdv) | `81.81% <66.66%> (-18.19%)` | :arrow_down: |
| [sitemap\_fetcher.go](https://codecov.io/gh/raviqqe/muffet/pull/138/diff?src=pr&el=tree#diff-c2l0ZW1hcF9mZXRjaGVyLmdv) | `88.88% <66.66%> (-11.12%)` | :arrow_down: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/raviqqe/muffet/pull/138?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/raviqqe/muffet/pull/138?src=pr&el=footer). Last update [5dd0697...370376b](https://codecov.io/gh/raviqqe/muffet/pull/138?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/configuration.go b/configuration.go
index 90b6c74..eb7ccdf 100644
--- a/configuration.go
+++ b/configuration.go
@@ -3,7 +3,7 @@ package main
 import "time"
 
 const (
-	version     = "2.3.0"
+	version     = "2.3.1"
 	agentName   = "muffet"
 	concurrency = 1024
 	tcpTimeout  = 5 * time.Second
diff --git a/fasthttp_http_client.go b/fasthttp_http_client.go
index e977425..e58dfba 100644
--- a/fasthttp_http_client.go
+++ b/fasthttp_http_client.go
@@ -24,6 +24,9 @@ func (c *fasthttpHTTPClient) Get(u *url.URL, headers map[string]string) (httpRes
 	req.SetRequestURI(u.String())
 	req.SetConnectionClose()
 
+	// Some HTTP servers require "Accept" headers to be set explicitly.
+	req.Header.Add("Accept", "*/*")
+
 	for k, v := range headers {
 		req.Header.Add(k, v)
 	}
diff --git a/fasthttp_http_response.go b/fasthttp_http_response.go
index 4dc3477..481144a 100644
--- a/fasthttp_http_response.go
+++ b/fasthttp_http_response.go
@@ -25,6 +25,11 @@ func (r fasthttpHTTPResponse) Header(key string) string {
 	return string(r.response.Header.Peek(key))
 }
 
-func (r fasthttpHTTPResponse) Body() []byte {
-	return r.response.Body()
+func (r fasthttpHTTPResponse) Body() ([]byte, error) {
+	switch string(r.response.Header.Peek("Content-Encoding")) {
+	case "gzip":
+		return r.response.BodyGunzip()
+	}
+
+	return r.response.Body(), nil
 }
diff --git a/go.sum b/go.sum
index 315cba0..a1dfcdf 100644
--- a/go.sum
+++ b/go.sum
@@ -39,8 +39,7 @@ github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++Ja
 github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAkqg=
-github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
+github.com/valyala/fasthttp v1.18.0 h1:IV0DdMlatq9QO1Cr6wGJPVW1sV1Q8HvZXAIcjorylyM=
 github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945 h1:6Ju8pZBYFTN9FaV/JvNBiIHcsgEmP4z4laciqjfjY8E=
diff --git a/http_response.go b/http_response.go
index c4caf00..9e4c997 100644
--- a/http_response.go
+++ b/http_response.go
@@ -4,5 +4,5 @@ type httpResponse interface {
 	URL() string
 	StatusCode() int
 	Header(string) string
-	Body() []byte
+	Body() ([]byte, error)
 }
diff --git a/link_fetcher.go b/link_fetcher.go
index c160dae..7d16c14 100644
--- a/link_fetcher.go
+++ b/link_fetcher.go
@@ -86,7 +86,12 @@ func (f *linkFetcher) sendRequest(s string) (int, *page, error) {
 		}
 	}
 
-	p, err := f.pageParser.Parse(r.URL(), r.Body())
+	bs, err := r.Body()
+	if err != nil {
+		return 0, nil, err
+	}
+
+	p, err := f.pageParser.Parse(r.URL(), bs)
 	if err != nil {
 		return 0, nil, err
 	}
diff --git a/robots_txt_fetcher.go b/robots_txt_fetcher.go
index eccda11..74df3c4 100644
--- a/robots_txt_fetcher.go
+++ b/robots_txt_fetcher.go
@@ -18,10 +18,20 @@ func newRobotsTxtFetcher(c httpClient) *robotsTxtFetcher {
 func (f *robotsTxtFetcher) Fetch(uu *url.URL) (*robotstxt.RobotsData, error) {
 	u := *uu
 	u.Path = "robots.txt"
+
 	r, err := f.client.Get(&u, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to fetch robots.txt: %v", err)
+		return nil, f.formatError(err)
+	}
+
+	bs, err := r.Body()
+	if err != nil {
+		return nil, f.formatError(err)
 	}
 
-	return robotstxt.FromBytes(r.Body())
+	return robotstxt.FromBytes(bs)
+}
+
+func (*robotsTxtFetcher) formatError(err error) error {
+	return fmt.Errorf("failed to fetch robots.txt: %v", err)
 }
diff --git a/sitemap_fetcher.go b/sitemap_fetcher.go
index 0ce313d..c038443 100644
--- a/sitemap_fetcher.go
+++ b/sitemap_fetcher.go
@@ -22,12 +22,17 @@ func (f *sitemapFetcher) Fetch(uu *url.URL) (map[string]struct{}, error) {
 
 	r, err := f.client.Get(&u, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to GET sitemap.xml: %v", err)
+		return nil, f.formatGetError(err)
 	}
 
 	us := map[string]struct{}{}
 
-	err = sitemap.Parse(bytes.NewReader(r.Body()), func(e sitemap.Entry) error {
+	bs, err := r.Body()
+	if err != nil {
+		return nil, f.formatGetError(err)
+	}
+
+	err = sitemap.Parse(bytes.NewReader(bs), func(e sitemap.Entry) error {
 		us[e.GetLocation()] = struct{}{}
 
 		return nil
@@ -39,3 +44,7 @@ func (f *sitemapFetcher) Fetch(uu *url.URL) (map[string]struct{}, error) {
 
 	return us, nil
 }
+
+func (*sitemapFetcher) formatGetError(err error) error {
+	return fmt.Errorf("failed to GET sitemap.xml: %v", err)
+}


Test patch:
diff --git a/fake_http_response_test.go b/fake_http_response_test.go
index 9ad6162..3e263ae 100644
--- a/fake_http_response_test.go
+++ b/fake_http_response_test.go
@@ -29,6 +29,6 @@ func (r *fakeHTTPResponse) Header(name string) string {
 	return ""
 }
 
-func (r *fakeHTTPResponse) Body() []byte {
-	return r.body
+func (r *fakeHTTPResponse) Body() ([]byte, error) {
+	return r.body, nil
 }