Repository: raviqqe/muffet
Pull request #108 (2020-09-27 03:00:56)

Fix duplicate visits

codecov[bot]: # [Codecov](https://codecov.io/gh/raviqqe/muffet/pull/108?src=pr&el=h1) Report
> Merging [#108](https://codecov.io/gh/raviqqe/muffet/pull/108?src=pr&el=desc) into [master](https://codecov.io/gh/raviqqe/muffet/commit/e38867027966238ef2e97c11fdd21d29e9c90a1f?el=desc) will **decrease** coverage by `0.12%`.
> The diff coverage is `100.00%`.

[![Impacted file tree graph](https://codecov.io/gh/raviqqe/muffet/pull/108/graphs/tree.svg?width=650&height=150&src=pr&token=qXJG42bGNw)](https://codecov.io/gh/raviqqe/muffet/pull/108?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #108      +/-   ##
==========================================
- Coverage   77.77%   77.65%   -0.13%     
==========================================
  Files          21       21              
  Lines         360      358       -2     
==========================================
- Hits          280      278       -2     
  Misses         66       66              
  Partials       14       14              
```


| [Impacted Files](https://codecov.io/gh/raviqqe/muffet/pull/108?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [checker.go](https://codecov.io/gh/raviqqe/muffet/pull/108/diff?src=pr&el=tree#diff-Y2hlY2tlci5nbw==) | `92.10% <100.00%> (-0.40%)` | :arrow_down: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/raviqqe/muffet/pull/108?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/raviqqe/muffet/pull/108?src=pr&el=footer). Last update [e388670...1003ef3](https://codecov.io/gh/raviqqe/muffet/pull/108?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/arguments.go b/arguments.go
index 3fee516..c537acd 100644
--- a/arguments.go
+++ b/arguments.go
@@ -45,7 +45,6 @@ func getArguments(ss []string) (*arguments, error) {
 	args.URL = ss[0]
 
 	rs, err := compileRegexps(args.RawExcludedPatterns)
-
 	if err != nil {
 		return nil, err
 	}
@@ -53,7 +52,6 @@ func getArguments(ss []string) (*arguments, error) {
 	args.ExcludedPatterns = rs
 
 	hs, err := parseHeaders(args.RawHeaders)
-
 	if err != nil {
 		return nil, err
 	}
diff --git a/checker.go b/checker.go
index 5dda938..f4c854a 100644
--- a/checker.go
+++ b/checker.go
@@ -61,10 +61,7 @@ func (c *checker) checkPage(p *page) {
 				ec <- &errorLinkResult{u, err}
 			}
 
-			if !c.onePageOnly &&
-				p != nil &&
-				c.linkValidator.Validate(p.URL()) &&
-				!c.donePages.Add(p.URL().String()) {
+			if !c.onePageOnly && p != nil && c.linkValidator.Validate(p.URL()) {
 				c.addPage(p)
 			}
 		}(u)
@@ -91,5 +88,7 @@ func (c *checker) checkPage(p *page) {
 }
 
 func (c *checker) addPage(p *page) {
-	c.daemonManager.Add(func() { c.checkPage(p) })
+	if !c.donePages.Add(p.URL().String()) {
+		c.daemonManager.Add(func() { c.checkPage(p) })
+	}
 }
diff --git a/command.go b/command.go
index 11147ef..3111a60 100644
--- a/command.go
+++ b/command.go
@@ -22,7 +22,6 @@ func newCommand(stdout, stderr io.Writer, terminal bool, f httpClientFactory) *c
 
 func (c *command) Run(args []string) bool {
 	ok, err := c.runWithError(args)
-
 	if err != nil {
 		c.printError(err)
 	}
diff --git a/configuration.go b/configuration.go
index 3c164c2..61f767e 100644
--- a/configuration.go
+++ b/configuration.go
@@ -3,7 +3,7 @@ package main
 import "time"
 
 const (
-	version               = "2.0.0"
+	version               = "2.0.1"
 	agentName             = "muffet"
 	concurrency           = 1024
 	tcpTimeout            = time.Minute
diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go
index 6620056..f8bcfec 100644
--- a/fasthttp_http_client_factory.go
+++ b/fasthttp_http_client_factory.go
@@ -24,7 +24,8 @@ func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient {
 			},
 			Dial: func(addr string) (net.Conn, error) {
 				return fasthttp.DialTimeout(addr, tcpTimeout)
-			}},
+			},
+		},
 		o.MaxRedirections,
 		o.Timeout,
 	)
diff --git a/robots_txt_fetcher.go b/robots_txt_fetcher.go
index c0592e3..eccda11 100644
--- a/robots_txt_fetcher.go
+++ b/robots_txt_fetcher.go
@@ -19,7 +19,6 @@ func (f *robotsTxtFetcher) Fetch(uu *url.URL) (*robotstxt.RobotsData, error) {
 	u := *uu
 	u.Path = "robots.txt"
 	r, err := f.client.Get(&u, nil)
-
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch robots.txt: %v", err)
 	}
diff --git a/sitemap_fetcher.go b/sitemap_fetcher.go
index a34fc26..0ce313d 100644
--- a/sitemap_fetcher.go
+++ b/sitemap_fetcher.go
@@ -21,7 +21,6 @@ func (f *sitemapFetcher) Fetch(uu *url.URL) (map[string]struct{}, error) {
 	u.Path = "sitemap.xml"
 
 	r, err := f.client.Get(&u, nil)
-
 	if err != nil {
 		return nil, fmt.Errorf("failed to GET sitemap.xml: %v", err)
 	}


Test patch:
diff --git a/checker_test.go b/checker_test.go
index 4904884..c3f5833 100644
--- a/checker_test.go
+++ b/checker_test.go
@@ -92,3 +92,23 @@ func TestCheckerFailToCheckPage(t *testing.T) {
 
 	assert.False(t, (<-c.Results()).OK())
 }
+
+func TestCheckerDoNotCheckSamePageTwice(t *testing.T) {
+	c := newTestChecker(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				return newFakeHTTPResponse(200, "http://foo.com", "text/html", nil), nil
+			},
+		),
+	)
+
+	go c.Check(newTestPage(t, nil, map[string]error{"http://foo.com": nil}))
+
+	i := 0
+
+	for range c.Results() {
+		i++
+	}
+
+	assert.Equal(t, 1, i)
+}