Repository: raviqqe/muffet
Pull request #319 (2023-06-29 02:38:56)

Fix link filtering for sitemap files

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/raviqqe/muffet/pull/319?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama) Report
> Merging [#319](https://app.codecov.io/gh/raviqqe/muffet/pull/319?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama) (7a4e845) into [main](https://app.codecov.io/gh/raviqqe/muffet/commit/06b7f4bfb291e59a9fd2e445ed1ec494813a7fb3?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama) (06b7f4b) will **decrease** coverage by `0.19%`.
> The diff coverage is `88.88%`.

```diff
@@            Coverage Diff             @@
##             main     #319      +/-   ##
==========================================
- Coverage   87.13%   86.94%   -0.19%     
==========================================
  Files          29       30       +1     
  Lines         847      858      +11     
==========================================
+ Hits          738      746       +8     
- Misses         88       90       +2     
- Partials       21       22       +1     
```


| [Impacted Files](https://app.codecov.io/gh/raviqqe/muffet/pull/319?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama) | Coverage Δ | |
|---|---|---|
| [link\_filterer.go](https://app.codecov.io/gh/raviqqe/muffet/pull/319?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-bGlua19maWx0ZXJlci5nbw==) | `84.21% <84.21%> (ø)` | |
| [command.go](https://app.codecov.io/gh/raviqqe/muffet/pull/319?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-Y29tbWFuZC5nbw==) | `85.98% <100.00%> (+0.18%)` | :arrow_up: |
| [link\_finder.go](https://app.codecov.io/gh/raviqqe/muffet/pull/319?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama#diff-bGlua19maW5kZXIuZ28=) | `100.00% <100.00%> (ø)` | |

:mega: We’re building smart automated test selection to slash your CI/CD build times. [Learn more](https://about.codecov.io/iterative-testing/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Yota+Toyama)


Patch:
diff --git a/command.go b/command.go
index 1b0fa49..167d55b 100644
--- a/command.go
+++ b/command.go
@@ -64,11 +64,13 @@ func (c *command) runWithError(ss []string) (bool, error) {
 		args.MaxRedirections,
 	)
 
+	fl := newLinkFilterer(args.ExcludedPatterns, args.IncludePatterns)
+
 	f := newLinkFetcher(
 		client,
 		[]pageParser{
-			newSitemapPageParser(),
-			newHtmlPageParser(newLinkFinder(args.ExcludedPatterns, args.IncludePatterns)),
+			newSitemapPageParser(fl),
+			newHtmlPageParser(newLinkFinder(fl)),
 		},
 		linkFetcherOptions{
 			args.IgnoreFragments,
diff --git a/link_filterer.go b/link_filterer.go
new file mode 100644
index 0000000..f0e3f29
--- /dev/null
+++ b/link_filterer.go
@@ -0,0 +1,49 @@
+package main
+
+import (
+	"net/url"
+	"regexp"
+)
+
+var validSchemes = map[string]struct{}{
+	"":      {},
+	"http":  {},
+	"https": {},
+}
+
+type linkFilterer struct {
+	excludedPatterns []*regexp.Regexp
+	includedPatterns []*regexp.Regexp
+}
+
+func newLinkFilterer(es []*regexp.Regexp, is []*regexp.Regexp) linkFilterer {
+	return linkFilterer{excludedPatterns: es, includedPatterns: is}
+}
+
+func (f linkFilterer) IsValid(u *url.URL) bool {
+	s := u.String()
+
+	if _, ok := validSchemes[u.Scheme]; !ok {
+		return false
+	}
+
+	return !f.isLinkExcluded(s) && f.isLinkIncluded(s)
+}
+
+func (f linkFilterer) isLinkExcluded(u string) bool {
+	return f.matches(u, f.excludedPatterns)
+}
+
+func (f linkFilterer) isLinkIncluded(u string) bool {
+	return len(f.includedPatterns) == 0 || f.matches(u, f.includedPatterns)
+}
+
+func (f linkFilterer) matches(u string, rs []*regexp.Regexp) bool {
+	for _, r := range rs {
+		if r.MatchString(u) {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/link_finder.go b/link_finder.go
index 69a72ac..ca2f804 100644
--- a/link_finder.go
+++ b/link_finder.go
@@ -10,12 +10,6 @@ import (
 	"golang.org/x/net/html/atom"
 )
 
-var validSchemes = map[string]struct{}{
-	"":      {},
-	"http":  {},
-	"https": {},
-}
-
 var atomToAttributes = map[atom.Atom][]string{
 	atom.A:      {"href"},
 	atom.Frame:  {"src"},
@@ -31,12 +25,11 @@ var atomToAttributes = map[atom.Atom][]string{
 var imageDescriptorPattern = regexp.MustCompile(" [^ ]*$")
 
 type linkFinder struct {
-	excludedPatterns []*regexp.Regexp
-	includedPatterns []*regexp.Regexp
+	linkFilterer linkFilterer
 }
 
-func newLinkFinder(es []*regexp.Regexp, is []*regexp.Regexp) linkFinder {
-	return linkFinder{excludedPatterns: es, includedPatterns: is}
+func newLinkFinder(f linkFilterer) linkFinder {
+	return linkFinder{f}
 }
 
 func (f linkFinder) Find(n *html.Node, base *url.URL) map[string]error {
@@ -62,10 +55,10 @@ func (f linkFinder) Find(n *html.Node, base *url.URL) map[string]error {
 					continue
 				}
 
-				s = base.ResolveReference(u).String()
+				u = base.ResolveReference(u)
 
-				if _, ok := validSchemes[u.Scheme]; ok && !f.isLinkExcluded(s) && f.isLinkIncluded(s) {
-					ls[s] = nil
+				if f.linkFilterer.IsValid(u) {
+					ls[u.String()] = nil
 				}
 			}
 		}
@@ -94,21 +87,3 @@ func (linkFinder) parseLinks(n *html.Node, a string) []string {
 
 	return ss
 }
-
-func (f linkFinder) isLinkExcluded(u string) bool {
-	return f.matches(u, f.excludedPatterns)
-}
-
-func (f linkFinder) isLinkIncluded(u string) bool {
-	return len(f.includedPatterns) == 0 || f.matches(u, f.includedPatterns)
-}
-
-func (f linkFinder) matches(u string, rs []*regexp.Regexp) bool {
-	for _, r := range rs {
-		if r.MatchString(u) {
-			return true
-		}
-	}
-
-	return false
-}
diff --git a/sitemap_page_parser.go b/sitemap_page_parser.go
index e7a1851..c8c09ad 100644
--- a/sitemap_page_parser.go
+++ b/sitemap_page_parser.go
@@ -7,20 +7,27 @@ import (
 	sitemap "github.com/oxffaa/gopher-parse-sitemap"
 )
 
-type sitemapPageParser struct{}
+type sitemapPageParser struct {
+	linkFilterer linkFilterer
+}
 
-func newSitemapPageParser() *sitemapPageParser {
-	return &sitemapPageParser{}
+func newSitemapPageParser(f linkFilterer) *sitemapPageParser {
+	return &sitemapPageParser{f}
 }
 
-func (f *sitemapPageParser) Parse(u *url.URL, typ string, bs []byte) (page, error) {
+func (p *sitemapPageParser) Parse(u *url.URL, typ string, bs []byte) (page, error) {
 	if typ != "application/xml" {
 		return nil, nil
 	}
 
 	ls := map[string]error{}
 	c := func(e interface{ GetLocation() string }) error {
-		ls[e.GetLocation()] = nil
+		u, err := url.Parse(e.GetLocation())
+
+		if p.linkFilterer.IsValid(u) {
+			ls[u.String()] = err
+		}
+
 		return nil
 	}
 


Test patch:
diff --git a/html_page_parser_test.go b/html_page_parser_test.go
index 23c363a..75c980d 100644
--- a/html_page_parser_test.go
+++ b/html_page_parser_test.go
@@ -18,20 +18,20 @@ func parseURL(t *testing.T, s string) *url.URL {
 }
 
 func TestHtmlPageParserParsePage(t *testing.T) {
-	_, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(parseURL(t, "http://foo.com"), HTML_MIME_TYPE, nil)
+	_, err := newHtmlPageParser(newTestLinkFinder()).Parse(parseURL(t, "http://foo.com"), HTML_MIME_TYPE, nil)
 	assert.Nil(t, err)
 }
 
 func TestHtmlPageParserSetCorrectURL(t *testing.T) {
 	u := parseURL(t, "http://foo.com")
 
-	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(u, HTML_MIME_TYPE, nil)
+	p, err := newHtmlPageParser(newTestLinkFinder()).Parse(u, HTML_MIME_TYPE, nil)
 	assert.Nil(t, err)
 	assert.Equal(t, u.String(), p.URL().String())
 }
 
 func TestHtmlPageParserIgnorePageURLFragment(t *testing.T) {
-	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(parseURL(t, "http://foo.com#id"), HTML_MIME_TYPE, nil)
+	p, err := newHtmlPageParser(newTestLinkFinder()).Parse(parseURL(t, "http://foo.com#id"), HTML_MIME_TYPE, nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "http://foo.com", p.URL().String())
 }
@@ -39,13 +39,13 @@ func TestHtmlPageParserIgnorePageURLFragment(t *testing.T) {
 func TestHtmlPageParserKeepQuery(t *testing.T) {
 	s := "http://foo.com?bar=baz"
 
-	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(parseURL(t, s), HTML_MIME_TYPE, nil)
+	p, err := newHtmlPageParser(newTestLinkFinder()).Parse(parseURL(t, s), HTML_MIME_TYPE, nil)
 	assert.Nil(t, err)
 	assert.Equal(t, s, p.URL().String())
 }
 
 func TestHtmlPageParserResolveLinksWithBaseTag(t *testing.T) {
-	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+	p, err := newHtmlPageParser(newTestLinkFinder()).Parse(
 		parseURL(t, "http://foo.com"),
 		HTML_MIME_TYPE,
 		[]byte(`
@@ -64,7 +64,7 @@ func TestHtmlPageParserResolveLinksWithBaseTag(t *testing.T) {
 }
 
 func TestHtmlPageParserResolveLinksWithBlankBaseTag(t *testing.T) {
-	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+	p, err := newHtmlPageParser(newTestLinkFinder()).Parse(
 		parseURL(t, "http://foo.com"),
 		HTML_MIME_TYPE,
 		[]byte(`
@@ -83,7 +83,7 @@ func TestHtmlPageParserResolveLinksWithBlankBaseTag(t *testing.T) {
 }
 
 func TestHtmlPageParserFailToParseWithInvalidBaseTag(t *testing.T) {
-	_, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+	_, err := newHtmlPageParser(newTestLinkFinder()).Parse(
 		parseURL(t, "http://foo.com"),
 		HTML_MIME_TYPE,
 		[]byte(`
@@ -100,7 +100,7 @@ func TestHtmlPageParserFailToParseWithInvalidBaseTag(t *testing.T) {
 }
 
 func TestHtmlPageParserParseID(t *testing.T) {
-	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+	p, err := newHtmlPageParser(newTestLinkFinder()).Parse(
 		parseURL(t, "http://foo.com"),
 		HTML_MIME_TYPE,
 		[]byte(`<p id="foo" />`),
@@ -110,7 +110,7 @@ func TestHtmlPageParserParseID(t *testing.T) {
 }
 
 func TestHtmlPageParserParseName(t *testing.T) {
-	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+	p, err := newHtmlPageParser(newTestLinkFinder()).Parse(
 		parseURL(t, "http://foo.com"),
 		HTML_MIME_TYPE,
 		[]byte(`<p name="foo" />`),
@@ -120,7 +120,7 @@ func TestHtmlPageParserParseName(t *testing.T) {
 }
 
 func TestHtmlPageParserParseIDAndName(t *testing.T) {
-	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+	p, err := newHtmlPageParser(newTestLinkFinder()).Parse(
 		parseURL(t, "http://foo.com"),
 		HTML_MIME_TYPE,
 		[]byte(`<p id="foo" name="bar" />`),
@@ -140,7 +140,7 @@ func TestHtmlPageParserParseLinks(t *testing.T) {
 			"http://foo.com/foo.img",
 		},
 	} {
-		p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+		p, err := newHtmlPageParser(newTestLinkFinder()).Parse(
 			parseURL(t, "http://foo.com"),
 			HTML_MIME_TYPE,
 			[]byte(ss[0]),
diff --git a/link_fetcher_test.go b/link_fetcher_test.go
index df8ed17..10f4e3e 100644
--- a/link_fetcher_test.go
+++ b/link_fetcher_test.go
@@ -15,7 +15,7 @@ func newTestLinkFetcher(c *fakeHttpClient) *linkFetcher {
 }
 
 func newTestLinkFetcherWithOptions(c *fakeHttpClient, o linkFetcherOptions) *linkFetcher {
-	return newLinkFetcher(c, []pageParser{newSitemapPageParser(), newHtmlPageParser(newLinkFinder(nil, nil))}, o)
+	return newLinkFetcher(c, []pageParser{newSitemapPageParser(newTestLinkFilterer()), newHtmlPageParser(newTestLinkFinder())}, o)
 }
 
 func TestNewFetcher(t *testing.T) {
diff --git a/link_filterer_test.go b/link_filterer_test.go
new file mode 100644
index 0000000..de4a3b7
--- /dev/null
+++ b/link_filterer_test.go
@@ -0,0 +1,108 @@
+package main
+
+import (
+	"fmt"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func newTestLinkFilterer() linkFilterer {
+	return newLinkFilterer(nil, nil)
+}
+
+func TestLinkFiltererIsLinkExcluded(t *testing.T) {
+	u, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	for _, x := range []struct {
+		regexps []string
+		answer  bool
+	}{
+		{
+			[]string{"foo\\.com"},
+			false,
+		},
+		{
+			[]string{"foo"},
+			false,
+		},
+		{
+			[]string{"bar", "foo"},
+			false,
+		},
+		{
+			[]string{"bar"},
+			true,
+		},
+	} {
+		t.Run(fmt.Sprint(x.regexps), func(t *testing.T) {
+			rs, err := compileRegexps(x.regexps)
+			assert.Nil(t, err)
+
+			assert.Equal(t, x.answer, newLinkFilterer(rs, nil).IsValid(u))
+		})
+	}
+}
+
+func TestLinkFiltererIsLinkIncluded(t *testing.T) {
+	u, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	for _, x := range []struct {
+		regexps []string
+		answer  bool
+	}{
+		{
+			[]string{"foo\\.com"},
+			true,
+		},
+		{
+			[]string{"foo"},
+			true,
+		},
+		{
+			[]string{"bar", "foo"},
+			true,
+		},
+		{
+			[]string{"bar"},
+			false,
+		},
+	} {
+		t.Run(fmt.Sprint(x.regexps), func(t *testing.T) {
+			rs, err := compileRegexps(x.regexps)
+			assert.Nil(t, err)
+
+			assert.Equal(t, x.answer, newLinkFilterer(nil, rs).IsValid(u))
+		})
+	}
+}
+
+func TestLinkFiltererExcludeEntireUrl(t *testing.T) {
+	b, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	rs, err := compileRegexps([]string{"foo"})
+	assert.Nil(t, err)
+
+	assert.False(t, newLinkFilterer(rs, nil).IsValid(b))
+}
+
+func TestLinkFiltererIncludeEntireUrl(t *testing.T) {
+	b, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	rs, err := compileRegexps([]string{"foo"})
+	assert.Nil(t, err)
+
+	assert.True(t, newLinkFilterer(nil, rs).IsValid(b))
+}
+
+func TestLinkFiltererExcludeInvalidScheme(t *testing.T) {
+	b, err := url.Parse("mailto:foo@bar.baz")
+	assert.Nil(t, err)
+
+	assert.False(t, newLinkFilterer(nil, nil).IsValid(b))
+}
diff --git a/link_finder_test.go b/link_finder_test.go
index 4da52e8..cfd241c 100644
--- a/link_finder_test.go
+++ b/link_finder_test.go
@@ -10,6 +10,10 @@ import (
 	"golang.org/x/net/html"
 )
 
+func newTestLinkFinder() linkFinder {
+	return newLinkFinder(newTestLinkFilterer())
+}
+
 func TestLinkFinderFindLinks(t *testing.T) {
 	b, err := url.Parse("https://localhost")
 	assert.Nil(t, err)
@@ -39,7 +43,7 @@ func TestLinkFinderFindLinks(t *testing.T) {
 
 		s, e := 0, 0
 
-		for _, err := range newLinkFinder(nil, nil).Find(n, b) {
+		for _, err := range newTestLinkFinder().Find(n, b) {
 			if err == nil {
 				s++
 			} else {
@@ -61,7 +65,7 @@ func TestLinkFinderFindLinkWithoutEncodedSpaces(t *testing.T) {
 	)
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	err, ok := ls["http://foo.com/a%20b"]
 	assert.True(t, ok)
@@ -77,7 +81,7 @@ func TestLinkFinderFindLinkWithoutSpacesNotEncoded(t *testing.T) {
 	)
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	err, ok := ls["http://foo.com/a%20b"]
 	assert.True(t, ok)
@@ -93,7 +97,7 @@ func TestLinkFinderFindLinkWithLeadingAndTrailingSpaces(t *testing.T) {
 	)
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	err, ok := ls["http://foo.com"]
 	assert.True(t, ok)
@@ -107,96 +111,12 @@ func TestLinkFinderFailWithInvalidURL(t *testing.T) {
 	n, err := html.Parse(strings.NewReader(htmlWithBody(`<a href=":" />`)))
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	assert.Equal(t, 1, len(ls))
 	assert.NotNil(t, ls[":"])
 }
 
-func TestLinkFinderIsLinkExcluded(t *testing.T) {
-	for _, x := range []struct {
-		regexps []string
-		answer  bool
-	}{
-		{
-			[]string{"foo\\.com"},
-			true,
-		},
-		{
-			[]string{"foo"},
-			true,
-		},
-		{
-			[]string{"bar", "foo"},
-			true,
-		},
-		{
-			[]string{"bar"},
-			false,
-		},
-	} {
-		rs, err := compileRegexps(x.regexps)
-		assert.Nil(t, err)
-
-		assert.Equal(t, x.answer, newLinkFinder(rs, nil).isLinkExcluded("http://foo.com"))
-	}
-}
-
-func TestLinkFinderIsLinkIncluded(t *testing.T) {
-	for _, x := range []struct {
-		regexps []string
-		answer  bool
-	}{
-		{
-			[]string{"foo\\.com"},
-			true,
-		},
-		{
-			[]string{"foo"},
-			true,
-		},
-		{
-			[]string{"bar", "foo"},
-			true,
-		},
-		{
-			[]string{"bar"},
-			false,
-		},
-	} {
-		rs, err := compileRegexps(x.regexps)
-		assert.Nil(t, err)
-
-		assert.Equal(t, x.answer, newLinkFinder(nil, rs).isLinkIncluded("http://foo.com"))
-	}
-}
-
-func TestLinkFinderExcludeEntireUrl(t *testing.T) {
-	b, err := url.Parse("http://foo.com")
-	assert.Nil(t, err)
-
-	n, err := html.Parse(strings.NewReader(htmlWithBody(`<a href="/bar" />`)))
-	assert.Nil(t, err)
-
-	rs, err := compileRegexps([]string{"foo"})
-	assert.Nil(t, err)
-
-	assert.Equal(t, map[string]error{}, newLinkFinder(rs, nil).Find(n, b))
-}
-
-func TestLinkFinderIncludeEntireUrl(t *testing.T) {
-	b, err := url.Parse("http://foo.com")
-	assert.Nil(t, err)
-
-	n, err := html.Parse(strings.NewReader(htmlWithBody(`<a href="/bar" />`)))
-	assert.Nil(t, err)
-
-	rs, err := compileRegexps([]string{"foo"})
-	assert.Nil(t, err)
-
-	assert.Equal(t, map[string]error{"http://foo.com/bar": nil}, newLinkFinder(nil, rs).Find(n, b))
-}
-
 func TestLinkFinderFindLinkInSrcSet(t *testing.T) {
 	b, err := url.Parse("http://foo.com")
 	assert.Nil(t, err)
@@ -206,7 +126,7 @@ func TestLinkFinderFindLinkInSrcSet(t *testing.T) {
 	)
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	err, ok := ls["http://foo.com/foo.png"]
 	assert.True(t, ok)
@@ -222,7 +142,7 @@ func TestLinkFinderFindMultipleLinksInSrcSet(t *testing.T) {
 	)
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	err, ok := ls["http://foo.com/foo.png"]
 	assert.True(t, ok)
@@ -242,7 +162,7 @@ func TestLinkFinderFindMultipleLinksInSrcSetWithDescriptors(t *testing.T) {
 	)
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	err, ok := ls["http://foo.com/foo.png"]
 	assert.True(t, ok)
@@ -262,7 +182,7 @@ func TestLinkFinderFindMetaTags(t *testing.T) {
 	)
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	err, ok := ls["http://foo.com/foo.png"]
 	assert.True(t, ok)
@@ -278,7 +198,7 @@ func TestLinkFinderIgnoreMetaTags(t *testing.T) {
 	)
 	assert.Nil(t, err)
 
-	ls := newLinkFinder(nil, nil).Find(n, b)
+	ls := newTestLinkFinder().Find(n, b)
 
 	assert.Len(t, ls, 0)
 }
diff --git a/page_checker_test.go b/page_checker_test.go
index 19640f9..82a2a86 100644
--- a/page_checker_test.go
+++ b/page_checker_test.go
@@ -12,7 +12,7 @@ func newTestPageChecker(c *fakeHttpClient) *pageChecker {
 	return newPageChecker(
 		newLinkFetcher(
 			c,
-			[]pageParser{newHtmlPageParser(newLinkFinder(nil, nil))},
+			[]pageParser{newHtmlPageParser(newTestLinkFinder())},
 			linkFetcherOptions{},
 		),
 		newLinkValidator("foo.com", nil, nil),
diff --git a/sitemap_page_parser_test.go b/sitemap_page_parser_test.go
index 9481d53..1cc3a77 100644
--- a/sitemap_page_parser_test.go
+++ b/sitemap_page_parser_test.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -9,7 +10,7 @@ import (
 const SITEMAP_MIME_TYPE = "application/xml"
 
 func TestSitemapPageParserParsePage(t *testing.T) {
-	p, err := newSitemapPageParser().Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(`
+	p, err := newSitemapPageParser(newTestLinkFilterer()).Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(`
 		<?xml version="1.0" encoding="UTF-8"?>
 		<urlset
 			xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
@@ -31,7 +32,7 @@ func TestSitemapPageParserParsePage(t *testing.T) {
 }
 
 func TestSitemapPageParserParseIndexPage(t *testing.T) {
-	p, err := newSitemapPageParser().Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(`
+	p, err := newSitemapPageParser(newTestLinkFilterer()).Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(`
 		<?xml version="1.0" encoding="UTF-8"?>
 		<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 			<sitemap>
@@ -46,15 +47,36 @@ func TestSitemapPageParserParseIndexPage(t *testing.T) {
 	assert.Equal(t, map[string]struct{}(nil), p.Fragments())
 }
 
+func TestSitemapPageParserExcludeLink(t *testing.T) {
+	p, err := newSitemapPageParser(
+		newLinkFilterer([]*regexp.Regexp{regexp.MustCompile("private")}, nil),
+	).Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+			<sitemap>
+				<loc>https://foo.com/private-sitemap.xml</loc>
+			</sitemap>
+			<sitemap>
+				<loc>https://foo.com/public-sitemap.xml</loc>
+			</sitemap>
+		</sitemapindex>
+	`))
+
+	assert.Nil(t, err)
+	assert.Equal(t, "https://foo.com/sitemap.xml", p.URL().String())
+	assert.Equal(t, map[string]error{"https://foo.com/public-sitemap.xml": nil}, p.Links())
+	assert.Equal(t, map[string]struct{}(nil), p.Fragments())
+}
+
 func TestSitemapPageParserFailToParseXML(t *testing.T) {
-	p, err := newSitemapPageParser().Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(""))
+	p, err := newSitemapPageParser(newTestLinkFilterer()).Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(""))
 
 	assert.Nil(t, err)
 	assert.Nil(t, p)
 }
 
 func TestSitemapPageParserFailToParseHTML(t *testing.T) {
-	p, err := newSitemapPageParser().Parse(parseURL(t, "https://foo.com/sitemap.xml"), "text/html", []byte(""))
+	p, err := newSitemapPageParser(newTestLinkFilterer()).Parse(parseURL(t, "https://foo.com/sitemap.xml"), "text/html", []byte(""))
 
 	assert.Nil(t, err)
 	assert.Nil(t, p)