Repository: raviqqe/muffet (go)

Issue:
--follow-sitemap doesn't follow the sitemap
Originally added after #32, the `--follow-sitemap` option should read the `sitemap.xml` from a site to determine which pages to check. 

This doesn't appear to work as expected.

Consider a very simple site with three main pages: `/index.html`, `/page1.html` and `/secret.html`.  All three are listed in the `/sitemap.xml`,  and `index.html` links to `page1.html` - however none of the pages link directly to `secret.html`.

When muffet is invoked with `--follow-sitemap` I expect it to read the sitemap and visit each page that is listed there, including `secret.html`. What it seems to do instead, is to start with `/index.html` and scan linked pages recursively _only if_ they're also in the sitemap.  

Is this the intended behavior of this option? Is there a way to achieve our use case? 

Pull request description:
Sitemap file as root page
Close #316.

# Examples

```sh
> muffet --buffer-size 10000 -v https://raviqqe.com/sitemap-index.xml
https://raviqqe.com/sitemap-index.xml
        200     https://raviqqe.com/sitemap-0.xml
https://raviqqe.com/sitemap-0.xml
        200     https://raviqqe.com/
https://raviqqe.com/
        200     https://alacritty.org/
        200     https://amanita-design.net/
        200     https://cloe-lang.org
        200     https://ein-lang.org
        200     https://englia.app/
        200     https://github.com/cloe-lang/cloe
        200     https://github.com/ein-lang/ein
        200     https://github.com/pen-lang/pen
        200     https://github.com/raviqqe
        200     https://github.com/raviqqe/bdwgc-alloc
        200     https://github.com/raviqqe/bstie
        200     https://github.com/raviqqe/dotfiles
        200     https://github.com/raviqqe/eslint-config
        200     https://github.com/raviqqe/fmm
        200     https://github.com/raviqqe/gherkin2markdown
        200     https://github.com/raviqqe/hamt
        200     https://github.com/raviqqe/hamt-rs
        200     https://github.com/raviqqe/melior
        200     https://github.com/raviqqe/muffet
        200     https://github.com/raviqqe/oneRPC
        200     https://github.com/raviqqe/parsdown
        200     https://github.com/raviqqe/pomodoro
        200     https://github.com/raviqqe/react-tsconfig
        200     https://github.com/raviqqe/rnm
        200     https://github.com/raviqqe/self-talk
        200     https://github.com/raviqqe/ssf
        200     https://github.com/raviqqe/tasks
        200     https://github.com/raviqqe/tsconfig
        200     https://github.com/raviqqe/turtle-build
        200     https://img.shields.io/badge/editor-neovim-brightgreen.svg?style=flat-square
        200     https://img.shields.io/badge/github-raviqqe-red.svg?style=flat-square
        200     https://img.shields.io/badge/shell-zsh-rebeccapurple.svg?style=flat-square
        200     https://img.shields.io/badge/terminal-alacritty-darkorange.svg?style=flat-square
        200     https://img.shields.io/badge/twitter-raviqqe-blue.svg?style=flat-square
        200     https://neovim.io/
        200     https://notes.code2d.org
        200     https://pen-lang.org
        200     https://plausible.io/js/plausible.js
        200     https://pomodoro.code2d.org
        200     https://raviqqe.com/
        200     https://raviqqe.com/_astro/index.53ff2848.css
        200     https://raviqqe.com/icon.svg
        200     https://raviqqe.com/manifest.json
        200     https://raviqqe.github.io/melior/melior/
        200     https://tasks.code2d.org
        200     https://twitter.com/raviqqe
        200     https://unlicense.org
        200     https://webassembly.org
        200     https://www.archlinux.org
        200     https://www.freebsd.org
        200     https://www.nyan.cat/
        200     https://zsh.sourceforge.io/
        200     https://zty.pe
```

Patch:
diff --git a/.cspell.json b/.cspell.json
index e97f5d2..449b8b8 100644
--- a/.cspell.json
+++ b/.cspell.json
@@ -23,6 +23,7 @@
     "redirections",
     "redirections",
     "robotstxt",
+    "sitemapindex",
     "stretchr",
     "temoto",
     "unbrotli",
diff --git a/arguments.go b/arguments.go
index 2fb8620..52927fc 100644
--- a/arguments.go
+++ b/arguments.go
@@ -18,7 +18,7 @@ type arguments struct {
 	RawExcludedPatterns   []string `short:"e" long:"exclude" value-name:"<pattern>..." description:"Exclude URLs matched with given regular expressions"`
 	RawIncludedPatterns   []string `short:"i" long:"include" value-name:"<pattern>..." description:"Include URLs matched with given regular expressions"`
 	FollowRobotsTxt       bool     `long:"follow-robots-txt" description:"Follow robots.txt when scraping pages"`
-	FollowSitemapXML      bool     `long:"follow-sitemap-xml" description:"Scrape only pages listed in sitemap.xml"`
+	FollowSitemapXML      bool     `long:"follow-sitemap-xml" description:"Scrape only pages listed in sitemap.xml (deprecated)"`
 	RawHeaders            []string `long:"header" value-name:"<header>..." description:"Custom headers"`
 	// TODO Remove a short option.
 	IgnoreFragments bool   `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"`
diff --git a/command.go b/command.go
index b78dbeb..1b0fa49 100644
--- a/command.go
+++ b/command.go
@@ -64,11 +64,12 @@ func (c *command) runWithError(ss []string) (bool, error) {
 		args.MaxRedirections,
 	)
 
-	pp := newPageParser(newLinkFinder(args.ExcludedPatterns, args.IncludePatterns))
-
 	f := newLinkFetcher(
 		client,
-		pp,
+		[]pageParser{
+			newSitemapPageParser(),
+			newHtmlPageParser(newLinkFinder(args.ExcludedPatterns, args.IncludePatterns)),
+		},
 		linkFetcherOptions{
 			args.IgnoreFragments,
 		},
@@ -78,7 +79,7 @@ func (c *command) runWithError(ss []string) (bool, error) {
 	if err != nil {
 		return false, fmt.Errorf("failed to fetch root page: %v", err)
 	} else if p == nil {
-		return false, errors.New("root page is not HTML")
+		return false, errors.New("root page has invalid content type")
 	}
 
 	rd := (*robotstxt.RobotsData)(nil)
diff --git a/html_page.go b/html_page.go
new file mode 100644
index 0000000..a9b5f01
--- /dev/null
+++ b/html_page.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+	"net/url"
+)
+
+type htmlPage struct {
+	url       *url.URL
+	fragments map[string]struct{}
+	links     map[string]error
+}
+
+func newHtmlPage(u *url.URL, fragments map[string]struct{}, links map[string]error) *htmlPage {
+	return &htmlPage{u, fragments, links}
+}
+
+func (p *htmlPage) URL() *url.URL {
+	return p.url
+}
+
+func (p *htmlPage) Fragments() map[string]struct{} {
+	return p.fragments
+}
+
+func (p *htmlPage) Links() map[string]error {
+	return p.links
+}
diff --git a/html_page_parser.go b/html_page_parser.go
new file mode 100644
index 0000000..6f9c138
--- /dev/null
+++ b/html_page_parser.go
@@ -0,0 +1,58 @@
+package main
+
+import (
+	"bytes"
+	"net/url"
+
+	"github.com/yhat/scrape"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+type htmlPageParser struct {
+	linkFinder linkFinder
+}
+
+func newHtmlPageParser(f linkFinder) *htmlPageParser {
+	return &htmlPageParser{f}
+}
+
+func (p htmlPageParser) Parse(u *url.URL, typ string, body []byte) (page, error) {
+	if typ != "text/html" {
+		return nil, nil
+	}
+
+	n, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+
+	u.Fragment = ""
+
+	frs := map[string]struct{}{}
+
+	scrape.FindAllNested(n, func(n *html.Node) bool {
+		for _, a := range []string{"id", "name"} {
+			if s := scrape.Attr(n, a); s != "" {
+				frs[s] = struct{}{}
+			}
+		}
+
+		return false
+	})
+
+	base := u
+
+	if n, ok := scrape.Find(n, func(n *html.Node) bool {
+		return n.DataAtom == atom.Base
+	}); ok {
+		u, err := url.Parse(scrape.Attr(n, "href"))
+		if err != nil {
+			return nil, err
+		}
+
+		base = base.ResolveReference(u)
+	}
+
+	return newHtmlPage(u, frs, p.linkFinder.Find(n, base)), nil
+}
diff --git a/link_fetcher.go b/link_fetcher.go
index ad1b27e..b7a2d2b 100644
--- a/link_fetcher.go
+++ b/link_fetcher.go
@@ -8,23 +8,23 @@ import (
 )
 
 type linkFetcher struct {
-	client     httpClient
-	pageParser *pageParser
-	cache      cache
-	options    linkFetcherOptions
+	client      httpClient
+	pageParsers []pageParser
+	cache       cache
+	options     linkFetcherOptions
 }
 
 type fetchResult struct {
 	StatusCode int
-	Page       *page
+	Page       page
 }
 
-func newLinkFetcher(c httpClient, pp *pageParser, o linkFetcherOptions) *linkFetcher {
-	return &linkFetcher{c, pp, newCache(), o}
+func newLinkFetcher(c httpClient, ps []pageParser, o linkFetcherOptions) *linkFetcher {
+	return &linkFetcher{c, ps, newCache(), o}
 }
 
 // Fetch fetches a link and returns a successful status code and optionally HTML page, or an error.
-func (f *linkFetcher) Fetch(u string) (int, *page, error) {
+func (f *linkFetcher) Fetch(u string) (int, page, error) {
 	u, fr, err := separateFragment(u)
 	if err != nil {
 		return 0, nil, err
@@ -43,7 +43,7 @@ func (f *linkFetcher) Fetch(u string) (int, *page, error) {
 	return s, p, nil
 }
 
-func (f *linkFetcher) sendRequestWithCache(u string) (int, *page, error) {
+func (f *linkFetcher) sendRequestWithCache(u string) (int, page, error) {
 	x, store := f.cache.LoadOrStore(u)
 
 	if store == nil {
@@ -67,7 +67,7 @@ func (f *linkFetcher) sendRequestWithCache(u string) (int, *page, error) {
 	return s, p, err
 }
 
-func (f *linkFetcher) sendRequest(s string) (int, *page, error) {
+func (f *linkFetcher) sendRequest(s string) (int, page, error) {
 	u, err := url.Parse(s)
 	if err != nil {
 		return 0, nil, err
@@ -77,13 +77,15 @@ func (f *linkFetcher) sendRequest(s string) (int, *page, error) {
 
 	if err != nil {
 		return 0, nil, err
-	} else if s := strings.TrimSpace(r.Header("Content-Type")); s != "" {
-		t, _, err := mime.ParseMediaType(s)
+	}
+
+	t := ""
+
+	if s := strings.TrimSpace(r.Header("Content-Type")); s != "" {
+		t, _, err = mime.ParseMediaType(s)
 
 		if err != nil {
 			return 0, nil, err
-		} else if t != "text/html" {
-			return r.StatusCode(), nil, nil
 		}
 	}
 
@@ -92,12 +94,21 @@ func (f *linkFetcher) sendRequest(s string) (int, *page, error) {
 		return 0, nil, err
 	}
 
-	p, err := f.pageParser.Parse(r.URL(), bs)
-	if err != nil {
-		return 0, nil, err
+	for _, pp := range f.pageParsers {
+		u, err := url.Parse(r.URL())
+		if err != nil {
+			return 0, nil, err
+		}
+
+		p, err := pp.Parse(u, t, bs)
+		if err != nil {
+			return 0, nil, err
+		} else if p != nil {
+			return r.StatusCode(), p, nil
+		}
 	}
 
-	return r.StatusCode(), p, nil
+	return r.StatusCode(), nil, nil
 }
 
 func separateFragment(s string) (string, string, error) {
diff --git a/page.go b/page.go
index 2e23d05..aa06763 100644
--- a/page.go
+++ b/page.go
@@ -4,24 +4,8 @@ import (
 	"net/url"
 )
 
-type page struct {
-	url       *url.URL
-	fragments map[string]struct{}
-	links     map[string]error
-}
-
-func newPage(u *url.URL, fragments map[string]struct{}, links map[string]error) *page {
-	return &page{u, fragments, links}
-}
-
-func (p *page) URL() *url.URL {
-	return p.url
-}
-
-func (p *page) Fragments() map[string]struct{} {
-	return p.fragments
-}
-
-func (p *page) Links() map[string]error {
-	return p.links
+type page interface {
+	URL() *url.URL
+	Fragments() map[string]struct{}
+	Links() map[string]error
 }
diff --git a/page_checker.go b/page_checker.go
index 3a724cd..7205df0 100644
--- a/page_checker.go
+++ b/page_checker.go
@@ -26,14 +26,14 @@ func (c *pageChecker) Results() <-chan *pageResult {
 	return c.results
 }
 
-func (c *pageChecker) Check(page *page) {
+func (c *pageChecker) Check(page page) {
 	c.addPage(page)
 	c.daemonManager.Run()
 
 	close(c.results)
 }
 
-func (c *pageChecker) checkPage(p *page) {
+func (c *pageChecker) checkPage(p page) {
 	us := p.Links()
 
 	sc := make(chan *successLinkResult, len(us))
@@ -85,7 +85,7 @@ func (c *pageChecker) checkPage(p *page) {
 	c.results <- &pageResult{p.URL().String(), ss, es}
 }
 
-func (c *pageChecker) addPage(p *page) {
+func (c *pageChecker) addPage(p page) {
 	if !c.donePages.Add(p.URL().String()) {
 		c.daemonManager.Add(func() { c.checkPage(p) })
 	}
diff --git a/page_parser.go b/page_parser.go
index 896650f..b7b9557 100644
--- a/page_parser.go
+++ b/page_parser.go
@@ -1,59 +1,8 @@
 package main
 
-import (
-	"bytes"
-	"net/url"
+import "net/url"
 
-	"github.com/yhat/scrape"
-	"golang.org/x/net/html"
-	"golang.org/x/net/html/atom"
-)
-
-type pageParser struct {
-	linkFinder linkFinder
-}
-
-func newPageParser(f linkFinder) *pageParser {
-	return &pageParser{f}
-}
-
-func (p pageParser) Parse(rawURL string, body []byte) (*page, error) {
-	n, err := html.Parse(bytes.NewReader(body))
-	if err != nil {
-		return nil, err
-	}
-
-	u, err := url.Parse(rawURL)
-	if err != nil {
-		return nil, err
-	}
-
-	u.Fragment = ""
-
-	frs := map[string]struct{}{}
-
-	scrape.FindAllNested(n, func(n *html.Node) bool {
-		for _, a := range []string{"id", "name"} {
-			if s := scrape.Attr(n, a); s != "" {
-				frs[s] = struct{}{}
-			}
-		}
-
-		return false
-	})
-
-	base := u
-
-	if n, ok := scrape.Find(n, func(n *html.Node) bool {
-		return n.DataAtom == atom.Base
-	}); ok {
-		u, err := url.Parse(scrape.Attr(n, "href"))
-		if err != nil {
-			return nil, err
-		}
-
-		base = base.ResolveReference(u)
-	}
-
-	return newPage(u, frs, p.linkFinder.Find(n, base)), nil
+type pageParser interface {
+	// Returned pages can be nil to indicate unrecognized file formats.
+	Parse(*url.URL, string, []byte) (page, error)
 }
diff --git a/sitemap_page.go b/sitemap_page.go
new file mode 100644
index 0000000..0b443e8
--- /dev/null
+++ b/sitemap_page.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"net/url"
+)
+
+type sitemapPage struct {
+	url   *url.URL
+	links map[string]error
+}
+
+func newSitemapPage(u *url.URL, links map[string]error) *sitemapPage {
+	return &sitemapPage{u, links}
+}
+
+func (p *sitemapPage) URL() *url.URL {
+	return p.url
+}
+
+func (p *sitemapPage) Fragments() map[string]struct{} {
+	return nil
+}
+
+func (p *sitemapPage) Links() map[string]error {
+	return p.links
+}
diff --git a/sitemap_page_parser.go b/sitemap_page_parser.go
new file mode 100644
index 0000000..e7a1851
--- /dev/null
+++ b/sitemap_page_parser.go
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"bytes"
+	"net/url"
+
+	sitemap "github.com/oxffaa/gopher-parse-sitemap"
+)
+
+type sitemapPageParser struct{}
+
+func newSitemapPageParser() *sitemapPageParser {
+	return &sitemapPageParser{}
+}
+
+func (f *sitemapPageParser) Parse(u *url.URL, typ string, bs []byte) (page, error) {
+	if typ != "application/xml" {
+		return nil, nil
+	}
+
+	ls := map[string]error{}
+	c := func(e interface{ GetLocation() string }) error {
+		ls[e.GetLocation()] = nil
+		return nil
+	}
+
+	err := sitemap.Parse(bytes.NewReader(bs), func(e sitemap.Entry) error {
+		return c(e)
+	})
+
+	// TODO Detect XML files as sitemaps.
+	if err == nil && len(ls) != 0 {
+		return newSitemapPage(u, ls), nil
+	}
+
+	err = sitemap.ParseIndex(bytes.NewReader(bs), func(e sitemap.IndexEntry) error {
+		return c(e)
+	})
+
+	// TODO Detect XML files as sitemap indices.
+	if err == nil && len(ls) != 0 {
+		return newSitemapPage(u, ls), nil
+	}
+
+	return nil, nil
+}


Test patch:
diff --git a/.snapshots/TestCommandFailToGetHTMLRootPage b/.snapshots/TestCommandFailToGetHTMLRootPage
index 7747b16..699c045 100644
--- a/.snapshots/TestCommandFailToGetHTMLRootPage
+++ b/.snapshots/TestCommandFailToGetHTMLRootPage
@@ -1,2 +1,2 @@
-root page is not HTML
+root page has invalid content type
 
diff --git a/.snapshots/TestHelp b/.snapshots/TestHelp
index 8cd6ee2..e352c53 100644
--- a/.snapshots/TestHelp
+++ b/.snapshots/TestHelp
@@ -17,7 +17,7 @@ Application Options:
       --follow-robots-txt                   Follow robots.txt when scraping
                                             pages
       --follow-sitemap-xml                  Scrape only pages listed in
-                                            sitemap.xml
+                                            sitemap.xml (deprecated)
       --header=<header>...                  Custom headers
   -f, --ignore-fragments                    Ignore URL fragments
       --format=[text|json|junit]            Output format (default: text)
diff --git a/fake_http_response_test.go b/fake_http_response_test.go
index 1639f35..c1e61d3 100644
--- a/fake_http_response_test.go
+++ b/fake_http_response_test.go
@@ -28,6 +28,15 @@ func newFakeHtmlResponse(location string, body string) *fakeHttpResponse {
 	)
 }
 
+func newFakeXmlResponse(location string, body string) *fakeHttpResponse {
+	return newFakeHttpResponse(
+		200,
+		location,
+		[]byte(body),
+		map[string]string{"content-type": "application/xml"},
+	)
+}
+
 func (r *fakeHttpResponse) URL() string {
 	return r.location
 }
diff --git a/html_page_parser_test.go b/html_page_parser_test.go
new file mode 100644
index 0000000..23c363a
--- /dev/null
+++ b/html_page_parser_test.go
@@ -0,0 +1,154 @@
+package main
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const HTML_MIME_TYPE = "text/html"
+
+func parseURL(t *testing.T, s string) *url.URL {
+	u, err := url.Parse(s)
+
+	assert.Nil(t, err)
+
+	return u
+}
+
+func TestHtmlPageParserParsePage(t *testing.T) {
+	_, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(parseURL(t, "http://foo.com"), HTML_MIME_TYPE, nil)
+	assert.Nil(t, err)
+}
+
+func TestHtmlPageParserSetCorrectURL(t *testing.T) {
+	u := parseURL(t, "http://foo.com")
+
+	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(u, HTML_MIME_TYPE, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, u.String(), p.URL().String())
+}
+
+func TestHtmlPageParserIgnorePageURLFragment(t *testing.T) {
+	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(parseURL(t, "http://foo.com#id"), HTML_MIME_TYPE, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, "http://foo.com", p.URL().String())
+}
+
+func TestHtmlPageParserKeepQuery(t *testing.T) {
+	s := "http://foo.com?bar=baz"
+
+	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(parseURL(t, s), HTML_MIME_TYPE, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, s, p.URL().String())
+}
+
+func TestHtmlPageParserResolveLinksWithBaseTag(t *testing.T) {
+	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+		parseURL(t, "http://foo.com"),
+		HTML_MIME_TYPE,
+		[]byte(`
+			<html>
+			  <head>
+					<base href="foo/" />
+				</head>
+				<body>
+				  <a href="bar" />
+				</body>
+			</html>
+		`),
+	)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]error{"http://foo.com/foo/bar": nil}, p.Links())
+}
+
+func TestHtmlPageParserResolveLinksWithBlankBaseTag(t *testing.T) {
+	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+		parseURL(t, "http://foo.com"),
+		HTML_MIME_TYPE,
+		[]byte(`
+			<html>
+			  <head>
+					<base href="_blank" />
+				</head>
+				<body>
+				  <a href="bar" />
+				</body>
+			</html>
+		`),
+	)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]error{"http://foo.com/bar": nil}, p.Links())
+}
+
+func TestHtmlPageParserFailToParseWithInvalidBaseTag(t *testing.T) {
+	_, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+		parseURL(t, "http://foo.com"),
+		HTML_MIME_TYPE,
+		[]byte(`
+			<html>
+			  <head>
+					<base href=":" />
+				</head>
+				<body>
+				</body>
+			</html>
+		`),
+	)
+	assert.NotNil(t, err)
+}
+
+func TestHtmlPageParserParseID(t *testing.T) {
+	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+		parseURL(t, "http://foo.com"),
+		HTML_MIME_TYPE,
+		[]byte(`<p id="foo" />`),
+	)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]struct{}{"foo": {}}, p.Fragments())
+}
+
+func TestHtmlPageParserParseName(t *testing.T) {
+	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+		parseURL(t, "http://foo.com"),
+		HTML_MIME_TYPE,
+		[]byte(`<p name="foo" />`),
+	)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]struct{}{"foo": {}}, p.Fragments())
+}
+
+func TestHtmlPageParserParseIDAndName(t *testing.T) {
+	p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+		parseURL(t, "http://foo.com"),
+		HTML_MIME_TYPE,
+		[]byte(`<p id="foo" name="bar" />`),
+	)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]struct{}{"foo": {}, "bar": {}}, p.Fragments())
+}
+
+func TestHtmlPageParserParseLinks(t *testing.T) {
+	for _, ss := range [][2]string{
+		{
+			`<a href="foo">bar</a>`,
+			"http://foo.com/foo",
+		},
+		{
+			`<img src="foo.img" />`,
+			"http://foo.com/foo.img",
+		},
+	} {
+		p, err := newHtmlPageParser(newLinkFinder(nil, nil)).Parse(
+			parseURL(t, "http://foo.com"),
+			HTML_MIME_TYPE,
+			[]byte(ss[0]),
+		)
+
+		assert.Nil(t, err)
+		assert.Equal(t, 1, len(p.Links()))
+		_, ok := p.Links()[ss[1]]
+		assert.True(t, ok)
+	}
+}
diff --git a/link_fetcher_test.go b/link_fetcher_test.go
index d332e6b..df8ed17 100644
--- a/link_fetcher_test.go
+++ b/link_fetcher_test.go
@@ -15,7 +15,7 @@ func newTestLinkFetcher(c *fakeHttpClient) *linkFetcher {
 }
 
 func newTestLinkFetcherWithOptions(c *fakeHttpClient, o linkFetcherOptions) *linkFetcher {
-	return newLinkFetcher(c, newPageParser(newLinkFinder(nil, nil)), o)
+	return newLinkFetcher(c, []pageParser{newSitemapPageParser(), newHtmlPageParser(newLinkFinder(nil, nil))}, o)
 }
 
 func TestNewFetcher(t *testing.T) {
@@ -162,6 +162,66 @@ func TestLinkFetcherFetchSkippingTextFragment(t *testing.T) {
 	assert.Nil(t, err)
 }
 
+func TestLinkFetcherFetchSitemap(t *testing.T) {
+	f := newTestLinkFetcher(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				if u.String() != "http://foo.com/sitemap.xml" {
+					return nil, errors.New("")
+				}
+
+				return newFakeXmlResponse("http://foo.com/sitemap.xml", `
+					<?xml version="1.0" encoding="UTF-8"?>
+					<urlset
+						xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+						xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"
+						xmlns:xhtml="http://www.w3.org/1999/xhtml"
+						xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"
+						xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"
+					>
+						<url>
+							<loc>https://foo.com/</loc>
+						</url>
+					</urlset>
+				`), nil
+			}),
+	)
+
+	s, p, err := f.Fetch("http://foo.com/sitemap.xml")
+
+	assert.Equal(t, 200, s)
+	assert.NotNil(t, p)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]error{"https://foo.com/": nil}, p.Links())
+}
+
+func TestLinkFetcherFetchSitemapIndex(t *testing.T) {
+	f := newTestLinkFetcher(
+		newFakeHttpClient(
+			func(u *url.URL) (*fakeHttpResponse, error) {
+				if u.String() != "http://foo.com/sitemap-index.xml" {
+					return nil, errors.New("")
+				}
+
+				return newFakeXmlResponse("http://foo.com/sitemap-index.xml", `
+					<?xml version="1.0" encoding="UTF-8"?>
+					<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+						<sitemap>
+							<loc>https://foo.com/sitemap-0.xml</loc>
+						</sitemap>
+					</sitemapindex>
+				`), nil
+			}),
+	)
+
+	s, p, err := f.Fetch("http://foo.com/sitemap-index.xml")
+
+	assert.Equal(t, 200, s)
+	assert.NotNil(t, p)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]error{"https://foo.com/sitemap-0.xml": nil}, p.Links())
+}
+
 func TestLinkFetcherFailToFetch(t *testing.T) {
 	f := newTestLinkFetcher(
 		newFakeHttpClient(func(*url.URL) (*fakeHttpResponse, error) {
diff --git a/page_checker_test.go b/page_checker_test.go
index 199c6fe..19640f9 100644
--- a/page_checker_test.go
+++ b/page_checker_test.go
@@ -12,7 +12,7 @@ func newTestPageChecker(c *fakeHttpClient) *pageChecker {
 	return newPageChecker(
 		newLinkFetcher(
 			c,
-			newPageParser(newLinkFinder(nil, nil)),
+			[]pageParser{newHtmlPageParser(newLinkFinder(nil, nil))},
 			linkFetcherOptions{},
 		),
 		newLinkValidator("foo.com", nil, nil),
@@ -20,11 +20,11 @@ func newTestPageChecker(c *fakeHttpClient) *pageChecker {
 	)
 }
 
-func newTestPage(t *testing.T, fragments map[string]struct{}, links map[string]error) *page {
+func newTestPage(t *testing.T, fragments map[string]struct{}, links map[string]error) page {
 	u, err := url.Parse("http://foo.com")
 	assert.Nil(t, err)
 
-	return newPage(u, fragments, links)
+	return newHtmlPage(u, fragments, links)
 }
 
 func TestPageCheckerCheckOnePage(t *testing.T) {
diff --git a/page_parser_test.go b/page_parser_test.go
deleted file mode 100644
index 370c049..0000000
--- a/page_parser_test.go
+++ /dev/null
@@ -1,143 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestPageParserParsePage(t *testing.T) {
-	_, err := newPageParser(newLinkFinder(nil, nil)).Parse("http://foo.com", nil)
-	assert.Nil(t, err)
-}
-
-func TestPageParserFailWithInvalidURL(t *testing.T) {
-	_, err := newPageParser(newLinkFinder(nil, nil)).Parse(":", nil)
-	assert.NotNil(t, err)
-}
-
-func TestPageParserSetCorrectURL(t *testing.T) {
-	s := "http://foo.com"
-
-	p, err := newPageParser(newLinkFinder(nil, nil)).Parse(s, nil)
-	assert.Nil(t, err)
-	assert.Equal(t, s, p.URL().String())
-}
-
-func TestPageParserIgnorePageURLFragment(t *testing.T) {
-	s := "http://foo.com#id"
-
-	p, err := newPageParser(newLinkFinder(nil, nil)).Parse(s, nil)
-	assert.Nil(t, err)
-	assert.Equal(t, "http://foo.com", p.URL().String())
-}
-
-func TestPageParserKeepQuery(t *testing.T) {
-	s := "http://foo.com?bar=baz"
-
-	p, err := newPageParser(newLinkFinder(nil, nil)).Parse(s, nil)
-	assert.Nil(t, err)
-	assert.Equal(t, s, p.URL().String())
-}
-
-func TestPageParserResolveLinksWithBaseTag(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil, nil)).Parse(
-		"http://foo.com",
-		[]byte(`
-			<html>
-			  <head>
-					<base href="foo/" />
-				</head>
-				<body>
-				  <a href="bar" />
-				</body>
-			</html>
-		`),
-	)
-	assert.Nil(t, err)
-	assert.Equal(t, map[string]error{"http://foo.com/foo/bar": nil}, p.Links())
-}
-
-func TestPageParserResolveLinksWithBlankBaseTag(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil, nil)).Parse(
-		"http://foo.com",
-		[]byte(`
-			<html>
-			  <head>
-					<base href="_blank" />
-				</head>
-				<body>
-				  <a href="bar" />
-				</body>
-			</html>
-		`),
-	)
-	assert.Nil(t, err)
-	assert.Equal(t, map[string]error{"http://foo.com/bar": nil}, p.Links())
-}
-
-func TestPageParserFailToParseWithInvalidBaseTag(t *testing.T) {
-	_, err := newPageParser(newLinkFinder(nil, nil)).Parse(
-		"http://foo.com",
-		[]byte(`
-			<html>
-			  <head>
-					<base href=":" />
-				</head>
-				<body>
-				</body>
-			</html>
-		`),
-	)
-	assert.NotNil(t, err)
-}
-
-func TestPageParserParseID(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil, nil)).Parse(
-		"http://foo.com",
-		[]byte(`<p id="foo" />`),
-	)
-	assert.Nil(t, err)
-	assert.Equal(t, map[string]struct{}{"foo": {}}, p.Fragments())
-}
-
-func TestPageParserParseName(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil, nil)).Parse(
-		"http://foo.com",
-		[]byte(`<p name="foo" />`),
-	)
-	assert.Nil(t, err)
-	assert.Equal(t, map[string]struct{}{"foo": {}}, p.Fragments())
-}
-
-func TestPageParserParseIDAndName(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil, nil)).Parse(
-		"http://foo.com",
-		[]byte(`<p id="foo" name="bar" />`),
-	)
-	assert.Nil(t, err)
-	assert.Equal(t, map[string]struct{}{"foo": {}, "bar": {}}, p.Fragments())
-}
-
-func TestPageParserParseLinks(t *testing.T) {
-	for _, ss := range [][2]string{
-		{
-			`<a href="foo">bar</a>`,
-			"http://foo.com/foo",
-		},
-		{
-			`<img src="foo.img" />`,
-			"http://foo.com/foo.img",
-		},
-	} {
-		p, err := newPageParser(newLinkFinder(nil, nil)).Parse(
-			"http://foo.com",
-			[]byte(ss[0]),
-		)
-
-		assert.Nil(t, err)
-		assert.Equal(t, 1, len(p.Links()))
-		_, ok := p.Links()[ss[1]]
-		assert.True(t, ok)
-	}
-}
diff --git a/sitemap_page_parser_test.go b/sitemap_page_parser_test.go
new file mode 100644
index 0000000..9481d53
--- /dev/null
+++ b/sitemap_page_parser_test.go
@@ -0,0 +1,61 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const SITEMAP_MIME_TYPE = "application/xml"
+
+func TestSitemapPageParserParsePage(t *testing.T) {
+	p, err := newSitemapPageParser().Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<urlset
+			xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+			xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"
+			xmlns:xhtml="http://www.w3.org/1999/xhtml"
+			xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"
+			xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"
+		>
+			<url>
+				<loc>https://foo.com/</loc>
+			</url>
+		</urlset>
+	`))
+
+	assert.Nil(t, err)
+	assert.Equal(t, "https://foo.com/sitemap.xml", p.URL().String())
+	assert.Equal(t, map[string]error{"https://foo.com/": nil}, p.Links())
+	assert.Equal(t, map[string]struct{}(nil), p.Fragments())
+}
+
+func TestSitemapPageParserParseIndexPage(t *testing.T) {
+	p, err := newSitemapPageParser().Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+			<sitemap>
+				<loc>https://foo.com/sitemap-0.xml</loc>
+			</sitemap>
+		</sitemapindex>
+	`))
+
+	assert.Nil(t, err)
+	assert.Equal(t, "https://foo.com/sitemap.xml", p.URL().String())
+	assert.Equal(t, map[string]error{"https://foo.com/sitemap-0.xml": nil}, p.Links())
+	assert.Equal(t, map[string]struct{}(nil), p.Fragments())
+}
+
+func TestSitemapPageParserFailToParseXML(t *testing.T) {
+	p, err := newSitemapPageParser().Parse(parseURL(t, "https://foo.com/sitemap.xml"), SITEMAP_MIME_TYPE, []byte(""))
+
+	assert.Nil(t, err)
+	assert.Nil(t, p)
+}
+
+func TestSitemapPageParserFailToParseHTML(t *testing.T) {
+	p, err := newSitemapPageParser().Parse(parseURL(t, "https://foo.com/sitemap.xml"), "text/html", []byte(""))
+
+	assert.Nil(t, err)
+	assert.Nil(t, p)
+}