Repository: raviqqe/muffet
Pull request #88 (2020-09-19 02:46:10)

Refactor for v2
# Refactored files

- [x] arguments.go  
- [x] arguments_test.go  
- [x] cache.go 
- [x] cache_test.go        
- [x] checker.go                
- [x] checker_test.go
- [x] command.go       
- [x] command_test.go
- [x] concurrent_string_set.go
- [x] concurrent_string_set_test.go
- [x] configuration.go
- [x] daemon_manager.go
- [x] daemon_manager_test.go
- [x] Dockerfile
- [x] fake_http_client.go
- [x] fake_http_response.go
- [x] fasthttp_http_client.go
- [x] fasthttp_http_response.go
- [x] go.mod
- [x] go.sum
- [x] http_client.go
- [x] http_response.go
- [x] img
- [x] LICENSE
- [x] link_fetcher.go
- [x] link_fetcher_options.go
- [x] link_fetcher_options_test.go
- [x] link_fetcher_test.go
- [x] link_finder.go
- [x] link_finder_test.go
- [x] link_validator.go
- [x] link_validator_test.go
- [x] main.go
- [x] page.go
- [x] page_parser.go
- [x] page_parser_test.go
- [x] page_result.go
- [x] page_result_test.go
- [x] README.md
- [x] robots_txt_fetcher.go
- [x] robots_txt_fetcher_test.go
- [x] semaphore.go
- [x] semaphore_test.go
- [x] setup_test.go
- [x] sitemap_fetcher.go
- [x] sitemap_fetcher_test.go
- [x] test
- [x] tools

codecov[bot]: # [Codecov](https://codecov.io/gh/raviqqe/muffet/pull/88?src=pr&el=h1) Report
> Merging [#88](https://codecov.io/gh/raviqqe/muffet/pull/88?src=pr&el=desc) into [master](https://codecov.io/gh/raviqqe/muffet/commit/49a18d21511d4975656ae3d81499b15c4fa66e36?el=desc) will **not change** coverage.
> The diff coverage is `62.50%`.

[![Impacted file tree graph](https://codecov.io/gh/raviqqe/muffet/pull/88/graphs/tree.svg?width=650&height=150&src=pr&token=qXJG42bGNw)](https://codecov.io/gh/raviqqe/muffet/pull/88?src=pr&el=tree)

```diff
@@           Coverage Diff           @@
##           master      #88   +/-   ##
=======================================
  Coverage   97.37%   97.37%           
=======================================
  Files          14       14           
  Lines         343      343           
=======================================
  Hits          334      334           
  Misses          7        7           
  Partials        2        2           
```


| [Impacted Files](https://codecov.io/gh/raviqqe/muffet/pull/88?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [main.go](https://codecov.io/gh/raviqqe/muffet/pull/88/diff?src=pr&el=tree#diff-bWFpbi5nbw==) | `79.41% <62.50%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/raviqqe/muffet/pull/88?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/raviqqe/muffet/pull/88?src=pr&el=footer). Last update [49a18d2...8df6f45](https://codecov.io/gh/raviqqe/muffet/pull/88?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index a75121c..57aae85 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -25,11 +25,9 @@ jobs:
     working_directory: /go/src/github.com/raviqqe/muffet
     steps:
       - checkout
-      - run: tools/dependencies.sh
-      - run: tools/lint.sh
       - run: tools/build.sh
-      - run: tools/unit_test.sh
-      - run: tools/integration_test.sh
+      - run: tools/lint.sh
+      - run: tools/test.sh
       - persist_to_workspace:
           root: .
           paths:
diff --git a/arguments.go b/arguments.go
index 928f16f..18ec7a3 100644
--- a/arguments.go
+++ b/arguments.go
@@ -50,46 +50,37 @@ type arguments struct {
 	OnePageOnly bool
 }
 
-func getArguments(ss []string) (arguments, error) {
-	args := parseArguments(usage, ss)
+func getArguments(regexps []string) (arguments, error) {
+	args := parseArguments(usage, regexps)
 
 	b, err := parseInt(args["--buffer-size"].(string))
-
 	if err != nil {
 		return arguments{}, err
 	}
 
 	c, err := parseInt(args["--concurrency"].(string))
-
 	if err != nil {
 		return arguments{}, err
 	}
 
-	ss, _ = args["--exclude"].([]string)
+	ss, _ := args["--exclude"].([]string)
 	rs, err := compileRegexps(ss)
-
 	if err != nil {
 		return arguments{}, err
 	}
 
-	hs := map[string]string(nil)
-
-	if ss := args["--header"]; ss != nil {
-		hs, err = parseHeaders(ss.([]string))
-
-		if err != nil {
-			return arguments{}, err
-		}
+	ss, _ = args["--header"].([]string)
+	hs, err := parseHeaders(ss)
+	if err != nil {
+		return arguments{}, err
 	}
 
 	r, err := parseInt(args["--limit-redirections"].(string))
-
 	if err != nil {
 		return arguments{}, err
 	}
 
 	t, err := parseInt(args["--timeout"].(string))
-
 	if err != nil {
 		return arguments{}, err
 	}
@@ -113,8 +104,7 @@ func getArguments(ss []string) (arguments, error) {
 }
 
 func parseArguments(u string, ss []string) map[string]interface{} {
-	args, err := docopt.ParseArgs(u, ss, "1.3.3")
-
+	args, err := docopt.ParseArgs(u, ss, version)
 	if err != nil {
 		panic(err)
 	}
@@ -127,12 +117,11 @@ func parseInt(s string) (int, error) {
 	return int(i), err
 }
 
-func compileRegexps(ss []string) ([]*regexp.Regexp, error) {
-	rs := make([]*regexp.Regexp, 0, len(ss))
+func compileRegexps(regexps []string) ([]*regexp.Regexp, error) {
+	rs := make([]*regexp.Regexp, 0, len(regexps))
 
-	for _, s := range ss {
+	for _, s := range regexps {
 		r, err := regexp.Compile(s)
-
 		if err != nil {
 			return nil, err
 		}
@@ -143,10 +132,10 @@ func compileRegexps(ss []string) ([]*regexp.Regexp, error) {
 	return rs, nil
 }
 
-func parseHeaders(ss []string) (map[string]string, error) {
-	m := make(map[string]string, len(ss))
+func parseHeaders(headers []string) (map[string]string, error) {
+	m := make(map[string]string, len(headers))
 
-	for _, s := range ss {
+	for _, s := range headers {
 		i := strings.IndexRune(s, ':')
 
 		if i < 0 {
diff --git a/cache.go b/cache.go
index ea2242d..f747bf8 100644
--- a/cache.go
+++ b/cache.go
@@ -11,23 +11,23 @@ func newCache() cache {
 	return cache{&sync.Map{}, &sync.Map{}}
 }
 
-func (c cache) LoadOrStore(s string) (interface{}, func(interface{}), bool) {
-	if x, ok := c.values.Load(s); ok {
-		return x, nil, true
+func (c cache) LoadOrStore(key string) (interface{}, func(interface{})) {
+	if x, ok := c.values.Load(key); ok {
+		return x, nil
 	}
 
 	g := &sync.WaitGroup{}
 	g.Add(1)
 
-	if g, ok := c.locks.LoadOrStore(s, g); ok {
+	if g, ok := c.locks.LoadOrStore(key, g); ok {
 		g.(*sync.WaitGroup).Wait()
-		x, _ := c.values.Load(s)
+		x, _ := c.values.Load(key)
 
-		return x, nil, true
+		return x, nil
 	}
 
 	return nil, func(x interface{}) {
-		c.values.Store(s, x)
+		c.values.Store(key, x)
 		g.Done()
-	}, false
+	}
 }
diff --git a/checker.go b/checker.go
index 5b947a2..b6b47eb 100644
--- a/checker.go
+++ b/checker.go
@@ -1,75 +1,43 @@
 package main
 
 import (
-	"crypto/tls"
-	"errors"
 	"sync"
 
 	"github.com/fatih/color"
-	"github.com/valyala/fasthttp"
 )
 
 type checker struct {
-	fetcher
-	daemons      daemons
-	urlInspector urlInspector
-	results      chan pageResult
-	donePages    concurrentStringSet
+	fetcher       *linkFetcher
+	linkValidator *linkValidator
+	daemonManager *daemonManager
+	results       chan pageResult
+	donePages     concurrentStringSet
+	onePageOnly   bool
 }
 
-func newChecker(s string, o checkerOptions) (checker, error) {
-	o.Initialize()
-
-	c := &fasthttp.Client{
-		MaxConnsPerHost: o.Concurrency,
-		ReadBufferSize:  o.BufferSize,
-		TLSConfig: &tls.Config{
-			InsecureSkipVerify: o.SkipTLSVerification,
-		},
-	}
-	f := newFetcher(c, o.fetcherOptions)
-	r, err := f.Fetch(s)
-
-	if err != nil {
-		return checker{}, err
-	}
-
-	p, ok := r.Page()
-
-	if !ok {
-		return checker{}, errors.New("non-HTML page")
-	}
-
-	ui, err := newURLInspector(c, p.URL().String(), o.FollowRobotsTxt, o.FollowSitemapXML)
-
-	if err != nil {
-		return checker{}, err
-	}
-
-	ch := checker{
+func newChecker(f *linkFetcher, v *linkValidator, concurrency int, onePageOnly bool) *checker {
+	return &checker{
 		f,
-		newDaemons(o.Concurrency),
-		ui,
-		make(chan pageResult, o.Concurrency),
+		v,
+		newDaemonManager(concurrency),
+		make(chan pageResult, concurrency),
 		newConcurrentStringSet(),
+		onePageOnly,
 	}
-
-	ch.addPage(p)
-
-	return ch, nil
 }
 
-func (c checker) Results() <-chan pageResult {
+func (c *checker) Results() <-chan pageResult {
 	return c.results
 }
 
-func (c checker) Check() {
-	c.daemons.Run()
+func (c *checker) Check(page *page) {
+	c.addPage(page)
+	c.daemonManager.Run()
 
 	close(c.results)
 }
 
-func (c checker) checkPage(p *page) {
+func (c *checker) checkPage(p *page) {
 	us := p.Links()
 
 	sc := make(chan string, len(us))
@@ -87,39 +55,39 @@ func (c checker) checkPage(p *page) {
 		go func(u string) {
 			defer w.Done()
 
-			r, err := c.fetcher.Fetch(u)
+			status, p, err := c.fetcher.Fetch(u)
 
 			if err == nil {
-				sc <- formatLinkSuccess(u, r.StatusCode())
+				sc <- formatLinkSuccess(u, status)
 			} else {
 				ec <- formatLinkError(u, err)
 			}
 
-			// only consider adding the page to the list if we're recursing
-			if !c.fetcher.options.OnePageOnly {
-				if p, ok := r.Page(); ok && c.urlInspector.Inspect(p.URL()) {
-					c.addPage(p)
-				}
+			if !c.onePageOnly && p != nil && c.linkValidator.Validate(p.URL()) {
+				c.addPage(p)
 			}
 		}(u)
 	}
 
 	w.Wait()
 
+	close(sc)
+	close(ec)
+
 	c.results <- newPageResult(p.URL().String(), stringChannelToSlice(sc), stringChannelToSlice(ec))
 }
 
-func (c checker) addPage(p *page) {
+func (c *checker) addPage(p *page) {
 	if !c.donePages.Add(p.URL().String()) {
-		c.daemons.Add(func() { c.checkPage(p) })
+		c.daemonManager.Add(func() { c.checkPage(p) })
 	}
 }
 
 func stringChannelToSlice(sc <-chan string) []string {
 	ss := make([]string, 0, len(sc))
 
-	for i := 0; i < cap(ss); i++ {
-		ss = append(ss, <-sc)
+	for s := range sc {
+		ss = append(ss, s)
 	}
 
 	return ss
diff --git a/checker_option.go b/checker_option.go
deleted file mode 100644
index 0776bc3..0000000
--- a/checker_option.go
+++ /dev/null
@@ -1,10 +0,0 @@
-package main
-
-type checkerOptions struct {
-	fetcherOptions
-	BufferSize int
-	FollowRobotsTxt,
-	FollowSitemapXML,
-	FollowURLParams,
-	SkipTLSVerification bool
-}
diff --git a/command.go b/command.go
new file mode 100644
index 0000000..4d32eab
--- /dev/null
+++ b/command.go
@@ -0,0 +1,121 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/temoto/robotstxt"
+)
+
+type command struct {
+	stdout, stderr    io.Writer
+	httpClientFactory httpClientFactory
+}
+
+func newCommand(stdout, stderr io.Writer, f httpClientFactory) *command {
+	return &command{stdout, stderr, f}
+}
+
+func (c *command) Run(args []string) bool {
+	ok, err := c.runWithError(args)
+
+	if err != nil {
+		c.printError(err)
+	}
+
+	return ok
+}
+
+func (c *command) runWithError(ss []string) (bool, error) {
+	args, err := getArguments(ss)
+	if err != nil {
+		return false, err
+	}
+
+	client := newThrottledHTTPClient(
+		c.httpClientFactory.Create(
+			httpClientOptions{
+				Concurrency:         args.Concurrency,
+				BufferSize:          args.BufferSize,
+				MaxRedirections:     args.MaxRedirections,
+				SkipTLSVerification: args.SkipTLSVerification,
+				Timeout:             args.Timeout,
+			},
+		),
+		args.Concurrency,
+	)
+
+	pp := newPageParser(newLinkFinder(args.ExcludedPatterns), args.FollowURLParams)
+
+	f := newLinkFetcher(
+		client,
+		pp,
+		linkFetcherOptions{
+			args.Headers,
+			args.IgnoreFragments,
+		},
+	)
+
+	_, p, err := f.Fetch(args.URL)
+	if err != nil {
+		return false, err
+	} else if p == nil {
+		return false, errors.New("non-HTML page")
+	}
+
+	rd := (*robotstxt.RobotsData)(nil)
+
+	if args.FollowRobotsTxt {
+		rd, err = newRobotsTxtFetcher(client).Fetch(p.URL())
+
+		if err != nil {
+			return false, err
+		}
+	}
+
+	sm := (map[string]struct{})(nil)
+
+	if args.FollowSitemapXML {
+		sm, err = newSitemapFetcher(client).Fetch(p.URL())
+
+		if err != nil {
+			return false, err
+		}
+	}
+
+	checker := newChecker(
+		f,
+		newLinkValidator(p.URL().Hostname(), rd, sm),
+		args.Concurrency,
+		args.OnePageOnly,
+	)
+
+	go checker.Check(p)
+
+	ok := true
+
+	for r := range checker.Results() {
+		if !r.OK() || args.Verbose {
+			c.print(r.String(args.Verbose))
+		}
+
+		if !r.OK() {
+			ok = false
+		}
+	}
+
+	return ok, nil
+}
+
+func (c command) print(xs ...interface{}) {
+	if _, err := fmt.Fprintln(c.stdout, xs...); err != nil {
+		panic(err)
+	}
+}
+
+func (c command) printError(xs ...interface{}) {
+	if _, err := fmt.Fprintln(c.stderr, xs...); err != nil {
+		panic(err)
+	}
+}
diff --git a/defaults.go b/configuration.go
similarity index 71%
rename from defaults.go
rename to configuration.go
index 60d79b8..081189e 100644
--- a/defaults.go
+++ b/configuration.go
@@ -3,8 +3,10 @@ package main
 import "time"
 
 const (
+	version                = "1.4.0"
 	defaultBufferSize      = 4096
 	defaultConcurrency     = 512
 	defaultMaxRedirections = 64
 	defaultTimeout         = 10 * time.Second
+	agentName              = "muffet"
 )
diff --git a/daemon_manager.go b/daemon_manager.go
new file mode 100644
index 0000000..1a7bc2d
--- /dev/null
+++ b/daemon_manager.go
@@ -0,0 +1,31 @@
+package main
+
+import "sync"
+
+type daemonManager struct {
+	daemons   chan func()
+	waitGroup *sync.WaitGroup
+}
+
+func newDaemonManager(capacity int) *daemonManager {
+	return &daemonManager{make(chan func(), capacity), &sync.WaitGroup{}}
+}
+
+func (m daemonManager) Add(f func()) {
+	m.waitGroup.Add(1)
+
+	m.daemons <- func() {
+		f()
+		m.waitGroup.Done()
+	}
+}
+
+func (m daemonManager) Run() {
+	go func() {
+		for f := range m.daemons {
+			go f()
+		}
+	}()
+
+	m.waitGroup.Wait()
+}
diff --git a/daemons.go b/daemons.go
deleted file mode 100644
index d72f467..0000000
--- a/daemons.go
+++ /dev/null
@@ -1,31 +0,0 @@
-package main
-
-import "sync"
-
-type daemons struct {
-	daemons   chan func()
-	waitGroup *sync.WaitGroup
-}
-
-func newDaemons(c int) daemons {
-	return daemons{make(chan func(), c), &sync.WaitGroup{}}
-}
-
-func (ds daemons) Add(f func()) {
-	ds.waitGroup.Add(1)
-
-	ds.daemons <- func() {
-		f()
-		ds.waitGroup.Done()
-	}
-}
-
-func (ds daemons) Run() {
-	go func() {
-		for f := range ds.daemons {
-			go f()
-		}
-	}()
-
-	ds.waitGroup.Wait()
-}
diff --git a/fasthttp_http_client.go b/fasthttp_http_client.go
new file mode 100644
index 0000000..e977425
--- /dev/null
+++ b/fasthttp_http_client.go
@@ -0,0 +1,60 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/valyala/fasthttp"
+)
+
+type fasthttpHTTPClient struct {
+	client          *fasthttp.Client
+	maxRedirections int
+	timeout         time.Duration
+}
+
+func newFasthttpHTTPClient(c *fasthttp.Client, maxRedirections int, timeout time.Duration) httpClient {
+	return &fasthttpHTTPClient{c, maxRedirections, timeout}
+}
+
+func (c *fasthttpHTTPClient) Get(u *url.URL, headers map[string]string) (httpResponse, error) {
+	req, res := fasthttp.Request{}, fasthttp.Response{}
+	req.SetRequestURI(u.String())
+	req.SetConnectionClose()
+
+	for k, v := range headers {
+		req.Header.Add(k, v)
+	}
+
+	i := 0
+
+	for {
+		err := c.client.DoTimeout(&req, &res, c.timeout)
+		if err != nil {
+			return nil, err
+		}
+
+		switch res.StatusCode() / 100 {
+		case 2:
+			return newFasthttpHTTPResponse(req.URI(), &res), nil
+		case 3:
+			i++
+
+			if i > c.maxRedirections {
+				return nil, errors.New("too many redirections")
+			}
+
+			u := res.Header.Peek("Location")
+
+			if len(u) == 0 {
+				return nil, errors.New("location header not found")
+			}
+
+			req.URI().UpdateBytes(u)
+		default:
+			return nil, fmt.Errorf("%v", res.StatusCode())
+		}
+	}
+}
diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go
new file mode 100644
index 0000000..20fd8ce
--- /dev/null
+++ b/fasthttp_http_client_factory.go
@@ -0,0 +1,28 @@
+package main
+
+import (
+	"crypto/tls"
+
+	"github.com/valyala/fasthttp"
+)
+
+type fasthttpHTTPClientFactory struct {
+}
+
+func newFasthttpHTTPClientFactory() *fasthttpHTTPClientFactory {
+	return &fasthttpHTTPClientFactory{}
+}
+
+func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient {
+	return newFasthttpHTTPClient(
+		&fasthttp.Client{
+			MaxConnsPerHost: o.Concurrency,
+			ReadBufferSize:  o.BufferSize,
+			TLSConfig: &tls.Config{
+				InsecureSkipVerify: o.SkipTLSVerification,
+			},
+		},
+		o.MaxRedirections,
+		o.Timeout,
+	)
+}
diff --git a/fasthttp_http_response.go b/fasthttp_http_response.go
new file mode 100644
index 0000000..4dc3477
--- /dev/null
+++ b/fasthttp_http_response.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"github.com/valyala/fasthttp"
+)
+
+type fasthttpHTTPResponse struct {
+	url      *fasthttp.URI
+	response *fasthttp.Response
+}
+
+func newFasthttpHTTPResponse(u *fasthttp.URI, r *fasthttp.Response) httpResponse {
+	return fasthttpHTTPResponse{u, r}
+}
+
+func (r fasthttpHTTPResponse) URL() string {
+	return r.url.String()
+}
+
+func (r fasthttpHTTPResponse) StatusCode() int {
+	return r.response.StatusCode()
+}
+
+func (r fasthttpHTTPResponse) Header(key string) string {
+	return string(r.response.Header.Peek(key))
+}
+
+func (r fasthttpHTTPResponse) Body() []byte {
+	return r.response.Body()
+}
diff --git a/fetch_result.go b/fetch_result.go
deleted file mode 100644
index 4752c24..0000000
--- a/fetch_result.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package main
-
-type fetchResult struct {
-	statusCode int
-	page       *page
-}
-
-func newFetchResult(s int, p *page) fetchResult {
-	return fetchResult{s, p}
-}
-
-func (r fetchResult) StatusCode() int {
-	return r.statusCode
-}
-
-func (r fetchResult) Page() (*page, bool) {
-	return r.page, r.page != nil
-}
diff --git a/fetcher.go b/fetcher.go
deleted file mode 100644
index 75094fb..0000000
--- a/fetcher.go
+++ /dev/null
@@ -1,159 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"mime"
-	"net/url"
-	"strings"
-
-	"github.com/valyala/fasthttp"
-	"golang.org/x/net/html"
-)
-
-type fetcher struct {
-	client              *fasthttp.Client
-	connectionSemaphore semaphore
-	cache               cache
-	options             fetcherOptions
-	scraper
-}
-
-func newFetcher(c *fasthttp.Client, o fetcherOptions) fetcher {
-	o.Initialize()
-
-	return fetcher{
-		c,
-		newSemaphore(o.Concurrency),
-		newCache(),
-		o,
-		newScraper(o.ExcludedPatterns, o.FollowURLParams),
-	}
-}
-
-func (f fetcher) Fetch(u string) (fetchResult, error) {
-	u, fr, err := separateFragment(u)
-
-	if err != nil {
-		return fetchResult{}, err
-	}
-
-	r, err := f.sendRequestWithCache(u)
-
-	if err != nil {
-		return fetchResult{}, err
-	}
-
-	if p, ok := r.Page(); ok && !f.options.IgnoreFragments && fr != "" {
-		if _, ok := p.IDs()[fr]; !ok {
-			return fetchResult{}, fmt.Errorf("id #%v not found", fr)
-		}
-	}
-
-	return r, nil
-}
-
-func (f fetcher) sendRequestWithCache(u string) (fetchResult, error) {
-	x, s, ok := f.cache.LoadOrStore(u)
-
-	if ok {
-		if err, ok := x.(error); ok {
-			return fetchResult{}, err
-		}
-
-		return x.(fetchResult), nil
-	}
-
-	r, err := f.sendRequest(u)
-
-	if err == nil {
-		s(r)
-	} else {
-		s(err)
-	}
-
-	return r, err
-}
-
-func (f fetcher) sendRequest(u string) (fetchResult, error) {
-	f.connectionSemaphore.Request()
-	defer f.connectionSemaphore.Release()
-
-	req, res := fasthttp.Request{}, fasthttp.Response{}
-	req.SetRequestURI(u)
-	req.SetConnectionClose()
-
-	for k, v := range f.options.Headers {
-		req.Header.Add(k, v)
-	}
-
-	r := 0
-
-redirects:
-	for {
-		err := f.client.DoTimeout(&req, &res, f.options.Timeout)
-
-		if err != nil {
-			return fetchResult{}, err
-		}
-
-		switch res.StatusCode() / 100 {
-		case 2:
-			break redirects
-		case 3:
-			r++
-
-			if r > f.options.MaxRedirections {
-				return fetchResult{}, errors.New("too many redirections")
-			}
-
-			bs := res.Header.Peek("Location")
-
-			if len(bs) == 0 {
-				return fetchResult{}, errors.New("location header not found")
-			}
-
-			req.URI().UpdateBytes(bs)
-		default:
-			return fetchResult{}, fmt.Errorf("%v", res.StatusCode())
-		}
-	}
-
-	if s := strings.TrimSpace(string(res.Header.Peek("Content-Type"))); s != "" {
-		t, _, err := mime.ParseMediaType(s)
-
-		if err != nil {
-			return fetchResult{}, err
-		} else if t != "text/html" {
-			return newFetchResult(res.StatusCode(), nil), nil
-		}
-	}
-
-	n, err := html.Parse(bytes.NewReader(res.Body()))
-
-	if err != nil {
-		return fetchResult{}, err
-	}
-
-	p, err := newPage(req.URI().String(), n, f.scraper)
-
-	if err != nil {
-		return fetchResult{}, err
-	}
-
-	return newFetchResult(res.StatusCode(), p), nil
-}
-
-func separateFragment(s string) (string, string, error) {
-	u, err := url.Parse(s)
-
-	if err != nil {
-		return "", "", err
-	}
-
-	f := u.Fragment
-	u.Fragment = ""
-
-	return u.String(), f, nil
-}
diff --git a/fetcher_option.go b/fetcher_option.go
deleted file mode 100644
index 9ca964a..0000000
--- a/fetcher_option.go
+++ /dev/null
@@ -1,31 +0,0 @@
-package main
-
-import (
-	"regexp"
-	"time"
-)
-
-type fetcherOptions struct {
-	Concurrency      int
-	ExcludedPatterns []*regexp.Regexp
-	Headers          map[string]string
-	IgnoreFragments  bool
-	FollowURLParams  bool
-	MaxRedirections  int
-	Timeout          time.Duration
-	OnePageOnly      bool
-}
-
-func (o *fetcherOptions) Initialize() {
-	if o.Concurrency <= 0 {
-		o.Concurrency = defaultConcurrency
-	}
-
-	if o.MaxRedirections <= 0 {
-		o.MaxRedirections = defaultMaxRedirections
-	}
-
-	if o.Timeout <= 0 {
-		o.Timeout = defaultTimeout
-	}
-}
diff --git a/go.mod b/go.mod
index 2cff53d..ee891f3 100644
--- a/go.mod
+++ b/go.mod
@@ -32,7 +32,7 @@ require (
 	github.com/m3ng9i/go-utils v0.0.0-20160811013010-f9b7dc669fde // indirect
 	github.com/m3ng9i/ran v0.1.4 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/mattn/go-colorable v0.1.4
+	github.com/mattn/go-colorable v0.1.7
 	github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b // indirect
 	github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
diff --git a/go.sum b/go.sum
index 487c305..7d82a3a 100644
--- a/go.sum
+++ b/go.sum
@@ -67,6 +67,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
 github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
 github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
+github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540 h1:djv/qAomOVj8voCHt0M0OYwR/4vfDq1zNKSPKjJCexs=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -108,6 +109,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
 github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -133,6 +135,7 @@ github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2N
 github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
 github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
 github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c h1:/7detzz5stiXWPzkTlPTzkBEIIE4WGpppBJYjKqBiPI=
 github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
@@ -144,17 +147,21 @@ github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiD
 github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
 github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
+github.com/golangci/golangci-lint v1.18.0 h1:XmQgfcLofSG/6AsQuQqmLizB+3GggD+o6ObBG9L+VMM=
 github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
 github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
 github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
 github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
+github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547 h1:fUdgm/BdKvwOHxg5AhNbkNRp2mSy8sxTXyBVs/laQHo=
 github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
 github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc h1:XRFao922N8F3EcIXBSNX8Iywk+GI0dxD/8FicMX2D/c=
 github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
 github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
+github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE=
 github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
@@ -186,6 +193,7 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gostaticanalysis/analysisutil v0.0.2 h1:OZ4/Q9Lt9bzdyyjAgAWzJfL5dSwPrbkN+6UOHwYeJDM=
 github.com/gostaticanalysis/analysisutil v0.0.2/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gostaticanalysis/analysisutil v0.0.3 h1:iwp+5/UAyzQSFgQ4uR2sni99sJ8Eo9DEacKWM5pekIg=
 github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE=
@@ -259,6 +267,8 @@ github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw=
+github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -268,6 +278,8 @@ github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW1
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -403,6 +415,7 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@@ -553,6 +566,8 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191119195528-f068ffe820e4 h1:FjhQftcbpdYXneEYSWZO7+6Bu+Bi1A8VPvGYWOIzIbw=
 golang.org/x/sys v0.0.0-20191119195528-f068ffe820e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
@@ -594,6 +609,7 @@ golang.org/x/tools v0.0.0-20190802003818-e9bb7d36c060/go.mod h1:jcCCGcm9btYwXyDq
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190909214602-067311248421/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190912132005-44811c017489 h1:XM34pzJ/v8knkjxpdMPIU2rFBPd3/jVjDrtFmTZT90g=
 golang.org/x/tools v0.0.0-20190912132005-44811c017489/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
diff --git a/http_client.go b/http_client.go
new file mode 100644
index 0000000..0695e5a
--- /dev/null
+++ b/http_client.go
@@ -0,0 +1,9 @@
+package main
+
+import (
+	"net/url"
+)
+
+type httpClient interface {
+	Get(url *url.URL, headers map[string]string) (httpResponse, error)
+}
diff --git a/http_client_factory.go b/http_client_factory.go
new file mode 100644
index 0000000..0eb1695
--- /dev/null
+++ b/http_client_factory.go
@@ -0,0 +1,15 @@
+package main
+
+import "time"
+
+type httpClientOptions struct {
+	Concurrency,
+	BufferSize,
+	MaxRedirections int
+	SkipTLSVerification bool
+	Timeout             time.Duration
+}
+
+type httpClientFactory interface {
+	Create(httpClientOptions) httpClient
+}
diff --git a/http_response.go b/http_response.go
new file mode 100644
index 0000000..c4caf00
--- /dev/null
+++ b/http_response.go
@@ -0,0 +1,8 @@
+package main
+
+type httpResponse interface {
+	URL() string
+	StatusCode() int
+	Header(string) string
+	Body() []byte
+}
diff --git a/link_fetcher.go b/link_fetcher.go
new file mode 100644
index 0000000..31c9bfa
--- /dev/null
+++ b/link_fetcher.go
@@ -0,0 +1,109 @@
+package main
+
+import (
+	"fmt"
+	"mime"
+	"net/url"
+	"strings"
+)
+
+type linkFetcher struct {
+	client     httpClient
+	pageParser *pageParser
+	cache      cache
+	options    linkFetcherOptions
+}
+
+type fetchResult struct {
+	StatusCode int
+	Page       *page
+}
+
+func newLinkFetcher(c httpClient, pp *pageParser, o linkFetcherOptions) *linkFetcher {
+	return &linkFetcher{c, pp, newCache(), o}
+}
+
+// Fetch fetches a link and returns a successful status code and optionally HTML page, or an error.
+func (f *linkFetcher) Fetch(u string) (int, *page, error) {
+	u, fr, err := separateFragment(u)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	s, p, err := f.sendRequestWithCache(u)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	if p != nil && !f.options.IgnoreFragments && fr != "" {
+		if _, ok := p.IDs()[fr]; !ok {
+			return 0, nil, fmt.Errorf("id #%v not found", fr)
+		}
+	}
+
+	return s, p, nil
+}
+
+func (f *linkFetcher) sendRequestWithCache(u string) (int, *page, error) {
+	x, store := f.cache.LoadOrStore(u)
+
+	if store == nil {
+		if err, ok := x.(error); ok {
+			return 0, nil, err
+		}
+
+		r := x.(fetchResult)
+
+		return r.StatusCode, r.Page, nil
+	}
+
+	s, p, err := f.sendRequest(u)
+
+	if err == nil {
+		store(fetchResult{s, p})
+	} else {
+		store(err)
+	}
+
+	return s, p, err
+}
+
+func (f *linkFetcher) sendRequest(s string) (int, *page, error) {
+	u, err := url.Parse(s)
+	if err != nil {
+		return 0, nil, err
+	}
+
+	r, err := f.client.Get(u, f.options.Headers)
+
+	if err != nil {
+		return 0, nil, err
+	} else if s := strings.TrimSpace(r.Header("Content-Type")); s != "" {
+		t, _, err := mime.ParseMediaType(s)
+
+		if err != nil {
+			return 0, nil, err
+		} else if t != "text/html" {
+			return r.StatusCode(), nil, nil
+		}
+	}
+
+	p, err := f.pageParser.Parse(r.URL(), r.Body())
+	if err != nil {
+		return 0, nil, err
+	}
+
+	return r.StatusCode(), p, nil
+}
+
+func separateFragment(s string) (string, string, error) {
+	u, err := url.Parse(s)
+	if err != nil {
+		return "", "", err
+	}
+
+	f := u.Fragment
+	u.Fragment = ""
+
+	return u.String(), f, nil
+}
diff --git a/link_fetcher_options.go b/link_fetcher_options.go
new file mode 100644
index 0000000..e241b27
--- /dev/null
+++ b/link_fetcher_options.go
@@ -0,0 +1,7 @@
+package main
+
+type linkFetcherOptions struct {
+	// TODO Move to httpClient?
+	Headers         map[string]string
+	IgnoreFragments bool
+}
diff --git a/scraper.go b/link_finder.go
similarity index 67%
rename from scraper.go
rename to link_finder.go
index 36ccfd5..06cd028 100644
--- a/scraper.go
+++ b/link_finder.go
@@ -28,17 +28,16 @@ var atomToAttributes = map[atom.Atom][]string{
 	atom.Track:  {"src"},
 }
 
-type scraper struct {
+type linkFinder struct {
 	excludedPatterns []*regexp.Regexp
-	followURLParams  bool
 }
 
-func newScraper(rs []*regexp.Regexp, followURLParams bool) scraper {
-	return scraper{rs, followURLParams}
+func newLinkFinder(rs []*regexp.Regexp) linkFinder {
+	return linkFinder{rs}
 }
 
-func (sc scraper) Scrape(n *html.Node, base *url.URL) map[string]error {
-	us := map[string]error{}
+func (f linkFinder) Find(n *html.Node, base *url.URL) map[string]error {
+	ls := map[string]error{}
 
 	for _, n := range scrape.FindAllNested(n, func(n *html.Node) bool {
 		_, ok := atomToAttributes[n.DataAtom]
@@ -47,30 +46,25 @@ func (sc scraper) Scrape(n *html.Node, base *url.URL) map[string]error {
 		for _, a := range atomToAttributes[n.DataAtom] {
 			s := normalizeURL(scrape.Attr(n, a))
 
-			if s == "" || sc.isURLExcluded(s) {
+			if s == "" || f.isLinkExcluded(s) {
 				continue
 			}
 
 			u, err := url.Parse(s)
-
 			if err != nil {
-				us[s] = err
+				ls[s] = err
 				continue
+			} else if _, ok := validSchemes[u.Scheme]; ok {
+				ls[base.ResolveReference(u).String()] = nil
 			}
-
-			if _, ok := validSchemes[u.Scheme]; !ok {
-				continue
-			}
-
-			us[base.ResolveReference(u).String()] = nil
 		}
 	}
 
-	return us
+	return ls
 }
 
-func (sc scraper) isURLExcluded(u string) bool {
-	for _, r := range sc.excludedPatterns {
+func (f linkFinder) isLinkExcluded(u string) bool {
+	for _, r := range f.excludedPatterns {
 		if r.MatchString(u) {
 			return true
 		}
diff --git a/link_validator.go b/link_validator.go
new file mode 100644
index 0000000..380a45d
--- /dev/null
+++ b/link_validator.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+	"net/url"
+
+	"github.com/temoto/robotstxt"
+)
+
+type linkValidator struct {
+	hostname    string
+	sitemapURLs map[string]struct{}
+	robotsData  *robotstxt.RobotsData
+}
+
+func newLinkValidator(hostname string, robotsData *robotstxt.RobotsData, sitemap map[string]struct{}) *linkValidator {
+	return &linkValidator{hostname, sitemap, robotsData}
+}
+
+// Validate validates a link and returns true if it is valid as one of an HTML page.
+func (v *linkValidator) Validate(u *url.URL) bool {
+	if v.sitemapURLs != nil {
+		if _, ok := v.sitemapURLs[u.String()]; !ok {
+			return false
+		}
+	}
+
+	if v.robotsData != nil && !v.robotsData.TestAgent(u.Path, agentName) {
+		return false
+	}
+
+	return u.Hostname() == v.hostname
+}
diff --git a/main.go b/main.go
index aa1b690..736ae15 100644
--- a/main.go
+++ b/main.go
@@ -1,72 +1,19 @@
 package main
 
 import (
-	"fmt"
-	"io"
 	"os"
 
 	"github.com/mattn/go-colorable"
 )
 
 func main() {
-	s, err := command(os.Args[1:], colorable.NewColorableStdout())
+	ok := newCommand(
+		colorable.NewColorableStdout(),
+		os.Stderr,
+		newFasthttpHTTPClientFactory(),
+	).Run(os.Args[1:])
 
-	if err != nil {
-		fprintln(os.Stderr, err)
+	if !ok {
 		os.Exit(1)
 	}
-
-	os.Exit(s)
-}
-
-func command(ss []string, w io.Writer) (int, error) {
-	args, err := getArguments(ss)
-
-	if err != nil {
-		return 0, err
-	}
-
-	c, err := newChecker(args.URL, checkerOptions{
-		fetcherOptions{
-			args.Concurrency,
-			args.ExcludedPatterns,
-			args.Headers,
-			args.IgnoreFragments,
-			args.FollowURLParams,
-			args.MaxRedirections,
-			args.Timeout,
-			args.OnePageOnly,
-		},
-		args.BufferSize,
-		args.FollowRobotsTxt,
-		args.FollowSitemapXML,
-		args.FollowURLParams,
-		args.SkipTLSVerification,
-	})
-
-	if err != nil {
-		return 0, err
-	}
-
-	go c.Check()
-
-	s := 0
-
-	for r := range c.Results() {
-		if !r.OK() || args.Verbose {
-			fprintln(w, r.String(args.Verbose))
-		}
-
-		if !r.OK() {
-			s = 1
-		}
-	}
-
-	return s, nil
-}
-
-func fprintln(w io.Writer, xs ...interface{}) {
-	if _, err := fmt.Fprintln(w, xs...); err != nil {
-		panic(err)
-	}
 }
diff --git a/page.go b/page.go
index 3dda41e..b29bb1c 100644
--- a/page.go
+++ b/page.go
@@ -2,10 +2,6 @@ package main
 
 import (
 	"net/url"
-
-	"github.com/yhat/scrape"
-	"golang.org/x/net/html"
-	"golang.org/x/net/html/atom"
 )
 
 type page struct {
@@ -14,53 +10,18 @@ type page struct {
 	links map[string]error
 }
 
-func newPage(s string, n *html.Node, sc scraper) (*page, error) {
-	u, err := url.Parse(s)
-
-	if err != nil {
-		return nil, err
-	}
-
-	u.Fragment = ""
-	if !sc.followURLParams {
-		u.RawQuery = ""
-	}
-
-	ids := map[string]struct{}{}
-
-	scrape.FindAllNested(n, func(n *html.Node) bool {
-		if s := scrape.Attr(n, "id"); s != "" {
-			ids[s] = struct{}{}
-		}
-
-		return false
-	})
-
-	b := u
-
-	if n, ok := scrape.Find(n, func(n *html.Node) bool {
-		return n.DataAtom == atom.Base
-	}); ok {
-		u, err := url.Parse(scrape.Attr(n, "href"))
-
-		if err != nil {
-			return nil, err
-		}
-
-		b = b.ResolveReference(u)
-	}
-
-	return &page{u, ids, sc.Scrape(n, b)}, nil
+func newPage(u *url.URL, ids map[string]struct{}, links map[string]error) *page {
+	return &page{u, ids, links}
 }
 
-func (p page) URL() *url.URL {
+func (p *page) URL() *url.URL {
 	return p.url
 }
 
-func (p page) IDs() map[string]struct{} {
+func (p *page) IDs() map[string]struct{} {
 	return p.ids
 }
 
-func (p page) Links() map[string]error {
+func (p *page) Links() map[string]error {
 	return p.links
 }
diff --git a/page_parser.go b/page_parser.go
new file mode 100644
index 0000000..80d5770
--- /dev/null
+++ b/page_parser.go
@@ -0,0 +1,62 @@
+package main
+
+import (
+	"bytes"
+	"net/url"
+
+	"github.com/yhat/scrape"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+type pageParser struct {
+	linkFinder      linkFinder
+	followURLParams bool
+}
+
+func newPageParser(s linkFinder, followURLParams bool) *pageParser {
+	return &pageParser{s, followURLParams}
+}
+
+func (p pageParser) Parse(rawURL string, body []byte) (*page, error) {
+	n, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return nil, err
+	}
+
+	u.Fragment = ""
+
+	if !p.followURLParams {
+		u.RawQuery = ""
+	}
+
+	ids := map[string]struct{}{}
+
+	scrape.FindAllNested(n, func(n *html.Node) bool {
+		if s := scrape.Attr(n, "id"); s != "" {
+			ids[s] = struct{}{}
+		}
+
+		return false
+	})
+
+	base := u
+
+	if n, ok := scrape.Find(n, func(n *html.Node) bool {
+		return n.DataAtom == atom.Base
+	}); ok {
+		u, err := url.Parse(scrape.Attr(n, "href"))
+		if err != nil {
+			return nil, err
+		}
+
+		base = base.ResolveReference(u)
+	}
+
+	return newPage(u, ids, p.linkFinder.Find(n, base)), nil
+}
diff --git a/page_result.go b/page_result.go
index 689f248..f660395 100644
--- a/page_result.go
+++ b/page_result.go
@@ -28,9 +28,10 @@ func (r pageResult) String(v bool) string {
 	}
 
 	return strings.Join(
-		append(append([]string{color.YellowString(r.url)},
-			ss...),
-			formatMessages(r.errorMessages)...),
+		append(
+			append([]string{color.YellowString(r.url)}, ss...),
+			formatMessages(r.errorMessages)...,
+		),
 		"\n")
 }
 
diff --git a/robots_txt_fetcher.go b/robots_txt_fetcher.go
new file mode 100644
index 0000000..173ba8b
--- /dev/null
+++ b/robots_txt_fetcher.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+
+	"github.com/temoto/robotstxt"
+)
+
+type robotsTxtFetcher struct {
+	client httpClient
+}
+
+func newRobotsTxtFetcher(c httpClient) *robotsTxtFetcher {
+	return &robotsTxtFetcher{c}
+}
+
+func (f *robotsTxtFetcher) Fetch(uu *url.URL) (*robotstxt.RobotsData, error) {
+	u := *uu
+	u.Path = "robots.txt"
+	r, err := f.client.Get(&u, nil)
+
+	if err != nil {
+		return nil, err
+	} else if r.StatusCode() != 200 {
+		return nil, errors.New("failed to load robots.txt")
+	}
+
+	return robotstxt.FromBytes(r.Body())
+}
diff --git a/sitemap_fetcher.go b/sitemap_fetcher.go
new file mode 100644
index 0000000..8361871
--- /dev/null
+++ b/sitemap_fetcher.go
@@ -0,0 +1,51 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+
+	"github.com/yterajima/go-sitemap"
+)
+
+type sitemapFetcher struct {
+	client httpClient
+}
+
+func newSitemapFetcher(c httpClient) *sitemapFetcher {
+	sitemap.SetFetch(func(s string, _ interface{}) ([]byte, error) {
+		u, err := url.Parse(s)
+		if err != nil {
+			return nil, err
+		}
+
+		r, err := c.Get(u, nil)
+
+		if err != nil {
+			return nil, err
+		} else if r.StatusCode() != 200 {
+			return nil, errors.New("failed to load sitemap")
+		}
+
+		return r.Body(), err
+	})
+
+	return &sitemapFetcher{c}
+}
+
+func (f *sitemapFetcher) Fetch(uu *url.URL) (map[string]struct{}, error) {
+	u := *uu
+	u.Path = "sitemap.xml"
+
+	us := map[string]struct{}{}
+
+	sm, err := sitemap.Get(u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, u := range sm.URL {
+		us[u.Loc] = struct{}{}
+	}
+
+	return us, nil
+}
diff --git a/throttled_http_client.go b/throttled_http_client.go
new file mode 100644
index 0000000..9daff40
--- /dev/null
+++ b/throttled_http_client.go
@@ -0,0 +1,23 @@
+package main
+
+import "net/url"
+
+type throttledHTTPClient struct {
+	client    httpClient
+	semaphore semaphore
+}
+
+func newThrottledHTTPClient(c httpClient, concurrency int) httpClient {
+	if concurrency < 1 {
+		concurrency = defaultConcurrency
+	}
+
+	return &throttledHTTPClient{c, newSemaphore(concurrency)}
+}
+
+func (c *throttledHTTPClient) Get(u *url.URL, hs map[string]string) (httpResponse, error) {
+	c.semaphore.Request()
+	defer c.semaphore.Release()
+
+	return c.client.Get(u, hs)
+}
diff --git a/tools/dependencies.sh b/tools/dependencies.sh
deleted file mode 100755
index 9b0092d..0000000
--- a/tools/dependencies.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-go get \
-  github.com/golangci/golangci-lint/cmd/golangci-lint \
-  github.com/m3ng9i/ran
-
-go get -d ./...
diff --git a/tools/lint.sh b/tools/lint.sh
index 9e115d1..5600919 100755
--- a/tools/lint.sh
+++ b/tools/lint.sh
@@ -2,4 +2,4 @@
 
 set -ex
 
-golangci-lint run
+go run github.com/golangci/golangci-lint/cmd/golangci-lint run
diff --git a/tools/unit_test.sh b/tools/test.sh
similarity index 100%
rename from tools/unit_test.sh
rename to tools/test.sh
diff --git a/url_inspector.go b/url_inspector.go
deleted file mode 100644
index 778621e..0000000
--- a/url_inspector.go
+++ /dev/null
@@ -1,85 +0,0 @@
-package main
-
-import (
-	"errors"
-	"net/url"
-
-	"github.com/temoto/robotstxt"
-	"github.com/valyala/fasthttp"
-	"github.com/yterajima/go-sitemap"
-)
-
-type urlInspector struct {
-	hostname     string
-	includedURLs map[string]struct{}
-	robotsTxt    *robotstxt.RobotsData
-}
-
-func newURLInspector(c *fasthttp.Client, s string, r, sm bool) (urlInspector, error) {
-	u, err := url.Parse(s)
-
-	if err != nil {
-		return urlInspector{}, err
-	}
-
-	rd := (*robotstxt.RobotsData)(nil)
-
-	if r {
-		u.Path = "robots.txt"
-		c, bs, err := c.Get(nil, u.String())
-
-		if err != nil {
-			return urlInspector{}, err
-		} else if c != 200 {
-			return urlInspector{}, errors.New("robots.txt not found")
-		}
-
-		rd, err = robotstxt.FromBytes(bs)
-
-		if err != nil {
-			return urlInspector{}, err
-		}
-	}
-
-	us := map[string]struct{}{}
-
-	if sm {
-		u.Path = "sitemap.xml"
-
-		sitemap.SetFetch(func(s string, _ interface{}) ([]byte, error) {
-			c, bs, err := c.Get(nil, s)
-
-			if c != 200 {
-				return nil, errors.New("sitemap not found")
-			}
-
-			return bs, err
-		})
-
-		m, err := sitemap.Get(u.String(), nil)
-
-		if err != nil {
-			return urlInspector{}, err
-		}
-
-		for _, u := range m.URL {
-			us[u.Loc] = struct{}{}
-		}
-	}
-
-	return urlInspector{u.Hostname(), us, rd}, nil
-}
-
-func (i urlInspector) Inspect(u *url.URL) bool {
-	if len(i.includedURLs) != 0 {
-		if _, ok := i.includedURLs[u.String()]; !ok {
-			return false
-		}
-	}
-
-	if i.robotsTxt != nil && !i.robotsTxt.TestAgent(u.Path, "muffet") {
-		return false
-	}
-
-	return u.Hostname() == i.hostname
-}


Test patch:
diff --git a/cache_test.go b/cache_test.go
index 06b2c52..b26ec7a 100644
--- a/cache_test.go
+++ b/cache_test.go
@@ -15,19 +15,17 @@ func TestNewCache(t *testing.T) {
 func TestCacheLoadOrStore(t *testing.T) {
 	c := newCache()
 
-	x, f, ok := c.LoadOrStore("https://foo.com")
+	x, f := c.LoadOrStore("https://foo.com")
 
 	assert.Nil(t, x)
 	assert.NotNil(t, f)
-	assert.False(t, ok)
 
 	f(42)
 
-	x, f, ok = c.LoadOrStore("https://foo.com")
+	x, f = c.LoadOrStore("https://foo.com")
 
 	assert.Equal(t, 42, x)
 	assert.Nil(t, f)
-	assert.True(t, ok)
 }
 
 func TestCacheLoadOrStoreConcurrency(t *testing.T) {
@@ -40,9 +38,9 @@ func TestCacheLoadOrStoreConcurrency(t *testing.T) {
 		g.Add(1)
 
 		go func() {
-			x, f, ok := c.LoadOrStore("https://foo.com")
+			x, f := c.LoadOrStore("https://foo.com")
 
-			if ok {
+			if f == nil {
 				assert.Equal(t, 42, x)
 				atomic.AddInt32(&l, 1)
 			} else {
@@ -65,8 +63,7 @@ func BenchmarkCacheLoadOrStore(b *testing.B) {
 	c := newCache()
 	g := &sync.WaitGroup{}
 
-	_, f, ok := c.LoadOrStore("https://foo.com")
-	assert.False(b, ok)
+	_, f := c.LoadOrStore("https://foo.com")
 	f(42)
 
 	b.ResetTimer()
diff --git a/checker_option_test.go b/checker_option_test.go
deleted file mode 100644
index 0f13902..0000000
--- a/checker_option_test.go
+++ /dev/null
@@ -1,14 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCheckerOptionsInitialize(t *testing.T) {
-	o := checkerOptions{}
-	o.Initialize()
-
-	assert.Equal(t, defaultConcurrency, o.Concurrency)
-}
diff --git a/checker_test.go b/checker_test.go
index fb83bc1..9383129 100644
--- a/checker_test.go
+++ b/checker_test.go
@@ -1,103 +1,97 @@
 package main
 
 import (
-	"regexp"
-	"strings"
+	"errors"
+	"net/url"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestNewChecker(t *testing.T) {
-	_, err := newChecker(rootURL, checkerOptions{})
-	assert.Nil(t, err)
-}
-
-func TestNewCheckerError(t *testing.T) {
-	for _, s := range []string{":", invalidBaseURL} {
-		_, err := newChecker(s, checkerOptions{})
-		assert.NotNil(t, err)
-	}
-}
-
-func TestNewCheckerWithNonHTMLPage(t *testing.T) {
-	_, err := newChecker(robotsTxtURL, checkerOptions{})
-	assert.Equal(t, "non-HTML page", err.Error())
+func createTestChecker(c *fakeHTTPClient) *checker {
+	return newChecker(
+		newLinkFetcher(
+			c,
+			newPageParser(newLinkFinder(nil), false),
+			linkFetcherOptions{},
+		),
+		newLinkValidator("foo.com", nil, nil),
+		512,
+		false,
+	)
 }
 
-func TestNewCheckerWithMissingSitemapXML(t *testing.T) {
-	_, err := newChecker(missingMetadataURL, checkerOptions{FollowSitemapXML: true})
-	assert.Equal(t, "sitemap not found", err.Error())
-}
-
-func TestCheckerCheck(t *testing.T) {
-	for _, s := range []string{rootURL, fragmentURL, baseURL, redirectURL} {
-		c, err := newChecker(s, checkerOptions{})
-		assert.Nil(t, err)
-
-		go c.Check()
+func createTestPage(t *testing.T, ids map[string]struct{}, links map[string]error) *page {
+	u, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
 
-		for r := range c.Results() {
-			assert.True(t, r.OK())
-		}
-	}
+	return newPage(u, ids, links)
 }
 
-func TestCheckerCheckMultiplePages(t *testing.T) {
-	c, _ := newChecker(rootURL, checkerOptions{})
+func TestCheckerCheckOnePage(t *testing.T) {
+	c := createTestChecker(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				return nil, errors.New("")
+			},
+		),
+	)
 
-	go c.Check()
+	go c.Check(createTestPage(t, nil, nil))
 
 	i := 0
 
 	for r := range c.Results() {
-		i += strings.Count(r.String(true), "\n") + 1
+		i++
+		assert.True(t, r.OK())
 	}
 
-	assert.Equal(t, 4, i)
+	assert.Equal(t, 1, i)
 }
 
-func TestCheckerCheckPage(t *testing.T) {
-	c, _ := newChecker(rootURL, checkerOptions{})
+func TestCheckerCheckTwoPages(t *testing.T) {
+	c := createTestChecker(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				s := "http://foo.com/foo"
 
-	r, err := c.fetcher.Fetch(existentURL)
-	assert.Nil(t, err)
+				if u.String() != s {
+					return nil, errors.New("")
+				}
 
-	p, ok := r.Page()
-	assert.True(t, ok)
+				return newFakeHTTPResponse(200, s, "text/html", nil), nil
+			},
+		),
+	)
 
-	go c.checkPage(p)
+	go c.Check(
+		createTestPage(t, nil, map[string]error{"http://foo.com/foo": nil}),
+	)
 
-	assert.True(t, (<-c.Results()).OK())
-}
-
-func TestCheckerCheckWithExcludedURLs(t *testing.T) {
-	r, err := regexp.Compile("bar")
-	assert.Nil(t, err)
-
-	c, _ := newChecker(erroneousURL, checkerOptions{
-		fetcherOptions: fetcherOptions{ExcludedPatterns: []*regexp.Regexp{r}},
-	})
+	i := 0
 
-	go c.Check()
+	for r := range c.Results() {
+		i++
+		assert.True(t, r.OK())
+	}
 
-	assert.Equal(t, 2, strings.Count((<-c.Results()).String(true), "\n"))
+	assert.Equal(t, 2, i)
 }
 
-func TestCheckerCheckPageError(t *testing.T) {
-	for _, s := range []string{erroneousURL} {
-		c, _ := newChecker(rootURL, checkerOptions{})
-
-		r, err := c.fetcher.Fetch(s)
-		assert.Nil(t, err)
-
-		p, ok := r.Page()
-		assert.True(t, ok)
+func TestCheckerFailToCheckPage(t *testing.T) {
+	c := createTestChecker(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				return nil, errors.New("")
+			},
+		),
+	)
 
-		go c.checkPage(p)
+	go c.Check(
+		createTestPage(t, nil, map[string]error{"http://foo.com/foo": nil}),
+	)
 
-		assert.False(t, (<-c.Results()).OK())
-	}
+	assert.False(t, (<-c.Results()).OK())
 }
 
 func TestStringChannelToSlice(t *testing.T) {
@@ -137,6 +131,8 @@ func TestStringChannelToSlice(t *testing.T) {
 			[]string{"foo", "bar", "baz"},
 		},
 	} {
+		close(c.channel)
+
 		assert.Equal(t, c.slice, stringChannelToSlice(c.channel))
 	}
 }
diff --git a/command_test.go b/command_test.go
new file mode 100644
index 0000000..67d9059
--- /dev/null
+++ b/command_test.go
@@ -0,0 +1,44 @@
+package main
+
+import (
+	"errors"
+	"io/ioutil"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func createTestCommand(h func(*url.URL) (*fakeHTTPResponse, error)) *command {
+	return newCommand(
+		ioutil.Discard,
+		ioutil.Discard,
+		newFakeHTTPClientFactory(h),
+	)
+}
+
+func TestCommand(t *testing.T) {
+	ok := createTestCommand(
+		func(u *url.URL) (*fakeHTTPResponse, error) {
+			s := "http://foo.com"
+
+			if u.String() != s {
+				return nil, errors.New("")
+			}
+
+			return newFakeHTTPResponse(200, s, "text/html", nil), nil
+		},
+	).Run([]string{"http://foo.com"})
+
+	assert.True(t, ok)
+}
+
+func TestCommandFailToRun(t *testing.T) {
+	ok := createTestCommand(
+		func(u *url.URL) (*fakeHTTPResponse, error) {
+			return nil, errors.New("")
+		},
+	).Run([]string{"http://foo.com"})
+
+	assert.False(t, ok)
+}
diff --git a/daemon_manager_test.go b/daemon_manager_test.go
new file mode 100644
index 0000000..44239bd
--- /dev/null
+++ b/daemon_manager_test.go
@@ -0,0 +1,22 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewDaemonManager(t *testing.T) {
+	newDaemonManager(1)
+}
+
+func TestDaemonsRun(t *testing.T) {
+	x := 0
+
+	m := newDaemonManager(42)
+	m.Add(func() { x++ })
+	m.Run()
+
+	assert.Equal(t, 1, x)
+	assert.Zero(t, len(m.daemons))
+}
diff --git a/daemons_test.go b/daemons_test.go
deleted file mode 100644
index 6ade7da..0000000
--- a/daemons_test.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewDaemons(t *testing.T) {
-	newDaemons(1)
-}
-
-func TestDaemonsAdd(t *testing.T) {
-	ds := newDaemons(42)
-	ds.Add(func() {})
-
-	assert.Equal(t, 1, len(ds.daemons))
-}
-
-func TestDaemonsRun(t *testing.T) {
-	x := 0
-
-	ds := newDaemons(42)
-	ds.Add(func() { x++ })
-	ds.Run()
-
-	assert.Equal(t, 1, x)
-	assert.Zero(t, len(ds.daemons))
-}
diff --git a/fake_http_client_factory_test.go b/fake_http_client_factory_test.go
new file mode 100644
index 0000000..fe99fb0
--- /dev/null
+++ b/fake_http_client_factory_test.go
@@ -0,0 +1,15 @@
+package main
+
+import "net/url"
+
+type fakeHTTPClientFactory struct {
+	handler func(*url.URL) (*fakeHTTPResponse, error)
+}
+
+func newFakeHTTPClientFactory(h func(*url.URL) (*fakeHTTPResponse, error)) httpClientFactory {
+	return &fakeHTTPClientFactory{h}
+}
+
+func (f *fakeHTTPClientFactory) Create(httpClientOptions) httpClient {
+	return newFakeHTTPClient(f.handler)
+}
diff --git a/fake_http_client_test.go b/fake_http_client_test.go
new file mode 100644
index 0000000..6db81e8
--- /dev/null
+++ b/fake_http_client_test.go
@@ -0,0 +1,17 @@
+package main
+
+import (
+	"net/url"
+)
+
+type fakeHTTPClient struct {
+	handler func(*url.URL) (*fakeHTTPResponse, error)
+}
+
+func newFakeHTTPClient(h func(*url.URL) (*fakeHTTPResponse, error)) *fakeHTTPClient {
+	return &fakeHTTPClient{h}
+}
+
+func (c *fakeHTTPClient) Get(u *url.URL, headers map[string]string) (httpResponse, error) {
+	return c.handler(u)
+}
diff --git a/fake_http_response_test.go b/fake_http_response_test.go
new file mode 100644
index 0000000..9ad6162
--- /dev/null
+++ b/fake_http_response_test.go
@@ -0,0 +1,34 @@
+package main
+
+import "strings"
+
+type fakeHTTPResponse struct {
+	statusCode  int
+	location    string
+	contentType string
+	body        []byte
+}
+
+func newFakeHTTPResponse(statusCode int, location string, contentType string, body []byte) *fakeHTTPResponse {
+	return &fakeHTTPResponse{statusCode, location, contentType, body}
+}
+
+func (r *fakeHTTPResponse) URL() string {
+	return r.location
+}
+
+func (r *fakeHTTPResponse) StatusCode() int {
+	return r.statusCode
+}
+
+func (r *fakeHTTPResponse) Header(name string) string {
+	if strings.ToLower(name) == "content-type" {
+		return r.contentType
+	}
+
+	return ""
+}
+
+func (r *fakeHTTPResponse) Body() []byte {
+	return r.body
+}
diff --git a/fetch_result_test.go b/fetch_result_test.go
deleted file mode 100644
index bebe965..0000000
--- a/fetch_result_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewFetchResult(t *testing.T) {
-	newFetchResult(200, nil)
-}
-
-func TestNewFetchResultWithPage(t *testing.T) {
-	p, err := newPage("", dummyHTML(t), newScraper(nil, false))
-	assert.Nil(t, err)
-
-	newFetchResult(200, p)
-}
-
-func TestFetchResultStatusCode(t *testing.T) {
-	assert.Equal(t, 200, newFetchResult(200, nil).StatusCode())
-}
-
-func TestFetchResultPage(t *testing.T) {
-	p, ok := newFetchResult(200, nil).Page()
-
-	assert.False(t, ok)
-	assert.Equal(t, (*page)(nil), p)
-
-	q, err := newPage("", dummyHTML(t), newScraper(nil, false))
-	assert.Nil(t, err)
-
-	p, ok = newFetchResult(200, q).Page()
-
-	assert.True(t, ok)
-	assert.Equal(t, q, p)
-}
diff --git a/fetcher_option_test.go b/fetcher_option_test.go
deleted file mode 100644
index d9dd68b..0000000
--- a/fetcher_option_test.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestFetcherOptionsInitialize(t *testing.T) {
-	o := fetcherOptions{}
-	o.Initialize()
-
-	assert.Equal(t, defaultConcurrency, o.Concurrency)
-	assert.Equal(t, defaultMaxRedirections, o.MaxRedirections)
-	assert.Equal(t, defaultTimeout, o.Timeout)
-}
diff --git a/fetcher_test.go b/fetcher_test.go
deleted file mode 100644
index 9640f63..0000000
--- a/fetcher_test.go
+++ /dev/null
@@ -1,190 +0,0 @@
-package main
-
-import (
-	"crypto/tls"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/valyala/fasthttp"
-)
-
-func TestNewFetcher(t *testing.T) {
-	newFetcher(&fasthttp.Client{}, fetcherOptions{})
-}
-
-func TestFetcherFetch(t *testing.T) {
-	f := newFetcher(&fasthttp.Client{}, fetcherOptions{})
-
-	for _, s := range []string{rootURL, existentURL, fragmentURL, erroneousURL} {
-		r, err := f.Fetch(s)
-		_, ok := r.Page()
-
-		assert.Equal(t, 200, r.StatusCode())
-		assert.True(t, ok)
-		assert.Nil(t, err)
-	}
-}
-
-func TestFetcherFetchCache(t *testing.T) {
-	f := newFetcher(&fasthttp.Client{}, fetcherOptions{})
-
-	r, err := f.Fetch(rootURL)
-	assert.NotEqual(t, fetchResult{}, r)
-	assert.Nil(t, err)
-	_, ok := r.Page()
-	assert.True(t, ok)
-
-	_, err = f.Fetch(nonExistentURL)
-	assert.NotNil(t, err)
-
-	r, err = f.Fetch(rootURL)
-	assert.NotEqual(t, fetchResult{}, r)
-	assert.Nil(t, err)
-	_, ok = r.Page()
-	assert.True(t, ok)
-
-	_, err = f.Fetch(nonExistentURL)
-	assert.NotNil(t, err)
-}
-
-func TestFetcherFetchCacheConcurrency(t *testing.T) {
-	g := &sync.WaitGroup{}
-	f := newFetcher(&fasthttp.Client{}, fetcherOptions{})
-
-	for i := 0; i < 1000; i++ {
-		g.Add(1)
-		go func() {
-			_, err := f.Fetch(countingURL)
-			assert.Nil(t, err)
-			g.Done()
-		}()
-	}
-
-	g.Wait()
-
-	assert.Equal(t, 1, testCountingHandler.Count())
-}
-
-func TestFetcherFetchWithFragments(t *testing.T) {
-	f := newFetcher(&fasthttp.Client{}, fetcherOptions{})
-
-	r, err := f.Fetch(existentIDURL)
-	_, ok := r.Page()
-
-	assert.Equal(t, 200, r.StatusCode())
-	assert.True(t, ok)
-	assert.Nil(t, err)
-
-	_, err = f.Fetch(nonExistentIDURL)
-	assert.Equal(t, "id #bar not found", err.Error())
-}
-
-func TestFetcherFetchIgnoreFragments(t *testing.T) {
-	_, err := newFetcher(&fasthttp.Client{}, fetcherOptions{}).Fetch(nonExistentIDURL)
-
-	assert.NotNil(t, err)
-
-	r, err := newFetcher(&fasthttp.Client{}, fetcherOptions{IgnoreFragments: true}).Fetch(nonExistentIDURL)
-
-	assert.NotEqual(t, fetchResult{}, r)
-	assert.Nil(t, err)
-}
-
-func TestFetcherFetchWithTLSVerification(t *testing.T) {
-	_, err := newFetcher(&fasthttp.Client{}, fetcherOptions{}).Fetch(selfCertificateURL)
-	assert.NotNil(t, err)
-
-	p, err := newFetcher(
-		&fasthttp.Client{TLSConfig: &tls.Config{InsecureSkipVerify: true}},
-		fetcherOptions{}).Fetch(selfCertificateURL)
-
-	assert.NotEqual(t, page{}, p)
-	assert.Nil(t, err)
-}
-
-func TestFetcherFetchWithInfiniteRedirections(t *testing.T) {
-	_, err := newFetcher(&fasthttp.Client{}, fetcherOptions{}).Fetch(infiniteRedirectURL)
-	assert.NotNil(t, err)
-}
-
-func TestFetcherFetchError(t *testing.T) {
-	f := newFetcher(&fasthttp.Client{}, fetcherOptions{})
-
-	for _, s := range []string{nonExistentURL, ":"} {
-		_, err := f.Fetch(s)
-
-		assert.NotNil(t, err)
-	}
-}
-
-func TestFetcherSendRequest(t *testing.T) {
-	f := newFetcher(&fasthttp.Client{}, fetcherOptions{})
-
-	for _, s := range []string{rootURL, existentURL, fragmentURL, erroneousURL, redirectURL} {
-		r, err := f.sendRequest(s)
-		_, ok := r.Page()
-
-		assert.Equal(t, 200, r.StatusCode())
-		assert.True(t, ok)
-		assert.Nil(t, err)
-	}
-}
-
-func TestFetcherSendRequestWithMissingLocationHeader(t *testing.T) {
-	_, err := newFetcher(&fasthttp.Client{}, fetcherOptions{}).sendRequest(invalidRedirectURL)
-	assert.NotNil(t, err)
-}
-
-func TestFetcherSendRequestWithInvalidMIMEType(t *testing.T) {
-	_, err := newFetcher(&fasthttp.Client{}, fetcherOptions{}).sendRequest(invalidMIMETypeURL)
-	assert.Equal(t, "mime: no media type", err.Error())
-}
-
-func TestFetcherSendRequestWithTimeout(t *testing.T) {
-	_, err := newFetcher(&fasthttp.Client{}, fetcherOptions{Timeout: 1 * time.Second}).sendRequest(timeoutURL)
-	assert.NotNil(t, err)
-
-	r, err := newFetcher(&fasthttp.Client{}, fetcherOptions{Timeout: 60 * time.Second}).sendRequest(timeoutURL)
-	assert.Equal(t, 200, r.StatusCode())
-	assert.Nil(t, err)
-}
-
-// TODO Eanble it fixing its flakiness.
-// func TestFetcherSendRequestConcurrency(t *testing.T) {
-// 	c := 900
-// 	f := newFetcher(&fasthttp.Client{MaxConnsPerHost: c}, fetcherOptions{Concurrency: c})
-
-// 	g := sync.WaitGroup{}
-
-// 	for i := 0; i < 10000; i++ {
-// 		g.Add(1)
-// 		go func() {
-// 			_, err := f.sendRequest("http://httpbin.org/get")
-// 			assert.Nil(t, err)
-// 			g.Done()
-// 		}()
-// 	}
-
-// 	g.Wait()
-// }
-
-func TestSeparateFragment(t *testing.T) {
-	for _, ss := range [][3]string{
-		{"http://foo.com#bar", "http://foo.com", "bar"},
-		{"#bar", "", "bar"},
-	} {
-		u, id, err := separateFragment(ss[0])
-
-		assert.Nil(t, err)
-		assert.Equal(t, ss[1], u)
-		assert.Equal(t, ss[2], id)
-	}
-}
-
-func TestSeparateFragmentError(t *testing.T) {
-	_, _, err := separateFragment(":")
-
-	assert.NotNil(t, err)
-}
diff --git a/link_fetcher_test.go b/link_fetcher_test.go
new file mode 100644
index 0000000..f98ae02
--- /dev/null
+++ b/link_fetcher_test.go
@@ -0,0 +1,185 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func createTestLinkFetcher(c *fakeHTTPClient) *linkFetcher {
+	return createTestLinkFetcherWithOptions(c, linkFetcherOptions{})
+}
+
+func createTestLinkFetcherWithOptions(c *fakeHTTPClient, o linkFetcherOptions) *linkFetcher {
+	return newLinkFetcher(c, newPageParser(newLinkFinder(nil), false), o)
+}
+
+func TestNewFetcher(t *testing.T) {
+	createTestLinkFetcher(newFakeHTTPClient(nil))
+}
+
+func TestLinkFetcherFetch(t *testing.T) {
+	f := createTestLinkFetcher(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				if u.String() != "http://foo.com" {
+					return nil, errors.New("")
+				}
+
+				return newFakeHTTPResponse(
+					200,
+					"http://foo.com",
+					"text/html",
+					nil,
+				), nil
+			}),
+	)
+
+	s, p, err := f.Fetch("http://foo.com")
+
+	assert.Equal(t, 200, s)
+	assert.NotNil(t, p)
+	assert.Nil(t, err)
+}
+
+func TestLinkFetcherFetchFromCache(t *testing.T) {
+	ok := true
+	s := "http://foo.com"
+
+	f := createTestLinkFetcher(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				if !ok {
+					return nil, errors.New("")
+				}
+
+				ok = false
+
+				return newFakeHTTPResponse(
+					200,
+					s,
+					"text/html",
+					nil,
+				), nil
+			}),
+	)
+
+	sc, p, err := f.Fetch(s)
+	assert.Equal(t, 200, sc)
+	assert.NotNil(t, p)
+	assert.Nil(t, err)
+
+	sc, p, err = f.Fetch(s)
+	assert.Equal(t, 200, sc)
+	assert.NotNil(t, p)
+	assert.Nil(t, err)
+}
+
+func TestLinkFetcherFetchCacheConcurrency(t *testing.T) {
+	c := 0
+
+	f := createTestLinkFetcher(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				c++
+
+				return newFakeHTTPResponse(200, "http://foo.com", "text/html", nil), nil
+			}),
+	)
+
+	g := &sync.WaitGroup{}
+
+	for i := 0; i < 1000; i++ {
+		g.Add(1)
+		go func() {
+			defer g.Done()
+
+			time.Sleep(time.Millisecond)
+
+			_, _, err := f.Fetch("http://foo.com")
+			assert.Nil(t, err)
+		}()
+	}
+
+	g.Wait()
+
+	assert.Equal(t, 1, c)
+}
+
+func TestLinkFetcherFetchWithFragments(t *testing.T) {
+	s := "http://foo.com"
+	f := createTestLinkFetcher(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				if u.String() != s {
+					return nil, errors.New("")
+				}
+
+				return newFakeHTTPResponse(200, s, "text/html", []byte(`<p id="foo" />`)), nil
+			},
+		),
+	)
+
+	sc, p, err := f.Fetch(s + "#foo")
+
+	assert.Equal(t, 200, sc)
+	assert.NotNil(t, p)
+	assert.Nil(t, err)
+
+	_, _, err = f.Fetch(s + "#bar")
+
+	assert.Equal(t, "id #bar not found", err.Error())
+}
+
+func TestLinkFetcherFetchIgnoringFragments(t *testing.T) {
+	s := "http://foo.com"
+	f := createTestLinkFetcherWithOptions(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				if u.String() != s {
+					return nil, errors.New("")
+				}
+
+				return newFakeHTTPResponse(200, s, "text/html", nil), nil
+			},
+		),
+		linkFetcherOptions{IgnoreFragments: true},
+	)
+
+	_, _, err := f.Fetch(s + "#bar")
+	assert.Nil(t, err)
+}
+
+func TestLinkFetcherFailToFetch(t *testing.T) {
+	f := createTestLinkFetcher(
+		newFakeHTTPClient(func(*url.URL) (*fakeHTTPResponse, error) {
+			return nil, errors.New("")
+		}))
+
+	_, _, err := f.Fetch("http://foo.com")
+
+	assert.NotNil(t, err)
+}
+
+func TestSeparateFragment(t *testing.T) {
+	for _, ss := range [][3]string{
+		{"http://foo.com#bar", "http://foo.com", "bar"},
+		{"#bar", "", "bar"},
+	} {
+		u, id, err := separateFragment(ss[0])
+
+		assert.Nil(t, err)
+		assert.Equal(t, ss[1], u)
+		assert.Equal(t, ss[2], id)
+	}
+}
+
+func TestFailToSeparateFragment(t *testing.T) {
+	_, _, err := separateFragment(":")
+
+	assert.NotNil(t, err)
+}
diff --git a/scraper_test.go b/link_finder_test.go
similarity index 65%
rename from scraper_test.go
rename to link_finder_test.go
index 0a7a939..aecd17b 100644
--- a/scraper_test.go
+++ b/link_finder_test.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"net/url"
 	"strings"
 	"testing"
@@ -9,13 +10,13 @@ import (
 	"golang.org/x/net/html"
 )
 
-func TestScrapePage(t *testing.T) {
+func TestLinkFinderFindLinks(t *testing.T) {
 	b, err := url.Parse("https://localhost")
 	assert.Nil(t, err)
 
 	for _, c := range []struct {
-		html  string
-		links int
+		html      string
+		linkCount int
 	}{
 		{``, 0},
 		{`<a href="/" />`, 1},
@@ -38,7 +39,7 @@ func TestScrapePage(t *testing.T) {
 
 		s, e := 0, 0
 
-		for _, err := range newScraper(nil, false).Scrape(n, b) {
+		for _, err := range newLinkFinder(nil).Find(n, b) {
 			if err == nil {
 				s++
 			} else {
@@ -46,62 +47,53 @@ func TestScrapePage(t *testing.T) {
 			}
 		}
 
-		assert.Equal(t, c.links, s)
+		assert.Equal(t, c.linkCount, s)
 		assert.Equal(t, 0, e)
 	}
 }
 
-func TestScrapePageError(t *testing.T) {
-	b, err := url.Parse("https://localhost")
+func TestLinkFinderScrapePageError(t *testing.T) {
+	b, err := url.Parse("http://foo.com")
 	assert.Nil(t, err)
 
 	n, err := html.Parse(strings.NewReader(htmlWithBody(`<a href=":" />`)))
 	assert.Nil(t, err)
 
-	s, e := 0, 0
-
-	for _, err := range newScraper(nil, false).Scrape(n, b) {
-		if err == nil {
-			s++
-		} else {
-			e++
-		}
-	}
+	ls := newLinkFinder(nil).Find(n, b)
 
-	assert.Equal(t, 0, s)
-	assert.Equal(t, 1, e)
+	assert.Equal(t, 1, len(ls))
+	assert.NotNil(t, ls[":"])
 }
 
-func TestScraperIsURLExcluded(t *testing.T) {
+func TestLinkFinderIsLinkExcluded(t *testing.T) {
 	for _, x := range []struct {
-		url     string
 		regexps []string
 		answer  bool
 	}{
 		{
-			rootURL,
-			[]string{"localhost"},
+			[]string{"foo\\.com"},
 			true,
 		},
 		{
-			rootURL,
-			[]string{"localhost", "foo"},
+			[]string{"foo"},
 			true,
 		},
 		{
-			rootURL,
-			[]string{"foo", "localhost"},
+			[]string{"bar", "foo"},
 			true,
 		},
 		{
-			rootURL,
-			[]string{"foo"},
+			[]string{"bar"},
 			false,
 		},
 	} {
 		rs, err := compileRegexps(x.regexps)
 		assert.Nil(t, err)
 
-		assert.Equal(t, x.answer, newScraper(rs, false).isURLExcluded(x.url))
+		assert.Equal(t, x.answer, newLinkFinder(rs).isLinkExcluded("http://foo.com"))
 	}
 }
+
+func htmlWithBody(b string) string {
+	return fmt.Sprintf(`<html><body>%v</body></html>`, b)
+}
diff --git a/link_validator_test.go b/link_validator_test.go
new file mode 100644
index 0000000..e16f1e8
--- /dev/null
+++ b/link_validator_test.go
@@ -0,0 +1,65 @@
+package main
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/temoto/robotstxt"
+)
+
+func TestLinkValidatorReturnTrueForSameHostname(t *testing.T) {
+	i := newLinkValidator("foo.com", nil, nil)
+
+	for _, s := range []string{
+		"http://foo.com",
+		"http://foo.com/bar",
+		"https://foo.com",
+	} {
+		u, err := url.Parse(s)
+		assert.Nil(t, err)
+		assert.True(t, i.Validate(u))
+	}
+}
+
+func TestLinkValidatorReturnFalseForDifferentHostname(t *testing.T) {
+	i := newLinkValidator("foo.com", nil, nil)
+
+	u, err := url.Parse("http://bar.com")
+	assert.Nil(t, err)
+	assert.False(t, i.Validate(u))
+}
+
+func TestLinkValidatorValidateWithSitemap(t *testing.T) {
+	i := newLinkValidator(
+		"foo.com",
+		nil,
+		map[string]struct{}{"http://foo.com/foo": {}},
+	)
+
+	u, err := url.Parse("http://foo.com/foo")
+	assert.Nil(t, err)
+	assert.True(t, i.Validate(u))
+
+	u, err = url.Parse("http://foo.com/bar")
+	assert.Nil(t, err)
+	assert.False(t, i.Validate(u))
+}
+
+func TestLinkValidatorValidateWithRobotsTxt(t *testing.T) {
+	r, err := robotstxt.FromString(`
+		User-Agent: *
+		Disallow: /bar
+	`)
+	assert.Nil(t, err)
+
+	i := newLinkValidator("foo.com", r, nil)
+
+	u, err := url.Parse("http://foo.com/foo")
+	assert.Nil(t, err)
+	assert.True(t, i.Validate(u))
+
+	u, err = url.Parse("http://foo.com/bar")
+	assert.Nil(t, err)
+	assert.False(t, i.Validate(u))
+}
diff --git a/main_test.go b/main_test.go
deleted file mode 100644
index 33cb314..0000000
--- a/main_test.go
+++ /dev/null
@@ -1,49 +0,0 @@
-package main
-
-import (
-	"encoding/base64"
-	"io/ioutil"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCommand(t *testing.T) {
-	for _, ss := range [][]string{
-		{"-x", rootURL},
-		{"-j", authorizationHeader("me:password"), basicAuthURL},
-		{"-e", ".*", erroneousURL},
-	} {
-		s, err := command(ss, ioutil.Discard)
-
-		assert.Zero(t, s)
-		assert.Nil(t, err)
-	}
-}
-
-func TestCommandErroneousResult(t *testing.T) {
-	for _, ss := range [][]string{
-		{erroneousURL},
-	} {
-		s, err := command(ss, ioutil.Discard)
-
-		assert.Equal(t, 1, s)
-		assert.Nil(t, err)
-	}
-}
-
-func TestCommandError(t *testing.T) {
-	for _, ss := range [][]string{
-		{":"},
-		{"-t", "foo", rootURL},
-		{"-j", authorizationHeader("you:password"), basicAuthURL},
-	} {
-		_, err := command(ss, ioutil.Discard)
-
-		assert.NotNil(t, err)
-	}
-}
-
-func authorizationHeader(s string) string {
-	return "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte(s))
-}
diff --git a/page_parser_test.go b/page_parser_test.go
new file mode 100644
index 0000000..371a017
--- /dev/null
+++ b/page_parser_test.go
@@ -0,0 +1,117 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPageParserParsePage(t *testing.T) {
+	_, err := newPageParser(newLinkFinder(nil), false).Parse("http://foo.com", nil)
+	assert.Nil(t, err)
+}
+
+func TestPageParserFailWithInvalidURL(t *testing.T) {
+	_, err := newPageParser(newLinkFinder(nil), false).Parse(":", nil)
+	assert.NotNil(t, err)
+}
+
+func TestPageParserSetCorrectURL(t *testing.T) {
+	s := "http://foo.com"
+
+	p, err := newPageParser(newLinkFinder(nil), false).Parse(s, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, s, p.URL().String())
+}
+
+func TestPageParserIgnorePageURLFragment(t *testing.T) {
+	s := "http://foo.com#id"
+
+	p, err := newPageParser(newLinkFinder(nil), false).Parse(s, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, "http://foo.com", p.URL().String())
+}
+
+func TestPageParserIgnoreQueryParameters(t *testing.T) {
+	s := "http://foo.com?bar=baz"
+
+	p, err := newPageParser(newLinkFinder(nil), false).Parse(s, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, "http://foo.com", p.URL().String())
+}
+
+func TestPageParserKeepQueryParameters(t *testing.T) {
+	s := "http://foo.com?bar=baz"
+
+	p, err := newPageParser(newLinkFinder(nil), true).Parse(s, nil)
+	assert.Nil(t, err)
+	assert.Equal(t, s, p.URL().String())
+}
+
+func TestPageParserResolveLinksWithBaseTag(t *testing.T) {
+	p, err := newPageParser(newLinkFinder(nil), false).Parse(
+		"http://foo.com",
+		[]byte(`
+			<html>
+			  <head>
+					<base href="foo/" />
+				</head>
+				<body>
+				  <a href="bar" />
+				</body>
+			</html>
+		`),
+	)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]error{"http://foo.com/foo/bar": nil}, p.Links())
+}
+
+func TestPageParserResolveLinksWithBlankBaseTag(t *testing.T) {
+	p, err := newPageParser(newLinkFinder(nil), false).Parse(
+		"http://foo.com",
+		[]byte(`
+			<html>
+			  <head>
+					<base href="_blank" />
+				</head>
+				<body>
+				  <a href="bar" />
+				</body>
+			</html>
+		`),
+	)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]error{"http://foo.com/bar": nil}, p.Links())
+}
+
+func TestPageParserParseIDs(t *testing.T) {
+	p, err := newPageParser(newLinkFinder(nil), false).Parse(
+		"http://foo.com",
+		[]byte(`<p id="foo" />`),
+	)
+	assert.Nil(t, err)
+	assert.Equal(t, map[string]struct{}{"foo": {}}, p.IDs())
+}
+
+func TestPageParserParseLinks(t *testing.T) {
+	for _, ss := range [][2]string{
+		{
+			`<a href="foo">bar</a>`,
+			"http://foo.com/foo",
+		},
+		{
+			`<img src="foo.img" />`,
+			"http://foo.com/foo.img",
+		},
+	} {
+		p, err := newPageParser(newLinkFinder(nil), false).Parse(
+			"http://foo.com",
+			[]byte(ss[0]),
+		)
+
+		assert.Nil(t, err)
+		assert.Equal(t, 1, len(p.Links()))
+		_, ok := p.Links()[ss[1]]
+		assert.True(t, ok)
+	}
+}
diff --git a/page_test.go b/page_test.go
deleted file mode 100644
index bd0dbb8..0000000
--- a/page_test.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package main
-
-import (
-	"net/url"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"golang.org/x/net/html"
-)
-
-func TestNewPage(t *testing.T) {
-	_, err := newPage("https://foo.com", dummyHTML(t), newScraper(nil, false))
-	assert.Nil(t, err)
-}
-
-func TestNewPageError(t *testing.T) {
-	_, err := newPage(":", dummyHTML(t), newScraper(nil, false))
-	assert.NotNil(t, err)
-}
-
-func TestPageURL(t *testing.T) {
-	s := "https://foo.com"
-	u, err := url.Parse(s)
-	assert.Nil(t, err)
-
-	p, err := newPage(s, dummyHTML(t), newScraper(nil, false))
-	assert.Nil(t, err)
-
-	assert.Equal(t, u, p.URL())
-}
-
-func TestPageURLWithBaseTag(t *testing.T) {
-	n, err := html.Parse(strings.NewReader(`<base href="_blank" />`))
-	assert.Nil(t, err)
-
-	p, err := newPage("https://foo.com", n, newScraper(nil, false))
-	assert.Nil(t, err)
-
-	assert.Equal(t, "https://foo.com", p.URL().String())
-}
-
-func TestPageIDs(t *testing.T) {
-	n, err := html.Parse(strings.NewReader(`<p id="foo">Hello!</p>`))
-	assert.Nil(t, err)
-
-	p, err := newPage("https://foo.com", n, newScraper(nil, false))
-	assert.Nil(t, err)
-
-	assert.Equal(t, 1, len(p.IDs()))
-}
-
-func TestPageLinks(t *testing.T) {
-	for _, ss := range [][2]string{
-		{
-			`<a href="foo">bar</a>`,
-			"https://foo.com/foo",
-		},
-		{
-			`<img src="foo.img" />`,
-			"https://foo.com/foo.img",
-		},
-		{
-			`
-				<html>
-					<head>
-						<base href="foo/" />
-					</head>
-					<body>
-						<a href="foo">bar</a>
-					</body>
-				</html>
-			`,
-			"https://foo.com/foo/foo",
-		},
-	} {
-		n, err := html.Parse(strings.NewReader(ss[0]))
-		assert.Nil(t, err)
-
-		p, err := newPage("https://foo.com", n, newScraper(nil, false))
-		assert.Nil(t, err)
-
-		assert.Equal(t, 1, len(p.Links()))
-
-		_, ok := p.Links()[ss[1]]
-
-		assert.True(t, ok)
-	}
-}
-
-func TestPageWithParams(t *testing.T) {
-	n, err := html.Parse(strings.NewReader(`<p id="foo">Hello!</p>`))
-	assert.Nil(t, err)
-
-	url := "https://foo.com/list?page=2"
-
-	p, err := newPage(url, n, newScraper(nil, true))
-	assert.Nil(t, err)
-
-	assert.Equal(t, p.url.String(), url)
-}
-
-func TestPageWithoutParams(t *testing.T) {
-	n, err := html.Parse(strings.NewReader(`<p id="foo">Hello!</p>`))
-	assert.Nil(t, err)
-
-	url := "https://foo.com/list?page=2"
-
-	p, err := newPage(url, n, newScraper(nil, false))
-	assert.Nil(t, err)
-
-	assert.Equal(t, p.url.String(), url[:strings.IndexByte(url, '?')])
-}
diff --git a/robots_txt_fetcher_test.go b/robots_txt_fetcher_test.go
new file mode 100644
index 0000000..2a34617
--- /dev/null
+++ b/robots_txt_fetcher_test.go
@@ -0,0 +1,36 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRobotsTxtFetcherFetchRobotsTxt(t *testing.T) {
+	s := "http://foo.com"
+	u, err := url.Parse(s)
+	assert.Nil(t, err)
+
+	r, err := newRobotsTxtFetcher(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				if u.String() != s+"/robots.txt" {
+					return nil, errors.New("")
+				}
+
+				return newFakeHTTPResponse(
+					200,
+					s,
+					"text/plain",
+					[]byte(`
+						User-Agent: *
+						Disallow: /bar
+					`),
+				), nil
+			})).Fetch(u)
+
+	assert.Nil(t, err)
+	assert.False(t, r.TestAgent("/bar", "foo"))
+}
diff --git a/setup_test.go b/setup_test.go
deleted file mode 100644
index 3d84478..0000000
--- a/setup_test.go
+++ /dev/null
@@ -1,265 +0,0 @@
-package main
-
-import (
-	"encoding/base64"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"os/exec"
-	"path"
-	"strings"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"golang.org/x/net/html"
-)
-
-const (
-	rootURL             = "http://localhost:8080"
-	existentURL         = "http://localhost:8080/foo"
-	nonExistentURL      = "http://localhost:8080/bar"
-	erroneousURL        = "http://localhost:8080/erroneous"
-	fragmentURL         = "http://localhost:8080/fragment"
-	existentIDURL       = "http://localhost:8080/fragment#foo"
-	nonExistentIDURL    = "http://localhost:8080/fragment#bar"
-	baseURL             = "http://localhost:8080/base"
-	invalidBaseURL      = "http://localhost:8080/invalid-base"
-	redirectURL         = "http://localhost:8080/redirect"
-	infiniteRedirectURL = "http://localhost:8080/infinite-redirect"
-	invalidRedirectURL  = "http://localhost:8080/invalid-redirect"
-	timeoutURL          = "http://localhost:8080/timeout"
-	basicAuthURL        = "http://localhost:8080/basic-auth"
-	robotsTxtURL        = "http://localhost:8080/robots.txt"
-	missingMetadataURL  = "http://localhost:8081"
-	invalidRobotsTxtURL = "http://localhost:8082"
-	invalidMIMETypeURL  = "http://localhost:8083"
-	countingURL         = "http://localhost:8084"
-	selfCertificateURL  = "https://localhost:8085"
-	noResponseURL       = "http://localhost:8086"
-)
-
-type handler struct{}
-
-// nolint:errcheck
-func (handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	w.Header().Add("Content-Type", "text/html")
-
-	switch r.URL.Path {
-	case "", "/":
-		w.Write([]byte(htmlWithBody(`<a href="/foo" />`)))
-	case "/foo":
-		w.Write([]byte(htmlWithBody(`<a href="/" />`)))
-	case "/erroneous":
-		w.Write([]byte(htmlWithBody(`
-			<a href=":" />
-			<a href="mailto:me@right.here" />
-			<a href="/bar" />
-			<a href="#foo" />
-		`)))
-	case "/fragment":
-		w.Write([]byte(htmlWithBody(`<a id="foo" href="#foo" />`)))
-	case "/base":
-		w.Write([]byte(`
-			<html>
-				<head>
-					<base href="/parent/" />
-				</head>
-				<body>
-					<a href="child" />
-				</body>
-			</html>
-		`))
-	case "/invalid-base":
-		w.Write([]byte(`
-			<html>
-				<head>
-					<base href=":" />
-				</head>
-				<body>
-					<a href="child" />
-				</body>
-			</html>
-		`))
-	case "/parent/child":
-	case "/redirect":
-		w.Header().Add("Location", "/")
-		w.WriteHeader(300)
-	case "/infinite-redirect":
-		w.Header().Add("Location", "/infinite-redirect")
-		w.WriteHeader(300)
-	case "/invalid-redirect":
-		w.WriteHeader(300)
-	case "/timeout":
-		time.Sleep(10 * time.Second)
-	case "/basic-auth":
-		ss := strings.Split(r.Header.Get("Authorization"), " ")
-
-		if ss[0] != "Basic" {
-			w.WriteHeader(401)
-			return
-		}
-
-		bs, err := base64.StdEncoding.DecodeString(ss[1])
-
-		if err != nil {
-			w.WriteHeader(401)
-			return
-		}
-
-		ss = strings.Split(string(bs), ":")
-
-		if ss[0] != "me" || ss[1] != "password" {
-			w.WriteHeader(401)
-			return
-		}
-	case "/robots.txt":
-		w.Header().Add("Content-Type", "text/plain")
-
-		u, err := url.Parse(erroneousURL)
-
-		if err != nil {
-			panic(err)
-		}
-
-		v, err := url.Parse(fragmentURL)
-
-		if err != nil {
-			panic(err)
-		}
-
-		w.Write([]byte(fmt.Sprintf(`
-			User-agent: *
-			Disallow: %v
-			Disallow: %v
-		`, u.Path, v.Path)))
-	case "/sitemap.xml":
-		w.Header().Add("Content-Type", "text/xml")
-
-		w.Write([]byte(fmt.Sprintf(`
-			<?xml version="1.0" encoding="UTF-8"?>
-			<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-				<url>
-					<loc>%v</loc>
-					<lastmod>896-5-4</lastmod>
-					<changefreq>monthly</changefreq>
-					<priority>0.8</priority>
-				</url>
-				<url>
-					<loc>%v</loc>
-				</url>
-			</urlset>
-		`, rootURL, existentURL)))
-	default:
-		w.WriteHeader(404)
-	}
-}
-
-type noMetadataHandler struct{}
-
-func (noMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	w.Header().Add("Content-Type", "text/html")
-
-	switch r.URL.Path {
-	case "", "/":
-	default:
-		w.WriteHeader(404)
-	}
-}
-
-type invalidRobotsTxtHandler struct{}
-
-// nolint:errcheck
-func (invalidRobotsTxtHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	w.Header().Add("Content-Type", "text/html")
-
-	switch r.URL.Path {
-	case "", "/":
-	case "/robots.txt":
-		w.Write([]byte(`
-			Disallow: /
-		`))
-	default:
-		w.WriteHeader(404)
-	}
-}
-
-type invalidMIMETypeHandler struct{}
-
-func (invalidMIMETypeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	w.Header().Add("Content-Type", ";")
-}
-
-func htmlWithBody(b string) string {
-	return fmt.Sprintf(`<html><body>%v</body></html>`, b)
-}
-
-type countingHandler struct{ count int32 }
-
-var testCountingHandler = &countingHandler{}
-
-func (h *countingHandler) Count() int {
-	return int(atomic.LoadInt32(&h.count))
-}
-
-func (h *countingHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
-	atomic.AddInt32(&h.count, 1)
-	w.WriteHeader(200)
-}
-
-// nolint:errcheck
-func TestMain(m *testing.M) {
-	go http.ListenAndServe(":8080", handler{})
-	go http.ListenAndServe(":8081", noMetadataHandler{})
-	go http.ListenAndServe(":8082", invalidRobotsTxtHandler{})
-	go http.ListenAndServe(":8083", invalidMIMETypeHandler{})
-	go http.ListenAndServe(":8084", testCountingHandler)
-
-	f, g, err := prepareTLSServer(":8085")
-	defer g()
-
-	if err != nil {
-		panic(err)
-	}
-
-	go f()
-
-	time.Sleep(time.Millisecond)
-
-	os.Exit(m.Run())
-}
-
-// nolint:errcheck
-func prepareTLSServer(a string) (func(), func(), error) {
-	d, err := ioutil.TempDir("", "")
-
-	if err != nil {
-		return nil, nil, err
-	}
-
-	c := path.Join(d, "foo.cert")
-	k := path.Join(d, "foo.pem")
-	err = exec.Command(
-		"openssl", "req", "-x509", "-newkey", "rsa:4096", "-nodes",
-		"-subj", "/CN=localhost",
-		"-out", c,
-		"-keyout", k,
-	).Run()
-
-	if err != nil {
-		return nil, nil, err
-	}
-
-	s := http.Server{Addr: a, ErrorLog: log.New(ioutil.Discard, "", 0), Handler: handler{}}
-	return func() { s.ListenAndServeTLS(c, k) }, func() { os.Remove(d) }, nil
-}
-
-func dummyHTML(t *testing.T) *html.Node {
-	n, err := html.Parse(strings.NewReader("hello"))
-	assert.Nil(t, err)
-	return n
-}
diff --git a/sitemap_fetcher_test.go b/sitemap_fetcher_test.go
new file mode 100644
index 0000000..523fd6c
--- /dev/null
+++ b/sitemap_fetcher_test.go
@@ -0,0 +1,42 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSitemapFetcherFetchSitemap(t *testing.T) {
+	s := "http://foo.com"
+	u, err := url.Parse(s)
+	assert.Nil(t, err)
+
+	sm, err := newSitemapFetcher(
+		newFakeHTTPClient(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				if u.String() != s+"/sitemap.xml" {
+					return nil, errors.New("")
+				}
+
+				return newFakeHTTPResponse(
+					200,
+					s,
+					"text/xml",
+					[]byte(`
+						<?xml version="1.0" encoding="UTF-8"?>
+						<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+							<url>
+								<loc>http://foo.com/bar</loc>
+							</url>
+						</urlset>
+					`),
+				), nil
+			})).Fetch(u)
+
+	assert.Nil(t, err)
+
+	_, ok := sm["http://foo.com/bar"]
+	assert.True(t, ok)
+}
diff --git a/test/dead_link/index.html b/test/dead_link/index.html
deleted file mode 100644
index 6af9d38..0000000
--- a/test/dead_link/index.html
+++ /dev/null
@@ -1,8 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<body>
-	<a href="https://shady.bakery.hotland" />
-</body>
-
-</html>
diff --git a/test/valid/foo.html b/test/valid/foo.html
deleted file mode 100644
index 4da0f76..0000000
--- a/test/valid/foo.html
+++ /dev/null
@@ -1,8 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<body>
-	<a href="/" />
-</body>
-
-</html>
diff --git a/test/valid/foo.png b/test/valid/foo.png
deleted file mode 100644
index e69de29..0000000
diff --git a/test/valid/index.css b/test/valid/index.css
deleted file mode 100644
index e69de29..0000000
diff --git a/test/valid/index.html b/test/valid/index.html
deleted file mode 100644
index b59e1d2..0000000
--- a/test/valid/index.html
+++ /dev/null
@@ -1,19 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-	<link href="/index.css" />
-</head>
-
-<body>
-	<a href="/" />
-	<a href="/foo.html" />
-	<a href="/f
-		oo
-		. html" />
-	<a href="#hello" />
-	<img src="/foo.png" />
-	<p id="hello">Hello!</p>
-</body>
-
-</html>
diff --git a/test/valid/robots.txt b/test/valid/robots.txt
deleted file mode 100644
index 869e764..0000000
--- a/test/valid/robots.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-User-Agent: *
-Disallow: /foo
diff --git a/test/valid/sitemap.xml b/test/valid/sitemap.xml
deleted file mode 100644
index b7e6100..0000000
--- a/test/valid/sitemap.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-	<url>
-		<loc>http://localhost:1111/</loc>
-	</url>
-</urlset>
diff --git a/tools/integration_test.sh b/tools/integration_test.sh
deleted file mode 100755
index 32b664c..0000000
--- a/tools/integration_test.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-ran -p=1111 -r=test/valid &
-pids=$!
-ran -p=2222 -r=test/dead_link &
-pids="$pids $!"
-
-sleep 1
-
-./muffet http://localhost:1111
-! ./muffet http://localhost:2222
-
-./muffet -c 1 http://localhost:1111
-./muffet --concurrency 1 http://localhost:1111
-
-./muffet --help
-
-./muffet -v http://localhost:1111 | grep 200
-[ $(./muffet -v http://localhost:1111 | wc -l) -eq 8 ]
-./muffet --verbose http://localhost:1111 | grep 200
-! ./muffet http://localhost:1111 | grep 200
-
-./muffet -v http://localhost:1111 | sort >/tmp/muffet_1.txt
-./muffet -v http://localhost:1111 | sort >/tmp/muffet_2.txt
-diff /tmp/muffet_1.txt /tmp/muffet_2.txt
-
-[ $(./muffet -rv http://localhost:1111 | wc -l) -eq 6 ]
-[ $(./muffet -sv http://localhost:1111 | wc -l) -eq 6 ]
-
-! ./muffet http://localhost:1111 | grep .
-
-kill $pids
diff --git a/url_inspector_test.go b/url_inspector_test.go
deleted file mode 100644
index a57fe22..0000000
--- a/url_inspector_test.go
+++ /dev/null
@@ -1,85 +0,0 @@
-package main
-
-import (
-	"crypto/tls"
-	"net/url"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/valyala/fasthttp"
-)
-
-func TestNewURLInspector(t *testing.T) {
-	_, err := newURLInspector(&fasthttp.Client{}, rootURL, false, false)
-	assert.Nil(t, err)
-}
-
-func TestNewURLInspectorError(t *testing.T) {
-	_, err := newURLInspector(&fasthttp.Client{}, ":", false, false)
-	assert.NotNil(t, err)
-}
-
-func TestNewURLInspectorWithSitemapXML(t *testing.T) {
-	_, err := newURLInspector(&fasthttp.Client{}, rootURL, false, true)
-	assert.Nil(t, err)
-}
-
-func TestNewURLInspectorErrorWithRobotsTxt(t *testing.T) {
-	for _, s := range []string{missingMetadataURL, invalidRobotsTxtURL, noResponseURL} {
-		_, err := newURLInspector(&fasthttp.Client{}, s, true, false)
-		assert.NotNil(t, err)
-	}
-}
-
-func TestNewURLInspectorWithMissingSitemapXML(t *testing.T) {
-	for _, s := range []string{missingMetadataURL, noResponseURL} {
-		_, err := newURLInspector(&fasthttp.Client{}, s, false, true)
-		assert.NotNil(t, err)
-	}
-}
-
-func TestNewURLInspectorWithSelfCertifiedServer(t *testing.T) {
-	for _, bs := range [][2]bool{{true, false}, {false, true}, {true, true}} {
-		_, err := newURLInspector(&fasthttp.Client{}, selfCertificateURL, bs[0], bs[1])
-		assert.NotNil(t, err)
-
-		_, err = newURLInspector(
-			&fasthttp.Client{TLSConfig: &tls.Config{InsecureSkipVerify: true}},
-			selfCertificateURL, bs[0], bs[1])
-		assert.Nil(t, err)
-	}
-}
-
-func TestURLInspectorInspectWithSitemapXML(t *testing.T) {
-	i, err := newURLInspector(&fasthttp.Client{}, rootURL, false, true)
-	assert.Nil(t, err)
-
-	for _, s := range []string{rootURL, existentURL} {
-		u, err := url.Parse(s)
-		assert.Nil(t, err)
-		assert.True(t, i.Inspect(u))
-	}
-
-	for _, s := range []string{nonExistentURL, erroneousURL, fragmentURL} {
-		u, err := url.Parse(s)
-		assert.Nil(t, err)
-		assert.False(t, i.Inspect(u))
-	}
-}
-
-func TestURLInspectorInspectWithRobotsTxt(t *testing.T) {
-	i, err := newURLInspector(&fasthttp.Client{}, rootURL, true, false)
-	assert.Nil(t, err)
-
-	for _, s := range []string{rootURL, existentURL} {
-		u, err := url.Parse(s)
-		assert.Nil(t, err)
-		assert.True(t, i.Inspect(u))
-	}
-
-	for _, s := range []string{erroneousURL, fragmentURL} {
-		u, err := url.Parse(s)
-		assert.Nil(t, err)
-		assert.False(t, i.Inspect(u))
-	}
-}