Repository: raviqqe/muffet (go)

Issue:
Split concurrency option into two
Currently it seems that the concurrency option defines how many HTTP requests are done at the same time by muffet overall.

This however is not super useful on it's own. I'm happy with muffet doing 512 concurrent HTTP requests but not if those requests all hit the same server.

Browsers typically have two settings (eg `network.http.max-connections` and `network.http.max-connections-per-server` in Firefox) to separate the two cases and I believe muffet should separate these cases as well.

Pull request:
v2
Close #71

Patch:
diff --git a/arguments.go b/arguments.go
index 18ec7a3..3fee516 100644
--- a/arguments.go
+++ b/arguments.go
@@ -2,119 +2,72 @@ package main
 
 import (
 	"errors"
-	"fmt"
+	"io"
 	"regexp"
-	"strconv"
 	"strings"
-	"time"
 
-	"github.com/docopt/docopt-go"
+	"github.com/jessevdk/go-flags"
 )
 
-var usage = fmt.Sprintf(`Muffet, the web repairgirl
-
-Usage:
-	muffet [-b <size>] [-c <concurrency>] [-e <pattern>...] [-f] [-j <header>...] [-l <times>] [-p] [-r] [-s] [-u] [-t <seconds>] [-v] [-x] <url>
-
-Options:
-	-b, --buffer-size <size>          Set HTTP response buffer size in bytes. [default: %v]
-	-c, --concurrency <concurrency>   Roughly maximum number of concurrent HTTP connections. [default: %v]
-	-e, --exclude <pattern>...        Exclude URLs matched with given regular expressions.
-	-f, --ignore-fragments            Ignore URL fragments.
-	-h, --help                        Show this help.
-	-j, --header <header>...          Set custom headers.
-	-l, --limit-redirections <times>  Limit a number of redirections. [default: %v]
-	-p, --one-page-only               Only check links found in the given URL, do not follow links.
-	-r, --follow-robots-txt           Follow robots.txt when scraping.
-	-s, --follow-sitemap-xml          Scrape only pages listed in sitemap.xml.
-	-u, --follow-url-params           Will not cut off the URL parameters.
-	-t, --timeout <seconds>           Set timeout for HTTP requests in seconds. [default: %v]
-	-v, --verbose                     Show successful results too.
-	-x, --skip-tls-verification       Skip TLS certificates verification.`,
-	defaultBufferSize, defaultConcurrency, defaultMaxRedirections, defaultTimeout.Seconds())
-
 type arguments struct {
-	BufferSize       int
-	Concurrency      int
-	ExcludedPatterns []*regexp.Regexp
-	FollowRobotsTxt,
-	FollowSitemapXML,
-	FollowURLParams bool
-	Headers         map[string]string
-	IgnoreFragments bool
-	MaxRedirections int
-	Timeout         time.Duration
-	URL             string
-	Verbose,
-	SkipTLSVerification bool
-	OnePageOnly bool
+	BufferSize            int      `short:"b" long:"buffer-size" default:"4096" description:"HTTP response buffer size in bytes"`
+	MaxConnections        int      `short:"c" long:"max-connections" default:"512" description:"Maximum number of HTTP connections"`
+	MaxConnectionsPerHost int      `long:"max-connections-per-host" default:"512" description:"Maximum number of HTTP connections per host"`
+	RawExcludedPatterns   []string `short:"e" long:"exclude" description:"Exclude URLs matched with given regular expressions"`
+	FollowRobotsTxt       bool     `long:"follow-robots-txt" description:"Follow robots.txt when scraping pages"`
+	FollowSitemapXML      bool     `long:"follow-sitemap-xml" description:"Scrape only pages listed in sitemap.xml"`
+	RawHeaders            []string `long:"header" description:"Custom headers"`
+	IgnoreFragments       bool     `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"`
+	MaxRedirections       int      `short:"r" long:"max-redirections" default:"64" description:"Maximum number of redirections"`
+	Timeout               int      `short:"t" long:"timeout" default:"10" description:"Timeout for HTTP requests in seconds"`
+	Verbose               bool     `short:"v" long:"verbose" description:"Show successful results too"`
+	SkipTLSVerification   bool     `long:"skip-tls-verification" description:"Skip TLS certificate verification"`
+	OnePageOnly           bool     `long:"one-page-only" description:"Only check links found in the given URL"`
+	Help                  bool     `short:"h" long:"help" description:"Show this help"`
+	Version               bool     `long:"version" description:"Show version"`
+	URL                   string
+	ExcludedPatterns      []*regexp.Regexp
+	Headers               map[string]string
 }
 
-func getArguments(regexps []string) (arguments, error) {
-	args := parseArguments(usage, regexps)
+func getArguments(ss []string) (*arguments, error) {
+	args := arguments{}
+	ss, err := flags.NewParser(&args, flags.PassDoubleDash).ParseArgs(ss)
 
-	b, err := parseInt(args["--buffer-size"].(string))
 	if err != nil {
-		return arguments{}, err
+		return nil, err
+	} else if args.Version || args.Help {
+		return &args, nil
+	} else if len(ss) != 1 {
+		return nil, errors.New("invalid number of arguments")
 	}
 
-	c, err := parseInt(args["--concurrency"].(string))
-	if err != nil {
-		return arguments{}, err
-	}
+	args.URL = ss[0]
 
-	ss, _ := args["--exclude"].([]string)
-	rs, err := compileRegexps(ss)
-	if err != nil {
-		return arguments{}, err
-	}
+	rs, err := compileRegexps(args.RawExcludedPatterns)
 
-	ss, _ = args["--header"].([]string)
-	hs, err := parseHeaders(ss)
 	if err != nil {
-		return arguments{}, err
+		return nil, err
 	}
 
-	r, err := parseInt(args["--limit-redirections"].(string))
-	if err != nil {
-		return arguments{}, err
-	}
+	args.ExcludedPatterns = rs
 
-	t, err := parseInt(args["--timeout"].(string))
-	if err != nil {
-		return arguments{}, err
-	}
-
-	return arguments{
-		b,
-		c,
-		rs,
-		args["--follow-robots-txt"].(bool),
-		args["--follow-sitemap-xml"].(bool),
-		args["--follow-url-params"].(bool),
-		hs,
-		args["--ignore-fragments"].(bool),
-		r,
-		time.Duration(t) * time.Second,
-		args["<url>"].(string),
-		args["--verbose"].(bool),
-		args["--skip-tls-verification"].(bool),
-		args["--one-page-only"].(bool),
-	}, nil
-}
+	hs, err := parseHeaders(args.RawHeaders)
 
-func parseArguments(u string, ss []string) map[string]interface{} {
-	args, err := docopt.ParseArgs(u, ss, version)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 
-	return args
+	args.Headers = hs
+
+	return &args, nil
 }
 
-func parseInt(s string) (int, error) {
-	i, err := strconv.ParseInt(s, 10, 32)
-	return int(i), err
+func printHelp(w io.Writer) {
+	p := flags.NewParser(&arguments{}, flags.PassDoubleDash)
+	p.Usage = "[options] <url>"
+
+	p.WriteHelp(w)
 }
 
 func compileRegexps(regexps []string) ([]*regexp.Regexp, error) {
diff --git a/checker.go b/checker.go
index 75bddf2..5dda938 100644
--- a/checker.go
+++ b/checker.go
@@ -13,7 +13,7 @@ type checker struct {
 	onePageOnly   bool
 }
 
-func newChecker(f *linkFetcher, v *linkValidator, concurrency int, onePageOnly bool) *checker {
+func newChecker(f *linkFetcher, v *linkValidator, onePageOnly bool) *checker {
 	return &checker{
 		f,
 		v,
diff --git a/command.go b/command.go
index ffafebc..11147ef 100644
--- a/command.go
+++ b/command.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/logrusorgru/aurora/v3"
 	"github.com/temoto/robotstxt"
@@ -33,22 +34,28 @@ func (c *command) runWithError(ss []string) (bool, error) {
 	args, err := getArguments(ss)
 	if err != nil {
 		return false, err
+	} else if args.Help {
+		printHelp(c.stdout)
+		return true, nil
+	} else if args.Version {
+		c.print(version)
+		return err == nil, err
 	}
 
 	client := newThrottledHTTPClient(
 		c.httpClientFactory.Create(
 			httpClientOptions{
-				Concurrency:         args.Concurrency,
-				BufferSize:          args.BufferSize,
-				MaxRedirections:     args.MaxRedirections,
-				SkipTLSVerification: args.SkipTLSVerification,
-				Timeout:             args.Timeout,
+				MaxConnectionsPerHost: args.MaxConnectionsPerHost,
+				BufferSize:            args.BufferSize,
+				MaxRedirections:       args.MaxRedirections,
+				SkipTLSVerification:   args.SkipTLSVerification,
+				Timeout:               time.Duration(args.Timeout) * time.Second,
 			},
 		),
-		args.Concurrency,
+		args.MaxConnections,
 	)
 
-	pp := newPageParser(newLinkFinder(args.ExcludedPatterns), args.FollowURLParams)
+	pp := newPageParser(newLinkFinder(args.ExcludedPatterns))
 
 	f := newLinkFetcher(
 		client,
@@ -89,7 +96,6 @@ func (c *command) runWithError(ss []string) (bool, error) {
 	checker := newChecker(
 		f,
 		newLinkValidator(p.URL().Hostname(), rd, sm),
-		args.Concurrency,
 		args.OnePageOnly,
 	)
 
diff --git a/configuration.go b/configuration.go
index 7d79d07..3c164c2 100644
--- a/configuration.go
+++ b/configuration.go
@@ -3,11 +3,9 @@ package main
 import "time"
 
 const (
-	version                = "1.5.7"
-	defaultBufferSize      = 4096
-	defaultConcurrency     = 512
-	defaultMaxRedirections = 64
-	defaultTimeout         = 10 * time.Second
-	agentName              = "muffet"
-	tcpTimeout             = time.Minute
+	version               = "2.0.0"
+	agentName             = "muffet"
+	concurrency           = 1024
+	tcpTimeout            = time.Minute
+	defaultMaxConnections = 512
 )
diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go
index b6b0987..6620056 100644
--- a/fasthttp_http_client_factory.go
+++ b/fasthttp_http_client_factory.go
@@ -17,7 +17,7 @@ func newFasthttpHTTPClientFactory() *fasthttpHTTPClientFactory {
 func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient {
 	return newFasthttpHTTPClient(
 		&fasthttp.Client{
-			MaxConnsPerHost: o.Concurrency,
+			MaxConnsPerHost: o.MaxConnectionsPerHost,
 			ReadBufferSize:  o.BufferSize,
 			TLSConfig: &tls.Config{
 				InsecureSkipVerify: o.SkipTLSVerification,
diff --git a/go.mod b/go.mod
index 017fa21..1ef0f59 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.12
 require (
 	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815
+	github.com/jessevdk/go-flags v1.4.0
 	github.com/klauspost/compress v1.11.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/logrusorgru/aurora/v3 v3.0.0
diff --git a/go.sum b/go.sum
index b085805..58ce84a 100644
--- a/go.sum
+++ b/go.sum
@@ -7,8 +7,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg=
 github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg=
diff --git a/http_client_factory.go b/http_client_factory.go
index 0eb1695..9301469 100644
--- a/http_client_factory.go
+++ b/http_client_factory.go
@@ -3,7 +3,7 @@ package main
 import "time"
 
 type httpClientOptions struct {
-	Concurrency,
+	MaxConnectionsPerHost,
 	BufferSize,
 	MaxRedirections int
 	SkipTLSVerification bool
diff --git a/page_parser.go b/page_parser.go
index 779ba11..896650f 100644
--- a/page_parser.go
+++ b/page_parser.go
@@ -10,12 +10,11 @@ import (
 )
 
 type pageParser struct {
-	linkFinder      linkFinder
-	followURLParams bool
+	linkFinder linkFinder
 }
 
-func newPageParser(s linkFinder, followURLParams bool) *pageParser {
-	return &pageParser{s, followURLParams}
+func newPageParser(f linkFinder) *pageParser {
+	return &pageParser{f}
 }
 
 func (p pageParser) Parse(rawURL string, body []byte) (*page, error) {
@@ -31,10 +30,6 @@ func (p pageParser) Parse(rawURL string, body []byte) (*page, error) {
 
 	u.Fragment = ""
 
-	if !p.followURLParams {
-		u.RawQuery = ""
-	}
-
 	frs := map[string]struct{}{}
 
 	scrape.FindAllNested(n, func(n *html.Node) bool {
diff --git a/throttled_http_client.go b/throttled_http_client.go
index 9daff40..f80d5b2 100644
--- a/throttled_http_client.go
+++ b/throttled_http_client.go
@@ -7,12 +7,12 @@ type throttledHTTPClient struct {
 	semaphore semaphore
 }
 
-func newThrottledHTTPClient(c httpClient, concurrency int) httpClient {
-	if concurrency < 1 {
-		concurrency = defaultConcurrency
+func newThrottledHTTPClient(c httpClient, maxConnections int) httpClient {
+	if maxConnections < 1 {
+		maxConnections = defaultMaxConnections
 	}
 
-	return &throttledHTTPClient{c, newSemaphore(concurrency)}
+	return &throttledHTTPClient{c, newSemaphore(maxConnections)}
 }
 
 func (c *throttledHTTPClient) Get(u *url.URL, hs map[string]string) (httpResponse, error) {
diff --git a/tools/build.sh b/tools/build.sh
deleted file mode 100755
index 8b3b577..0000000
--- a/tools/build.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-CGO_ENABLED=0 go build
diff --git a/tools/lint.sh b/tools/lint.sh
deleted file mode 100755
index 5600919..0000000
--- a/tools/lint.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-go run github.com/golangci/golangci-lint/cmd/golangci-lint run


Test patch:
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index db9429a..9b77ef4 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -8,7 +8,7 @@ jobs:
       image: golang
     steps:
       - uses: actions/checkout@v2
-      - run: tools/build.sh
-      - run: tools/lint.sh
-      - run: tools/test.sh
+      - run: go build
+      - uses: golangci/golangci-lint-action@v2
+      - run: go test -covermode atomic -coverprofile coverage.txt
       - uses: codecov/codecov-action@v1
diff --git a/.snapshots/TestCommandFailToParseRootPageAsHTML b/.snapshots/TestCommandFailToParseRootPageAsHTML
deleted file mode 100644
index 53c2162..0000000
--- a/.snapshots/TestCommandFailToParseRootPageAsHTML
+++ /dev/null
@@ -1,2 +0,0 @@
-failed to parse root page as HTML
-
diff --git a/.snapshots/TestPrintHelp b/.snapshots/TestPrintHelp
new file mode 100644
index 0000000..331fe92
--- /dev/null
+++ b/.snapshots/TestPrintHelp
@@ -0,0 +1,21 @@
+Usage:
+  muffet.test [options] <url>
+
+Application Options:
+  -b, --buffer-size=              HTTP response buffer size in bytes
+  -c, --max-connections=          Maximum number of HTTP connections
+      --max-connections-per-host= Maximum number of HTTP connections per host
+  -e, --exclude=                  Exclude URLs matched with given regular
+                                  expressions
+      --follow-robots-txt         Follow robots.txt when scraping pages
+      --follow-sitemap-xml        Scrape only pages listed in sitemap.xml
+      --header=                   Custom headers
+  -f, --ignore-fragments          Ignore URL fragments
+  -r, --max-redirections=         Maximum number of redirections
+  -t, --timeout=                  Timeout for HTTP requests in seconds
+  -v, --verbose                   Show successful results too
+      --skip-tls-verification     Skip TLS certificate verification
+      --one-page-only             Only check links found in the given URL
+  -h, --help                      Show this help
+      --version                   Show version
+
diff --git a/arguments_test.go b/arguments_test.go
index 02c3527..60a3973 100644
--- a/arguments_test.go
+++ b/arguments_test.go
@@ -1,8 +1,10 @@
 package main
 
 import (
+	"bytes"
 	"testing"
 
+	"github.com/bradleyjkemp/cupaloy"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -12,27 +14,26 @@ func TestGetArguments(t *testing.T) {
 		{"-b", "42", "https://foo.com"},
 		{"--buffer-size", "42", "https://foo.com"},
 		{"-c", "1", "https://foo.com"},
-		{"--concurrency", "1", "https://foo.com"},
+		{"--max-connections", "1", "https://foo.com"},
+		{"--max-connections-per-host", "1", "https://foo.com"},
 		{"-e", "regex1", "-e", "regex2", "https://foo.com"},
 		{"--exclude", "regex1", "--exclude", "regex2", "https://foo.com"},
-		{"-j", "MyHeader: foo", "-j", "YourHeader: bar", "https://foo.com"},
 		{"--header", "MyHeader: foo", "--header", "YourHeader: bar", "https://foo.com"},
-		{"-l", "4", "https://foo.com"},
-		{"--limit-redirections", "4", "https://foo.com"},
-		{"-s", "https://foo.com"},
+		{"-r", "4", "https://foo.com"},
+		{"--max-redirections", "4", "https://foo.com"},
+		{"--follow-robots-txt", "https://foo.com"},
 		{"--follow-sitemap-xml", "https://foo.com"},
-		{"-u", "https://foo.com"},
-		{"--follow-url-params", "https://foo.com"},
 		{"-t", "10", "https://foo.com"},
 		{"--timeout", "10", "https://foo.com"},
-		{"-x", "https://foo.com"},
 		{"--skip-tls-verification", "https://foo.com"},
 		{"-v", "https://foo.com"},
 		{"--verbose", "https://foo.com"},
 		{"-v", "-f", "https://foo.com"},
 		{"-v", "--ignore-fragments", "https://foo.com"},
-		{"-p", "https://foo.com"},
 		{"--one-page-only", "https://foo.com"},
+		{"-h"},
+		{"--help"},
+		{"--version"},
 	} {
 		_, err := getArguments(ss)
 		assert.Nil(t, err)
@@ -41,15 +42,16 @@ func TestGetArguments(t *testing.T) {
 
 func TestGetArgumentsError(t *testing.T) {
 	for _, ss := range [][]string{
+		{},
 		{"-b", "foo", "https://foo.com"},
 		{"--buffer-size", "foo", "https://foo.com"},
 		{"-c", "foo", "https://foo.com"},
-		{"--concurrency", "foo", "https://foo.com"},
+		{"--max-connections", "foo", "https://foo.com"},
 		{"-e", "(", "https://foo.com"},
 		{"-j", "MyHeader", "https://foo.com"},
 		{"--header", "MyHeader", "https://foo.com"},
 		{"-l", "foo", "https://foo.com"},
-		{"--limit-redirections", "foo", "https://foo.com"},
+		{"--max-redirections", "foo", "https://foo.com"},
 		{"-t", "foo", "https://foo.com"},
 		{"--timeout", "foo", "https://foo.com"},
 	} {
@@ -58,10 +60,10 @@ func TestGetArgumentsError(t *testing.T) {
 	}
 }
 
-func TestParseArguments(t *testing.T) {
-	assert.Panics(t, func() {
-		parseArguments("", nil)
-	})
+func TestPrintHelp(t *testing.T) {
+	b := &bytes.Buffer{}
+	printHelp(b)
+	cupaloy.SnapshotT(t, b.String())
 }
 
 func TestParseHeaders(t *testing.T) {
diff --git a/checker_test.go b/checker_test.go
index 740aa4f..4904884 100644
--- a/checker_test.go
+++ b/checker_test.go
@@ -12,11 +12,10 @@ func newTestChecker(c *fakeHTTPClient) *checker {
 	return newChecker(
 		newLinkFetcher(
 			c,
-			newPageParser(newLinkFinder(nil), false),
+			newPageParser(newLinkFinder(nil)),
 			linkFetcherOptions{},
 		),
 		newLinkValidator("foo.com", nil, nil),
-		512,
 		false,
 	)
 }
diff --git a/command_test.go b/command_test.go
index 5cd004a..74cb61e 100644
--- a/command_test.go
+++ b/command_test.go
@@ -6,6 +6,8 @@ import (
 	"io"
 	"io/ioutil"
 	"net/url"
+	"regexp"
+	"strings"
 	"testing"
 
 	"github.com/bradleyjkemp/cupaloy"
@@ -41,6 +43,32 @@ func TestCommandRun(t *testing.T) {
 	assert.True(t, ok)
 }
 
+func TestCommandRunWithLinks(t *testing.T) {
+	visited := false
+
+	ok := newTestCommand(
+		func(u *url.URL) (*fakeHTTPResponse, error) {
+			switch u.String() {
+			case "http://foo.com":
+				return newFakeHTTPResponse(
+					200,
+					"http://foo.com",
+					"text/html",
+					[]byte(`<html><body><a href="/foo" /></body></html>`),
+				), nil
+			case "http://foo.com/foo":
+				visited = true
+				return newFakeHTTPResponse(200, "http://foo.com", "text/html", nil), nil
+			}
+
+			return nil, errors.New("")
+		},
+	).Run([]string{"http://foo.com"})
+
+	assert.True(t, ok)
+	assert.True(t, visited)
+}
+
 func TestCommandFailToRun(t *testing.T) {
 	ok := newTestCommand(
 		func(u *url.URL) (*fakeHTTPResponse, error) {
@@ -51,6 +79,22 @@ func TestCommandFailToRun(t *testing.T) {
 	assert.False(t, ok)
 }
 
+func TestCommandFailToParseArguments(t *testing.T) {
+	b := &bytes.Buffer{}
+
+	c := newTestCommandWithStderr(
+		b,
+		func(u *url.URL) (*fakeHTTPResponse, error) {
+			return newFakeHTTPResponse(200, "", "text/html", nil), nil
+		},
+	)
+
+	ok := c.Run(nil)
+
+	assert.False(t, ok)
+	assert.Greater(t, b.Len(), 0)
+}
+
 func TestCommandFailToFetchRootPage(t *testing.T) {
 	b := &bytes.Buffer{}
 
@@ -62,7 +106,7 @@ func TestCommandFailToFetchRootPage(t *testing.T) {
 	).Run([]string{"http://foo.com"})
 
 	assert.False(t, ok)
-	cupaloy.SnapshotT(t, b.Bytes())
+	cupaloy.SnapshotT(t, b.String())
 }
 
 func TestCommandFailToGetHTMLRootPage(t *testing.T) {
@@ -76,7 +120,7 @@ func TestCommandFailToGetHTMLRootPage(t *testing.T) {
 	).Run([]string{"http://foo.com"})
 
 	assert.False(t, ok)
-	cupaloy.SnapshotT(t, b.Bytes())
+	cupaloy.SnapshotT(t, b.String())
 }
 
 func TestCommandColorErrorMessage(t *testing.T) {
@@ -94,5 +138,47 @@ func TestCommandColorErrorMessage(t *testing.T) {
 	ok := c.Run([]string{"http://foo.com"})
 
 	assert.False(t, ok)
-	cupaloy.SnapshotT(t, b.Bytes())
+	cupaloy.SnapshotT(t, b.String())
+}
+
+func TestCommandShowHelp(t *testing.T) {
+	b := &bytes.Buffer{}
+
+	c := newCommand(
+		b,
+		ioutil.Discard,
+		false,
+		newFakeHTTPClientFactory(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				return newFakeHTTPResponse(200, "", "text/html", nil), nil
+			},
+		),
+	)
+
+	ok := c.Run([]string{"--help"})
+
+	assert.True(t, ok)
+	assert.Greater(t, b.Len(), 0)
+}
+
+func TestCommandShowVersion(t *testing.T) {
+	b := &bytes.Buffer{}
+
+	c := newCommand(
+		b,
+		ioutil.Discard,
+		false,
+		newFakeHTTPClientFactory(
+			func(u *url.URL) (*fakeHTTPResponse, error) {
+				return newFakeHTTPResponse(200, "", "text/html", nil), nil
+			},
+		),
+	)
+
+	ok := c.Run([]string{"--version"})
+	assert.True(t, ok)
+
+	r, err := regexp.Compile(`^[0-9]+\.[0-9]+\.[0-9]+$`)
+	assert.Nil(t, err)
+	assert.True(t, r.MatchString(strings.TrimSpace(b.String())))
 }
diff --git a/link_fetcher_test.go b/link_fetcher_test.go
index fcaca41..bff6a4b 100644
--- a/link_fetcher_test.go
+++ b/link_fetcher_test.go
@@ -15,7 +15,7 @@ func newTestLinkFetcher(c *fakeHTTPClient) *linkFetcher {
 }
 
 func newTestLinkFetcherWithOptions(c *fakeHTTPClient, o linkFetcherOptions) *linkFetcher {
-	return newLinkFetcher(c, newPageParser(newLinkFinder(nil), false), o)
+	return newLinkFetcher(c, newPageParser(newLinkFinder(nil)), o)
 }
 
 func TestNewFetcher(t *testing.T) {
@@ -165,6 +165,17 @@ func TestLinkFetcherFailToFetch(t *testing.T) {
 	assert.NotNil(t, err)
 }
 
+func TestLinkFetcherFailToParseURL(t *testing.T) {
+	f := newTestLinkFetcher(
+		newFakeHTTPClient(func(*url.URL) (*fakeHTTPResponse, error) {
+			return newFakeHTTPResponse(200, "", "text/html", nil), nil
+		}))
+
+	_, _, err := f.Fetch(":")
+
+	assert.NotNil(t, err)
+}
+
 func TestSeparateFragment(t *testing.T) {
 	for _, ss := range [][3]string{
 		{"http://foo.com#bar", "http://foo.com", "bar"},
diff --git a/page_parser_test.go b/page_parser_test.go
index 5827997..a475942 100644
--- a/page_parser_test.go
+++ b/page_parser_test.go
@@ -7,19 +7,19 @@ import (
 )
 
 func TestPageParserParsePage(t *testing.T) {
-	_, err := newPageParser(newLinkFinder(nil), false).Parse("http://foo.com", nil)
+	_, err := newPageParser(newLinkFinder(nil)).Parse("http://foo.com", nil)
 	assert.Nil(t, err)
 }
 
 func TestPageParserFailWithInvalidURL(t *testing.T) {
-	_, err := newPageParser(newLinkFinder(nil), false).Parse(":", nil)
+	_, err := newPageParser(newLinkFinder(nil)).Parse(":", nil)
 	assert.NotNil(t, err)
 }
 
 func TestPageParserSetCorrectURL(t *testing.T) {
 	s := "http://foo.com"
 
-	p, err := newPageParser(newLinkFinder(nil), false).Parse(s, nil)
+	p, err := newPageParser(newLinkFinder(nil)).Parse(s, nil)
 	assert.Nil(t, err)
 	assert.Equal(t, s, p.URL().String())
 }
@@ -27,29 +27,21 @@ func TestPageParserSetCorrectURL(t *testing.T) {
 func TestPageParserIgnorePageURLFragment(t *testing.T) {
 	s := "http://foo.com#id"
 
-	p, err := newPageParser(newLinkFinder(nil), false).Parse(s, nil)
+	p, err := newPageParser(newLinkFinder(nil)).Parse(s, nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "http://foo.com", p.URL().String())
 }
 
-func TestPageParserIgnoreQueryParameters(t *testing.T) {
+func TestPageParserKeepQuery(t *testing.T) {
 	s := "http://foo.com?bar=baz"
 
-	p, err := newPageParser(newLinkFinder(nil), false).Parse(s, nil)
-	assert.Nil(t, err)
-	assert.Equal(t, "http://foo.com", p.URL().String())
-}
-
-func TestPageParserKeepQueryParameters(t *testing.T) {
-	s := "http://foo.com?bar=baz"
-
-	p, err := newPageParser(newLinkFinder(nil), true).Parse(s, nil)
+	p, err := newPageParser(newLinkFinder(nil)).Parse(s, nil)
 	assert.Nil(t, err)
 	assert.Equal(t, s, p.URL().String())
 }
 
 func TestPageParserResolveLinksWithBaseTag(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil), false).Parse(
+	p, err := newPageParser(newLinkFinder(nil)).Parse(
 		"http://foo.com",
 		[]byte(`
 			<html>
@@ -67,7 +59,7 @@ func TestPageParserResolveLinksWithBaseTag(t *testing.T) {
 }
 
 func TestPageParserResolveLinksWithBlankBaseTag(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil), false).Parse(
+	p, err := newPageParser(newLinkFinder(nil)).Parse(
 		"http://foo.com",
 		[]byte(`
 			<html>
@@ -84,8 +76,24 @@ func TestPageParserResolveLinksWithBlankBaseTag(t *testing.T) {
 	assert.Equal(t, map[string]error{"http://foo.com/bar": nil}, p.Links())
 }
 
+func TestPageParserFailToParseWithInvalidBaseTag(t *testing.T) {
+	_, err := newPageParser(newLinkFinder(nil)).Parse(
+		"http://foo.com",
+		[]byte(`
+			<html>
+			  <head>
+					<base href=":" />
+				</head>
+				<body>
+				</body>
+			</html>
+		`),
+	)
+	assert.NotNil(t, err)
+}
+
 func TestPageParserParseID(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil), false).Parse(
+	p, err := newPageParser(newLinkFinder(nil)).Parse(
 		"http://foo.com",
 		[]byte(`<p id="foo" />`),
 	)
@@ -94,7 +102,7 @@ func TestPageParserParseID(t *testing.T) {
 }
 
 func TestPageParserParseName(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil), false).Parse(
+	p, err := newPageParser(newLinkFinder(nil)).Parse(
 		"http://foo.com",
 		[]byte(`<p name="foo" />`),
 	)
@@ -103,7 +111,7 @@ func TestPageParserParseName(t *testing.T) {
 }
 
 func TestPageParserParseIDAndName(t *testing.T) {
-	p, err := newPageParser(newLinkFinder(nil), false).Parse(
+	p, err := newPageParser(newLinkFinder(nil)).Parse(
 		"http://foo.com",
 		[]byte(`<p id="foo" name="bar" />`),
 	)
@@ -122,7 +130,7 @@ func TestPageParserParseLinks(t *testing.T) {
 			"http://foo.com/foo.img",
 		},
 	} {
-		p, err := newPageParser(newLinkFinder(nil), false).Parse(
+		p, err := newPageParser(newLinkFinder(nil)).Parse(
 			"http://foo.com",
 			[]byte(ss[0]),
 		)
diff --git a/throttled_http_client_test.go b/throttled_http_client_test.go
new file mode 100644
index 0000000..50afee3
--- /dev/null
+++ b/throttled_http_client_test.go
@@ -0,0 +1,7 @@
+package main
+
+import "testing"
+
+func TestNewThrottledHTTPClientHandleInvalidConnectionsOptionGracefully(t *testing.T) {
+	newThrottledHTTPClient(nil, -1)
+}
diff --git a/tools/test.sh b/tools/test.sh
deleted file mode 100755
index dec3253..0000000
--- a/tools/test.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-go test -covermode atomic -coverprofile coverage.txt