func (c *Links) Prepare(*Test) (err error) {
	c.tags = nil
	which := strings.TrimSpace(c.Which)
	if which == "" {
		return fmt.Errorf("Missing value for Which")
	}
	for _, tag := range strings.Split(which, " ") {
		tag = strings.TrimSpace(tag)
		switch tag {
		case "", "-none-":  
		case "a", "img", "link", "script", "video", "audio", "source", "iframe":
			c.tags = append(c.tags, tag)
		default:
			fmt.Println("Bad", tag)
			return fmt.Errorf("Unknown link tag %q", tag)
		}
	}
	sort.Strings(c.tags)  
	return nil
}
Prepare implements Check's Prepare method.
package ht
import (
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"
	"github.com/andybalholm/cascadia"
	"github.com/vdobler/ht/errorlist"
	"golang.org/x/net/html"
)
func init() {
	RegisterCheck(&HTMLTag{})
	RegisterCheck(&HTMLContains{})
	RegisterCheck(W3CValidHTML{})
	RegisterCheck(&Links{})
}
type W3CValidHTML struct {
	AllowedErrors int `json:",omitempty"`
	IgnoredErrors []Condition `json:",omitempty"`
}
func (w W3CValidHTML) Execute(t *Test) error {
	file := "@file:@sample.html:" + t.Response.BodyStr
	test := &Test{
		Name: "W3CValidHTML",
		Request: Request{
			Method: "POST",
			URL:    "http://validator.w3.org/nu/",
			Params: url.Values{
				"file": {file},
			},
			ParamsAs: "multipart",
			Header: http.Header{
				"Accept":     {"text/html,application/xhtml+xml"},
				"User-Agent": {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36"},
			},
			Timeout: 20 * time.Second,
		},
		Checks: CheckList{
			StatusCode{Expect: 200},
		},
	}
	time.Sleep(100 * time.Millisecond)
	err := test.Run()
	if err != nil {
		return CantCheck{err}
	}
	if test.Result.Status != Pass {
		return CantCheck{test.Result.Error}
	}
	valStat := test.Response.Response.Header.Get("X-W3C-Validator-Status")
	if valStat == "Abort" {
		return CantCheck{fmt.Errorf("Validator service sent Abort")}
	} else if valStat == "Valid" {
		return nil
	}
	doc, err := html.Parse(test.Response.Body())
	if err != nil {
		return CantCheck{err}
	}
	validationErrors := []ValidationIssue{}
	noIgnored := 0
outer:
	for _, ve := range w3cValidatorErrSel.MatchAll(doc) {
		vi := extractValidationIssue(ve)
		for _, c := range w.IgnoredErrors {
			if c.Fulfilled(vi.Message) == nil {
				noIgnored++
				continue outer
			}
		}
		validationErrors = append(validationErrors, vi)
	}
	if len(validationErrors) > w.AllowedErrors {
		errmsg := fmt.Sprintf("Ignored %d errors", noIgnored)
		for i, e := range validationErrors {
			errmsg = fmt.Sprintf("%s\nError %d:\n  %s\n  %s\n  %s",
				errmsg, i, e.Position, e.Message, e.Input)
		}
		return fmt.Errorf("%s", errmsg)
	}
	return nil
}
type ValidationIssue struct {
	Position string
	Message  string
	Input    string
}
var (
	w3cValidatorErrSel   = cascadia.MustCompile("li.error")
	w3cValidatorLocSel   = cascadia.MustCompile("p.location")
	w3cValidatorMsgSel   = cascadia.MustCompile("p span")
	w3cValidatorInputSel = cascadia.MustCompile("p.extract")
)
func extractValidationIssue(node *html.Node) ValidationIssue {
	p := TextContent(w3cValidatorLocSel.MatchFirst(node), false)
	p = strings.Replace(p, "\n     ", "", -1)
	return ValidationIssue{
		Position: p,
		Message:  TextContent(w3cValidatorMsgSel.MatchFirst(node), false),
		Input:    TextContent(w3cValidatorInputSel.MatchFirst(node), false),
	}
}
type HTMLTag struct {
	Selector string
	Count int `json:",omitempty"`
	sel cascadia.Selector
}
func (c *HTMLTag) Execute(t *Test) error {
	if t.Response.BodyErr != nil {
		return ErrBadBody
	}
	doc, err := html.Parse(t.Response.Body())
	if err != nil {
		return CantCheck{err}
	}
	matches := c.sel.MatchAll(doc)
	switch {
	case c.Count < 0 && len(matches) > 0:
		return fmt.Errorf("Found forbidden %s", c.Selector)
	case c.Count == 0 && len(matches) == 0:
		return fmt.Errorf("Cannot find %s", c.Selector)
	case c.Count > 0:
		if cnt := len(matches); cnt != c.Count {
			return fmt.Errorf("Found %d occurrences of %s, want %d",
				cnt, c.Selector, c.Count)
		}
	}
	return nil
}
func (c *HTMLTag) Prepare(*Test) (err error) {
	c.sel, err = cascadia.Compile(c.Selector)
	if err != nil {
		c.sel = nil
		return err
	}
	return nil
}
var _ Preparable = &HTMLTag{}
type HTMLContains struct {
	Selector string
	Text []string `json:",omitempty"`
	Raw bool `json:",omitempty"`
	Complete bool `json:",omitempty"`
	InOrder bool `json:",omitempty"`
	sel cascadia.Selector
}
func (c *HTMLContains) Execute(t *Test) error {
	if c.sel == nil {
		if err := c.Prepare(t); err != nil {
			return err
		}
	}
	if t.Response.BodyErr != nil {
		return ErrBadBody
	}
	doc, err := html.Parse(t.Response.Body())
	if err != nil {
		return CantCheck{err}
	}
	matches := c.sel.MatchAll(doc)
	if len(matches) == 0 {
		return errTagNotFound
	}
	actual := make([]string, len(matches))
	for i, m := range matches {
		actual[i] = TextContent(m, c.Raw)
	}
	last := 0
	for _, want := range c.Text {
		found := -1
		for a := last; a < len(actual); a++ {
			if want == actual[a] {
				found = a
				break
			}
		}
		if found < 0 {
			return fmt.Errorf("Missing %q, have %q", want, actual[last:])
		}
		if c.InOrder {
			last = found + 1
		} else {
			last = 0
		}
	}
	if c.Complete && len(c.Text) != len(matches) {
		return WrongCount{Got: len(matches), Want: len(c.Text)}
	}
	return nil
}
var errTagNotFound = fmt.Errorf("tag not found")
func (c *HTMLContains) Prepare(*Test) (err error) {
	c.sel, err = cascadia.Compile(c.Selector)
	if err != nil {
		c.sel = nil
		return err
	}
	return nil
}
var _ Preparable = &HTMLContains{}
func TextContent(n *html.Node, raw bool) string {
	tc := textContentRec(n, raw)
	if !raw {
		tc = normalizeWhitespace(tc)
	}
	return tc
}
func normalizeWhitespace(s string) string {
	s = strings.TrimSpace(s)
	s = strings.Replace(strings.Replace(s, "\n", " ", -1), "\t", " ", -1)
	for strings.Contains(s, "  ") {
		s = strings.Replace(s, "  ", " ", -1)
	}
	return s
}
var inlineElement = map[string]bool{
	"b":        true,
	"big":      true,
	"i":        true,
	"small":    true,
	"tt":       true,
	"abbr":     true,
	"acronym":  true,
	"cite":     true,
	"code":     true,
	"dfn":      true,
	"em":       true,
	"kbd":      true,
	"strong":   true,
	"samp":     true,
	"var":      true,
	"a":        true,
	"bdo":      true,
	"br":       true,
	"img":      true,
	"map":      true,
	"object":   true,
	"q":        true,
	"script":   true,
	"span":     true,
	"sub":      true,
	"sup":      true,
	"button":   true,
	"input":    true,
	"label":    true,
	"select":   true,
	"true":     true,
	"textarea": true,
}
func textContentRec(n *html.Node, raw bool) string {
	if n == nil {
		return ""
	}
	switch n.Type {
	case html.TextNode:
		return n.Data
	case html.ElementNode:
		s := ""
		for child := n.FirstChild; child != nil; child = child.NextSibling {
			cs := textContentRec(child, raw)
			if !raw && child.Type == html.ElementNode &&
				!inlineElement[child.Data] {
				cs = "\n" + cs + "\n"
			}
			s += cs
		}
		return s
	}
	return ""
}
type Links struct {
	Which string
	Head bool `json:",omitempty"`
	Concurrency int `json:",omitempty"`
	Timeout time.Duration `json:",omitempty"`
	OnlyLinks, IgnoredLinks []Condition `json:",omitempty"`
	FailMixedContent bool `json:",omitempty"`
	MaxTime time.Duration `json:",omitempty"`
	tags []string
}
func (c *Links) collectURLs(t *Test) (map[string]bool, error) {
	if t.Response.BodyErr != nil {
		return nil, ErrBadBody
	}
	doc, err := html.Parse(t.Response.Body())
	if err != nil {
		return nil, CantCheck{err}
	}
	refs := make(map[string]bool)
	for _, tag := range c.tags {
		isAnchor := tag == "a"
	outer:
		for _, u := range c.linkURL(doc, tag, t.Request.Request.URL) {
			for i, cond := range c.OnlyLinks {
				if cond.Fulfilled(u) == nil {
					break
				}
				if i == len(c.OnlyLinks)-1 {
					continue outer
				}
			}
			for _, cond := range c.IgnoredLinks {
				if cond.Fulfilled(u) == nil {
					continue outer
				}
			}
			refs[u] = isAnchor
		}
	}
	return refs, nil
}
func (c *Links) Execute(t *Test) error {
	refs, err := c.collectURLs(t)
	if err != nil {
		return err
	}
	suite := &Collection{}
	method := http.MethodGet
	if c.Head {
		method = "HEAD"
	}
	timeout := t.Request.Timeout
	if c.Timeout > 0 {
		timeout = c.Timeout
	}
	upgrade := false
	if resp := t.Response.Response; resp != nil {
		csp := resp.Header.Get("Content-Security-Policy")
		upgrade = strings.Contains(csp, "upgrade-insecure-requests")  
	}
	broken := errorlist.List{}  
	var urefs map[string]bool  
	if !c.FailMixedContent || !strings.HasPrefix(t.Request.URL, "https") {
		urefs = refs
	} else {
		urefs = make(map[string]bool)
		for r, isA := range refs {
			if strings.HasPrefix(r, "http:") {
				if upgrade {
					r = "https:" + r[5:]
				} else if !isA {
					broken = append(broken,
						fmt.Errorf("%s  -->  un-upgraded mixed content", r))
				}
				urefs[r] = isA
			}
		}
	}
	for r := range urefs {
		test := &Test{
			Name: r,
			Request: Request{
				Method:          method,
				URL:             r,
				FollowRedirects: true,
				BasicAuthUser:   t.Request.BasicAuthUser,
				BasicAuthPass:   t.Request.BasicAuthPass,
				Timeout:         timeout,
			},
			Checks: CheckList{
				StatusCode{Expect: 200},
			},
			Execution: Execution{
				Verbosity: t.Execution.Verbosity - 1,
			},
		}
		test.PopulateCookies(t.Jar, t.Request.Request.URL)
		if ru, err := url.Parse(r); err == nil &&
			ru.Host == t.Request.Request.URL.Host {
			test.Request.BasicAuthUser = t.Request.BasicAuthUser
			test.Request.BasicAuthPass = t.Request.BasicAuthPass
		}
		suite.Tests = append(suite.Tests, test)
	}
	conc := 1
	if c.Concurrency > 1 {
		conc = c.Concurrency
	}
	started := time.Now()
	suite.ExecuteConcurrent(conc, nil)
	if suite.Status != Pass {
		for _, test := range suite.Tests {
			if test.Result.Status == Error || test.Result.Status == Bogus {
				broken = append(broken, fmt.Errorf("%s  -->  %s",
					test.Request.URL, test.Result.Error))
			} else if test.Result.Status == Fail {
				broken = append(broken, fmt.Errorf("%s  -->  %d",
					test.Request.URL,
					test.Response.Response.StatusCode))
			}
		}
	}
	if c.MaxTime > 0 {
		if took := time.Since(started); took > c.MaxTime {
			broken = append(broken,
				fmt.Errorf("retrieval of %d links took %s (allowed %s)",
					len(urefs), took, c.MaxTime))
		}
	}
	if len(broken) > 0 {
		return broken
	}
	return nil
}
func (Links) linkURL(document *html.Node, tag string, requestURL *url.URL) []string {
	href := linkURLattr[tag].sel
	matches := href.MatchAll(document)
	ak := linkURLattr[tag].attr
	refs := []string{}
	for _, m := range matches {
		for _, a := range m.Attr {
			if a.Key != ak || a.Val == "#" ||
				strings.HasPrefix(a.Val, "mailto:") {
				continue
			}
			u, err := requestURL.Parse(a.Val)
			if err != nil {
				refs = append(refs, "Error: "+err.Error())
			} else {
				u.Fragment = ""  
				refs = append(refs, u.String())
			}
		}
	}
	return refs
}
var linkURLattr = map[string]struct {
	attr string
	sel  cascadia.Selector
}{
	"a":      {"href", cascadia.MustCompile("a")},
	"img":    {"src", cascadia.MustCompile("img")},
	"script": {"src", cascadia.MustCompile("script")},
	"link":   {"href", cascadia.MustCompile("link")},
	"video":  {"src", cascadia.MustCompile("video")},
	"audio":  {"src", cascadia.MustCompile("audio")},
	"source": {"src", cascadia.MustCompile("source")},
	"iframe": {"src", cascadia.MustCompile("iframe")},
}