func (f *Fetcher) userAgent() string {
	if ua := f.UserAgent; ua != nil {
		return ua.String()
	}
	return UserAgent.String()
}
UserAgent returns the user agent to use.
package ant
import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)
type StaticAgent string
func (sa StaticAgent) String() string {
	return string(sa)
}
var (
	UserAgent = StaticAgent("antbot")
	DefaultFetcher = &Fetcher{
		Client:    DefaultClient,
		UserAgent: UserAgent,
	}
	minBackoff = 50 * time.Millisecond
	maxBackoff = 1 * time.Second
)
type FetchError struct {
	URL    *url.URL
	Status int
}
func (err FetchError) Error() string {
	return fmt.Sprintf("ant: fetch %q - %d %s",
		err.URL,
		err.Status,
		http.StatusText(err.Status),
	)
}
func (err FetchError) Temporary() bool {
	return err.Status == 503 ||  
		err.Status == 504 ||  
		err.Status == 429  
}
func Fetch(ctx context.Context, rawurl string) (*Page, error) {
	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	return DefaultFetcher.Fetch(ctx, u)
}
type Fetcher struct {
	Client Client
	UserAgent fmt.Stringer
	MaxAttempts int
	MinBackoff time.Duration
	MaxBackoff time.Duration
}
func (f *Fetcher) Fetch(ctx context.Context, url *URL) (*Page, error) {
	var maxAttempts = f.maxAttempts()
	var attempt int
	var resp *http.Response
	var err error
	for {
		if attempt++; attempt > maxAttempts {
			return nil, fmt.Errorf(
				"ant: max attempts of %d reached - %w",
				maxAttempts,
				err,
			)
		}
		if resp, err = f.fetch(ctx, url); err == nil {
			break
		}
		f.discard(resp)
		if isTemporary(err) {
			if err := f.backoff(ctx, attempt); err != nil {
				return nil, err
			}
			continue
		}
		if err, ok := err.(*FetchError); ok {
			if err.Status == 404 {
				return nil, nil
			}
		}
		return nil, err
	}
	return &Page{
		URL:    resp.Request.URL,
		Header: resp.Header,
		body:   resp.Body,
	}, nil
}
func (f *Fetcher) fetch(ctx context.Context, url *URL) (*http.Response, error) {
	var client = f.client()
	req, err := http.NewRequestWithContext(ctx, "GET", url.String(), nil)
	if err != nil {
		return nil, fmt.Errorf("ant: new request - %w", err)
	}
	for k, v := range f.headers() {
		req.Header[k] = v
	}
	resp, err := client.Do(req)
	if err != nil {
		return resp, fmt.Errorf("ant: %s %q - %w", req.Method, req.URL, err)
	}
	if resp.StatusCode >= 400 {
		return resp, &FetchError{
			URL:    resp.Request.URL,
			Status: resp.StatusCode,
		}
	}
	return resp, nil
}
func (f *Fetcher) discard(r *http.Response) {
	if r != nil {
		io.Copy(ioutil.Discard, r.Body)
		r.Body.Close()
	}
}
func (f *Fetcher) maxAttempts() int {
	if f.MaxAttempts > 0 {
		return f.MaxAttempts
	}
	return 5
}
func (f *Fetcher) headers() http.Header {
	var hdr = make(http.Header)
	hdr.Set("Accept", "text/html; charset=UTF-8")
	hdr.Set("User-Agent", f.userAgent())
	return hdr
}