func (f *Fetcher) processQueue() {
loop:
	for v := range f.q.ch {
		if v == nil {
			select {
			case <-f.q.closed:
				break loop
			default:
			}
		}
		select {
		case <-f.q.cancelled:
			continue
		default:
		}
		u := v.URL()
		f.mu.Lock()
		in, ok := f.hosts[u.Host]
		if !ok {
			var rob *url.URL
			if !f.DisablePoliteness {
				rob = u.ResolveReference(robotsTxtParsedPath)
			}
			var out chan Command
			in, out = make(chan Command, 1), make(chan Command, 1)
			f.hosts[u.Host] = in
			f.mu.Unlock()
			f.q.wg.Add(1)
			go sliceIQ(in, out)
			go f.processChan(out, u.Host)
			if !f.DisablePoliteness {
				in <- robotCommand{&Cmd{U: rob, M: "GET"}}
			}
		} else {
			f.mu.Unlock()
		}
		in <- v
		f.dbgmu.Lock()
		if f.debugging {
			f.mu.Lock()
			select {
			case f.dbg <- &DebugInfo{len(f.hosts)}:
			default:
			}
			f.mu.Unlock()
		}
		f.dbgmu.Unlock()
	}
	f.mu.Lock()
	for _, ch := range f.hosts {
		close(ch)
	}
	f.hosts = make(map[string]chan Command)
	f.mu.Unlock()
	f.q.wg.Done()
}
processQueue runs the queue in its own goroutine.
package fetchbot
import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
	"github.com/temoto/robotstxt"
)
var (
	ErrEmptyHost = errors.New("fetchbot: invalid empty host")
	ErrDisallowed = errors.New("fetchbot: disallowed by robots.txt")
	ErrQueueClosed = errors.New("fetchbot: send on a closed queue")
)
var robotsTxtParsedPath, _ = url.Parse("/robots.txt")
const (
	DefaultCrawlDelay = 5 * time.Second
	DefaultUserAgent = "Fetchbot (https://github.com/PuerkitoBio/fetchbot)"
	DefaultWorkerIdleTTL = 30 * time.Second
)
type Doer interface {
	Do(*http.Request) (*http.Response, error)
}
type Fetcher struct {
	Handler Handler
	DisablePoliteness bool
	CrawlDelay time.Duration
	HttpClient Doer
	UserAgent string
	WorkerIdleTTL time.Duration
	AutoClose bool
	q *Queue
	dbgmu     sync.Mutex
	dbg       chan *DebugInfo
	debugging bool
	mu    sync.Mutex
	hosts map[string]chan Command
}
type DebugInfo struct {
	NumHosts int
}
func New(h Handler) *Fetcher {
	return &Fetcher{
		Handler:       h,
		CrawlDelay:    DefaultCrawlDelay,
		HttpClient:    http.DefaultClient,
		UserAgent:     DefaultUserAgent,
		WorkerIdleTTL: DefaultWorkerIdleTTL,
		dbg:           make(chan *DebugInfo, 1),
	}
}
type Queue struct {
	ch chan Command
	closed, cancelled, done chan struct{}
	wg sync.WaitGroup
}
func (q *Queue) Close() error {
	select {
	case <-q.closed:
		return nil
	default:
		close(q.closed)
		q.ch <- nil
		q.wg.Wait()
		close(q.done)
		return nil
	}
}
func (q *Queue) Block() {
	<-q.done
}
func (q *Queue) Done() <-chan struct{} {
	return q.done
}
func (q *Queue) Cancel() error {
	select {
	case <-q.cancelled:
		return nil
	default:
		close(q.cancelled)
		return q.Close()
	}
}
func (q *Queue) Send(c Command) error {
	if c == nil {
		return ErrEmptyHost
	}
	if u := c.URL(); u == nil || u.Host == "" {
		return ErrEmptyHost
	}
	select {
	case <-q.closed:
		return ErrQueueClosed
	default:
		q.ch <- c
	}
	return nil
}
func (q *Queue) SendString(method string, rawurl ...string) (int, error) {
	return q.sendWithMethod(method, rawurl)
}
func (q *Queue) SendStringHead(rawurl ...string) (int, error) {
	return q.sendWithMethod("HEAD", rawurl)
}
func (q *Queue) SendStringGet(rawurl ...string) (int, error) {
	return q.sendWithMethod("GET", rawurl)
}
func (q *Queue) sendWithMethod(method string, rawurl []string) (int, error) {
	for i, v := range rawurl {
		parsed, err := url.Parse(v)
		if err != nil {
			return i, err
		}
		if err := q.Send(&Cmd{U: parsed, M: method}); err != nil {
			return i, err
		}
	}
	return len(rawurl), nil
}
func (f *Fetcher) Start() *Queue {
	f.hosts = make(map[string]chan Command)
	f.q = &Queue{
		ch:        make(chan Command, 1),
		closed:    make(chan struct{}),
		cancelled: make(chan struct{}),
		done:      make(chan struct{}),
	}
	f.q.wg.Add(1)
	go f.processQueue()
	return f.q
}
func (f *Fetcher) Debug() <-chan *DebugInfo {
	f.dbgmu.Lock()
	defer f.dbgmu.Unlock()
	f.debugging = true
	return f.dbg
}