func fetchRobots(req Request) (*http.Response, error) {
	fetcher := newBaseFetcher()
	return fetcher.response(req)
}
fetchRobots is used for getting robots.txt files.
package fetch
import (
	"net/http"
	"net/url"
	"strings"
	"time"
	"github.com/temoto/robotstxt"
)
func isRobotsTxt(url string) bool {
	return strings.HasSuffix(url, "/robots.txt")
}