func (this *MWin007Downloader) downloadFile(p *page.Page, req *request.Request) (*page.Page, string) {
	var err error
	var urlstr string
	req.Header = this.getHeader()
	if urlstr = req.GetUrl(); len(urlstr) == 0 {
		mlog.LogInst().LogError("url is empty")
		p.SetStatus(true, "url is empty")
		return p, ""
	}
	var resp *http.Response
	if proxystr := req.GetProxyHost(); len(proxystr) != 0 {
		resp, err = connectByHttpProxy(p, req)
	} else {
		resp, err = connectByHttp(p, req)
	}
	if err != nil {
		return p, ""
	}
	p.SetHeader(resp.Header)
	p.SetCookies(resp.Cookies())
	var bodyStr string
	if resp.Header.Get("Content-Encoding") == "gzip" {
		bodyStr = this.changeCharsetEncodingAutoGzipSupport(resp.Header.Get("Content-Type"), resp.Body)
	} else {
		bodyStr = this.changeCharsetEncodingAuto(resp.Header.Get("Content-Type"), resp.Body)
	}
	defer resp.Body.Close()
	return p, bodyStr
}
Download file and change the charset of page charset.
package down
import (
	"bytes"
	"github.com/PuerkitoBio/goquery"
	"github.com/bitly/go-simplejson"
	"github.com/hu17889/go_spider/core/common/mlog"
	"github.com/hu17889/go_spider/core/common/page"
	"github.com/hu17889/go_spider/core/common/request"
	"github.com/hu17889/go_spider/core/common/util"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"golang.org/x/net/html/charset"
	"compress/gzip"
	"strings"
)
type MWin007Downloader struct {
}
func NewMWin007Downloader() *MWin007Downloader {
	return &MWin007Downloader{}
}
func (this *MWin007Downloader) Download(req *request.Request) *page.Page {
	var mtype string
	var p = page.NewPage(req)
	mtype = req.GetResponceType()
	switch mtype {
	case "html":
		return this.downloadHtml(p, req)
	case "json":
		fallthrough
	case "jsonp":
		return this.downloadJson(p, req)
	case "text":
		return this.downloadText(p, req)
	default:
		mlog.LogInst().LogError("error request type:" + mtype)
	}
	return p
}
func (this *MWin007Downloader) changeCharsetEncodingAuto(contentTypeStr string, sor io.ReadCloser) string {
	var err error
	destReader, err := charset.NewReader(sor, contentTypeStr)
	if err != nil {
		mlog.LogInst().LogError(err.Error())
		destReader = sor
	}
	var sorbody []byte
	if sorbody, err = ioutil.ReadAll(destReader); err != nil {
		mlog.LogInst().LogError(err.Error())
	}
	bodystr := string(sorbody)
	return bodystr
}
func (this *MWin007Downloader) changeCharsetEncodingAutoGzipSupport(contentTypeStr string, sor io.ReadCloser) string {
	var err error
	gzipReader, err := gzip.NewReader(sor)
	if err != nil {
		mlog.LogInst().LogError(err.Error())
		return ""
	}
	defer gzipReader.Close()
	destReader, err := charset.NewReader(gzipReader, contentTypeStr)
	if err != nil {
		mlog.LogInst().LogError(err.Error())
		destReader = sor
	}
	var sorbody []byte
	if sorbody, err = ioutil.ReadAll(destReader); err != nil {
		mlog.LogInst().LogError(err.Error())
	}
	bodystr := string(sorbody)
	return bodystr
}
func (this *MWin007Downloader) getHeader() http.Header {
	header := http.Header{}
	header.Add("Host", "m.win007.com")
	header.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0")
	header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
	header.Add("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2")
	header.Add("Accept-Encoding", "gzip, deflate, br")
	header.Add("DNT", "1")
	header.Add("Connection", "keep-alive")
	header.Add("Referer", "http://m.win007.com/")
	header.Add("Cookie", "Hm_lvt_3c8ecbfa472e76b0340d7a701a04197e=1574622164,1574699164,1574900673,1575112214; Hm_lpvt_3c8ecbfa472e76b0340d7a701a04197e=1575188612")
	header.Add("Upgrade-Insecure-Requests", "1")
	header.Add("Pragma", "no-cache")
	header.Add("Cache-Control", "no-cache")
	return header
}
func connectByHttp(p *page.Page, req *request.Request) (*http.Response, error) {
	client := &http.Client{
		CheckRedirect: req.GetRedirectFunc(),
	}
	httpreq, err := http.NewRequest(req.GetMethod(), req.GetUrl(), strings.NewReader(req.GetPostdata()))
	if header := req.GetHeader(); header != nil {
		httpreq.Header = req.GetHeader()
	}
	if cookies := req.GetCookies(); cookies != nil {
		for i := range cookies {
			httpreq.AddCookie(cookies[i])
		}
	}
	var resp *http.Response
	if resp, err = client.Do(httpreq); err != nil {
		if e, ok := err.(*url.Error); ok && e.Err != nil && e.Err.Error() == "normal" {
		} else {
			mlog.LogInst().LogError(err.Error())
			p.SetStatus(true, err.Error())
			return nil, err
		}
	}
	return resp, nil
}
func connectByHttpProxy(p *page.Page, in_req *request.Request) (*http.Response, error) {
	request, _ := http.NewRequest("GET", in_req.GetUrl(), nil)
	proxy, err := url.Parse(in_req.GetProxyHost())
	if err != nil {
		return nil, err
	}
	client := &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyURL(proxy),
		},
	}
	resp, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	return resp, nil
}