src/main/java/com/doubleview/fastcrawler/exceptions/PageSizeOverException.java
public class PageSizeOverException extends  Exception {

    long pageSize;

    public PageSizeOverException(long pageSize) {
            super("Aborted fetching of this URL as it's size ( " + pageSize +
                    " ) exceeds the maximum size");
            this.pageSize = pageSize;
    }


    public long getPageSize() {
        return pageSize;
    }
}
src/main/java/com/doubleview/fastcrawler/fetcher/PageFetcher.java
public class PageFetcher extends CrawlerConfiguable {

    protected static final Logger logger = LoggerFactory.getLogger(PageFetcher.class);

    protected final ReentrantLock lock = new ReentrantLock();

    protected PoolingHttpClientConnectionManager connectionManager;

    protected CloseableHttpClient httpClient;

    protected IdleConnectionMonitorThread connectionMonitorThread = null;

    protected long lastFetchTime = 0;

    public PageFetcher(CrawlerConfig crawlerConfig) {
        super(crawlerConfig);
        initHttpClient(crawlerConfig);
    }


    /**
     * init the httpclient
     *
     * @param crawlerConfig
     */
    private void initHttpClient(CrawlerConfig crawlerConfig) {
        RequestConfig requestConfig = RequestConfig.custom()
                .setExpectContinueEnabled(true)
                .setCookieSpec(CookieSpecs.STANDARD)
                .setRedirectsEnabled(false)
                .setSocketTimeout(crawlerConfig.getSocketTimeout())
                .setConnectTimeout(crawlerConfig.getConnectionTimeout())
                .build();

        RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create();
        connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE);
        SSLConnectionSocketFactory sslConnectionSocketFactory = null;
        try {
            sslConnectionSocketFactory = new SSLConnectionSocketFactory(createIgnoreVerifySSL());
        } catch (Exception e) {
            logger.error("Error occurred : {}", e);
        }
        connRegistryBuilder.register("https", sslConnectionSocketFactory);
        Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build();
        connectionManager = new SniPoolingHttpClientConnectionManager(connRegistry);
        connectionManager.setMaxTotal(crawlerConfig.getMaxTotalConns());
        connectionManager.setDefaultMaxPerRoute(crawlerConfig.getMaxConnsPerHost());

        HttpClientBuilder clientBuilder = HttpClientBuilder.create()
                .setDefaultRequestConfig(requestConfig)
                .setConnectionManager(connectionManager)
                .setUserAgent(crawlerConfig.getUserAgent());

        if (crawlerConfig.getProxyHost() != null) {
            if (crawlerConfig.getProxyUsername() != null) {
                BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
                credentialsProvider.setCredentials(
                        new AuthScope(crawlerConfig.getProxyHost(), crawlerConfig.getProxyPort()),
                        new UsernamePasswordCredentials(crawlerConfig.getProxyUsername(), crawlerConfig.getProxyPassword()));
                clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }
            HttpHost proxy = new HttpHost(crawlerConfig.getProxyHost(), crawlerConfig.getProxyPort());
            clientBuilder.setProxy(proxy);
            logger.debug("Working through Proxy : {}", proxy.getHostName());
        }

        httpClient = clientBuilder.build();
        if (connectionMonitorThread == null) {
            connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
        }
        connectionMonitorThread.start();
    }

    private SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
        SSLContext sc = SSLContext.getInstance("SSLv3");

        // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
        X509TrustManager trustManager = new X509TrustManager() {
            @Override
            public void checkClientTrusted(java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
                                           String paramString) throws CertificateException {
            }

            @Override
            public void checkServerTrusted(java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
                                           String paramString) throws CertificateException {
            }

            @Override
            public java.security.cert.X509Certificate[] getAcceptedIssuers() {
                return null;
            }
        };

        sc.init(null, new TrustManager[]{trustManager}, null);
        return sc;
    }

    /**
     * @param crawlerRequest
     * @return
     * @throws InterruptedException
     * @throws IOException
     */
    public FetchResult fetchPage(CrawlerRequest crawlerRequest) throws InterruptedException, IOException, PageSizeOverException {
        logger.info("fetching url : {}", crawlerRequest.getUrl());
        FetchResult fetchResult = new FetchResult();
        String toFetchURL = crawlerRequest.getUrl();
        fetchResult.setFetchUrl(toFetchURL);

        HttpUriRequest request = null;
        try {
            request = new HttpGet(toFetchURL);
            try {
                lock.lock();
                long now = System.currentTimeMillis();
                if ((now - lastFetchTime) < crawlerConfig.getFetchTimeDelay()) {
                    Thread.sleep(crawlerConfig.getFetchTimeDelay() - (now - lastFetchTime));
                }
                lastFetchTime = System.currentTimeMillis();
            } finally {
                lock.unlock();
            }
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(request);
            } catch (IllegalStateException e) {
                logger.error(e.getMessage());
                return null;
            }

            fetchResult.setEntity(response.getEntity());
            fetchResult.setResponseHeaders(response.getAllHeaders());

            int statusCode = response.getStatusLine().getStatusCode();
            fetchResult.setStatusCode(statusCode);

            if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY ||
                    statusCode == HttpStatus.SC_MOVED_TEMPORARILY ||
                    statusCode == HttpStatus.SC_MULTIPLE_CHOICES ||
                    statusCode == HttpStatus.SC_SEE_OTHER ||
                    statusCode == HttpStatus.SC_TEMPORARY_REDIRECT || statusCode == 308) {
                Header header = response.getFirstHeader("Location");
                if (header != null) {
                    String movedToUrl = NetUtils.canonicalizeURL(header.getValue(), toFetchURL);
                    fetchResult.setMoveToUrl(movedToUrl);
                } else {
                    logger.warn("unexpected error, url: {} is redirected to nothing", crawlerRequest.getUrl());
                    return null;
                }
            } else if (statusCode >= 200 && statusCode <= 299) {
                if (fetchResult.getEntity() != null) {
                    long size = fetchResult.getEntity().getContentLength();
                    if (size == -1) {
                        Header length = response.getLastHeader("Content-Length");
                        if (length == null) {
                            length = response.getLastHeader("Content-length");
                        }
                        if (length != null) {
                            size = Integer.parseInt(length.getValue());
                        }
                    }
                    if (size > crawlerConfig.getMaxDownloadSize()) {
                        response.close();
                        throw new PageSizeOverException(size);
                    }
                }
            }
            return fetchResult;
        } finally {
            if (fetchResult.getEntity() == null && request != null) {
                request.abort();
            }
        }
    }


    /**
     * @param fetchResult
     * @return
     */
    public Page loadPage(FetchResult fetchResult, CrawlerRequest request) throws ContentFetchException {
        try {
            Page page = new Page(request);
            if (fetchResult.getMoveToUrl() != null) {
                page.setRedirect(true);
                CrawlerRequest redirectRequest = new CrawlerRequest(fetchResult.getMoveToUrl());
                redirectRequest.setParentUrl(fetchResult.getMoveToUrl());
                redirectRequest.setDepth(request.getDepth());
                page.setRedirectRequest(redirectRequest);
                return page;
            }
            try {
                page.load(fetchResult.getEntity(), crawlerConfig.getMaxDownloadSize());
                if (page.isTruncated()) {
                    logger.warn("Warning: unknown page size exceeded max-download-size, truncated to: " + "({}), at URL: {}", crawlerConfig.getMaxDownloadSize(), request.getUrl());
                }
            } catch (Exception e) {
                logger.error("exception while fetching content for: {} [{}]", request.getUrl(), e.getMessage());
                throw new ContentFetchException(e);
            }
            return page;
        } finally {
            if (fetchResult != null) {
                fetchResult.discardContent();
            }
        }

    }

    /**
     *
     */
    public void shutDown() {
        try {
            lock.lock();
            httpClient.close();
            if (connectionMonitorThread != null) {
                //connectionManager.shutdown();
                connectionMonitorThread.shutdown();
            }
        } catch (IOException e) {
            logger.error("error occurred while shutdown httpclient", e);
        } finally {
            lock.unlock();
        }
    }

}
src/main/java/com/doubleview/fastcrawler/parser/BinaryData.java
public class BinaryData extends AbstractResponseData {

    protected  byte[] binaryContent;

    protected  String extenstion;

    public static String STORE_PATH;

    public BinaryData(byte[] binaryContent) {
        this.binaryContent = binaryContent;
    }

    public byte[] getBinaryContent() {
        return binaryContent;
    }

    public void setBinaryContent(byte[] binaryContent) {
        this.binaryContent = binaryContent;
    }

    public String getExtenstion() {
        return extenstion;
    }

    public void setExtenstion(String extenstion) {
        this.extenstion = extenstion;
    }


    public InputStream getInputStream() {
        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(binaryContent);
        return byteArrayInputStream;
    }

    public long getLength() {
        return binaryContent.length;
    }


    public void store(OutputStream out) throws IOException {
        out.write(binaryContent);
    }


    public void store(File file) throws IOException {
        store(new FileOutputStream(file));
    }

    @Override
    protected List<String> toStrings() {
        return null;
    }
}
src/main/java/com/doubleview/fastcrawler/parser/PageParser.java
public class PageParser extends CrawlerConfiguable {

    protected static final Logger logger = LoggerFactory.getLogger(PageParser.class);

    public PageParser(CrawlerConfig crawlerConfig) {
        super(crawlerConfig);
    }

    /**
     * parse the Page object according to the page data
     *
     * @param page
     * @throws NotAllowBinaryDataException
     * @throws PageParserException
     */
    public void parse(Page page) throws NotAllowBinaryDataException, PageParserException {
        if (hasBinaryContent(page.getContentType())) {
            if (!getCrawlerConfig().isIncludeBinaryContent()) {
                throw new NotAllowBinaryDataException("not allow binary data");
            }
            BinaryData binaryData = new BinaryData(page.getContentData());
            binaryData.setExtenstion(getBinaryExtension(page.getCrawlerRequest().getUrl()));
            String binaryStorePath = getCrawlerConfig().getBinaryStorePath();
            if (binaryStorePath != null && BinaryData.STORE_PATH == null) {
                BinaryData.STORE_PATH = binaryStorePath;
            }
            page.setType(Page.TYPE.BINARY);
            page.setBinaryData(binaryData);
        } else if (hasPlainTextContent(page.getContentType())) {
            String contentCharset = page.getContentCharset();
            try {
                String plainText = new String(page.getContentData(), contentCharset);
                PlainTextData plainTextData = new PlainTextData(plainText);
                page.setType(Page.TYPE.PLAINTEXT);
                page.setPlainTextData(plainTextData);
            } catch (UnsupportedEncodingException e) {
                logger.error("{}, while parsing: {}", e.getMessage(), page.getCrawlerRequest().getUrl());
                throw new PageParserException();
            }
        } else if (hasJsonTextContent(page.getContentType())) {
            String contentCharset = page.getContentCharset();
            try {
                String jsonText = new String(page.getContentData(), contentCharset);
                JsonData jsonData = new JsonData(jsonText);
                page.setType(Page.TYPE.JSON);
                page.setJsonData(jsonData);
            } catch (UnsupportedEncodingException e) {
                logger.error("{} , while parsing , {}", e.getMessage(), page.getCrawlerRequest().getUrl());
                throw new PageParserException();
            }
        } else {
            parseHtml(page);
        }
    }

    /**
     * the url include the html
     * this will parse the html data to the page object
     *
     * @param page
     * @throws PageParserException
     */
    private void parseHtml(Page page) throws PageParserException {
        try {
            String rawText = new String(page.getContentData(), page.getContentCharset());
            HtmlData htmlData = new HtmlData(NetUtils.canonicalizeHrefs(rawText, page.getCrawlerRequest().getUrl()));
            page.setHtmlData(htmlData);
            page.setType(Page.TYPE.HTML);
            //not bigger than maxdepth
            int maxDepth = getCrawlerConfig().getMaxDepth();
            if (maxDepth == -1 || maxDepth > page.getCrawlerRequest().getDepth()) {
                List<String> links = page.getHtmlData().getAllLinks(crawlerConfig.isIncludeBinaryContent());
                page.setFollowRequest(getFollowRequests(page , links));
            }
        } catch (UnsupportedEncodingException e) {
            logger.error("{}, while parsing: {}", e.getMessage(), page.getCrawlerRequest().getUrl());
            throw new PageParserException();
        }
    }

    /**
     *
     * @param page
     * @param urls
     * @return
     */
    private List<CrawlerRequest> getFollowRequests(Page page, List<String> urls) {
        List<CrawlerRequest> crawlerRequestList = new ArrayList<>();
        if (CollectionUtils.isNotEmpty(urls)) {
            for (String url : urls) {
                try {
                    CrawlerRequest request = new CrawlerRequest(NetUtils.canonicalizeURL(url,
                            page.getCrawlerRequest().getUrl()));
                    request.setParentUrl(page.getCrawlerRequest().getUrl());
                    request.setDepth(page.getCrawlerRequest().getDepth() + 1);
                    crawlerRequestList.add(request);
                } catch (Exception e) {
                    logger.error("error occurred while construct request url : {}", url);
                }
            }
        }
        return crawlerRequestList;
    }

    public static String getBinaryExtension(String url) {
        int start = url.lastIndexOf('.');
        int end = url.lastIndexOf('?');
        if (end == -1) {
            return url.substring(start);
        } else {
            return url.substring(start, end);
        }
    }

    public static boolean hasBinaryContent(String contentType) {
        String typeStr = (contentType != null) ? contentType.toLowerCase() : "";
        return typeStr.contains("image") || typeStr.contains("audio") ||
                typeStr.contains("video") || typeStr.contains("application") ||
                typeStr.contains("pdf");
    }

    public static boolean hasPlainTextContent(String contentType) {
        String typeStr = (contentType != null) ? contentType.toLowerCase() : "";
        return typeStr.contains("text") && !typeStr.contains("html");
    }

    public static boolean hasJsonTextContent(String contentType) {
        String typeStr = (contentType != null) ? contentType.toLowerCase() : "";
        return typeStr.contains("json");
    }

}
src/main/java/com/doubleview/fastcrawler/dispatcher/SimpleDispatcher.java
@ThreadSafe
public class SimpleDispatcher extends AbstractPageDispather {

    private BlockingQueue<CrawlerRequest> queue = new LinkedBlockingQueue<>();

    @Override
    public void dispatch(CrawlerRequest request) {
        queue.add(request);
    }

    @Override
    public synchronized CrawlerRequest poll() {
        return queue.poll();
    }

    @Override
    public int getLeftRequestsCount() {
        return queue.size();
    }

    @Override
    public int getTotalRequestsCount() {
        return getDuplicateStrategy().getSize();
    }
}
src/main/java/com/doubleview/fastcrawler/exceptions/ContentFetchException.java
public class ContentFetchException extends Exception{


    public ContentFetchException() {
        super();
    }

    public ContentFetchException(Throwable e) {
        super(e);
    }

    public ContentFetchException(String message) {
        super(message);
    }

    public ContentFetchException(String message , Throwable cause) {
        super(message , cause);
    }
}
src/main/java/com/doubleview/fastcrawler/exceptions/PageParserException.java
public class PageParserException extends Exception{


    public PageParserException() {
        super();
    }

    public PageParserException(Throwable e) {
        super(e);
    }

    public PageParserException(String message) {
        super(message);
    }

    public PageParserException(String message , Throwable cause) {
        super(message , cause);
    }
}
src/main/java/com/doubleview/fastcrawler/fetcher/FetchResult.java
public class FetchResult {

    protected static final Logger logger = LoggerFactory.getLogger(FetchResult.class);

    protected int statusCode;

    protected HttpEntity entity;

    protected Header[] responseHeaders = null;

    protected  String fetchUrl = null;

    protected  String moveToUrl = null;

    public int getStatusCode() {
        return statusCode;
    }

    public void setStatusCode(int statusCode) {
        this.statusCode = statusCode;
    }

    public HttpEntity getEntity() {
        return entity;
    }

    public void setEntity(HttpEntity entity) {
        this.entity = entity;
    }

    public Header[] getResponseHeaders() {
        return responseHeaders;
    }

    public void setResponseHeaders(Header[] responseHeaders) {
        this.responseHeaders = responseHeaders;
    }

    public String getFetchUrl() {
        return fetchUrl;
    }

    public void setFetchUrl(String fetchUrl) {
        this.fetchUrl = fetchUrl;
    }

    public String getMoveToUrl() {
        return moveToUrl;
    }

    public void setMoveToUrl(String moveToUrl) {
        this.moveToUrl = moveToUrl;
    }

    public void discardContent(){
        try {
            if (entity != null) {
                EntityUtils.consume(entity);
            }
        }catch (Exception e){
            logger.warn("Unexpected error occurred while trying to discard content", e);
        }
    }
}
src/main/java/com/doubleview/fastcrawler/dispatcher/PageDispatcher.java
public interface PageDispatcher {

    /**
     * add a url
     * @param request
     */
     void push(CrawlerRequest request);

    /**
     * get an url to crawler
     * @return the url to crawl
     */
     CrawlerRequest poll();

    /**
     *
     * @return
     */
    int getLeftRequestsCount();

    /**
     *
     * @return
     */
    int getTotalRequestsCount();
}
src/main/java/com/doubleview/fastcrawler/exceptions/NotAllowBinaryDataException.java
public class NotAllowBinaryDataException extends Exception{

    public NotAllowBinaryDataException() {
        super();
    }

    public NotAllowBinaryDataException(Throwable e) {
        super(e);
    }

    public NotAllowBinaryDataException(String message) {
        super(message);
    }

    public NotAllowBinaryDataException(String message , Throwable cause) {
        super(message , cause);
    }

}


Question: package com.doubleview.fastcrawler;
import com.doubleview.fastcrawler.dispatcher.PageDispatcher;
import com.doubleview.fastcrawler.dispatcher.SimpleDispatcher;
import com.doubleview.fastcrawler.exceptions.ContentFetchException;
import com.doubleview.fastcrawler.exceptions.NotAllowBinaryDataException;
import com.doubleview.fastcrawler.exceptions.PageParserException;
import com.doubleview.fastcrawler.exceptions.PageSizeOverException;
import com.doubleview.fastcrawler.fetcher.FetchResult;
import com.doubleview.fastcrawler.fetcher.PageFetcher;
import com.doubleview.fastcrawler.handler.*;
import com.doubleview.fastcrawler.parser.BinaryData;
import com.doubleview.fastcrawler.parser.PageParser;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;




/**
 * The crawler with multi threads  manages a crawler session
 *
 * @author doubleview
 */
public class Crawler extends CrawlerConfiguable {

    protected static final Logger logger = LoggerFactory.getLogger(Crawler.class);

    private final AtomicLong handledPageCount = new AtomicLong(0);

    protected CrawlerConfig crawlerConfig;

    protected PageFetcher pageFetcher;

    protected PageDispatcher pageDispatcher;

    protected PageHandler pageHandler;

Answer:     protected PageParser pageParser;