private static void init() {
        LOG.info(String.format(".......... init to spider job ........."));
        //shutdown保存已抓取过的link
        Runtime.getRuntime().addShutdownHook(new CleanWorkThread());
        //已爬取过的新闻链接队列
        hasExtractLinks = Queues.newLinkedBlockingQueue(HAS_EXTRACT_LINKS_QUEUE_CAPACITY);
        //load已经爬取过得链接
        BufferedReader br = null;
        try {
            br = new BufferedReader(new FileReader(new File(CACHE_LINKS_FILE)));
            String link;
            while((link = br.readLine()) != null) {
                hasExtractLinks.offer(link);
            }
            LOG.info("init has extract size is " + hasExtractLinks.size());
        } catch (FileNotFoundException e) {
            LOG.warn("file not found, maybe this is first start. ", e);
        } catch (IOException e) {
            LOG.error("init error when load crawled links. ", e);
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    LOG.error("close buffered reader error.", e);
                }
            }
        }
        //当hasExtractLinks队列满则pop
        Thread popQueueAfterFullThread = new Thread(new Runnable() {
            @Override
            public void run() {
                boolean flag = true;
                while (flag) {
                    try {
                        if (hasExtractLinks.size() == HAS_EXTRACT_LINKS_QUEUE_CAPACITY) {
                            String link = hasExtractLinks.poll();
                            LOG.info(String.format("pop link(%s) which has extracted.", link));
                        }
                    } catch (Exception e) {
                        LOG.error("pop link error.", e);
                        flag = false;
                    }

                }
            }
        }, "popQueueThread");
        popQueueAfterFullThread.setDaemon(true);
        popQueueAfterFullThread.start();

    }