@Override
	public void run() {
        if(crawlURL == null || crawlURL.toString().equals("")) {
            return;
        }
        String path = crawlURL.getPath().equals("") ? "/" : crawlURL.getPath();
        String host = crawlURL.getHost();
        String htmlDoc = request(host, path);
        crawlerManager.finishCrawler(this);
        if(htmlDoc == null) {
            LOGGER.severe("Could not get URL " + crawlURL.toString());
            return;
        }

        Document doc = Jsoup.parse(htmlDoc);

        //Extract all the a links
        Elements links = doc.select("a");

        //go through all the links and extract the href
        for(Element link: links) {
            String l = link.attr("href");
            String html = link.html();

            //check from user implemented function if shouldCrawl the URL
            if(shouldCrawl(crawlURL, l, html)) {
                List<String> list = Util.getLinks(l);
                if(list !=null) {
                    for(String i: list) {
                        crawlerManager.queue(i);
                    }
                }

            }
        }

	}