public String getURL(String address){
        String URL = address;

        if(!URL.startsWith("http://")){
           URL = "http://" + address;
        }
        String domain = URL;
        if(!URL.endsWith("/") && !URL.endsWith(".html") && !URL.endsWith(".shtml") && !URL.endsWith(".htm") && !URL.endsWith(".aspx") && !URL.endsWith(".php")){
           URL = URL + "/";
        }
        if(domain.endsWith("/")){
           domain = domain.substring(0, domain.length()-1);
        }
        //directories.add(URL);
        return processRobots(URL + "robots.txt", domain);
    }