public synchronized boolean checkRobot(URL url) {
		boolean result = true;
		String host = url.getHost();
		ArrayList<String> disallowList = disallowListMap.get(host);
		if(disallowList == null) {
			disallowList = new ArrayList<String>();
			try {
				URL robotsFileUrl = new URL("http://" + host + "/robots.txt");
				BufferedReader robotReader = new BufferedReader(new InputStreamReader(robotsFileUrl.openStream()));
				String line;
			      while ((line = robotReader.readLine()) != null) {  //does not check for user-agent.
			        if (line.startsWith("Disallow:")) {
			          String disallowPath = line.substring("Disallow:".length());
			          int commentIndex = disallowPath.indexOf("#");
			          if (commentIndex != -1) {
			            disallowPath =
			              disallowPath.substring(0, commentIndex);
			          }
			          disallowPath = disallowPath.trim();
			          if(!disallowList.contains(disallowPath)) disallowList.add(disallowPath);
			        }
			      }
			      disallowListMap.put(host, disallowList);
			}catch(Exception e) {
				//robot.txt not found, every URL is allowed;
			}
		}
		for (String string : disallowList) {
			if(url.getFile().startsWith(string)) result = false;
		}
		return result;
	}