public boolean robotSafe(URL url) {
 String strHost = url.toString();

	// form URL of the robots.txt file
 String strRobot = strHost + "/robots.txt";
 URL urlRobot;
 try { urlRobot = new URL(strRobot);
	} catch (MalformedURLException e) {
	    // something weird is happening, so don't trust it
	    return false;
	}

 if (DEBUG) System.out.println("Checking robot protocol " + 
                                urlRobot.toString());
 String strCommands;
 try {
    InputStream urlRobotStream = urlRobot.openStream();

	    // read in entire file
    byte b[] = new byte[1000];
    int numRead = urlRobotStream.read(b);
    strCommands = new String(b, 0, numRead);
    while (numRead != -1) {
       numRead = urlRobotStream.read(b);
       if (numRead != -1) {
          String newCommands = new String(b, 0, numRead);
	         strCommands += newCommands;
		}
	    }
    urlRobotStream.close();
	} catch (IOException e) {
	    // if there is no robots.txt file, it is OK to search
	    return true;
	}
     if (DEBUG) System.out.println(strCommands);

	// assume that this robots.txt refers to us and 
	// search for "Disallow:" commands.
	String strURL = url.getFile();
	int index = 0;
	while ((index = strCommands.indexOf(index)) != -1) {
	    index += DISALLOW.length();
	    String strPath = strCommands.substring(index);
	    StringTokenizer st = new StringTokenizer(strPath);

	    if (!st.hasMoreTokens())
		break;
	    
	    String strBadPath = st.nextToken();

	    // if the URL starts with a disallowed path, it is not safe
	    if (strURL.indexOf(strBadPath) == 0)
		return false;
	}

	return true;
 }