public void parseRobots(InputStream in) {
        String line;
        String[] rule;

        BufferedReader br = new BufferedReader(new InputStreamReader(in));
        try {
            while((line = br.readLine()) != null) {
                if(this.isValidDirectiveLine(line)) {
                    // split our line on the colon
                    rule = line.split(":");
                    // strip the space
                    rule[0] = rule[0].replaceAll("\\s","").toLowerCase();
                    try {
                        rule[1] = rule[1].replaceAll("\\s","").toLowerCase();
                    } catch(ArrayIndexOutOfBoundsException e) {
                        // skip this rule, it's not correct
                        continue;
                    }
                    // parse the directives
                    switch(RobotRules.translateRule(rule[0])) {
                        case 1:
                            this.parseDisallow(rule[1]);
                            break;
                        case 2:
                            this.parseAllow(rule[1]);
                            break;
                        case 3:
                            this.parseUserAgent(rule[1]);
                            break;
                        case 4:
                            this.crawlSpeed = Integer.parseInt(rule[1]);
                            break;
                        case 5:
                            break;
                    }
                }
            }
        } catch(IOException e) {
            Log.d(TAG, String.format("IOException: %s", e.getMessage()), 3);
        }
    }