Source:
protected void initializeFromReader(BufferedReader reader) throws IOException { String read; RobotsDirectives current = null; boolean hasDirectivesYet = false; while (reader != null) { do { read = reader.readLine(); } while ((read != null) && ((read = read.trim()).startsWith("#") || read.length() == 0)); if (read == null) { reader.close(); reader = null; } else { read = read.replaceAll("<[^>]+>",""); int commentIndex = read.indexOf("#"); if (commentIndex > -1) { read = read.substring(0, commentIndex); } read = read.trim(); if (read.matches("(?i)^User-agent:.*")) { String ua = read.substring(11).trim().toLowerCase(); if (current == null || hasDirectivesYet ) { current = new RobotsDirectives(); hasDirectivesYet = false; } if (ua.equals("*")) { wildcardDirectives = current; } else { namedUserAgents.addLast(ua); agentsToDirectives.put(ua, current); } continue; } if (read.matches("(?i)Disallow:.*")) { if (current == null) { hasErrors = true; continue; } String path = read.substring(9).trim(); if(path.endsWith("*")) { path = path.substring(0,path.length()-1); } current.addDisallow(path); hasDirectivesYet = true; continue; } if (read.matches("(?i)Crawl-delay:.*")) { if (current == null) { hasErrors = true; continue; } hasDirectivesYet = true; String val = read.substring(12).trim(); val = val.split("[^\\d\\.]+")[0]; try { current.setCrawlDelay(Float.parseFloat(val)); } catch (NumberFormatException nfe) { } continue; } if (read.matches("(?i)Allow:.*")) { if (current == null) { hasErrors = true; continue; } String path = read.substring(6).trim(); if(path.endsWith("*")) { path = path.substring(0,path.length()-1); } current.addAllow(path); hasDirectivesYet = true; } } } }

Target:
protected void initializeFromReader(BufferedReader reader) throws IOException { String read; RobotsDirectives current = null; boolean hasDirectivesYet = false; while (reader != null) { do { read = reader.readLine(); } while ((read != null) && ((read = read.trim()).startsWith("#") || read.length() == 0)); if (read == null) { reader.close(); reader = null; } else { read = read.replaceAll("<[^>]+>",""); int commentIndex = read.indexOf("#"); if (commentIndex > -1) { read = read.substring(0, commentIndex); } read = read.trim(); if (read.matches("(?i)^User-agent:.*")) { String ua = read.substring(11).trim().toLowerCase(); if (current == null || hasDirectivesYet ) { current = new RobotsDirectives(); hasDirectivesYet = false; } if (ua.equals("*")) { wildcardDirectives = current; } else { namedUserAgents.addLast(ua); agentsToDirectives.put(ua, current); } continue; } if (read.matches("(?i)Disallow:.*")) { if (current == null) { hasErrors = true; continue; } String path = read.substring(9).trim(); if(path.endsWith("*")) { path = path.substring(0,path.length()-1); } current.addDisallow(path); hasDirectivesYet = true; continue; } if (read.matches("(?i)Crawl-delay:.*")) { if (current == null) { hasErrors = true; continue; } hasDirectivesYet = true; String val = read.substring(12).trim(); val = val.split("[^\\d\\.]+")[0]; try { current.setCrawlDelay(Float.parseFloat(val)); } catch (NumberFormatException nfe) { } continue; } if (read.matches("(?i)Allow:.*")) { if (current == null) { hasErrors = true; continue; } String path = read.substring(6).trim(); if(path.endsWith("*")) { path = path.substring(0,path.length()-1); } current.addAllow(path); hasDirectivesYet = true; continue; } } } }