Source:
public static HostDirectives parse(String content, RobotstxtConfig config) { String myUserAgent = config.getUserAgentName(); HostDirectives directives = new HostDirectives(); HostDirectives specificDirectives = new HostDirectives(); boolean inMatchingUserAgent = false; boolean inExactUserAgent = false; StringTokenizer st = new StringTokenizer(content, "\n\r"); while (st.hasMoreTokens()) { String line = st.nextToken(); int commentIndex = line.indexOf('#'); if (commentIndex > -1) { line = line.substring(0, commentIndex); } line = line.replaceAll("<[^>]+>", ""); line = line.trim(); if (line.isEmpty()) { continue; } if (line.matches(PATTERNS_USERAGENT)) { String ua = line.substring(PATTERNS_USERAGENT_LENGTH).trim().toLowerCase(); if (ua.equals("*") || ua.contains(myUserAgent)) { inMatchingUserAgent = true; } else { inMatchingUserAgent = false; } inExactUserAgent = ua.equals(myUserAgent); } else if (line.matches(PATTERNS_DISALLOW)) { if (!inMatchingUserAgent) { continue; } String path = line.substring(PATTERNS_DISALLOW_LENGTH).trim(); if (path.endsWith("*")) { path = path.substring(0, path.length() - 1); } path = path.trim(); if (!path.isEmpty()) { if (inExactUserAgent) { specificDirectives.addDisallow(path); } else { directives.addDisallow(path); } } } else if (line.matches(PATTERNS_ALLOW)) { if (!inMatchingUserAgent && !config.getIgnoreUserAgentInAllow()) { continue; } String path = line.substring(PATTERNS_ALLOW_LENGTH).trim(); if (path.endsWith("*")) { path = path.substring(0, path.length() - 1); } path = path.trim(); if (inExactUserAgent) { specificDirectives.addAllow(path); } else { directives.addAllow(path); } } } HostDirectives result = {directives, specificDirectives}; return result; }

Target:
public static HostDirectives parse(String content, RobotstxtConfig config) { String myUserAgent = config.getUserAgentName(); HostDirectives directives = new HostDirectives(); HostDirectives specificDirectives = new HostDirectives(); boolean inMatchingUserAgent = false; boolean inExactUserAgent = false; StringTokenizer st = new StringTokenizer(content, "\n\r"); while (st.hasMoreTokens()) { String line = st.nextToken(); int commentIndex = line.indexOf('#'); if (commentIndex > -1) { line = line.substring(0, commentIndex); } line = line.replaceAll("<[^>]+>", ""); line = line.trim(); if (line.isEmpty()) { continue; } if (line.matches(PATTERNS_USERAGENT)) { String ua = line.substring(PATTERNS_USERAGENT_LENGTH).trim().toLowerCase(); inMatchingUserAgent = "*".equals(ua) || ua.contains(myUserAgent); inExactUserAgent = ua.equals(myUserAgent); } else if (line.matches(PATTERNS_DISALLOW)) { if (!inMatchingUserAgent) { continue; } String path = line.substring(PATTERNS_DISALLOW_LENGTH).trim(); if (path.endsWith("*")) { path = path.substring(0, path.length() - 1); } path = path.trim(); if (!path.isEmpty()) { if (inExactUserAgent) { specificDirectives.addDisallow(path); } else { directives.addDisallow(path); } } } else if (line.matches(PATTERNS_ALLOW)) { if (!inMatchingUserAgent && !config.getIgnoreUserAgentInAllow()) { continue; } String path = line.substring(PATTERNS_ALLOW_LENGTH).trim(); if (path.endsWith("*")) { path = path.substring(0, path.length() - 1); } path = path.trim(); if (inExactUserAgent) { specificDirectives.addAllow(path); } else { directives.addAllow(path); } } } HostDirectives result = {directives, specificDirectives}; return result; }