Repository: crawler-commons/crawler-commons (java)

Issue:
[Robots.txt] Empty disallow statement not to clear other rules
In the current implementation of SimpleRobotRulesParser ([line 819](https://github.com/crawler-commons/crawler-commons/blob/0656e9c56137bc66bdd47238f2b6b45c36c30280/src/main/java/crawlercommons/robots/SimpleRobotRulesParser.java#L819)), an empty disallow statement (`Disallow: ` ) clears all rules read so far. It should not.

- the [initial robots.txt RFC draft](https://www.robotstxt.org/robotstxt.html) didn't include the `Allow:` statement and the empty disallow statement was required to express that all paths are allowed.

- the [formal syntax of RFC 9309](https://www.rfc-editor.org/rfc/rfc9309.html#section-2.2) allows empty patterns in allow/disallow statements. However, because longer rules take precedence, empty patterns would be matched last.

The issue was observed with [this robots.txt](https://agriexotic.com/robots.txt). Here the empty disallow statement in the second rule group, clears the already parsed rules. Both groups are addressed to the wildcard user-agent and therefore merged.

Pull request:
[Robots.txt] Empty disallow statement not to clear other rules, fixes #422
Also verified that Google's reference parser behaves the same:

```sh
$> cat empty-disallow-robots.txt 
User-agent: *
Disallow: /disallowed/
Allow: /allowed/
Disallow: 

$> for path in "" / /allowed/page.html /disallowed/page.html; do
    robots empty-disallow-robots.txt any https://www.example.com$path;
done
user-agent 'any' with URI 'https://www.example.com': ALLOWED
user-agent 'any' with URI 'https://www.example.com/': ALLOWED
user-agent 'any' with URI 'https://www.example.com/allowed/page.html': ALLOWED
user-agent 'any' with URI 'https://www.example.com/disallowed/page.html': DISALLOWED
```

Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index cf3720f..93aaeef 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,9 @@
 Crawler-Commons Change Log
 
 Current Development 1.4-SNAPSHOT (yyyy-mm-dd)
+- [Robots.txt] Document effect of rules merging in combination with multiple agent names (sebastian-nagel, Richard Zowalla) #423, #426
+- [Robots.txt] Pass empty collection of agent names to select rules for any robot (wildcard user-agent name) (sebastian-nagel, Richard Zowalla) #427
+- [Robots.txt] Rename default user-agent / robot name in unit tests (sebastian-nagel, Richard Zowalla) #429
 - [Robots.txt] Add units test based on examples in RFC 9309 (sebastian-nagel, Richard Zowalla) #420
 - [BasicNormalizer] Query parameters normalization in BasicURLNormalizer (aecio, sebastian-nagel, Richard Zowalla) #308, #421
 - [Robots.txt] Deduplicate robots rules before matching (sebastian-nagel, jnioche) #416
diff --git a/src/main/java/crawlercommons/robots/BaseRobotRules.java b/src/main/java/crawlercommons/robots/BaseRobotRules.java
index 00fdb8d..8d6c54b 100644
--- a/src/main/java/crawlercommons/robots/BaseRobotRules.java
+++ b/src/main/java/crawlercommons/robots/BaseRobotRules.java
@@ -22,8 +22,11 @@ import java.util.LinkedHashSet;
 import java.util.List;
 
 /**
- * Result from parsing a single robots.txt file - which means we get a set of
- * rules, and a crawl-delay.
+ * Result from parsing a single robots.txt file – a set of allow/disallow rules
+ * to check whether a given URL is allowed, and optionally a <a href=
+ * "https://en.wikipedia.org/wiki/Robots.txt#Crawl-delay_directive"
+ * >Crawl-delay</a> and <a
+ * href="https://www.sitemaps.org/protocol.html#submit_robots">Sitemap</a> URLs.
  */
 @SuppressWarnings("serial")
 public abstract class BaseRobotRules implements Serializable {
@@ -52,10 +55,17 @@ public abstract class BaseRobotRules implements Serializable {
         _crawlDelay = crawlDelay;
     }
 
+    /**
+     * @return whether to defer visits to the server
+     */
     public boolean isDeferVisits() {
         return _deferVisits;
     }
 
+    /**
+     * Indicate to defer visits to the server, e.g. to wait until the robots.txt
+     * becomes available.
+     */
     public void setDeferVisits(boolean deferVisits) {
         _deferVisits = deferVisits;
     }
@@ -103,7 +113,7 @@ public abstract class BaseRobotRules implements Serializable {
 
     /**
      * Returns a string with the crawl delay as well as a list of sitemaps if
-     * they exist (and aren't more than 10)
+     * they exist (and aren't more than 10).
      */
     @Override
     public String toString() {
diff --git a/src/main/java/crawlercommons/robots/BaseRobotsParser.java b/src/main/java/crawlercommons/robots/BaseRobotsParser.java
index 696f39b..737cc8a 100644
--- a/src/main/java/crawlercommons/robots/BaseRobotsParser.java
+++ b/src/main/java/crawlercommons/robots/BaseRobotsParser.java
@@ -19,15 +19,16 @@ package crawlercommons.robots;
 import java.io.Serializable;
 import java.util.Collection;
 
+/** Robots.txt parser definition. */
 @SuppressWarnings("serial")
 public abstract class BaseRobotsParser implements Serializable {
 
     /**
      * Parse the robots.txt file in <i>content</i>, and return rules appropriate
      * for processing paths by <i>userAgent</i>. Note that multiple agent names
-     * may be provided as comma-separated values; the order of these shouldn't
-     * matter, as the file is parsed in order, and each agent name found in the
-     * file will be compared to every agent name found in robotNames.
+     * may be provided as comma-separated values. How agent names are matched
+     * against user-agent lines in the robots.txt depends on the implementing
+     * class.
      * 
      * Also note that names are lower-cased before comparison, and that any
      * robot name you pass shouldn't contain commas or spaces; if the name has
@@ -76,7 +77,7 @@ public abstract class BaseRobotsParser implements Serializable {
      * @param content
      *            raw bytes from the site's robots.txt file
      * @param contentType
-     *            HTTP response header (mime-type)
+     *            content type (MIME type) from HTTP response header
      * @param robotNames
      *            name(s) of crawler, used to select rules from the robots.txt
      *            file by matching the names against the user-agent lines in the
diff --git a/src/main/java/crawlercommons/robots/SimpleRobotRules.java b/src/main/java/crawlercommons/robots/SimpleRobotRules.java
index 8204912..949bf95 100644
--- a/src/main/java/crawlercommons/robots/SimpleRobotRules.java
+++ b/src/main/java/crawlercommons/robots/SimpleRobotRules.java
@@ -25,22 +25,24 @@ import java.util.stream.Collectors;
 import crawlercommons.filters.basic.BasicURLNormalizer;
 
 /**
- * Result from parsing a single robots.txt file - set of rules, and optionally a
- * <a href=
- * "https://en.wikipedia.org/wiki/Robots.txt#Crawl-delay_directive">crawl
- * -delay</a> and <a
- * href="https://www.sitemaps.org/protocol.html#submit_robots">sitemap</a> URLs.
- * The <a href="https://www.rfc-editor.org/rfc/rfc9309.html">Robots Exclusion
- * Protocol RFC 9309</a> is fully supported. This includes <a href=
+ * {@inheritDoc}
+ * 
+ * <p>
+ * Allow/disallow rules are matched following the <a
+ * href="https://www.rfc-editor.org/rfc/rfc9309.html">Robots Exclusion Protocol
+ * RFC 9309</a>. This includes <a href=
  * "https://developers.google.com/search/reference/robots_txt">Google's
  * robots.txt extensions</a> to the <a
- * href="http://www.robotstxt.org/robotstxt.html">original RFC draft</a> are
- * covered: the <code>Allow</code> directive, <code>$</code>/<code>*</code>
- * special characters and precedence of more specific patterns
+ * href="http://www.robotstxt.org/robotstxt.html">original RFC draft</a>: the
+ * <code>Allow</code> directive, <code>$</code>/<code>*</code> special
+ * characters and precedence of longer (more specific) patterns.
+ * </p>
  * 
+ * <p>
  * See also: <a
  * href="https://en.wikipedia.org/wiki/Robots_exclusion_standard">Robots
  * Exclusion on Wikipedia</a>
+ * </p>
  */
 
 @SuppressWarnings("serial")
@@ -58,6 +60,10 @@ public class SimpleRobotRules extends BaseRobotRules {
         String _prefix;
         boolean _allow;
 
+        /**
+         * A allow/disallow rule: a path prefix or pattern and whether it is
+         * allowed or disallowed.
+         */
         public RobotRule(String prefix, boolean allow) {
             _prefix = prefix;
             _allow = allow;
@@ -88,11 +94,6 @@ public class SimpleRobotRules extends BaseRobotRules {
             }
         }
 
-        /*
-         * (non-Javadoc)
-         * 
-         * @see java.lang.Object#hashCode()
-         */
         @Override
         public int hashCode() {
             final int prime = 31;
@@ -102,11 +103,6 @@ public class SimpleRobotRules extends BaseRobotRules {
             return result;
         }
 
-        /*
-         * (non-Javadoc)
-         * 
-         * @see java.lang.Object#equals(java.lang.Object)
-         */
         @Override
         public boolean equals(Object obj) {
             if (this == obj)
@@ -163,6 +159,9 @@ public class SimpleRobotRules extends BaseRobotRules {
         _rules.add(new RobotRule(prefix, allow));
     }
 
+    /**
+     * @return the list of allow/disallow rules
+     */
     public List<RobotRule> getRobotRules() {
         return this._rules;
     }
@@ -377,6 +376,11 @@ public class SimpleRobotRules extends BaseRobotRules {
     /**
      * Is our ruleset set up to allow all access?
      * 
+     * <p>
+     * Note: This is decided only based on the {@link RobotRulesMode} without
+     * inspecting the set of allow/disallow rules.
+     * </p>
+     * 
      * @return true if all URLs are allowed.
      */
     @Override
@@ -387,6 +391,11 @@ public class SimpleRobotRules extends BaseRobotRules {
     /**
      * Is our ruleset set up to disallow all access?
      * 
+     * <p>
+     * Note: This is decided only based on the {@link RobotRulesMode} without
+     * inspecting the set of allow/disallow rules.
+     * </p>
+     * 
      * @return true if no URLs are allowed.
      */
     @Override
@@ -394,11 +403,6 @@ public class SimpleRobotRules extends BaseRobotRules {
         return _mode == RobotRulesMode.ALLOW_NONE;
     }
 
-    /*
-     * (non-Javadoc)
-     * 
-     * @see java.lang.Object#hashCode()
-     */
     @Override
     public int hashCode() {
         final int prime = 31;
@@ -408,11 +412,6 @@ public class SimpleRobotRules extends BaseRobotRules {
         return result;
     }
 
-    /*
-     * (non-Javadoc)
-     * 
-     * @see java.lang.Object#equals(java.lang.Object)
-     */
     @Override
     public boolean equals(Object obj) {
         if (this == obj)
@@ -433,9 +432,10 @@ public class SimpleRobotRules extends BaseRobotRules {
     }
 
     /*
-     * (non-Javadoc)
+     * {@inheritDoc}
      * 
-     * @see java.lang.Object#equals(java.lang.Object)
+     * In addition, the number of allow/disallow rules and the most specific
+     * rules by pattern length are shown (ten rules, at maximum).
      */
     @Override
     public String toString() {
diff --git a/src/main/java/crawlercommons/robots/SimpleRobotRulesParser.java b/src/main/java/crawlercommons/robots/SimpleRobotRulesParser.java
index bf7bf36..f7443f2 100644
--- a/src/main/java/crawlercommons/robots/SimpleRobotRulesParser.java
+++ b/src/main/java/crawlercommons/robots/SimpleRobotRulesParser.java
@@ -41,37 +41,62 @@ import org.slf4j.LoggerFactory;
 import crawlercommons.robots.SimpleRobotRules.RobotRulesMode;
 
 /**
+ * Robots.txt parser following RFC 9309, supporting the Sitemap and Crawl-delay
+ * extensions.
+ * 
  * <p>
  * This implementation of {@link BaseRobotsParser} retrieves a set of
  * {@link SimpleRobotRules rules} for an agent with the given name from the
  * <code>robots.txt</code> file of a given domain. The implementation follows
  * <a href="https://www.rfc-editor.org/rfc/rfc9309.html#name-access-method">RFC
- * 9309</a>.
- * </p>
+ * 9309</a>. The following robots.txt extensions are supported:</p>
+ * <ul>
+ * <li>the <a href=
+ * "https://en.wikipedia.org/wiki/Robots.txt#Crawl-delay_directive">Crawl-delay</a>
+ * directive</li>
+ * <li>the
+ * <a href="https://www.sitemaps.org/protocol.html#submit_robots">Sitemap
+ * directive</a></li>
+ * </ul>
  * 
  * <p>
  * The class fulfills two tasks. The first one is the parsing of the
- * <code>robots.txt</code> file done in
- * {@link #parseContent(String, byte[], String, String)}. During the parsing
+ * <code>robots.txt</code> file, done in
+ * {@link #parseContent(String, byte[], String, Collection)}. During the parsing
  * process the parser searches for the provided agent name(s). If the parser
- * finds a matching name, the set of rules for this name is parsed and returned
- * as the result. <b>Note</b> that if more than one agent name is given to the
- * parser, it parses the rules for the first matching agent name inside the file
- * and skips all following user agent groups. It doesn't matter which of the
- * other given agent names would match additional rules inside the file. Thus,
- * if more than one agent name is given to the parser, the result can be
- * influenced by the order of rule sets inside the <code>robots.txt</code> file.
+ * finds a matching name, the set of rules for this name is parsed and added to
+ * the list of rules returned later. If another group of rules is matched in the
+ * robots.txt file, also the rules of this group are added to the returned list
+ * of rules. See
+ * <a href="https://www.rfc-editor.org/rfc/rfc9309.html#section-2.2.1">RFC 9309,
+ * section 2.2.1</a> about the merging of groups of rules.
  * </p>
  * 
  * <p>
- * Note that the parser always parses the entire file, even if a matching agent
- * name group has been found, as it needs to collect all of the sitemap
- * directives.
+ * By default and following
+ * <a href="https://www.rfc-editor.org/rfc/rfc9309.html#section-2.2.1">RFC 9309,
+ * section 2.2.1</a>, the user-agent name (&quot;product token&quot;) is matched
+ * literally but case-insensitive over the full name. See
+ * {@link #setExactUserAgentMatching(boolean)} for details of agent name
+ * matching and a legacy substring prefix matching mode.
  * </p>
  * 
  * <p>
- * If no rule set matches any of the provided agent names, the rule set for the
- * <code>'*'</code> agent is returned. If there is no such rule set inside the
+ * {@link SimpleRobotRulesParser} allows to pass multiple agent names as a
+ * collection. All rule groups matching any of the agent names are followed and
+ * merged into one set of rules. The order of the agent names in the collection
+ * does not affect the selection of rules.
+ * </p>
+ * 
+ * <p>
+ * The parser always parses the entire file to select all matching rule groups,
+ * and also to collect all of the sitemap directives.
+ * </p>
+ * 
+ * <p>
+ * If no rule set matches any of the provided user-agent names, or if an empty
+ * collection of agent names is passed, the rule set for the <code>'*'</code>
+ * agent is returned. If there is no such rule set inside the
  * <code>robots.txt</code> file, a rule set allowing all resource to be crawled
  * is returned.
  * </p>
@@ -96,6 +121,14 @@ import crawlercommons.robots.SimpleRobotRules.RobotRulesMode;
  * or 5xx) the parser assumes a temporary error and a set of rules prohibiting
  * any crawling is returned.
  * </p>
+ * 
+ * <p>
+ * Note that fetching of the robots.txt file is outside the scope of this class.
+ * It must be implemented in the calling code, including the following of
+ * &quot;at least five consecutive redirects&quot; as required by
+ * <a href="https://www.rfc-editor.org/rfc/rfc9309.html#name-redirects">RFC
+ * 9309, section 2.3.1.2</a>.
+ * </p>
  */
 @SuppressWarnings("serial")
 public class SimpleRobotRulesParser extends BaseRobotsParser {
@@ -106,18 +139,26 @@ public class SimpleRobotRulesParser extends BaseRobotsParser {
 
         ALLOW, CRAWL_DELAY, SITEMAP,
 
-        // Russian-specific directive for mirror site?
-        // Used by the zerkalschik robot?
-        // See http://wataro.ur/en/web/robot.html
+        /**
+         * The &quot;Host&quot; directive was used by Yandex to indicate the
+         * main or canonical host of a set of mirrored web sites, see <a href=
+         * "https://web.archive.org/web/20130509230548/http://help.yandex.com/webmaster/?id=1113851#1113856">Yandex'
+         * Using robots.txt (archived version)</a>
+         */
         HOST,
 
         // Google extension
         NO_INDEX,
 
-        // ACAP directives all start with ACAP-
+        /**
+         * <a href=
+         * "https://en.wikipedia.org/wiki/Automated_Content_Access_Protocol">Automated
+         * Content Access Protocol</a> directives all start with
+         * <code>ACAP-</code>.
+         */
         ACAP_(true, false),
 
-        // Extended standard
+        // Extensions to the standard
         REQUEST_RATE, VISIT_TIME, ROBOT_VERSION, COMMENT,
 
         // Line starts with http:, which we treat as sitemap directive.
@@ -282,13 +323,31 @@ public class SimpleRobotRulesParser extends BaseRobotsParser {
         DIRECTIVE_PREFIX.put("useragent", RobotDirective.USER_AGENT);
         DIRECTIVE_PREFIX.put("useg-agent", RobotDirective.USER_AGENT);
         DIRECTIVE_PREFIX.put("ser-agent", RobotDirective.USER_AGENT);
+        DIRECTIVE_PREFIX.put("users-agent", RobotDirective.USER_AGENT);
+        DIRECTIVE_PREFIX.put("user agent", RobotDirective.USER_AGENT);
+        DIRECTIVE_PREFIX.put("user-agnet", RobotDirective.USER_AGENT);
+        DIRECTIVE_PREFIX.put("user-agents", RobotDirective.USER_AGENT);
 
         DIRECTIVE_PREFIX.put("desallow", RobotDirective.DISALLOW);
+        DIRECTIVE_PREFIX.put("dissallow", RobotDirective.DISALLOW);
         DIRECTIVE_PREFIX.put("dissalow", RobotDirective.DISALLOW);
+        DIRECTIVE_PREFIX.put("disalow", RobotDirective.DISALLOW);
         DIRECTIVE_PREFIX.put("dssalow", RobotDirective.DISALLOW);
         DIRECTIVE_PREFIX.put("dsallow", RobotDirective.DISALLOW);
+        DIRECTIVE_PREFIX.put("diasllow", RobotDirective.DISALLOW);
+        DIRECTIVE_PREFIX.put("disallaw", RobotDirective.DISALLOW);
+        DIRECTIVE_PREFIX.put("diallow", RobotDirective.DISALLOW);
+        DIRECTIVE_PREFIX.put("disallows", RobotDirective.DISALLOW);
+        DIRECTIVE_PREFIX.put("disllow", RobotDirective.DISALLOW);
 
         DIRECTIVE_PREFIX.put("crawl delay", RobotDirective.CRAWL_DELAY);
+        DIRECTIVE_PREFIX.put("clawl-delay", RobotDirective.CRAWL_DELAY);
+        DIRECTIVE_PREFIX.put("craw-delay", RobotDirective.CRAWL_DELAY);
+        DIRECTIVE_PREFIX.put("crawl-deley", RobotDirective.CRAWL_DELAY);
+
+        DIRECTIVE_PREFIX.put("sitemaps", RobotDirective.SITEMAP);
+
+        DIRECTIVE_PREFIX.put("https", RobotDirective.HTTP);
     }
 
     // separator is either one or more spaces/tabs, or a colon
@@ -402,6 +461,23 @@ public class SimpleRobotRulesParser extends BaseRobotsParser {
         return userAgent != null && USER_AGENT_PRODUCT_TOKEN_MATCHER.matcher(userAgent).matches();
     }
 
+    /**
+     * {@inheritDoc}
+     * 
+     * <p>
+     * Set rules for response status codes following <a href=
+     * "https://www.rfc-editor.org/rfc/rfc9309.html#name-access-results">RFC
+     * 9309, section 2.3.1</a></p>:
+     * <ul>
+     * <li>Success (HTTP 200-299): throws {@link IllegalStateException} because
+     * the response content needs to be parsed</li>
+     * <li>"Unavailable" Status (HTTP 400-499): allow all</li>
+     * <li>"Unreachable" Status (HTTP 500-599): disallow all</li>
+     * <li>every other HTTP status code is treated as "allow all", but further
+     * visits on the server are deferred (see
+     * {@link SimpleRobotRules#setDeferVisits(boolean)})</li>
+     * </ul>
+     */
     @Override
     public SimpleRobotRules failedFetch(int httpStatusCode) {
         SimpleRobotRules result;
@@ -459,25 +535,37 @@ public class SimpleRobotRulesParser extends BaseRobotsParser {
 
     /**
      * Parse the robots.txt file in <i>content</i>, and return rules appropriate
-     * for processing paths by <i>userAgent</i>. Multiple agent names are
-     * provided as collection. See {@link #setExactUserAgentMatching(boolean)}
-     * for details how agent names are matched.
+     * for processing paths by <i>userAgent</i>.
+     * 
+     * <p>
+     * Multiple agent names can be passed as a collection. See
+     * {@link #setExactUserAgentMatching(boolean)} for details how agent names
+     * are matched. If multiple agent names are passed, all matching rule groups
+     * are followed and merged into one set of rules. The order of the agent
+     * names in the collection does not affect the selection of rules.
+     * </p>
+     * 
+     * <p>
+     * If none of the provided agent names is matched, rules addressed to the
+     * wildcard user-agent (<code>*</code>) are selected.
+     * </p>
      * 
      * @param url
-     *            URL that robots.txt content was fetched from. A complete and
-     *            valid URL (e.g., https://example.com/robots.txt) is expected.
-     *            Used to resolve relative sitemap URLs and for
-     *            logging/reporting purposes.
+     *            {@inheritDoc}
      * @param content
-     *            raw bytes from the site's robots.txt file
+     *            {@inheritDoc}
      * @param contentType
-     *            HTTP response header (mime-type)
+     *            {@inheritDoc}
      * @param robotNames
-     *            name(s) of crawler, used to select rules from the robots.txt
-     *            file by matching the names against the user-agent lines in the
-     *            robots.txt file. Robot names should be single token names, w/o
-     *            version or other parts. Names should be lower-case, as the
-     *            user-agent line is also converted to lower-case for matching.
+     *            crawler (user-agent) name(s), used to select rules from the
+     *            robots.txt file by matching the names against the user-agent
+     *            lines in the robots.txt file. Robot names should be single
+     *            token names, without version or other parts. Names must be
+     *            lower-case, as the user-agent line is also converted to
+     *            lower-case for matching. If the collection is empty, the rules
+     *            for the wildcard user-agent (<code>*</code>) are selected. The
+     *            wildcard user-agent name should not be contained in
+     *            <i>robotNames</i>.
      * @return robot rules.
      */
     @Override
@@ -815,8 +903,20 @@ public class SimpleRobotRulesParser extends BaseRobotsParser {
         try {
             path = normalizePathDirective(path);
             if (path.length() == 0) {
-                // Disallow: <nothing> => allow all.
-                state.clearRules();
+                /*
+                 * "Disallow: <nothing>" meant "allow all" in the 1996 REP RFC
+                 * draft because there was no "Allow" directive.
+                 * 
+                 * RFC 9309 allows empty patterns in the formal syntax
+                 * (https://www.rfc-editor.org/rfc/rfc9309.html#section-2.2). No
+                 * specific handling of empty patterns is defined, as shortest
+                 * pattern they are matched with lowest priority.
+                 * 
+                 * Adding an extra rule with an empty pattern would have no
+                 * effect, because an "allow all" with lowest priority means
+                 * "allow everything else" which is the default anyway. So, we
+                 * ignore the empty disallow statement.
+                 */
             } else {
                 state.addRule(path, false);
             }
@@ -847,8 +947,11 @@ public class SimpleRobotRulesParser extends BaseRobotsParser {
         }
 
         if (path.length() == 0) {
-            // Allow: <nothing> => allow all.
-            state.clearRules();
+            /*
+             * Allow: <nothing> => allow all.
+             * 
+             * See handleDisallow(...): We ignore the empty allow statement.
+             */
         } else {
             state.addRule(path, true);
         }
@@ -1045,14 +1148,17 @@ public class SimpleRobotRulesParser extends BaseRobotsParser {
             System.err.println("              \ta single 'product token' as per RFC 9309.");
             System.err.println("              \tIf not defined check with '*'");
             System.err.println("  <URL>       \tcheck URL whether allowed or forbidden.");
-            System.err.println("              \tIf no URL is given show robots.txt rules");
+            System.err.println("              \tIf no URL is given show the robots.txt rules.");
             System.exit(1);
         }
 
         String url = args[0];
-        String agentName = "*";
+        // empty collection to select rules for wildcard user-agent (*)
+        Collection<String> agentNames = Set.of();
+        String agentName = "*"; // for logging
         if (args.length >= 2) {
-            agentName = args[1];
+            agentName = args[1].trim().toLowerCase(Locale.ROOT);
+            agentNames = Set.of(agentName);
         }
 
         SimpleRobotRulesParser parser = new SimpleRobotRulesParser();
@@ -1065,7 +1171,7 @@ public class SimpleRobotRulesParser extends BaseRobotsParser {
                 // sitemap paths for file:/ URLs
                 url = "http://example.com/robots.txt";
             }
-            rules = parser.parseContent(url, content, "text/plain", Set.of(agentName));
+            rules = parser.parseContent(url, content, "text/plain", agentNames);
         } catch (IOException e) {
             if (connection instanceof HttpURLConnection) {
                 int code = ((HttpURLConnection) connection).getResponseCode();


Test patch:
diff --git a/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java b/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
index 57641c5..18ef40e 100644
--- a/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
+++ b/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -60,15 +61,23 @@ public class SimpleRobotRulesParserTest {
         return createRobotRules(crawlerNames, content.getBytes(UTF_8), exactUserAgentMatching);
     }
 
+    private static BaseRobotRules createRobotRules(Collection<String> crawlerNames, String content, boolean exactUserAgentMatching) {
+        return createRobotRules(crawlerNames, content.getBytes(UTF_8), exactUserAgentMatching);
+    }
+
     private static BaseRobotRules createRobotRules(String[] crawlerNames, byte[] contentBytes, boolean exactUserAgentMatching) {
+        return createRobotRules(Arrays.asList(crawlerNames), contentBytes, exactUserAgentMatching);
+    }
+
+    private static BaseRobotRules createRobotRules(Collection<String> crawlerNames, byte[] contentBytes, boolean exactUserAgentMatching) {
         SimpleRobotRulesParser robotParser = new SimpleRobotRulesParser();
         robotParser.setExactUserAgentMatching(exactUserAgentMatching);
-        return robotParser.parseContent(FAKE_ROBOTS_URL, contentBytes, "text/plain", Arrays.asList(crawlerNames));
+        return robotParser.parseContent(FAKE_ROBOTS_URL, contentBytes, "text/plain", crawlerNames);
     }
 
     @Test
     void testEmptyRules() {
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", "");
+        BaseRobotRules rules = createRobotRules("anybot", "");
         assertTrue(rules.isAllowed("http://www.domain.com/anypage.html"));
     }
 
@@ -77,7 +86,7 @@ public class SimpleRobotRulesParserTest {
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow: /index.cfm?fuseaction=sitesearch.results*";
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertFalse(rules.isAllowed("http://searchservice.domain.com/index.cfm?fuseaction=sitesearch.results&type=People&qry=california&pg=2"));
     }
 
@@ -96,13 +105,13 @@ public class SimpleRobotRulesParserTest {
         // Test for /fish
         final String simpleRobotsTxt1 = "User-agent: *" + CRLF //
                         + "Disallow: /fish" + CRLF;
-        BaseRobotRules rule1 = createRobotRules("Any-darn-crawler", simpleRobotsTxt1);
+        BaseRobotRules rule1 = createRobotRules("anybot", simpleRobotsTxt1);
         assertEquals(isAllowed, rule1.isAllowed(urlStr));
 
         // Test for /fish*
         final String simpleRobotsTxt2 = "User-agent: *" + CRLF //
                         + "Disallow: /fish*" + CRLF;
-        BaseRobotRules rule2 = createRobotRules("Any-darn-crawler", simpleRobotsTxt2);
+        BaseRobotRules rule2 = createRobotRules("anybot", simpleRobotsTxt2);
         assertEquals(isAllowed, rule2.isAllowed(urlStr));
     }
 
@@ -117,7 +126,7 @@ public class SimpleRobotRulesParserTest {
         // Test for /fish
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow: /fish/" + CRLF;
-        BaseRobotRules rule = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rule = createRobotRules("anybot", simpleRobotsTxt);
         assertEquals(isAllowed, rule.isAllowed(urlStr));
     }
 
@@ -133,7 +142,7 @@ public class SimpleRobotRulesParserTest {
         // Test for /*.php
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow: /*.php" + CRLF;
-        BaseRobotRules rule = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rule = createRobotRules("anybot", simpleRobotsTxt);
         assertEquals(isAllowed, rule.isAllowed(urlStr));
     }
 
@@ -148,7 +157,7 @@ public class SimpleRobotRulesParserTest {
         // Test for /*.php$
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow: /*.php$" + CRLF;
-        BaseRobotRules rule = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rule = createRobotRules("anybot", simpleRobotsTxt);
         assertEquals(isAllowed, rule.isAllowed(urlStr));
     }
 
@@ -160,7 +169,7 @@ public class SimpleRobotRulesParserTest {
         // Test for /fish*.php
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow: /fish*.php" + CRLF;
-        BaseRobotRules rule = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rule = createRobotRules("anybot", simpleRobotsTxt);
         assertEquals(isAllowed, rule.isAllowed(urlStr));
     }
 
@@ -172,7 +181,7 @@ public class SimpleRobotRulesParserTest {
         // Test rule with multiple '*' characters
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow: /*fish*.php" + CRLF;
-        BaseRobotRules rule = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rule = createRobotRules("anybot", simpleRobotsTxt);
         assertEquals(isAllowed, rule.isAllowed(urlStr));
     }
 
@@ -190,7 +199,7 @@ public class SimpleRobotRulesParserTest {
                         + "#disallow: /index.html" + LF //
                         + "#disallow: /test" + LF + LF;
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertTrue(rules.isAllowed("http://www.domain.com/anypage.html"));
     }
 
@@ -223,7 +232,7 @@ public class SimpleRobotRulesParserTest {
         final String simpleRobotsTxt = "User-agent: *" + " # \u00A2 \u20B5" + CRLF //
                         + "Disallow:";
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertTrue(rules.isAllowed("http://www.domain.com/anypage.html"));
     }
 
@@ -291,10 +300,12 @@ public class SimpleRobotRulesParserTest {
                     "False, /search/%2a/, https://www.example.com/search/%2a/", //
                     "False, /search/%2a/, https://www.example.com/search/*/", //
                     "False, /search/*/, https://www.example.com/search/foobar/", //
-                    // examples from RFC 9309,  2.2.2. The "Allow" and "Disallow" Lines
+                    // examples from RFC 9309, 2.2.2. The "Allow" and "Disallow"
+                    // Lines
                     // https://www.rfc-editor.org/rfc/rfc9309.html#name-the-allow-and-disallow-line
                     "False, /foo/bar?baz=quz, https://www.example.com/foo/bar?baz=quz", //
-                    // See the comment in https://github.com/google/robotstxt/blob/master/robots_test.cc
+                    // See the comment in
+                    // https://github.com/google/robotstxt/blob/master/robots_test.cc
                     // "Percent encoding URIs in the rules is unnecessary."
                     // and "/foo/bar?baz=http://foo.bar stays unencoded."
                     "False, /foo/bar?baz=https://foo.bar, https://www.example.com/foo/bar?baz=https://foo.bar", //
@@ -325,7 +336,7 @@ public class SimpleRobotRulesParserTest {
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow:";
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertTrue(rules.isAllowed("http://www.domain.com/anypage.html"));
     }
 
@@ -335,7 +346,7 @@ public class SimpleRobotRulesParserTest {
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow: /";
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertFalse(rules.isAllowed("http://www.example.com"));
         assertFalse(rules.isAllowed("http://www.example.com?q=a"));
     }
@@ -613,13 +624,24 @@ public class SimpleRobotRulesParserTest {
 
     @Test
     void testHtmlMarkupInRobotsTxt() {
-        final String htmlRobotsTxt = "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\"><HTML>\n" + "<HEAD>\n" + "<TITLE>/robots.txt</TITLE>\n" + "</HEAD>\n" + "<BODY>\n"
-                        + "User-agent: anybot<BR>\n" + "Disallow: <BR>\n" + "Crawl-Delay: 10<BR>\n" + "\n" + "User-agent: *<BR>\n" + "Disallow: /<BR>\n" + "Crawl-Delay: 30<BR>\n" + "\n" + "</BODY>\n"
+        final String htmlRobotsTxt = "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\"><HTML>\n" //
+                        + "<HEAD>\n" //
+                        + "<TITLE>/robots.txt</TITLE>\n" //
+                        + "</HEAD>\n" //
+                        + "<BODY>\n" //
+                        + "User-agent: mybot<BR>\n" //
+                        + "Disallow: <BR>\n" //
+                        + "Crawl-Delay: 10<BR>\n" //
+                        + "\n" + "User-agent: *<BR>\n" //
+                        + "Disallow: /<BR>\n" //
+                        + "Crawl-Delay: 30<BR>\n" //
+                        + "\n" //
+                        + "</BODY>\n" //
                         + "</HTML>\n";
 
         BaseRobotRules rules;
 
-        rules = createRobotRules("anybot", htmlRobotsTxt);
+        rules = createRobotRules("mybot", htmlRobotsTxt);
         assertTrue(rules.isAllowed("http://www.domain.com/index.html"));
         assertEquals(10000, rules.getCrawlDelay());
 
@@ -688,7 +710,7 @@ public class SimpleRobotRulesParserTest {
                         + "Allow: /somepage.html" + CRLF //
                         + "Disallow: /";
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertTrue(rules.isAllowed("http://www.domain.com/AnyPage.html"));
         assertFalse(rules.isAllowed("http://www.domain.com/anypage.html"));
         assertTrue(rules.isAllowed("http://www.domain.com/somepage.html"));
@@ -700,7 +722,7 @@ public class SimpleRobotRulesParserTest {
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Disallow:";
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertTrue(rules.isAllowed("http://www.domain.com/anypage.html"));
     }
 
@@ -709,10 +731,37 @@ public class SimpleRobotRulesParserTest {
         final String simpleRobotsTxt = "User-agent: *" + CRLF //
                         + "Allow:";
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertTrue(rules.isAllowed("http://www.domain.com/anypage.html"));
     }
 
+    @Test
+    void testEmptyDisallowLowestPrecedence() {
+        String robotsTxt = "User-agent: *" + CRLF //
+                        + "Disallow: /disallowed/" + CRLF //
+                        + "Allow: /allowed/" + CRLF //
+                        + "Disallow: ";
+
+        BaseRobotRules rules = createRobotRules("anybot", robotsTxt);
+        assertTrue(rules.isAllowed("http://www.example.com/"));
+        assertTrue(rules.isAllowed("http://www.example.com/anypage.html"));
+        assertFalse(rules.isAllowed("http://www.example.com/disallowed/index.html"));
+        assertTrue(rules.isAllowed("http://www.example.com/allowed/index.html"));
+
+        // with merged groups
+        robotsTxt = "User-agent: *" + CRLF //
+                        + "Disallow: /disallowed/" + CRLF //
+                        + "Allow: /allowed/" + CRLF //
+                        + "User-agent: *" + CRLF //
+                        + "Disallow: ";
+
+        rules = createRobotRules("anybot", robotsTxt);
+        assertTrue(rules.isAllowed("http://www.example.com/"));
+        assertTrue(rules.isAllowed("http://www.example.com/anypage.html"));
+        assertFalse(rules.isAllowed("http://www.example.com/disallowed/index.html"));
+        assertTrue(rules.isAllowed("http://www.example.com/allowed/index.html"));
+    }
+
     @Test
     void testMultiWildcard() {
         // Make sure we only take the first wildcard entry.
@@ -723,7 +772,7 @@ public class SimpleRobotRulesParserTest {
                         + "User-agent: *" + CRLF //
                         + "Disallow: /";
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
         assertFalse(rules.isAllowed("http://www.domain.com/index.html"));
         assertTrue(rules.isAllowed("http://www.domain.com/anypage.html"));
     }
@@ -1058,7 +1107,7 @@ public class SimpleRobotRulesParserTest {
 
     @Test
     void testAmazonRobotsWithWildcards() throws Exception {
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", readFile("/robots/wildcards.txt"));
+        BaseRobotRules rules = createRobotRules("anybot", readFile("/robots/wildcards.txt"));
         assertFalse(rules.isAllowed("http://www.fict.com/wishlist/bogus"));
         assertTrue(rules.isAllowed("http://www.fict.com/wishlist/universal/page"));
         assertFalse(rules.isAllowed("http://www.fict.com/anydirectoryhere/gcrnsts"));
@@ -1070,7 +1119,7 @@ public class SimpleRobotRulesParserTest {
                         + "Disallow: /fish" + CRLF //
                         + "Allow: /fish" + CRLF;
 
-        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        BaseRobotRules rules = createRobotRules("anybot", simpleRobotsTxt);
 
         assertTrue(rules.isAllowed("http://www.fict.com/fish"));
     }
@@ -1092,7 +1141,7 @@ public class SimpleRobotRulesParserTest {
         rules = createRobotRules("Three", simpleRobotsTxt, false);
         assertFalse(rules.isAllowed("http://www.fict.com/fish"));
 
-        rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt);
+        rules = createRobotRules("anybot", simpleRobotsTxt);
         assertTrue(rules.isAllowed("http://www.fict.com/fish"));
     }
 
@@ -1231,14 +1280,14 @@ public class SimpleRobotRulesParserTest {
         assertTrue(rules.isAllowed("https://example.org/publications/doc1.html"));
         assertFalse(rules.isAllowed("https://example.org/example/page.html"));
         assertFalse(rules.isAllowed("https://example.org/example.gif"));
-        assertTrue(rules.isAllowed("https://example.org/")); // implicitly allowed
+        assertTrue(rules.isAllowed("https://example.org/"), "implicitly allowed");
 
         robotstxt = readFile("/robots/rfc9309-example-longest-match-robots.txt");
         rules = createRobotRules("foobot", robotstxt);
         assertTrue(rules.isAllowed("https://example.org/example/page/"));
         assertTrue(rules.isAllowed("https://example.org/example/page/index.html"));
         assertFalse(rules.isAllowed("https://example.org/example/page/disallowed.gif"));
-        assertTrue(rules.isAllowed("https://example.org/")); // implicitly allowed
+        assertTrue(rules.isAllowed("https://example.org/"), "implicitly allowed");
 
         robotstxt = readFile("/robots/rfc9309-example-rule-group-merging.txt");
         rules = createRobotRules("examplebot", robotstxt);
@@ -1246,23 +1295,46 @@ public class SimpleRobotRulesParserTest {
         assertFalse(rules.isAllowed("https://example.org/foo"));
         assertFalse(rules.isAllowed("https://example.org/bar"));
         assertFalse(rules.isAllowed("https://example.org/baz"));
-        assertTrue(rules.isAllowed("https://example.org/")); // implicitly allowed
+        assertTrue(rules.isAllowed("https://example.org/"), "implicitly allowed");
 
         rules = createRobotRules("anyotherbot", robotstxt);
         assertEquals(2, ((SimpleRobotRules) rules).getRobotRules().size());
         assertFalse(rules.isAllowed("https://example.org/foo"));
         assertFalse(rules.isAllowed("https://example.org/bar"));
         assertTrue(rules.isAllowed("https://example.org/baz"));
-        assertTrue(rules.isAllowed("https://example.org/")); // implicitly allowed
+        assertTrue(rules.isAllowed("https://example.org/"), "implicitly allowed");
 
         rules = createRobotRules("bazbot", robotstxt);
         assertEquals(1, ((SimpleRobotRules) rules).getRobotRules().size());
         assertTrue(rules.isAllowed("https://example.org/foo"));
         assertTrue(rules.isAllowed("https://example.org/bar"));
         assertFalse(rules.isAllowed("https://example.org/baz"));
-        assertTrue(rules.isAllowed("https://example.org/")); // implicitly allowed
+        assertTrue(rules.isAllowed("https://example.org/"), "implicitly allowed");
     }
 
+    @Test
+    void testAPIemptyUserAgentList() {
+        final String simpleRobotsTxt = "User-agent: *" + CRLF //
+                        + "Allow: /allowed/" + CRLF //
+                        + "Disallow: /" + CRLF //
+                        + "User-agent: allowedbot" + CRLF //
+                        + "Allow: /";
+
+        /*
+         * verify that the wildcard user-agent rules are selected if an empty
+         * list of user-agents is passed
+         */
+        BaseRobotRules rules = createRobotRules(Set.of(), simpleRobotsTxt, true);
+        assertTrue(rules.isAllowed("https://www.example.com/allowed/page.html"));
+        assertFalse(rules.isAllowed("https://www.example.com/"));
+
+        rules = createRobotRules(Set.of("anybot"), simpleRobotsTxt, true);
+        assertTrue(rules.isAllowed("https://www.example.com/allowed/page.html"));
+        assertFalse(rules.isAllowed("https://www.example.com/"));
+
+        rules = createRobotRules(Set.of("allowedbot"), simpleRobotsTxt, true);
+        assertTrue(rules.isAllowed("https://www.example.com/"));
+    }
 
     private byte[] readFile(String filename) throws Exception {
         byte[] bigBuffer = new byte[100000];