Repository: crawler-commons/crawler-commons
Pull request #216 (2018-08-01 00:12:55)

Fix for URL with only a query string (no path)
Fix for processing URLs which are just `domain.com?<query parameters>` (no path). These would get turned into a string that looks like `?<query parameters>`, instead of `/?<query parameters>`, which means they wouldn't match any rules (which all start with `/`).


Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index 247664d..9e0432a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,7 @@
 Crawler-Commons Change Log
 
 Current Development 0.11-SNAPSHOT (yyyy-mm-dd)
+- [Robots] Fix for handling URLs with query parameters but no path (kkrugler) #215
 
 Release 0.10 (2018-06-05)
 - Add JAX-B dependencies to POM (jnioche) #207
diff --git a/src/main/java/crawlercommons/robots/SimpleRobotRules.java b/src/main/java/crawlercommons/robots/SimpleRobotRules.java
index 61657c6..ab4ca3c 100644
--- a/src/main/java/crawlercommons/robots/SimpleRobotRules.java
+++ b/src/main/java/crawlercommons/robots/SimpleRobotRules.java
@@ -25,7 +25,12 @@ import java.util.List;
 
 /**
  * Result from parsing a single robots.txt file - which means we get a set of
- * rules, and a crawl-delay.
+ * rules, and an optional crawl-delay, and an optional sitemap URL. Note that we
+ * support Google's extensions (Allow directive and '$'/'*' special chars) plus
+ * the more widely used Sitemap directive.
+ * 
+ * See https://en.wikipedia.org/wiki/Robots_exclusion_standard See
+ * https://developers.google.com/search/reference/robots_txt
  */
 
 @SuppressWarnings("serial")
@@ -173,18 +178,18 @@ public class SimpleRobotRules extends BaseRobotRules {
         try {
             URL urlObj = new URL(url);
             String path = urlObj.getPath();
+            if ((path == null) || (path.equals(""))) {
+                path = "/";
+            }
+
             String query = urlObj.getQuery();
             if (getWithQuery && query != null) {
                 path += "?" + query;
             }
 
-            if ((path == null) || (path.equals(""))) {
-                return "/";
-            } else {
-                // We used to lower-case the path, but Google says we need to do
-                // case-sensitive matching.
-                return URLDecoder.decode(path, "UTF-8");
-            }
+            // We used to lower-case the path, but Google says we need to do
+            // case-sensitive matching.
+            return URLDecoder.decode(path, "UTF-8");
         } catch (Exception e) {
             // If the URL is invalid, we don't really care since the fetch
             // will fail, so return the root.


Test patch:
diff --git a/src/test/java/crawlercommons/domains/PaidLevelDomainTest.java b/src/test/java/crawlercommons/domains/PaidLevelDomainTest.java
index 7fbfc61..698c987 100644
--- a/src/test/java/crawlercommons/domains/PaidLevelDomainTest.java
+++ b/src/test/java/crawlercommons/domains/PaidLevelDomainTest.java
@@ -23,7 +23,6 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.UnknownHostException;
 
-import org.junit.Ignore;
 import org.junit.Test;
 
 import crawlercommons.domains.PaidLevelDomain;
diff --git a/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java b/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
index 281d8ce..ecf290d 100644
--- a/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
+++ b/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
@@ -191,6 +191,17 @@ public class SimpleRobotRulesParserTest {
         assertTrue(rules.isAllowed("http://www.domain.com/anypage.html"));
     }
 
+    // https://github.com/crawler-commons/crawler-commons/issues/215
+
+    @Test
+    public void testDisallowWithQueryOnly() {
+        final String simpleRobotsTxt = "User-agent: *" + CRLF + "Disallow: /";
+
+        BaseRobotRules rules = createRobotRules("Any-darn-crawler", simpleRobotsTxt.getBytes(UTF_8));
+        assertFalse(rules.isAllowed("http://www.example.com"));
+        assertFalse(rules.isAllowed("http://www.example.com?q=a"));
+    }
+
     @Test
     public void testMixedEndings() {
         final String mixedEndingsRobotsTxt = "# /robots.txt for http://www.fict.org/" + CRLF + "# comments to webmaster@fict.org" + CR + LF + "User-agent: unhipbot" + LF + "Disallow: /" + CR + ""