Repository: crawler-commons/crawler-commons (java)

Issue:
[BasicNormalizer] decode percent-encoded host names
BasicURLNormalizer should decode percent-encoded characters in host names, similar as done by web browsers. Examples: https://example%2Ecom/ or https://www.0251-sachverst%c3%a4ndiger.de/, see also [NUTCH-2824](https://issues.apache.org/jira/browse/NUTCH-2824).

Pull request description:
[BasicNormalizer] decode percent-encoded host names, fixes #303
- decoded percent-encoded sequences in host names
- extend unit tests to host names including IDNs
- add unit test to verify that runtime exception caused by invalid percent-encoded sequences are properly caught

Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index d2366ae..8fc02b6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,7 @@
 Crawler-Commons Change Log
 
 Current Development 1.2-SNAPSHOT (yyyy-mm-dd)
+- [BasicNormalizer] decode percent-encoded host names (sebastian-nagel) #303
 - [SiteMapParser] Document options *strict* and *allowPartial* in SiteMapParser constructors (sebastian-nagel) #267
 - [Robots] Maximum values (crawl-delay and warnings): document and make visible (sebastian-nagel, Avi Hayun) #276
 - [sitemaps] Replace priority "NaN" by default value (sebastian-nagel) #296
diff --git a/src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java b/src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java
index cf0108d..fadd970 100644
--- a/src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java
+++ b/src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java
@@ -21,10 +21,12 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
 import java.net.IDN;
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.URLDecoder;
 import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -162,7 +164,7 @@ public class BasicURLNormalizer extends URLFilter {
                 String newHost;
                 try {
                     newHost = normalizeHostName(host);
-                } catch (IllegalArgumentException | IndexOutOfBoundsException e) {
+                } catch (IllegalArgumentException | IndexOutOfBoundsException | UnsupportedEncodingException e) {
                     LOG.info("Invalid hostname: {}", host, e);
                     return null;
                 }
@@ -274,8 +276,8 @@ public class BasicURLNormalizer extends URLFilter {
 
     /**
      * Remove % encoding from path segment in URL for characters which should be
-     * unescaped according to
-     * <a href="https://tools.ietf.org/html/rfc3986#section-2.2">RFC3986</a>.
+     * unescaped according to <a
+     * href="https://tools.ietf.org/html/rfc3986#section-2.2">RFC3986</a>.
      */
     private String unescapePath(String path) {
         StringBuilder sb = new StringBuilder();
@@ -317,8 +319,8 @@ public class BasicURLNormalizer extends URLFilter {
 
     /**
      * Convert path segment of URL from Unicode to UTF-8 and escape all
-     * characters which should be escaped according to
-     * <a href="https://tools.ietf.org/html/rfc3986#section-2.2">RFC3986</a>..
+     * characters which should be escaped according to <a
+     * href="https://tools.ietf.org/html/rfc3986#section-2.2">RFC3986</a>.
      */
     private String escapePath(String path) {
         StringBuilder sb = new StringBuilder(path.length());
@@ -371,13 +373,22 @@ public class BasicURLNormalizer extends URLFilter {
         return sb.toString();
     }
 
-    private String normalizeHostName(String host) throws IllegalArgumentException, IndexOutOfBoundsException {
+    private String normalizeHostName(String host) throws IllegalArgumentException, IndexOutOfBoundsException, UnsupportedEncodingException {
 
-        /* 1. lowercase host name */
+        /* 1. unescape percent-encoded characters in host name */
+        if (host.indexOf('%') != -1) {
+            /*
+             * throws IllegalArgumentException on illegal percent-encoded
+             * sequences
+             */
+            host = URLDecoder.decode(host, UTF_8.toString());
+        }
+
+        /* 2. lowercase host name */
         host = host.toLowerCase(Locale.ROOT);
 
         /*
-         * 2. convert between Unicode and ASCII forms for Internationalized
+         * 3. convert between Unicode and ASCII forms for Internationalized
          * Domain Names (IDNs)
          */
         if (!isAscii(host)) {
@@ -392,7 +403,7 @@ public class BasicURLNormalizer extends URLFilter {
             host = IDN.toASCII(host);
         }
 
-        /* 3. trim a trailing dot */
+        /* 4. trim a trailing dot */
         if (host.endsWith(".")) {
             host = host.substring(0, host.length() - 1);
         }


Test patch:
diff --git a/src/test/java/crawlercommons/filters/basic/BasicURLNormalizerTest.java b/src/test/java/crawlercommons/filters/basic/BasicURLNormalizerTest.java
index 971f6d1..7568481 100644
--- a/src/test/java/crawlercommons/filters/basic/BasicURLNormalizerTest.java
+++ b/src/test/java/crawlercommons/filters/basic/BasicURLNormalizerTest.java
@@ -33,6 +33,12 @@ public class BasicURLNormalizerTest {
     @ParameterizedTest
     @CsvFileSource(resources = "/normalizer/weirdToNormalizedUrls.csv")
     void testBasicNormalizer(String weirdUrl, String expectedNormalizedUrl) throws Exception {
-      Assertions.assertEquals(expectedNormalizedUrl, normalizer.filter(weirdUrl), "normalizing: " + weirdUrl);
+        Assertions.assertEquals(expectedNormalizedUrl, normalizer.filter(weirdUrl), "normalizing: " + weirdUrl);
+    }
+
+    @ParameterizedTest
+    @CsvFileSource(resources = "/normalizer/invalidUrls.csv")
+    void testBasicNormalizerExceptionCaught(String weirdUrl) throws Exception {
+        Assertions.assertEquals(null, normalizer.filter(weirdUrl), "normalizing: " + weirdUrl);
     }
 }
\ No newline at end of file
diff --git a/src/test/resources/normalizer/invalidUrls.csv b/src/test/resources/normalizer/invalidUrls.csv
new file mode 100644
index 0000000..b0619b0
--- /dev/null
+++ b/src/test/resources/normalizer/invalidUrls.csv
@@ -0,0 +1,9 @@
+# Test with invalid URLs
+# - BasicURLNormalizer should return null
+# - but never throw an exception
+
+# invalid percent-encoded sequence in host name
+https://example%2Xcom/
+# not a valid UTF-8 sequence
+https://abc%FEdef.org/
+
diff --git a/src/test/resources/normalizer/weirdToNormalizedUrls.csv b/src/test/resources/normalizer/weirdToNormalizedUrls.csv
index e6cf27b..e5c0919 100644
--- a/src/test/resources/normalizer/weirdToNormalizedUrls.csv
+++ b/src/test/resources/normalizer/weirdToNormalizedUrls.csv
@@ -55,6 +55,9 @@ http://bücher.de/, http://xn--bcher-kva.de/
 http://êxample.com, http://xn--xample-hva.com/
 https://нэб.рф/, https://xn--90ax2c.xn--p1ai/
 
+# unescape percent-encoded characters also in IDNs
+https://www.0251-sachverst%c3%a4ndiger.de/, https://www.xn--0251-sachverstndiger-ozb.de/
+
 # test whether percent-encoding works together with other normalizations
 http://x.com/./a/../%66.html, http://x.com/f.html
 
@@ -78,6 +81,9 @@ http://foo.com/, http://foo.com/
 http://Foo.Com/index.html, http://foo.com/index.html
 http://Foo.Com/index.html, http://foo.com/index.html
 
+# unescape percent characters in host names
+https://example%2Ecom/, https://example.com/
+
 # check that port number is normalized
 http://foo.com:80/index.html, http://foo.com/index.html
 http://foo.com:81/, http://foo.com:81/