Repository: crawler-commons/crawler-commons (java)

Issue:
[BasicNormalizer] Provide builder to configure the normalizer
(suggested by @aecio in [#309](https://github.com/crawler-commons/crawler-commons/pull/309#discussion_r691340841))

With #246/#309 the class BasicURLNormalizer adds a normalization routine which is not safe in rare cases. It should be possible to enable this optimization on-demand only by providing a builder. This would also allow to configure other options (eg. the set of query params to be removed) and to keep the interface clean otherwise (no constructors to pass configurations in).

Pull request description:
Add a builder API for configuring the BasicURLNormalizer
Usage example:
```
normalizer = BasicURLNormalizer.newBuilder()
  .idnNormalization(IdnNormalization.PUNYCODE)
  .queryParamsToRemove(
    asList("sid", "phpsessid", "sessionid", "jsessionid")
  )
  .build();
```

Closes #321.

Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index 9b14065..e38cef5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,8 @@
 Crawler-Commons Change Log
 
 Current Development 1.2-SNAPSHOT (yyyy-mm-dd)
+- [URLs] Provide a builder class to configure the URL normalizer (aecio) #321, #324
+- [URLs] Make normalization of IDNs configurable (to ASCII or Unicode) via builder (aecio, sebastian-nagel) #324
 - [Sitemaps] Fix XXE vulnerability in Sitemap parser (kovyrin) #323
 - [URLs] Sorting the Query Parameters (aecio) #246, #309
 - [URLs] Allows to (optionally) remove common irrelevant query parameters (aecio) #309
diff --git a/src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java b/src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java
index a74d008..df6616d 100644
--- a/src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java
+++ b/src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java
@@ -29,13 +29,7 @@ import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -145,14 +139,17 @@ public class BasicURLNormalizer extends URLFilter {
         return true;
     }
 
-    final Set<String> queryElementsToRemove;
+    private final Set<String> queryParamsToRemove;
+    private final IdnNormalization idnNormalization;
+
 
     public BasicURLNormalizer() {
-        this(new TreeSet<>());
+        this(new Builder());
     }
 
-    public BasicURLNormalizer(Set<String> queryElementsToRemove) {
-        this.queryElementsToRemove = new TreeSet<>(queryElementsToRemove);
+    public BasicURLNormalizer(Builder builder) {
+        this.queryParamsToRemove = builder.queryParamsToRemove;
+        this.idnNormalization = builder.idnNormalization;
     }
 
     @Override
@@ -310,7 +307,7 @@ public class BasicURLNormalizer extends URLFilter {
         file = escapePath(unescapePath(file));
 
         List<NameValuePair> pairs =
-                parseQueryParameters(file, queryStartIdx, queryElementsToRemove);
+                parseQueryParameters(file, queryStartIdx, queryParamsToRemove);
 
         StringBuilder normalizedFile = new StringBuilder();
         String path = file.substring(0, endPathIdx);
@@ -612,7 +609,7 @@ public class BasicURLNormalizer extends URLFilter {
          * 3. convert between Unicode and ASCII forms for Internationalized
          * Domain Names (IDNs)
          */
-        if (!isAscii(host)) {
+        if (this.idnNormalization == IdnNormalization.PUNYCODE && !isAscii(host)) {
             /*
              * IllegalArgumentException: thrown if the input string contains
              * non-convertible Unicode codepoints
@@ -622,6 +619,8 @@ public class BasicURLNormalizer extends URLFilter {
              * cf. https://bugs.openjdk.java.net/browse/JDK-6806873
              */
             host = IDN.toASCII(host);
+        } else if (this.idnNormalization == IdnNormalization.UNICODE && host.contains("xn--")) {
+            host = IDN.toUnicode(host);
         }
 
         /* 4. trim a trailing dot */
@@ -632,6 +631,65 @@ public class BasicURLNormalizer extends URLFilter {
         return host;
     }
 
+    /**
+     * Create a new builder object for creating a customized {@link BasicURLNormalizer} object.
+     *
+     * @return
+     */
+    public static Builder newBuilder() {
+        return new Builder();
+    }
+
+    public enum IdnNormalization {
+        NONE,
+        PUNYCODE,
+        UNICODE
+    }
+
+    /**
+     * A builder class for the {@link BasicURLNormalizer}.
+     */
+    public static class Builder {
+
+        public IdnNormalization idnNormalization = IdnNormalization.PUNYCODE;
+        Set<String> queryParamsToRemove = new TreeSet<>();
+
+        private Builder() {
+        }
+
+        /**
+         * A collection of names of query parameters that should be removed from the URL query.
+         *
+         * @param queryParamsToRemove
+         * @return this builder
+         */
+        public Builder queryParamsToRemove(Collection<String> queryParamsToRemove) {
+            this.queryParamsToRemove = new TreeSet<>(queryParamsToRemove);
+            return this;
+        }
+
+        /**
+         * Configures whether internationalized domain names (IDNs) should be
+         * converted to ASCII/Punycode or Unicode.
+         *
+         * @param idnNormalization
+         * @return this builder
+         */
+        public Builder idnNormalization(IdnNormalization idnNormalization) {
+            this.idnNormalization = idnNormalization;
+            return this;
+        }
+
+        /**
+         * Constructs the custom URL normalizer instance.
+         *
+         * @return the constructed URL normalizer
+         */
+        public BasicURLNormalizer build() {
+            return new BasicURLNormalizer(this);
+        }
+    }
+
     public static void main(String args[]) throws IOException {
         BasicURLNormalizer normalizer = new BasicURLNormalizer();
         String line, normUrl;


Test patch:
diff --git a/src/test/java/crawlercommons/filters/basic/BasicURLNormalizerTest.java b/src/test/java/crawlercommons/filters/basic/BasicURLNormalizerTest.java
index a7b514f..f16a96b 100644
--- a/src/test/java/crawlercommons/filters/basic/BasicURLNormalizerTest.java
+++ b/src/test/java/crawlercommons/filters/basic/BasicURLNormalizerTest.java
@@ -16,6 +16,7 @@
 
 package crawlercommons.filters.basic;
 
+import static java.util.Arrays.asList;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
@@ -23,9 +24,6 @@ import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvFileSource;
-import java.util.Arrays;
-import java.util.List;
-import java.util.TreeSet;
 
 /** Unit tests for BasicURLNormalizer. */
 public class BasicURLNormalizerTest {
@@ -50,14 +48,26 @@ public class BasicURLNormalizerTest {
 
     @Test
     public void testRemoveSessionQueryParameters() {
-        List<String> invalidParameters = Arrays
-            .asList("sid", "phpsessid", "sessionid", "jsessionid");
-        normalizer = new BasicURLNormalizer(new TreeSet<>(invalidParameters));
+        normalizer = BasicURLNormalizer.newBuilder().queryParamsToRemove(asList("sid", "phpsessid", "sessionid", "jsessionid")).build();
         normalizeTest("http://foo.com/foo.php?phpsessid=2Aa3ASdfasfdadf&a=1", "http://foo.com/foo.php?a=1");
         normalizeTest("http://foo.com/foo.php?phpsessid=2Aa3ASdfasfdadf&a=1&b", "http://foo.com/foo.php?a=1&b");
         normalizeTest("http://foo.com/foo.php?phpsessid=2Aa3ASdfasfdadf", "http://foo.com/foo.php");
     }
 
+    @Test
+    public void testHostToUnicode() {
+        normalizer = BasicURLNormalizer.newBuilder().idnNormalization(BasicURLNormalizer.IdnNormalization.UNICODE).build();
+        normalizeTest("http://xn--schne-lua.xn--bcher-kva.de/", "http://schöne.bücher.de/");
+        normalizeTest("https://xn--90ax2c.xn--p1ai/", "https://нэб.рф/");
+    }
+
+    @Test
+    public void testNoIdnNormalization() {
+        normalizer = BasicURLNormalizer.newBuilder().idnNormalization(BasicURLNormalizer.IdnNormalization.NONE).build();
+        // leave the host name as is, even if it's mixed
+        normalizeTest("http://schöne.xn--bcher-kva.de/", "http://schöne.xn--bcher-kva.de/");
+    }
+
     private void normalizeTest(String weird, String normal) {
         assertEquals(normal, normalizer.filter(weird), "normalizing: " + weird);
     }