Repository: crawler-commons/crawler-commons (java)

Issue:
EffectiveTldFinder to validate returned domain names for length restrictions
In strict mode EffectiveTldFinder should validate the returned domain names also regarding length restrictions - 253 bytes (ASCII chars) in total, 63 bytes per dot-separated segment) as specified in [RFC1034](https://tools.ietf.org/html/rfc1034#section-3.1).

Pull request:
EffectiveTldFinder to validate returned domain names for length restrictions
fixes #251
- ensure that no dot-separated segment of a return domain name is longer than 63 chars
- implies that the entire domain name stays within the limit of 253 chars

Patch:
diff --git a/README.md b/README.md
index 6bdab69..0075364 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ If you find an issue, please file a report [here](https://github.com/crawler-com
 
 # Crawler-Commons News
 
-## 21st March 2019  - crawler-commons 1.0 released
+## 21st March 2018  - crawler-commons 1.0 released
 
 We are glad to announce the 1.0 release of Crawler-Commons. See the [CHANGES.txt](https://github.com/crawler-commons/crawler-commons/blob/crawler-commons-1.0/CHANGES.txt) file included with the release for a full list of details.
 Among other bug fixes and improvements this version adds support for parsing sitemap extensions (image, video, news, alternate links).
diff --git a/src/main/java/crawlercommons/domains/EffectiveTldFinder.java b/src/main/java/crawlercommons/domains/EffectiveTldFinder.java
index 3f0d33d..c939f75 100644
--- a/src/main/java/crawlercommons/domains/EffectiveTldFinder.java
+++ b/src/main/java/crawlercommons/domains/EffectiveTldFinder.java
@@ -94,6 +94,21 @@ public class EffectiveTldFinder {
     public static final String WILD_CARD = "*.";
     public static final char DOT = '.';
 
+    /**
+     * Max. length in ASCII characters of a dot-separated segment in host names
+     * (applies to domain names as well), cf.
+     * https://tools.ietf.org/html/rfc1034#section-3.1 and
+     * https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_hostnames
+     * 
+     * Note: We only have to validate domain names and not the host names passed
+     * as input. For domain names a verification of the segment length also
+     * implies that the entire domain names stays in the limit of 253
+     * characters. Wildcard suffixes only allow two additional segments (2*63+1
+     * = 127 chars) and all wildcard suffixes are far away from reaching the
+     * critical length of 126 characters.
+     */
+    public static final int MAX_DOMAIN_LENGTH_PART = 63;
+
     private static EffectiveTldFinder instance = null;
     private Map<String, EffectiveTLD> domains = null;
     private SuffixTrie<EffectiveTLD> domainTrie = new SuffixTrie<>();
@@ -332,9 +347,15 @@ public class EffectiveTldFinder {
             try {
                 IDN.toASCII(domainSegment);
             } catch (IllegalArgumentException e) {
-                // not a valid IDN segment
+                // not a valid IDN segment,
+                // includes check for max. length (63 chars)
                 return (strict ? null : hostname);
             }
+        } else if (strict) {
+            // (strict mode) check for max. length of segment (63 chars)
+            if (domainSegment.length() > MAX_DOMAIN_LENGTH_PART) {
+                return null;
+            }
         }
         return hostname.substring(start);
     }


Test patch:
diff --git a/src/test/java/crawlercommons/domains/EffectiveTldFinderTest.java b/src/test/java/crawlercommons/domains/EffectiveTldFinderTest.java
index 9a0e088..3cac005 100644
--- a/src/test/java/crawlercommons/domains/EffectiveTldFinderTest.java
+++ b/src/test/java/crawlercommons/domains/EffectiveTldFinderTest.java
@@ -233,6 +233,28 @@ public class EffectiveTldFinderTest {
         // => "example.com" is a valid domain name, no check whether the input
         // host name is valid
         assertEquals("example.com", EffectiveTldFinder.getAssignedDomain("\ufffd.example.com", true, false));
+        // (4) in strict mode, domain names (or dot-separated segments) should
+        // be checked for length restrictions
+        StringBuilder domain = new StringBuilder();
+        String part63chars = "123456789-abcdefghijklmnopqrstuvwxyz-abcdefghijklmnopqrstuvwxyz";
+        domain.append("www.");
+        domain.append(part63chars);
+        domain.append(".com");
+        assertEquals(domain.toString().substring(4), EffectiveTldFinder.getAssignedDomain(domain.toString(), true, false));
+        domain = new StringBuilder();
+        domain.append("www.");
+        domain.append('0');
+        domain.append(part63chars);
+        domain.append(".com");
+        // first part has 64 characters => not a valid domain name
+        assertNull(EffectiveTldFinder.getAssignedDomain(domain.toString(), true, false));
+        // IDNs: length limits apply (also) to the ASCII/Punycode representation
+        domain = new StringBuilder();
+        domain.append("www.");
+        domain.append(part63chars);
+        domain.setCharAt(4, '\u00e0'); // replace `1` by `à`
+        domain.append(".com");
+        assertNull(EffectiveTldFinder.getAssignedDomain(domain.toString(), true, false));
     }
 
 }