Repository: crawler-commons/crawler-commons (java)

Issue:
[Sitemaps] Add cross-submit feature into our project
In the official sitemaps documentation there is a reference about [Sitemaps & Cross Submits](http://www.sitemaps.org/protocol.html#location)

In a nutshell, it means that there is a way for a sitemap to reference urls from an other domain (as long as it gets validated in the other domain, and as long as it contains urls from one domain only per sitemap)

We should implement this functionality.


Pull request:
[Sitemaps] Add cross-submit feature, fixes #85
Add class SiteMapCrossSubmitValidator providing static methods to perform cross-submit validation of sitemaps, removing URLs which failed the cross-submit validation.

Patch:
diff --git a/src/main/java/crawlercommons/sitemaps/SiteMap.java b/src/main/java/crawlercommons/sitemaps/SiteMap.java
index e39b6c5..e86f499 100644
--- a/src/main/java/crawlercommons/sitemaps/SiteMap.java
+++ b/src/main/java/crawlercommons/sitemaps/SiteMap.java
@@ -26,7 +26,9 @@ public class SiteMap extends AbstractSiteMap {
     /**
      * The base URL for the Sitemap is where the Sitemap was found If found at
      * http://foo.org/abc/sitemap.xml then baseUrl is http://foo.org/abc/
-     * Sitemaps can only contain URLs that are under the base URL.
+     * Sitemaps can only contain URLs that are under the base URL, see <a
+     * href="https://www.sitemaps.org/protocol.html#location">sitemap file
+     * location</a>.
      */
     private String baseUrl;
 
diff --git a/src/main/java/crawlercommons/sitemaps/SiteMapCrossSubmitValidator.java b/src/main/java/crawlercommons/sitemaps/SiteMapCrossSubmitValidator.java
new file mode 100644
index 0000000..8df1e56
--- /dev/null
+++ b/src/main/java/crawlercommons/sitemaps/SiteMapCrossSubmitValidator.java
@@ -0,0 +1,226 @@
+/**
+ * Copyright 2025 Crawler-Commons
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+import java.net.URL;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.function.Predicate;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import crawlercommons.domains.EffectiveTldFinder;
+
+/**
+ * Validator for sitemap <a href=
+ * "https://www.sitemaps.org/protocol.html#sitemaps_cross_submits">cross
+ * submits</a>.
+ * 
+ * The sitemap protocol defines strict requirements regarding the <a
+ * href="https://www.sitemaps.org/protocol.html#location">location of a
+ * sitemap</a>:
+ * 
+ * <blockquote>
+ * <p>
+ * The location of a Sitemap file determines the set of URLs that can be
+ * included in that Sitemap. A Sitemap file located at
+ * http://example.com/catalog/sitemap.xml can include any URLs starting with
+ * http://example.com/catalog/ but can not include URLs starting with
+ * http://example.com/images/.
+ * </p>
+ * <p>
+ * If you have the permission to change http://example.org/path/sitemap.xml, it
+ * is assumed that you also have permission to provide information for URLs with
+ * the prefix http://example.org/path/.
+ * </p>
+ * </blockquote>
+ * 
+ * However, when the sitemap location (on host A) is specified in the robots.txt
+ * file of host B, this <q>proves the ownership</q> and the sitemap is allowed
+ * to <q><a href=
+ * "https://www.sitemaps.org/protocol.html#sitemaps_cross_submits"
+ * >cross-submit</a></q> URLs on host B.
+ * 
+ * Note: in order to use the validator, you need to create a sitemap parser
+ * without strict validation, see {@link SiteMapParser#isStrict()},
+ * {@link SiteMapParser#urlIsValid(String, String)} and the constructors of
+ * {@link SiteMapParser}.
+ */
+public class SiteMapCrossSubmitValidator {
+
+    public static final Logger LOG = LoggerFactory.getLogger(SiteMapCrossSubmitValidator.class);
+
+    public enum CrossSubmitValidationLevel {
+        /**
+         * Domain name below a suffix in the ICANN section of the public suffix
+         * list. Cf. {@link EffectiveTldFinder}
+         */
+        ICANN_DOMAIN,
+        /**
+         * Domain name below a public suffix, cf. {@link EffectiveTldFinder}
+         */
+        PRIVATE_DOMAIN,
+        /** Host name resp. full domain name */
+        HOST
+    }
+
+    private SiteMapCrossSubmitValidator() {
+    }
+
+    /**
+     * Validate the URLs submitted in a sitemap whether they are valid, that is
+     * below the same URL prefix as the location of the sitemap. Invalid URLs
+     * are removed.
+     * 
+     * Calling this method on a sitemap has the same effect as using a sitemap
+     * parser with <b>strict</b> validation, see
+     * {@link SiteMapParser#isStrict()}.
+     * 
+     * @param sitemap
+     *            sitemap holding the URLs and the base URL (the URL prefix)
+     */
+    public static void validateSiteMapURLs(SiteMap sitemap) {
+        LOG.debug("Validating sitemap URLs for prefix {}", sitemap.getBaseUrl());
+        validateSiteMapURLs(sitemap, (URL url) -> SiteMapParser.urlIsValid(sitemap.getBaseUrl(), url.toString()));
+    }
+
+    /**
+     * Validate the URLs in a sitemap against a single cross-submit host.
+     * Invalid URLs are removed.
+     * 
+     * This method implements the typical cross-submit check for a sitemap
+     * announced in the robots.txt of one host while located on a different
+     * host. Naturally, it also verifies that <q>all URLs in a Sitemap must be
+     * from a single host</q>.
+     * 
+     * @param sitemap
+     * 
+     * @param host
+     *            host name proved for cross-submits. Usually the host of the
+     *            robots.txt file the sitemap was announced.
+     */
+    public static void validateSiteMapURLs(SiteMap sitemap, String host) {
+        LOG.debug("Validating sitemap URLs for cross-submit host {}", host);
+        validateSiteMapURLs(sitemap, (URL url) -> url.getHost().equals(host));
+    }
+
+    /**
+     * Validate the URLs in a sitemap against a set of cross-submit hosts.
+     * Invalid URLs are removed.
+     * 
+     * This method is useful, if the same sitemap location is found in the
+     * robots.txt files of multiple hosts.
+     * 
+     * @param sitemap
+     *            sitemap holding the URLs to be validated
+     * @param hosts
+     *            set of host names proved for cross-submits
+     */
+    public static void validateSiteMapURLs(SiteMap sitemap, Collection<String> hosts) {
+        LOG.debug("Validating sitemap URLs against a set of cross-submit hosts {}", hosts);
+        validateSiteMapURLs(sitemap, (URL url) -> hosts.contains(url.getHost()));
+    }
+
+    /**
+     * Validate a single URL whether its host, ICANN or private domain is part
+     * of a list of domain names.
+     * 
+     * @param url
+     *            URL to validate
+     * @param domains
+     *            set of domain names proved for cross-submits
+     * @param domainValidationLevel
+     *            validation level for the domain names
+     */
+    protected static boolean validate(URL url, Collection<String> domains, CrossSubmitValidationLevel domainValidationLevel) {
+        String domain;
+        switch (domainValidationLevel) {
+            case ICANN_DOMAIN:
+            domain = EffectiveTldFinder.getAssignedDomain(url.getHost(), false, true);
+            return domains.contains(domain);
+            case PRIVATE_DOMAIN:
+            domain = EffectiveTldFinder.getAssignedDomain(url.getHost(), false, false);
+            return domains.contains(domain);
+            default:
+            return domains.contains(url.getHost());
+        }
+    }
+
+    /**
+     * Validate the URLs in a sitemap against a set of cross-submit domains.
+     * Invalid URLs are removed.
+     * 
+     * This method is useful, if a site owner is not sufficiently precise
+     * regarding sitemap location. It allows to <q>prove ownership</q> by
+     * verifying that both the sitemap and cross-submit host share the same
+     * domain holder. The public suffix list is used to determine the shared
+     * part below the suffix, cf. {@link EffectiveTldFinder}.
+     * 
+     * @param sitemap
+     *            sitemap holding the URLs to be validated
+     * @param domains
+     *            set of domain names proved for cross-submits
+     * @param domainValidationLevel
+     *            validation level for the domain names
+     */
+    public static void validateSiteMapURLs(SiteMap sitemap, Collection<String> domains, CrossSubmitValidationLevel domainValidationLevel) {
+        LOG.debug("Validating sitemap URLs against a set of cross-submit sites {} ({})", domains, domainValidationLevel);
+        validateSiteMapURLs(sitemap, (URL url) -> validate(url, domains, domainValidationLevel));
+    }
+
+    public static void validateSiteMapURLs(SiteMap sitemap, Predicate<URL> validator) {
+        Iterator<SiteMapURL> iter = sitemap.getSiteMapUrls().iterator();
+        int validated = 0, invalid = 0;
+        while (iter.hasNext()) {
+            SiteMapURL u = iter.next();
+            validated++;
+            if (!validator.test(u.getUrl())) {
+                LOG.debug("Removing URL: {}", u.toString());
+                iter.remove();
+                invalid++;
+            }
+        }
+        LOG.debug("Validated {} sitemap URLs: {} valid, {} invalid", validated, (validated - invalid), invalid);
+    }
+
+    /**
+     * Validation of a sitemap or recursive validation of a sitemap index.
+     * 
+     * See
+     * {@link #validateSiteMapURLs(SiteMap, Collection, CrossSubmitValidationLevel)}
+     * .
+     * 
+     * @param sitemap
+     *            sitemap or sitemap index, holding the URLs to be validated
+     * @param domains
+     *            set of domain names proved for cross-submits
+     * @param domainValidationLevel
+     *            validation level for the domain names
+     */
+    public static void validateSiteMapURLs(AbstractSiteMap sitemap, Collection<String> domains, CrossSubmitValidationLevel domainValidationLevel) {
+        if (sitemap.isIndex()) {
+            Collection<AbstractSiteMap> sitemaps = ((SiteMapIndex) sitemap).getSitemaps();
+            for (AbstractSiteMap asm : sitemaps) {
+                // recursive call
+                validateSiteMapURLs(asm, domains, domainValidationLevel);
+            }
+        } else {
+            validateSiteMapURLs((SiteMap) sitemap, domains, domainValidationLevel);
+        }
+    }
+}
diff --git a/src/main/java/crawlercommons/sitemaps/SiteMapParser.java b/src/main/java/crawlercommons/sitemaps/SiteMapParser.java
index 91aa6cb..0c56c44 100644
--- a/src/main/java/crawlercommons/sitemaps/SiteMapParser.java
+++ b/src/main/java/crawlercommons/sitemaps/SiteMapParser.java
@@ -73,7 +73,7 @@ public class SiteMapParser {
 
     /**
      * True (by default) meaning that invalid URLs should be rejected, as the
-     * official docs allow the siteMapURLs to be only under the base url:
+     * official docs allow the siteMapURLs to be only under the base URL:
      * https://www.sitemaps.org/protocol.html#location
      */
     protected boolean strict = true;
@@ -159,7 +159,7 @@ public class SiteMapParser {
      * @return whether invalid URLs will be rejected (where invalid means that
      *         the URL is not under the base URL, see <a href=
      *         "https://www.sitemaps.org/protocol.html#location">sitemap file
-     *         location</a>)
+     *         location</a>). See also {@link #urlIsValid(String, String)}.
      */
     public boolean isStrict() {
         return strict;
@@ -668,8 +668,18 @@ public class SiteMapParser {
     }
 
     /**
-     * See if testUrl is under sitemapBaseUrl. Only URLs under sitemapBaseUrl
-     * are valid.
+     * Verify whether the <code>testUrl</code> is under the
+     * <code>sitemapBaseUrl</code>. Only URLs under sitemapBaseUrl are valid,
+     * see <a href="https://www.sitemaps.org/protocol.html#location">sitemap
+     * file location</a>. This is method is used when the parser is configured
+     * with <b>strict</b> checking, see {@link #isStrict()}.
+     * 
+     * Note that this method does not allow to check for <a href=
+     * "https://www.sitemaps.org/protocol.html#sitemaps_cross_submits">cross
+     * submits</a>: if a sitemap, located on host A, is referenced in the
+     * robots.txt file of host B, then this is considered as
+     * <q>prove of ownership</q> and the sitemap is allowed to list URLs on host
+     * B. For cross-submit validation, see {@link SiteMapCrossSubmitValidator}.
      * 
      * @param sitemapBaseUrl
      *            the base URL of the sitemap


Test patch:
diff --git a/src/test/java/crawlercommons/sitemaps/SiteMapCrossSubmitValidatorTest.java b/src/test/java/crawlercommons/sitemaps/SiteMapCrossSubmitValidatorTest.java
new file mode 100644
index 0000000..a1b5f9b
--- /dev/null
+++ b/src/test/java/crawlercommons/sitemaps/SiteMapCrossSubmitValidatorTest.java
@@ -0,0 +1,118 @@
+/**
+ * Copyright 2025 Crawler-Commons
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class SiteMapCrossSubmitValidatorTest {
+
+    protected static SiteMap getSiteMap(String url, String content) throws UnknownFormatException, IOException {
+        SiteMapParser parser = new SiteMapParser(false);
+        String contentType = "text/plain";
+        byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
+        AbstractSiteMap asm = parser.parseSiteMap(contentType, bytes, new URL(url));
+        AbstractSiteMapTest.testSerializable(asm);
+        assertEquals(false, asm.isIndex());
+        assertEquals(true, asm instanceof SiteMap);
+        return (SiteMap) asm;
+    }
+
+    @Test
+    public void testValidateURLs() throws Exception {
+        SiteMap sm = getSiteMap("https://blog.example.com/sitemap.txt", //
+                        "https://blog.example.com/blog-1.html\n" //
+                                        + "https://blog.example.com/blog-2.html");
+        assertEquals(2, sm.getSiteMapUrls().size());
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm);
+        assertEquals(2, sm.getSiteMapUrls().size());
+    }
+
+    @Test
+    public void testValidateCrossSubmitHost() throws Exception {
+        SiteMap sm = getSiteMap("https://www.example.com/sitemap.txt", //
+                        "https://blog.example.com/blog-1.html\n" //
+                                        + "https://blog.example.com/blog-2.html");
+        assertEquals(2, sm.getSiteMapUrls().size());
+        // sitemap was announced in https://blog.example.com/robots.txt
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, "blog.example.com");
+        assertEquals(2, sm.getSiteMapUrls().size());
+        // but not in https://www.otherdomain.com/robots.txt
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, "www.otherdomain.com");
+        assertEquals(0, sm.getSiteMapUrls().size());
+    }
+
+    @Test
+    public void testValidateCrossSubmitSites() throws Exception {
+        String url = "https://www.example.com/sitemap.txt";
+        /*
+         * Note: for simplicity, in this test we use sitemaps with URLs from
+         * multiple hosts, although the sitemap protocol specifies that
+         * "all URLs in a Sitemap must be from a single host"
+         * (https://www.sitemaps.org/protocol.html)
+         */
+        String content = "https://blog.example.com/blog-1.html\n" //
+                        + "https://blog.example.com/blog-2.html\n" //
+                        + "https://docs.example.com/index.html";
+        SiteMap sm = getSiteMap(url, content);
+        assertEquals(3, sm.getSiteMapUrls().size());
+        // sitemap was announced in https://blog.example.com/robots.txt
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, Set.of("blog.example.com"));
+        assertEquals(2, sm.getSiteMapUrls().size());
+        // but not in https://www.otherdomain.com/robots.txt
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, Set.of("www.otherdomain.com"));
+        assertEquals(0, sm.getSiteMapUrls().size());
+        // test with multiple hosts
+        sm = getSiteMap(url, content);
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, Set.of("www.example.com", "blog.example.com", "docs.example.com"));
+        assertEquals(3, sm.getSiteMapUrls().size());
+        // equivalent call
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, Set.of("www.example.com", "blog.example.com", "docs.example.com"), SiteMapCrossSubmitValidator.CrossSubmitValidationLevel.HOST);
+        assertEquals(3, sm.getSiteMapUrls().size());
+        // test with ICANN / private domain
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, Set.of("example.com"), SiteMapCrossSubmitValidator.CrossSubmitValidationLevel.PRIVATE_DOMAIN);
+        assertEquals(3, sm.getSiteMapUrls().size());
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, Set.of("example.com"), SiteMapCrossSubmitValidator.CrossSubmitValidationLevel.ICANN_DOMAIN);
+        assertEquals(3, sm.getSiteMapUrls().size());
+        /*
+         * a more realistic test for a domain name below a suffix in the private
+         * section of the public suffix list
+         */
+        content = "https://this.github.io/this.html\n" //
+                        + "https://that.github.io/that-1.html\n" //
+                        + "https://that.github.io/that-2.html";
+        sm = getSiteMap(url, content);
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, Set.of("this.github.io"), SiteMapCrossSubmitValidator.CrossSubmitValidationLevel.PRIVATE_DOMAIN);
+        assertEquals(1, sm.getSiteMapUrls().size());
+        sm = getSiteMap(url, content);
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(sm, Set.of("that.github.io"), SiteMapCrossSubmitValidator.CrossSubmitValidationLevel.PRIVATE_DOMAIN);
+        assertEquals(2, sm.getSiteMapUrls().size());
+        /** validate sitemap index */
+        sm = getSiteMap(url, content);
+        SiteMapIndex smi = new SiteMapIndex();
+        smi.addSitemap(sm);
+        SiteMapCrossSubmitValidator.validateSiteMapURLs(smi, Set.of("this.github.io"), SiteMapCrossSubmitValidator.CrossSubmitValidationLevel.PRIVATE_DOMAIN);
+        assertEquals(1, sm.getSiteMapUrls().size());
+    }
+
+}