Repository: crawler-commons/crawler-commons (java)

Issue:
[sitemaps] Allow to deduplicate sitemap links in sitemap indexes
(see [NUTCH-2730](https://issues.apache.org/jira/browse/NUTCH-2730))

A sitemap index may list the same sitemap multiple times (erroneously or maliciously). There should be an easy way to deduplicate the list of subsitemaps
- provide a method to that (eg. `getSitemaps(boolean unify)`)
- or overwrite the "equals" and "hashCode" methods in AbstractSitemap to make the deduplication easy

Pull request description:
[sitemaps] Allow to deduplicate sitemap links in sitemap indexes, fixes #262

Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index 81455be..28072b1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,7 @@
 Crawler-Commons Change Log
 
 Current Development 1.1-SNAPSHOT (yyyy-mm-dd)
+- [sitemaps] Allow to deduplicate sitemap links in sitemap indexes (sebastian-nagel) #262
 - [Robots] Upgrade the toString() method of the Base/Simple RobotRules (Avi Hayun) #264
 - Upgrade GitIgnore (Avi Hayun) #260
 - [Robots] Deduplicate sitemap links (sebastian-nagel) #261
diff --git a/src/main/java/crawlercommons/sitemaps/SiteMapIndex.java b/src/main/java/crawlercommons/sitemaps/SiteMapIndex.java
index 2406f78..95d8607 100644
--- a/src/main/java/crawlercommons/sitemaps/SiteMapIndex.java
+++ b/src/main/java/crawlercommons/sitemaps/SiteMapIndex.java
@@ -19,7 +19,10 @@ package crawlercommons.sitemaps;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 public class SiteMapIndex extends AbstractSiteMap {
 
@@ -42,6 +45,21 @@ public class SiteMapIndex extends AbstractSiteMap {
         return sitemaps;
     }
 
+    /**
+     * @param deduplicate
+     *            deduplicate sitemaps by URL: from two or more sitemaps
+     *            pointing to the same URL only the first is kept in the
+     *            returned collection
+     * @return the (deduplicated) Collection of Sitemaps in this Sitemap Index.
+     */
+    public Collection<AbstractSiteMap> getSitemaps(boolean deduplicate) {
+        if (deduplicate) {
+            Set<String> urls = new HashSet<>(sitemaps.size());
+            return sitemaps.stream().filter(s -> urls.add(s.url.toString())).collect(Collectors.toList());
+        }
+        return sitemaps;
+    }
+
     /**
      * Add this Sitemap to the list of Sitemaps,
      * 


Test patch:
diff --git a/src/test/java/crawlercommons/sitemaps/SiteMapIndexTest.java b/src/test/java/crawlercommons/sitemaps/SiteMapIndexTest.java
new file mode 100644
index 0000000..cff59bc
--- /dev/null
+++ b/src/test/java/crawlercommons/sitemaps/SiteMapIndexTest.java
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2020 Crawler-Commons
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class SiteMapIndexTest {
+
+    @Test
+    public void testDeduplication() {
+        SiteMapIndex index = new SiteMapIndex();
+        index.addSitemap(new SiteMap("https://example.com/sitemap1.xml", "2020-06-18"));
+        index.addSitemap(new SiteMap("https://example.com/sitemap2.xml", "2020-06-19"));
+        index.addSitemap(new SiteMap("https://example.com/sitemap1.xml", "2020-06-19"));
+        assertEquals(3, index.getSitemaps().size());
+        assertEquals(2, index.getSitemaps(true).size());
+        assertEquals(3, index.getSitemaps(false).size());
+    }
+}