Repository: crawler-commons/crawler-commons (java)

Issue:
[robots.txt parser] Deduplicate sitemap links
Deduplicate sitemap links contained in a robots.txt, see [NUTCH-2730](https://issues.apache.org/jira/browse/NUTCH-2730). Ideally, the collection should be still ordered with duplicates (optionally) removed.

Pull request:
Robots parser: deduplicate sitemap links, fixes #261
- use a LinkedHashSet to store and unify sitemap URLs

Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index cf0135b..b6cf963 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,7 @@
 Crawler-Commons Change Log
 
 Current Development 1.1-SNAPSHOT (yyyy-mm-dd)
+- [Robots] Deduplicate sitemap links (sebastian-nagel) #261
 - EffectiveTldFinder to log loading of public suffix list (sebastian-nagel) #284
 - SiteMapParser getPublicationDate in VideoAttributes may throw NPE (panthony, sebastian-nagel) #283
 - SimpleRobotRulesParser: Trim log messages (jnioche, sebastian-nagel) #281
diff --git a/src/main/java/crawlercommons/robots/BaseRobotRules.java b/src/main/java/crawlercommons/robots/BaseRobotRules.java
index 6e02f29..de3c501 100644
--- a/src/main/java/crawlercommons/robots/BaseRobotRules.java
+++ b/src/main/java/crawlercommons/robots/BaseRobotRules.java
@@ -18,6 +18,7 @@ package crawlercommons.robots;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 
 /**
@@ -38,10 +39,10 @@ public abstract class BaseRobotRules implements Serializable {
 
     private long _crawlDelay = UNSET_CRAWL_DELAY;
     private boolean _deferVisits = false;
-    private List<String> _sitemaps;
+    private LinkedHashSet<String> _sitemaps;
 
     public BaseRobotRules() {
-        _sitemaps = new ArrayList<String>();
+        _sitemaps = new LinkedHashSet<String>();
     }
 
     public long getCrawlDelay() {
@@ -60,12 +61,14 @@ public abstract class BaseRobotRules implements Serializable {
         _deferVisits = deferVisits;
     }
 
+    /** Add sitemap URL to rules if not a duplicate */
     public void addSitemap(String sitemap) {
         _sitemaps.add(sitemap);
     }
 
+    /** Get URLs of sitemap links found in robots.txt */
     public List<String> getSitemaps() {
-        return _sitemaps;
+        return new ArrayList<>(_sitemaps);
     }
 
     /*


Test patch:
diff --git a/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java b/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
index 56e766c..22afa54 100644
--- a/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
+++ b/src/test/java/crawlercommons/robots/SimpleRobotRulesParserTest.java
@@ -697,6 +697,12 @@ public class SimpleRobotRulesParserTest {
         assertEquals(1, rules.getSitemaps().size(), "Found sitemap");
     }
 
+    @Test
+    void testSitemapDedup() throws Exception {
+        BaseRobotRules rules = createRobotRules("bot1", readFile("/robots/sitemap-robots-dedup.txt"));
+        assertEquals(1, rules.getSitemaps().size(), "Sitemaps deduped");
+    }
+
     @Test
     void testManyUserAgents() throws Exception {
         BaseRobotRules rules = createRobotRules("wget", readFile("/robots/many-user-agents.txt"));
diff --git a/src/test/resources/robots/sitemap-robots-dedup.txt b/src/test/resources/robots/sitemap-robots-dedup.txt
new file mode 100644
index 0000000..356919e
--- /dev/null
+++ b/src/test/resources/robots/sitemap-robots-dedup.txt
@@ -0,0 +1,5 @@
+User-agent: *
+Disallow: 
+
+sitemap: http://example.org/sitemap.xml
+Sitemap: http://example.org/sitemap.xml