Repository: crawler-commons/crawler-commons (java)

Issue:
[Sitemaps] Sitemap index: stop URL at closing </loc>
(cf. [Nutch mailing list](https://lists.apache.org/thread.html/8ebcffbe2bd8edafb6030e4f28fceee07aee08a1ce06a94755ee8d74@%3Cuser.nutch.apache.org%3E)

With #153 the sitemaps SAX parser handles sitemaps with missing or not properly closed <url> elements. This should be also done for sitemap indexes, e.g.:
```
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex>
<sitemap>
<loc>https://www.example.orgl/sitemap1.xml</loc>
<loc>https://www.example.org/sitemap2.xml</loc>
</sitemap>
</sitemapindex>
```


Pull request description:
Sitemap index: stop URL at closing </loc>
(fixes #213)
At start of a `<loc>` element auto-close any "unclosed" `<sitemap>` element and add the sitemap if there is a valid URL from the previous `<loc>` element.

Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index 3baa9bc..848fb68 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,7 @@
 Crawler-Commons Change Log
 
 Current Development 0.11-SNAPSHOT (yyyy-mm-dd)
+- [Sitemaps] Sitemap index: stop URL at closing </loc> (sebastian-nagel, kkrugler) #213
 - [Sitemaps] Allow empty price in video sitemaps (sebastian-nagel) #221
 - [Sitemaps] In case of the use of a different locale, price tag can be formatted with ',' instead of '.' leading to a NPE (goldenlink) #220
 - [Sitemaps] Add support for sitemap extensions (tuxnco, sebastian-nagel) #35, #36, #149, #162
diff --git a/src/main/java/crawlercommons/sitemaps/sax/DelegatorHandler.java b/src/main/java/crawlercommons/sitemaps/sax/DelegatorHandler.java
index 32767ef..d42b9c6 100644
--- a/src/main/java/crawlercommons/sitemaps/sax/DelegatorHandler.java
+++ b/src/main/java/crawlercommons/sitemaps/sax/DelegatorHandler.java
@@ -208,4 +208,14 @@ public class DelegatorHandler extends DefaultHandler {
             delegate.fatalError(e);
         }
     }
+
+    public static boolean isAllBlank(CharSequence charSeq) {
+        for (int i = 0; i < charSeq.length(); i++) {
+            if (!Character.isWhitespace(charSeq.charAt(i))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
 }
diff --git a/src/main/java/crawlercommons/sitemaps/sax/XMLHandler.java b/src/main/java/crawlercommons/sitemaps/sax/XMLHandler.java
index d1fcfa0..91b6c5c 100644
--- a/src/main/java/crawlercommons/sitemaps/sax/XMLHandler.java
+++ b/src/main/java/crawlercommons/sitemaps/sax/XMLHandler.java
@@ -94,12 +94,9 @@ class XMLHandler extends DelegatorHandler {
 
         // flush any unclosed or missing URL element
         if (loc.length() > 0 && ("loc".equals(localName) || "url".equals(localName))) {
-            // check whether loc isn't white space only
-            for (int i = 0; i < loc.length(); i++) {
-                if (!Character.isWhitespace(loc.charAt(i))) {
-                    maybeAddSiteMapUrl();
-                    return;
-                }
+            if (!isAllBlank(loc)) {
+                maybeAddSiteMapUrl();
+                return;
             }
             loc = new StringBuilder();
             if ("url".equals(localName)) {
diff --git a/src/main/java/crawlercommons/sitemaps/sax/XMLIndexHandler.java b/src/main/java/crawlercommons/sitemaps/sax/XMLIndexHandler.java
index 84d87ca..6c73d43 100644
--- a/src/main/java/crawlercommons/sitemaps/sax/XMLIndexHandler.java
+++ b/src/main/java/crawlercommons/sitemaps/sax/XMLIndexHandler.java
@@ -67,6 +67,19 @@ class XMLIndexHandler extends DelegatorHandler {
     }
 
     public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
+        // flush any unclosed or missing <sitemap> element
+        if (loc.length() > 0 && ("loc".equals(localName) || "sitemap".equals(localName))) {
+            if (!isAllBlank(loc)) {
+                maybeAddSiteMap();
+                return;
+            }
+            loc = new StringBuilder();
+            if ("sitemap".equals(localName)) {
+                // reset also attributes
+                locClosed = false;
+                lastMod = null;
+            }
+        }
     }
 
     public void endElement(String uri, String localName, String qName) throws SAXException {


Test patch:
diff --git a/src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java b/src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java
index 10fc53c..270ea0a 100644
--- a/src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java
+++ b/src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java
@@ -274,6 +274,53 @@ public class SiteMapParserTest {
         assertNotNull("Sitemap " + sitemap + " not found in sitemap index", sm.getSitemap(new URL(sitemap)));
     }
 
+    @Test
+    public void testUnclosedSitemapIndexFile() throws UnknownFormatException, IOException {
+        SiteMapParser parser = new SiteMapParser();
+        StringBuilder scontent = new StringBuilder();
+        scontent.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>") //
+                        .append("<sitemapindex>") //
+                        .append(" <sitemap>") //
+                        .append("  <loc>https://www.example.org/sitemap1.xml</loc>") //
+                        .append("  <loc>https://www.example.org/sitemap2.xml</loc>") //
+                        .append(" </sitemap>") //
+                        .append("</sitemapindex>");
+        byte[] content = scontent.toString().getBytes(UTF_8);
+        URL url = new URL("http://www.example.com/sitemapindex.xml");
+
+        AbstractSiteMap asm = parser.parseSiteMap(content, url);
+        assertEquals(true, asm.isIndex());
+        assertEquals(true, asm instanceof SiteMapIndex);
+        SiteMapIndex sm = (SiteMapIndex) asm;
+        assertEquals(2, sm.getSitemaps().size());
+        String urlSecondSitemap = "https://www.example.org/sitemap2.xml";
+        AbstractSiteMap secondSitemap = sm.getSitemap(new URL(urlSecondSitemap));
+        assertNotNull("Sitemap " + urlSecondSitemap + " not found in sitemap index", secondSitemap);
+
+        // check reset of attributes (lastmod) when "autoclosing" <sitemap>
+        // elements
+        scontent = new StringBuilder();
+        scontent.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n") //
+                        .append("<sitemapindex>\n") //
+                        .append(" <sitemap>\n") //
+                        .append("  <loc>https://www.example.org/sitemap1.xml</loc>\n") //
+                        .append("  <lastmod>2004-10-01T18:23:17+00:00</lastmod>\n") //
+                        .append("  <loc>https://www.example.org/sitemap2.xml</loc>\n") //
+                        .append("  <loc>https://www.example.org/sitemap3.xml</loc>\n") //
+                        .append("  <lastmod>2005-11-02T19:24:18+00:00</lastmod>\n") //
+                        .append(" </sitemap>\n") //
+                        .append("</sitemapindex>");
+        content = scontent.toString().getBytes(UTF_8);
+        asm = parser.parseSiteMap(content, url);
+        assertEquals(true, asm.isIndex());
+        assertEquals(true, asm instanceof SiteMapIndex);
+        sm = (SiteMapIndex) asm;
+        assertEquals(3, sm.getSitemaps().size());
+        secondSitemap = sm.getSitemap(new URL(urlSecondSitemap));
+        assertNotNull("Sitemap " + urlSecondSitemap + " not found in sitemap index", secondSitemap);
+        assertNull("Sitemap " + urlSecondSitemap + " without modification date", secondSitemap.getLastModified());
+    }
+
     @Test
     public void testSitemapGZ() throws UnknownFormatException, IOException {
         SiteMapParser parser = new SiteMapParser();