Repository: crawler-commons/crawler-commons
Pull request #234 (2019-02-22 12:03:26)

Detection and parsing of XML sitemaps fails with whitespace before XML declaration #144
- implement InputStream skipping over white space at beginning of file
- use for XML sitemaps in combination with BOMInputStream, so that white space or empty lines before  `<?xml ...>` do not cause the parser to fail


Patch:
diff --git a/CHANGES.txt b/CHANGES.txt
index 84eaea0..ecb07a6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,7 @@
 Crawler-Commons Change Log
 
 Current Development 0.11-SNAPSHOT (yyyy-mm-dd)
+- [Sitemaps] Detection and parsing of XML sitemaps fails with whitespace before XML declaration (sebastian-nagel, jnioche) #144
 - [Sitemaps] XMLHandler needs to append text in characters() vs. immediately processing (kkrugler, sebastian-nagel) #226
 - [Sitemaps] XMLIndexHandler needs to accumulate the lastmod date string before parsing (kkrugler, sebastian-nagel) #225
 - EffectiveTldFinder throws IllegalArgumentException on IDN domain names containing prohibited characters (sebastian-nagel) #231
diff --git a/src/main/java/crawlercommons/sitemaps/SiteMapParser.java b/src/main/java/crawlercommons/sitemaps/SiteMapParser.java
index 77a8138..e9d86e2 100644
--- a/src/main/java/crawlercommons/sitemaps/SiteMapParser.java
+++ b/src/main/java/crawlercommons/sitemaps/SiteMapParser.java
@@ -408,9 +408,9 @@ public class SiteMapParser {
      */
     protected AbstractSiteMap processXml(URL sitemapUrl, byte[] xmlContent) throws UnknownFormatException {
 
-        BOMInputStream bomIs = new BOMInputStream(new ByteArrayInputStream(xmlContent));
+        InputStream in = new SkipLeadingWhiteSpaceInputStream(new BOMInputStream(new ByteArrayInputStream(xmlContent)));
         InputSource is = new InputSource();
-        is.setCharacterStream(new BufferedReader(new InputStreamReader(bomIs, UTF_8)));
+        is.setCharacterStream(new BufferedReader(new InputStreamReader(in, UTF_8)));
 
         return processXml(sitemapUrl, is);
     }
@@ -502,7 +502,7 @@ public class SiteMapParser {
         String xmlUrl = url.toString().replaceFirst("\\.gz$", "");
         LOG.debug("XML url = {}", xmlUrl);
 
-        BOMInputStream decompressed = new BOMInputStream(new GZIPInputStream(is));
+        InputStream decompressed = new SkipLeadingWhiteSpaceInputStream(new BOMInputStream(new GZIPInputStream(is)));
         InputSource in = new InputSource(decompressed);
         in.setSystemId(xmlUrl);
         return processXml(url, in);
diff --git a/src/main/java/crawlercommons/sitemaps/SkipLeadingWhiteSpaceInputStream.java b/src/main/java/crawlercommons/sitemaps/SkipLeadingWhiteSpaceInputStream.java
new file mode 100644
index 0000000..1abe07c
--- /dev/null
+++ b/src/main/java/crawlercommons/sitemaps/SkipLeadingWhiteSpaceInputStream.java
@@ -0,0 +1,53 @@
+/**
+ * Copyright 2019 Crawler-Commons
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.io.input.ProxyInputStream;
+
+/**
+ * Wraps a stream and skips over leading whitespace (at beginning of file) in
+ * the wrapped stream. The wrapped stream must support to mark the current
+ * position (see {@link InputStream#mark(int)}.
+ * 
+ * Only ASCII whitespace (space/U+0020, <code>\t</code>/U+0009, <code>\n</code>
+ * /U+000A, vertical tab/U+000b, <code>\r</code>/U+000D) is skipped.
+ */
+public class SkipLeadingWhiteSpaceInputStream extends ProxyInputStream {
+
+    private boolean inLeadingWhiteSpace = true;
+
+    public SkipLeadingWhiteSpaceInputStream(InputStream proxy) {
+        super(proxy);
+        inLeadingWhiteSpace = proxy.markSupported();
+    }
+
+    protected void beforeRead(final int n) throws IOException {
+        while (inLeadingWhiteSpace) {
+            in.mark(1);
+            int c = in.read();
+            if (c == ' ' || c == '\t' || c == '\n' || c == 0x0b || c == '\r') {
+                continue;
+            } else {
+                in.reset();
+                inLeadingWhiteSpace = false;
+            }
+        }
+    }
+}
\ No newline at end of file


Test patch:
diff --git a/src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java b/src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java
index 480b708..54a1fb5 100644
--- a/src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java
+++ b/src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java
@@ -228,6 +228,21 @@ public class SiteMapParserTest {
         }
     }
 
+    @Test
+    public void testSitemapXMLleadingWhiteSpace() throws UnknownFormatException, IOException {
+        SiteMapParser parser = new SiteMapParser();
+        String contentType = "text/xml";
+        StringBuilder scontent = new StringBuilder();
+        scontent.append("\ufeff"); // leading BOM
+        scontent.append("\n \t\r\n"); // and leading white space
+        byte[] content = getXMLSitemapAsBytes(scontent);
+        URL url = new URL("http://www.example.com/sitemap.xml");
+
+        AbstractSiteMap asm = parser.parseSiteMap(contentType, content, url);
+        assertEquals(false, asm.isIndex());
+        assertEquals(true, asm instanceof SiteMap);
+    }
+
     @Test
     public void testSitemapXMLMediaTypes() throws UnknownFormatException, IOException {
         SiteMapParser parser = new SiteMapParser();
@@ -663,10 +678,25 @@ public class SiteMapParserTest {
     }
 
     /**
-     * Returns a good simple default XML sitemap as a byte array
+     * @return good simple default XML sitemap as UTF-8 encoded byte array
      */
     private byte[] getXMLSitemapAsBytes() {
         StringBuilder scontent = new StringBuilder(1024);
+        return getXMLSitemapAsBytes(scontent);
+    }
+
+    /**
+     * See {@link #getXMLSitemapAsBytes()}.
+     * 
+     * @param scontent
+     *            The sitemap content is appended to the passed StringBuilder
+     *            which allows to prefix/suffix the sitemap content or get the
+     *            encoding from the StringBuilder in a different encoding (not
+     *            UTF-8).
+     * @return content of passed StringBuilder plus appended sitemap content as
+     *         UTF-8 encoded bytes
+     */
+    private byte[] getXMLSitemapAsBytes(StringBuilder scontent) {
         scontent.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>") //
                         .append("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
         for (String[] surl : SITEMAP_URLS) {