Repository: LangStream/langstream (language: python)

Issue:
Cannot crawl website with bad sitemap at all
I tried to crawl the docs.datastax.com website with follow robots enabled. This site has a number of nested sitemaps. One of the listed sitemaps is a redirect to a regular HTML page, which is incorrect. The crawler fails to parse the sitemap (because it's not really a sitemap) and then crashes. This crash blocks the crawler from making any forward progress on the website, since it is only just parsing the sitemaps.

It would be better if the crawler could handle this error and continue crawling the site. The error should be clearly logged, but forward progress should not be stopped.

To reproduced configure a crawler to crawl https://docs.datastax.com


Discussion / hints:

Fix patch:
diff --git a/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/crawler/WebCrawler.java b/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/crawler/WebCrawler.java
index 43ae24158..8405b2aeb 100644
--- a/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/crawler/WebCrawler.java
+++ b/langstream-agents/langstream-agent-webcrawler/src/main/java/ai/langstream/agents/webcrawler/crawler/WebCrawler.java
@@ -357,34 +357,45 @@ private void applyRobotsRules(String url, SimpleRobotRules simpleRobotRules) {
     }
 
     private void handleSitemapsFile(String url) throws Exception {
-        HttpResponse<byte[]> response = downloadUrl(url);
-        SiteMapParser siteMapParser = new SiteMapParser();
-        AbstractSiteMap abstractSiteMap = siteMapParser.parseSiteMap(response.body(), new URL(url));
-        if (abstractSiteMap instanceof SiteMap siteMap) {
-
-            Collection<SiteMapURL> siteMapUrls = siteMap.getSiteMapUrls();
-
-            siteMapUrls.forEach(
-                    siteMapUrl -> {
-                        log.info("Adding url from sitemap : {}", siteMapUrl.getUrl());
-                        String loc = siteMapUrl.getUrl().toString();
-                        if (configuration.isAllowedUrl(loc)) {
-                            addPageUrl(loc, null);
-                        } else {
-                            log.debug("Ignoring not allowed url: {}", loc);
-                            discardUrl(loc, new URLReference(loc, URLReference.Type.PAGE, 0));
-                        }
-                    });
-        } else if (abstractSiteMap instanceof SiteMapIndex siteMapIndex) {
-            log.info("It is a sitemap index");
-            Collection<AbstractSiteMap> sitemaps = siteMapIndex.getSitemaps(true);
-            sitemaps.forEach(
-                    s -> {
-                        log.info("Adding this sitemap : {}", s.getUrl());
-                        forceAddUrl(s.getUrl().toString(), URLReference.Type.SITEMAP, 0);
-                    });
-        } else {
-            log.warn("Unknown sitemap type: {}", abstractSiteMap.getClass().getName());
+
+        if (isUrlForbidden(url)) {
+            log.info("Sitemap {} is forbidden. Skipping", url);
+            return;
+        }
+
+        try {
+            HttpResponse<byte[]> response = downloadUrl(url);
+            SiteMapParser siteMapParser = new SiteMapParser();
+            AbstractSiteMap abstractSiteMap =
+                    siteMapParser.parseSiteMap(response.body(), new URL(url));
+            if (abstractSiteMap instanceof SiteMap siteMap) {
+
+                Collection<SiteMapURL> siteMapUrls = siteMap.getSiteMapUrls();
+
+                siteMapUrls.forEach(
+                        siteMapUrl -> {
+                            log.info("Adding url from sitemap : {}", siteMapUrl.getUrl());
+                            String loc = siteMapUrl.getUrl().toString();
+                            if (configuration.isAllowedUrl(loc)) {
+                                addPageUrl(loc, null);
+                            } else {
+                                log.debug("Ignoring not allowed url: {}", loc);
+                                discardUrl(loc, new URLReference(loc, URLReference.Type.PAGE, 0));
+                            }
+                        });
+            } else if (abstractSiteMap instanceof SiteMapIndex siteMapIndex) {
+                log.info("It is a sitemap index");
+                Collection<AbstractSiteMap> sitemaps = siteMapIndex.getSitemaps(true);
+                sitemaps.forEach(
+                        s -> {
+                            log.info("Adding this sitemap : {}", s.getUrl());
+                            forceAddUrl(s.getUrl().toString(), URLReference.Type.SITEMAP, 0);
+                        });
+            } else {
+                log.warn("Unknown sitemap type: {}", abstractSiteMap.getClass().getName());
+            }
+        } catch (crawlercommons.sitemaps.UnknownFormatException e) {
+            log.warn("Error while parsing the sitemap file at {}", url, e);
         }
     }
 


Test patch:
diff --git a/langstream-agents/langstream-agent-webcrawler/src/test/java/ai/langstream/agents/webcrawler/WebCrawlerSourceTest.java b/langstream-agents/langstream-agent-webcrawler/src/test/java/ai/langstream/agents/webcrawler/WebCrawlerSourceTest.java
index baecde97e..481e6a70e 100644
--- a/langstream-agents/langstream-agent-webcrawler/src/test/java/ai/langstream/agents/webcrawler/WebCrawlerSourceTest.java
+++ b/langstream-agents/langstream-agent-webcrawler/src/test/java/ai/langstream/agents/webcrawler/WebCrawlerSourceTest.java
@@ -155,6 +155,40 @@ void testReadLangStreamDocs() throws Exception {
         agentSource.close();
     }
 
+    @Test
+    @Disabled("This test is disabled because it connects to a real live website")
+    void testReadDataStaxDocs() throws Exception {
+        String bucket = "langstream-test-" + UUID.randomUUID();
+        String url = "https://docs.datastax.com/";
+        String allowed = "https://docs.datastax.com/";
+
+        WebCrawlerSource agentSource =
+                buildAgentSource(
+                        bucket,
+                        allowed,
+                        Set.of("/en/dse/6.8/dse-admin/sitemap.xml"),
+                        url,
+                        Map.of("reindex-interval-seconds", "30", "max-urls", 5));
+        List<Record> read = agentSource.read();
+        Set<String> urls = new HashSet<>();
+        agentSource.setOnReindexStart(
+                () -> {
+                    urls.clear();
+                });
+        int count = 0;
+        while (count < 30000) {
+            log.info("read: {}", read);
+            for (Record r : read) {
+                String docUrl = r.key().toString();
+                assertTrue(urls.add(docUrl), "Read twice the same url: " + docUrl);
+            }
+            count += read.size();
+            agentSource.commit(read);
+            read = agentSource.read();
+        }
+        agentSource.close();
+    }
+
     @Test
     @Disabled("This test is disabled because it connects to a real live website")
     void testReadLangStreamGithubRepo() throws Exception {