mapleeit commited on
Commit
5a4b35e
·
1 Parent(s): ee29be5

fix(adaptive-crawler): if no sitemap, use recursive instead

Browse files
backend/functions/src/cloud-functions/adaptive-crawler.ts CHANGED
@@ -120,9 +120,12 @@ export class AdaptiveCrawlerHost extends RPCHost {
120
  failed: {},
121
  });
122
 
 
123
  if (useSitemap) {
124
- const urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages);
 
125
 
 
126
  await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({
127
  status: AdaptiveCrawlTaskStatus.PROCESSING,
128
  statusText: `Processing 0/${urls.length}`,
 
120
  failed: {},
121
  });
122
 
123
+ let urls: string[] = [];
124
  if (useSitemap) {
125
+ urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages);
126
+ }
127
 
128
+ if (urls.length > 0) {
129
  await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({
130
  status: AdaptiveCrawlTaskStatus.PROCESSING,
131
  statusText: `Processing 0/${urls.length}`,