Spaces:
Build error
Build error
fix(adaptive-crawler): if no sitemap, use recursive instead
Browse files
backend/functions/src/cloud-functions/adaptive-crawler.ts
CHANGED
|
@@ -120,9 +120,12 @@ export class AdaptiveCrawlerHost extends RPCHost {
|
|
| 120 |
failed: {},
|
| 121 |
});
|
| 122 |
|
|
|
|
| 123 |
if (useSitemap) {
|
| 124 |
-
|
|
|
|
| 125 |
|
|
|
|
| 126 |
await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({
|
| 127 |
status: AdaptiveCrawlTaskStatus.PROCESSING,
|
| 128 |
statusText: `Processing 0/${urls.length}`,
|
|
|
|
| 120 |
failed: {},
|
| 121 |
});
|
| 122 |
|
| 123 |
+
let urls: string[] = [];
|
| 124 |
if (useSitemap) {
|
| 125 |
+
urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages);
|
| 126 |
+
}
|
| 127 |
|
| 128 |
+
if (urls.length > 0) {
|
| 129 |
await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({
|
| 130 |
status: AdaptiveCrawlTaskStatus.PROCESSING,
|
| 131 |
statusText: `Processing 0/${urls.length}`,
|