GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-20 16:39:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(adaptive-crawler): if no sitemap, use recursive instead

Browse Source
This commit is contained in:
Zhaofeng Miao 2024-10-08 11:50:50 +08:00
parent ee29be58f1
commit 5a4b35e4b9
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/functions/src/cloud-functions/adaptive-crawler.ts
View File

@ -120,9 +120,12 @@ export class AdaptiveCrawlerHost extends RPCHost {
failed: {}, failed: {},
}); });
let urls: string[] = [];
if (useSitemap) { if (useSitemap) {
const urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages); urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages);
}
if (urls.length > 0) {
await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({ await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({
status: AdaptiveCrawlTaskStatus.PROCESSING, status: AdaptiveCrawlTaskStatus.PROCESSING,
statusText: `Processing 0/${urls.length}`, statusText: `Processing 0/${urls.length}`,