AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 20:16:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(sitemap): change sitemap logging

Browse Source
This commit is contained in:
Móricz Gergő 2025-01-23 12:06:50 +01:00
parent d3518e85a8
commit 6557365149
2 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -632,6 +632,10 @@ export class WebCrawler {
}
}
if (this.sitemapsHit.size >= 20) {
this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });
}
return sitemapCount;
}
}

1
apps/api/src/scraper/WebScraper/sitemap.ts
View File

@ -22,7 +22,6 @@ export async function getLinksFromSitemap(
sitemapsHit: Set<string>,
): Promise<number> {
if (sitemapsHit.size >= 20) {
logger.warn("Sitemap limit hit, not hitting this one.");
return 0;
}