AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 07:29:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(sitemap): increase limit to 20

Browse Source
This commit is contained in:
Móricz Gergő 2025-01-23 11:29:49 +01:00
parent 1e28ba291e
commit 434a435a4b
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/sitemap.ts
View File

@ -21,7 +21,7 @@ export async function getLinksFromSitemap(
crawlId: string,
sitemapsHit: Set<string>,
): Promise<number> {
if (sitemapsHit.size >= 10) {
if (sitemapsHit.size >= 20) {
logger.warn("Sitemap limit hit, not hitting this one.");
return 0;
}