AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-06-23 01:37:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(sitemap): increase limit

Browse Source
This commit is contained in:
Móricz Gergő 2025-01-23 09:21:38 +01:00
parent bee2b2873e
commit 1e28ba291e
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/sitemap.ts
View File

@ -21,8 +21,8 @@ export async function getLinksFromSitemap(
crawlId: string,
sitemapsHit: Set<string>,
): Promise<number> {
if (sitemapsHit.size >= 5) {
logger.warn("Sitemap limit of 5 hit, not hitting this one.");
if (sitemapsHit.size >= 10) {
logger.warn("Sitemap limit hit, not hitting this one.");
return 0;
}