diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts index 7e0c6f8a..77db0db2 100644 --- a/apps/api/src/scraper/WebScraper/sitemap.ts +++ b/apps/api/src/scraper/WebScraper/sitemap.ts @@ -21,7 +21,7 @@ export async function getLinksFromSitemap( crawlId: string, sitemapsHit: Set, ): Promise { - if (sitemapsHit.size >= 10) { + if (sitemapsHit.size >= 20) { logger.warn("Sitemap limit hit, not hitting this one."); return 0; }