diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index 643b274e..9ceaf434 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -632,6 +632,10 @@ export class WebCrawler { } } + if (this.sitemapsHit.size >= 20) { + this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl }); + } + return sitemapCount; } } diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts index 77db0db2..d55efeea 100644 --- a/apps/api/src/scraper/WebScraper/sitemap.ts +++ b/apps/api/src/scraper/WebScraper/sitemap.ts @@ -22,7 +22,6 @@ export async function getLinksFromSitemap( sitemapsHit: Set, ): Promise { if (sitemapsHit.size >= 20) { - logger.warn("Sitemap limit hit, not hitting this one."); return 0; }