From 6557365149fcb3fdbd58b98663edfb014bd9c7aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=B3ricz=20Gerg=C5=91?= <mo.geryy@gmail.com>
Date: Thu, 23 Jan 2025 12:06:50 +0100
Subject: [PATCH] feat(sitemap): change sitemap logging

---
 apps/api/src/scraper/WebScraper/crawler.ts | 4 ++++
 apps/api/src/scraper/WebScraper/sitemap.ts | 1 -
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 643b274e..9ceaf434 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -632,6 +632,10 @@ export class WebCrawler {
       }
     }
 
+    if (this.sitemapsHit.size >= 20) {
+      this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });
+    }
+
     return sitemapCount;
   }
 }
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 77db0db2..d55efeea 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -22,7 +22,6 @@ export async function getLinksFromSitemap(
   sitemapsHit: Set<string>,
 ): Promise<number> {
   if (sitemapsHit.size >= 20) {
-    logger.warn("Sitemap limit hit, not hitting this one.");
     return 0;
   }