From 474e5a0543737a697cc78bcee4dccb06450db3b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 27 May 2025 15:39:31 +0200
Subject: [PATCH] fix(crawler): always set expiry on sitemap links in redis

---
 apps/api/src/scraper/WebScraper/crawler.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 34d1426a..56c46faf 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -334,6 +334,12 @@ export class WebCrawler {
         count++;
       }
 
+      await redisConnection.expire(
+        "sitemap:" + this.jobId + ":links",
+        3600,
+        "NX",
+      );
+
       return count;
     } catch (error) {
       if (error.message === "Sitemap fetch timeout") {