diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index 34d1426a..56c46faf 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -334,6 +334,12 @@ export class WebCrawler { count++; } + await redisConnection.expire( + "sitemap:" + this.jobId + ":links", + 3600, + "NX", + ); + return count; } catch (error) { if (error.message === "Sitemap fetch timeout") {