From c22c87ab0a9f3a0ade34e8957ca23c13a99c11d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Wed, 5 Mar 2025 18:48:05 +0100
Subject: [PATCH] fix(crawler): check for more strings

---
 apps/api/src/scraper/WebScraper/crawler.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 5e91b46e..126520e2 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -179,7 +179,7 @@ export class WebCrawler {
 
         const isAllowed = this.ignoreRobotsTxt
           ? true
-          : (this.robots.isAllowed(link, "FireCrawlAgent") ?? true);
+          : ((this.robots.isAllowed(link, "FireCrawlAgent") || this.robots.isAllowed(link, "FirecrawlAgent")) ?? true);
         // Check if the link is disallowed by robots.txt
         if (!isAllowed) {
           this.logger.debug(`Link disallowed by robots.txt: ${link}`, {
@@ -453,7 +453,7 @@ export class WebCrawler {
     return ignoreRobotsTxt
       ? true
       : this.robots
-        ? (this.robots.isAllowed(url, "FireCrawlAgent") ?? true)
+        ? ((this.robots.isAllowed(url, "FireCrawlAgent") || this.robots.isAllowed(url, "FirecrawlAgent")) ?? true)
         : true;
   }