AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 15:49:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawler): check for more strings

Browse Source
This commit is contained in:
Gergő Móricz 2025-03-05 18:48:05 +01:00
parent 4902d0ac64
commit c22c87ab0a
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -179,7 +179,7 @@ export class WebCrawler {
const isAllowed = this.ignoreRobotsTxt const isAllowed = this.ignoreRobotsTxt
? true ? true
: (this.robots.isAllowed(link, "FireCrawlAgent") ?? true); : ((this.robots.isAllowed(link, "FireCrawlAgent") || this.robots.isAllowed(link, "FirecrawlAgent")) ?? true);
// Check if the link is disallowed by robots.txt // Check if the link is disallowed by robots.txt
if (!isAllowed) { if (!isAllowed) {
this.logger.debug(`Link disallowed by robots.txt: ${link}`, { this.logger.debug(`Link disallowed by robots.txt: ${link}`, {
@ -453,7 +453,7 @@ export class WebCrawler {
return ignoreRobotsTxt return ignoreRobotsTxt
? true ? true
: this.robots : this.robots
? (this.robots.isAllowed(url, "FireCrawlAgent") ?? true) ? ((this.robots.isAllowed(url, "FireCrawlAgent") || this.robots.isAllowed(url, "FirecrawlAgent")) ?? true)
: true; : true;
} }