AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 20:19:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawler): sitemaps poisoning crawls with unrelated links (#1334)

Browse Source
This commit is contained in:
Gergő Móricz 2025-03-13 19:31:05 +01:00 committed by GitHub
parent da6b7505f3
commit c7ae50d2d0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -271,7 +271,7 @@ export class WebCrawler {
return urlsHandler(urls); return urlsHandler(urls);
} else { } else {
let filteredLinks = this.filterLinks( let filteredLinks = this.filterLinks(
[...new Set(urls)], [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl) !== null),
leftOfLimit, leftOfLimit,
this.maxCrawledDepth, this.maxCrawledDepth,
fromMap, fromMap,