AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 17:59:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawler): sitemaps poisoning crawls with unrelated links (#1334)

Browse Source
This commit is contained in:
Gergő Móricz 2025-03-13 19:31:05 +01:00 committed by GitHub
parent da6b7505f3
commit c7ae50d2d0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -271,7 +271,7 @@ export class WebCrawler {
return urlsHandler(urls);
} else {
let filteredLinks = this.filterLinks(
[...new Set(urls)],
[...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl) !== null),
leftOfLimit,
this.maxCrawledDepth,
fromMap,