AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 20:39:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawler): also take the hostname of the base url when determining isInternalLink

Browse Source
This commit is contained in:
Móricz Gergő 2025-01-07 09:29:58 +01:00
parent b82cfa8540
commit 35d1d85978
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -397,8 +397,7 @@ export class WebCrawler {
private isInternalLink(link: string): boolean { private isInternalLink(link: string): boolean {
const urlObj = new URL(link, this.baseUrl); const urlObj = new URL(link, this.baseUrl);
const baseDomain = this.baseUrl const baseDomain = new URL(this.baseUrl).hostname
.replace(/^https?:\/\//, "")
.replace(/^www\./, "") .replace(/^www\./, "")
.trim(); .trim();
const linkDomain = urlObj.hostname.replace(/^www\./, "").trim(); const linkDomain = urlObj.hostname.replace(/^www\./, "").trim();