AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 04:36:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(crawler): verify URL

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-22 23:30:19 +02:00
parent e690a6fda7
commit 8e3c2b2855
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -108,7 +108,12 @@ export class WebCrawler {
// Normalize the initial URL and the link to account for www and non-www versions // Normalize the initial URL and the link to account for www and non-www versions
const normalizedInitialUrl = new URL(this.initialUrl); const normalizedInitialUrl = new URL(this.initialUrl);
const normalizedLink = new URL(link); let normalizedLink;
try {
normalizedLink = new URL(link);
} catch (_) {
return false;
}
const initialHostname = normalizedInitialUrl.hostname.replace(/^www\./, ''); const initialHostname = normalizedInitialUrl.hostname.replace(/^www\./, '');
const linkHostname = normalizedLink.hostname.replace(/^www\./, ''); const linkHostname = normalizedLink.hostname.replace(/^www\./, '');