AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 08:49:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: filter out invalid URLs from crawl links

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-21 20:49:25 +02:00
parent dae1408e66
commit 55009e51f5
1 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -267,9 +267,18 @@ export class WebCrawler {
public filterURL(href: string, url: string): string | null { public filterURL(href: string, url: string): string | null {
let fullUrl = href; let fullUrl = href;
if (!href.startsWith("http")) { if (!href.startsWith("http")) {
fullUrl = new URL(href, this.baseUrl).toString(); try {
fullUrl = new URL(href, this.baseUrl).toString();
} catch (_) {
return null;
}
}
let urlObj;
try {
urlObj = new URL(fullUrl);
} catch (_) {
return null;
} }
const urlObj = new URL(fullUrl);
const path = urlObj.pathname; const path = urlObj.pathname;
if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS