AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 08:06:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update apps/api/src/scraper/WebScraper/crawler.ts

Browse Source 
no need for regex

Co-authored-by: Gergő Móricz <mo.geryy@gmail.com>
This commit is contained in:
Rafael Miller 2024-07-24 08:33:00 -03:00 committed by GitHub
parent a684bd3c5d
commit 5e728c1a4d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -64,15 +64,7 @@ export class WebCrawler {
private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] { private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
return sitemapLinks return sitemapLinks
.filter((link) => { .filter((link) => {
const url = new URL(link.trim(), this.baseUrl);
// if link is not a complete url, add the base url
link = link.trim();
const isCompleteUrl = new RegExp('^(?:[a-z+]+:)?//', 'i');
if (!isCompleteUrl.test(link)){
link = this.baseUrl + link;
}
const url = new URL(link);
const path = url.pathname; const path = url.pathname;
const depth = getURLDepth(url.toString()); const depth = getURLDepth(url.toString());