AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 20:16:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

added regex for links in sitemap

Browse Source
This commit is contained in:
rafaelsideguide 2024-07-23 09:07:23 -03:00
parent 252bc09ee2
commit a684bd3c5d
1 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -64,6 +64,14 @@ export class WebCrawler {
private filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] {
return sitemapLinks
.filter((link) => {
// if link is not a complete url, add the base url
link = link.trim();
const isCompleteUrl = new RegExp('^(?:[a-z+]+:)?//', 'i');
if (!isCompleteUrl.test(link)){
link = this.baseUrl + link;
}
const url = new URL(link);
const path = url.pathname;