diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index ba4793d8..29a35e3b 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -271,7 +271,7 @@ export class WebCrawler { return urlsHandler(urls); } else { let filteredLinks = this.filterLinks( - [...new Set(urls)], + [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl) !== null), leftOfLimit, this.maxCrawledDepth, fromMap,