diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index 79e4bf18..af3a9d69 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -69,7 +69,13 @@ export class WebCrawler { public filterLinks(sitemapLinks: string[], limit: number, maxDepth: number): string[] { return sitemapLinks .filter((link) => { - const url = new URL(link.trim(), this.baseUrl); + let url: URL; + try { + url = new URL(link.trim(), this.baseUrl); + } catch (error) { + Logger.debug(`Error processing link: ${link} | Error: ${error.message}`); + return false; + } const path = url.pathname; const depth = getURLDepth(url.toString()); diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index e427f582..7c24fab4 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -73,7 +73,6 @@ export async function scrapWithFireEngine({ ); if (pageOptions?.useFastMode) { - console.log('using tlsclient') fireEngineOptionsParam.engine = "tlsclient"; engine = "tlsclient"; }