From 677faa27f3e3da4fad6c59300609800110c3a8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Mon, 23 Sep 2024 18:47:07 +0200 Subject: [PATCH] fix(WebScraper): explicitly ignore 404s --- apps/api/src/scraper/WebScraper/single_url.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index d0b7269f..75c5da2c 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -423,7 +423,7 @@ export async function scrapSingleUrl( // NOTE: This exception for status codes may only work with fire-engine. In lieu of better error management, // it's the best we can do. - mogery - if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code "))) { + if (!text && !Object.values(errors).some(x => x.startsWith("Request failed with status code ") || x === "NOT FOUND")) { throw new Error(`All scraping methods failed for URL: ${urlToScrap}`); }