diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index c4496ce0..b7d6fc12 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -518,12 +518,13 @@ export async function scrapSingleUrl( html = attempt.html ?? ""; rawHtml = attempt.rawHtml ?? ""; screenshot = attempt.screenshot ?? ""; + if (attempt.pageStatusCode) { pageStatusCode = attempt.pageStatusCode; } - if (attempt.pageError && attempt.pageStatusCode != 200) { + if (attempt.pageError && attempt.pageStatusCode >= 400) { pageError = attempt.pageError; - } else { + } else if (attempt.pageStatusCode < 400) { pageError = undefined; }