From 42980c899d8b9cb13c63e841665b42b7c7c3bb61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Thu, 28 Nov 2024 18:41:48 +0100
Subject: [PATCH] fix(scrapeURL/fire-engine): fast fail on chrome error

---
 .../scrapeURL/engines/fire-engine/checkStatus.ts | 16 ++++++++++------
 .../scrapeURL/engines/fire-engine/index.ts       |  4 ++--
 apps/api/src/scraper/scrapeURL/error.ts          | 10 +++++++++-
 apps/api/src/scraper/scrapeURL/index.ts          |  6 +++++-
 4 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 2c67e196..53c19f3c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -3,7 +3,7 @@ import * as Sentry from "@sentry/node";
 import { z } from "zod";
 
 import { robustFetch } from "../../lib/fetch";
-import { EngineError } from "../../error";
+import { EngineError, SiteError } from "../../error";
 
 const successSchema = z.object({
     jobId: z.string(),
@@ -90,11 +90,15 @@ export async function fireEngineCheckStatus(logger: Logger, jobId: string): Prom
         throw new StillProcessingError(jobId);
     } else if (failedParse.success) {
         logger.debug("Scrape job failed", { status, jobId });
-        throw new EngineError("Scrape job failed", {
-            cause: {
-                status, jobId
-            }
-        });
+        if (typeof status.error === "string" && status.error.includes("Chrome error: ")) {
+            throw new SiteError(status.error.split("Chrome error: ")[1]);
+        } else {
+            throw new EngineError("Scrape job failed", {
+                cause: {
+                    status, jobId
+                }
+            });
+        }
     } else {
         logger.debug("Check status returned response not matched by any schema", { status, jobId });
         throw new Error("Check status returned response not matched by any schema", {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 17bb40f2..106803ac 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -3,7 +3,7 @@ import { Meta } from "../..";
 import { fireEngineScrape, FireEngineScrapeRequestChromeCDP, FireEngineScrapeRequestCommon, FireEngineScrapeRequestPlaywright, FireEngineScrapeRequestTLSClient } from "./scrape";
 import { EngineScrapeResult } from "..";
 import { fireEngineCheckStatus, FireEngineCheckStatusSuccess, StillProcessingError } from "./checkStatus";
-import { EngineError, TimeoutError } from "../../error";
+import { EngineError, SiteError, TimeoutError } from "../../error";
 import * as Sentry from "@sentry/node";
 import { Action } from "../../../../lib/entities";
 import { specialtyScrapeCheck } from "../utils/specialtyHandler";
@@ -41,7 +41,7 @@ async function performFireEngineScrape<Engine extends FireEngineScrapeRequestChr
         } catch (error) {
             if (error instanceof StillProcessingError) {
                 // nop
-            } else if (error instanceof EngineError) {
+            } else if (error instanceof EngineError || error instanceof SiteError) {
                 logger.debug("Fire-engine scrape job failed.", { error, jobId: scrape.jobId });
                 throw error;
             } else {
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index c78c812d..3eb46033 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -31,4 +31,12 @@ export class AddFeatureError extends Error {
         super("New feature flags have been discovered: " + featureFlags.join(", "));
         this.featureFlags = featureFlags;
     }
-}
\ No newline at end of file
+}
+
+export class SiteError extends Error {
+    public code: string;
+    constructor(code: string) {
+        super("Specified URL is failing to load in the browser. Error code: " + code)
+        this.code = code;
+    }
+}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 7be8b67a..eaa84bf7 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -5,7 +5,7 @@ import { Document, ScrapeOptions } from "../../controllers/v1/types";
 import { logger } from "../../lib/logger";
 import { buildFallbackList, Engine, EngineScrapeResult, FeatureFlag, scrapeURLWithEngine } from "./engines";
 import { parseMarkdown } from "../../lib/html-to-markdown";
-import { AddFeatureError, EngineError, NoEnginesLeftError, TimeoutError } from "./error";
+import { AddFeatureError, EngineError, NoEnginesLeftError, SiteError, TimeoutError } from "./error";
 import { executeTransformers } from "./transformers";
 import { LLMRefusalError } from "./transformers/llmExtract";
 import { urlSpecificParams } from "./lib/urlSpecificParams";
@@ -227,6 +227,8 @@ async function scrapeURLLoop(
                 error.results = results;
                 meta.logger.warn("LLM refusal encountered", { error });
                 throw error;
+            } else if (error instanceof SiteError) {
+                throw error;
             } else {
                 Sentry.captureException(error);
                 meta.logger.info("An unexpected error happened while scraping with " + engine + ".", { error });
@@ -306,6 +308,8 @@ export async function scrapeURL(
         } else if (error instanceof Error && error.message.includes("Invalid schema for response_format")) { // TODO: seperate into custom error
             meta.logger.warn("scrapeURL: LLM schema error", { error });
             // TODO: results?
+        } else if (error instanceof SiteError) {
+            meta.logger.warn("scrapeURL: Site failed to load in browser", { error });
         } else {
             Sentry.captureException(error);
             meta.logger.error("scrapeURL: Unexpected error happened", { error });