diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts index fadce779..4e64f74e 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts @@ -7,6 +7,7 @@ import { ActionError, EngineError, SiteError, + SSLError, UnsupportedFileError, } from "../../error"; import { MockState } from "../../lib/mock"; @@ -169,7 +170,13 @@ export async function fireEngineCheckStatus( typeof status.error === "string" && status.error.includes("Chrome error: ") ) { - throw new SiteError(status.error.split("Chrome error: ")[1]); + const code = status.error.split("Chrome error: ")[1]; + + if (code.includes("ERR_CERT_") || code.includes("ERR_SSL_") || code.includes("ERR_BAD_SSL_")) { + throw new SSLError(); + } else { + throw new SiteError(code); + } } else if ( typeof status.error === "string" && status.error.includes("File size exceeds") diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts index ff445f8d..29b8970e 100644 --- a/apps/api/src/scraper/scrapeURL/error.ts +++ b/apps/api/src/scraper/scrapeURL/error.ts @@ -49,6 +49,12 @@ export class RemoveFeatureError extends Error { } } +export class SSLError extends Error { + constructor() { + super("An SSL error occurred while scraping the URL. If you're not inputting any sensitive data, try scraping with `skipTlsVerification: true`."); + } +} + export class SiteError extends Error { public code: string; constructor(code: string) { diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index 7dfc821f..baf29752 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -21,6 +21,7 @@ import { SiteError, TimeoutError, UnsupportedFileError, + SSLError, } from "./error"; import { executeTransformers } from "./transformers"; import { LLMRefusalError } from "./transformers/llmExtract"; @@ -323,6 +324,8 @@ async function scrapeURLLoop(meta: Meta): Promise { throw error; } else if (error instanceof SiteError) { throw error; + } else if (error instanceof SSLError) { + throw error; } else if (error instanceof ActionError) { throw error; } else if (error instanceof UnsupportedFileError) { @@ -470,6 +473,8 @@ export async function scrapeURL( // TODO: results? } else if (error instanceof SiteError) { meta.logger.warn("scrapeURL: Site failed to load in browser", { error }); + } else if (error instanceof SSLError) { + meta.logger.warn("scrapeURL: SSL error", { error }); } else if (error instanceof ActionError) { meta.logger.warn("scrapeURL: Action(s) failed to complete", { error }); } else if (error instanceof UnsupportedFileError) {