AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 02:09:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(scrapeURL): better error for SSL failures (#1552)

Browse Source
This commit is contained in:
Gergő Móricz 2025-05-14 23:34:59 +02:00 committed by GitHub
parent 06189b9646
commit 3db2294b97
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 19 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
View File

@ -7,6 +7,7 @@ import {
ActionError, ActionError,
EngineError, EngineError,
SiteError, SiteError,
SSLError,
UnsupportedFileError, UnsupportedFileError,
} from "../../error"; } from "../../error";
import { MockState } from "../../lib/mock"; import { MockState } from "../../lib/mock";
@ -169,7 +170,13 @@ export async function fireEngineCheckStatus(
typeof status.error === "string" && typeof status.error === "string" &&
status.error.includes("Chrome error: ") status.error.includes("Chrome error: ")
) { ) {
throw new SiteError(status.error.split("Chrome error: ")[1]); const code = status.error.split("Chrome error: ")[1];
if (code.includes("ERR_CERT_") || code.includes("ERR_SSL_") || code.includes("ERR_BAD_SSL_")) {
throw new SSLError();
} else {
throw new SiteError(code);
}
} else if ( } else if (
typeof status.error === "string" && typeof status.error === "string" &&
status.error.includes("File size exceeds") status.error.includes("File size exceeds")

6
apps/api/src/scraper/scrapeURL/error.ts
View File

@ -49,6 +49,12 @@ export class RemoveFeatureError extends Error {
} }
} }
export class SSLError extends Error {
constructor() {
super("An SSL error occurred while scraping the URL. If you're not inputting any sensitive data, try scraping with `skipTlsVerification: true`.");
}
}
export class SiteError extends Error { export class SiteError extends Error {
public code: string; public code: string;
constructor(code: string) { constructor(code: string) {

5
apps/api/src/scraper/scrapeURL/index.ts
View File

@ -21,6 +21,7 @@ import {
SiteError, SiteError,
TimeoutError, TimeoutError,
UnsupportedFileError, UnsupportedFileError,
SSLError,
} from "./error"; } from "./error";
import { executeTransformers } from "./transformers"; import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract"; import { LLMRefusalError } from "./transformers/llmExtract";
@ -323,6 +324,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
throw error; throw error;
} else if (error instanceof SiteError) { } else if (error instanceof SiteError) {
throw error; throw error;
} else if (error instanceof SSLError) {
throw error;
} else if (error instanceof ActionError) { } else if (error instanceof ActionError) {
throw error; throw error;
} else if (error instanceof UnsupportedFileError) { } else if (error instanceof UnsupportedFileError) {
@ -470,6 +473,8 @@ export async function scrapeURL(
// TODO: results? // TODO: results?
} else if (error instanceof SiteError) { } else if (error instanceof SiteError) {
meta.logger.warn("scrapeURL: Site failed to load in browser", { error }); meta.logger.warn("scrapeURL: Site failed to load in browser", { error });
} else if (error instanceof SSLError) {
meta.logger.warn("scrapeURL: SSL error", { error });
} else if (error instanceof ActionError) { } else if (error instanceof ActionError) {
meta.logger.warn("scrapeURL: Action(s) failed to complete", { error }); meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });
} else if (error instanceof UnsupportedFileError) { } else if (error instanceof UnsupportedFileError) {