AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 22:18:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(scrapeURL): better error for SSL failures (#1552)

Browse Source
This commit is contained in:
Gergő Móricz 2025-05-14 23:34:59 +02:00 committed by GitHub
parent 06189b9646
commit 3db2294b97
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 19 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
View File

@ -7,6 +7,7 @@ import {
ActionError,
EngineError,
SiteError,
SSLError,
UnsupportedFileError,
} from "../../error";
import { MockState } from "../../lib/mock";
@ -169,7 +170,13 @@ export async function fireEngineCheckStatus(
typeof status.error === "string" &&
status.error.includes("Chrome error: ")
) {
throw new SiteError(status.error.split("Chrome error: ")[1]);
const code = status.error.split("Chrome error: ")[1];
if (code.includes("ERR_CERT_") || code.includes("ERR_SSL_") || code.includes("ERR_BAD_SSL_")) {
throw new SSLError();
} else {
throw new SiteError(code);
}
} else if (
typeof status.error === "string" &&
status.error.includes("File size exceeds")

6
apps/api/src/scraper/scrapeURL/error.ts
View File

@ -49,6 +49,12 @@ export class RemoveFeatureError extends Error {
}
}
export class SSLError extends Error {
constructor() {
super("An SSL error occurred while scraping the URL. If you're not inputting any sensitive data, try scraping with `skipTlsVerification: true`.");
}
}
export class SiteError extends Error {
public code: string;
constructor(code: string) {

5
apps/api/src/scraper/scrapeURL/index.ts
View File

@ -21,6 +21,7 @@ import {
SiteError,
TimeoutError,
UnsupportedFileError,
SSLError,
} from "./error";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
@ -323,6 +324,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
throw error;
} else if (error instanceof SiteError) {
throw error;
} else if (error instanceof SSLError) {
throw error;
} else if (error instanceof ActionError) {
throw error;
} else if (error instanceof UnsupportedFileError) {
@ -470,6 +473,8 @@ export async function scrapeURL(
// TODO: results?
} else if (error instanceof SiteError) {
meta.logger.warn("scrapeURL: Site failed to load in browser", { error });
} else if (error instanceof SSLError) {
meta.logger.warn("scrapeURL: SSL error", { error });
} else if (error instanceof ActionError) {
meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });
} else if (error instanceof UnsupportedFileError) {