AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 13:59:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] added second scrapeURLWithFireEngine (#1494)

Browse Source
This commit is contained in:
Rafael Miller 2025-04-23 11:36:36 -07:00 committed by GitHub
parent 9435c800c0
commit 37dabce1ed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 21 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
apps/api/src/scraper/scrapeURL/engines/index.ts
View File

@ -13,6 +13,7 @@ import { scrapeCache } from "./cache";
export type Engine =
| "fire-engine;chrome-cdp"
| "fire-engine(retry);chrome-cdp"
| "fire-engine;playwright"
| "fire-engine;tlsclient"
| "playwright"
@ -36,6 +37,7 @@ export const engines: Engine[] = [
...(useFireEngine
? [
"fire-engine;chrome-cdp" as const,
"fire-engine(retry);chrome-cdp" as const,
"fire-engine;playwright" as const,
"fire-engine;tlsclient" as const,
]
@ -109,6 +111,7 @@ const engineHandlers: {
} = {
cache: scrapeCache,
"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,
"fire-engine(retry);chrome-cdp": scrapeURLWithFireEngineChromeCDP,
"fire-engine;playwright": scrapeURLWithFireEnginePlaywright,
"fire-engine;tlsclient": scrapeURLWithFireEngineTLSClient,
playwright: scrapeURLWithPlaywright,
@ -161,6 +164,23 @@ export const engineOptions: {
},
quality: 50,
},
"fire-engine(retry);chrome-cdp": {
features: {
actions: true,
waitFor: true, // through actions transform
screenshot: true, // through actions transform
"screenshot@fullScreen": true, // through actions transform
pdf: false,
docx: false,
atsv: false,
location: true,
mobile: true,
skipTlsVerification: true,
useFastMode: false,
stealthProxy: true,
},
quality: 45,
},
"fire-engine;playwright": {
features: {
actions: false,
@ -273,7 +293,7 @@ export function buildFallbackList(meta: Meta): {
...engines,
// enable fire-engine in self-hosted testing environment when mocks are supplied
...((!useFireEngine && meta.mock !== null) ? ["fire-engine;chrome-cdp", "fire-engine;playwright", "fire-engine;tlsclient"] as Engine[] : [])
...((!useFireEngine && meta.mock !== null) ? ["fire-engine;chrome-cdp", "fire-engine(retry);chrome-cdp", "fire-engine;playwright", "fire-engine;tlsclient"] as Engine[] : [])
];
if (meta.internalOptions.useCache !== true) {