AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 06:29:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrapeURL/fire-engine): timeout handling

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-15 16:04:17 +01:00
parent afbd01299a
commit b4a5e1a6e9
1 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
View File

@ -120,6 +120,8 @@ export async function scrapeURLWithFireEngineChromeCDP(
// Include specified actions // Include specified actions
...(meta.options.actions ?? []), ...(meta.options.actions ?? []),
]; ];
const timeout = (meta.options.timeout === undefined ? 300000 : Math.round(meta.options.timeout / 3));
const request: FireEngineScrapeRequestCommon & const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestChromeCDP = { FireEngineScrapeRequestChromeCDP = {
@ -136,7 +138,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
priority: meta.internalOptions.priority, priority: meta.internalOptions.priority,
geolocation: meta.options.geolocation, geolocation: meta.options.geolocation,
mobile: meta.options.mobile, mobile: meta.options.mobile,
timeout: meta.options.timeout === undefined ? 300000 : meta.options.timeout, // TODO: better timeout logic timeout, // TODO: better timeout logic
disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache, disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,
// TODO: scrollXPaths // TODO: scrollXPaths
}; };
@ -152,7 +154,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
request, request,
}), }),
request, request,
meta.options.timeout !== undefined ? defaultTimeout + totalWait : Infinity, // TODO: better timeout handling timeout + totalWait,
); );
specialtyScrapeCheck( specialtyScrapeCheck(
@ -207,6 +209,8 @@ export async function scrapeURLWithFireEngineChromeCDP(
export async function scrapeURLWithFireEnginePlaywright( export async function scrapeURLWithFireEnginePlaywright(
meta: Meta, meta: Meta,
): Promise<EngineScrapeResult> { ): Promise<EngineScrapeResult> {
const timeout = meta.options.timeout === undefined ? 300000 : Math.round(meta.options.timeout / 3);
const request: FireEngineScrapeRequestCommon & const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestPlaywright = { FireEngineScrapeRequestPlaywright = {
url: meta.url, url: meta.url,
@ -220,7 +224,7 @@ export async function scrapeURLWithFireEnginePlaywright(
wait: meta.options.waitFor, wait: meta.options.waitFor,
geolocation: meta.options.geolocation, geolocation: meta.options.geolocation,
timeout: meta.options.timeout === undefined ? 300000 : meta.options.timeout, // TODO: better timeout logic timeout,
}; };
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
@ -229,9 +233,7 @@ export async function scrapeURLWithFireEnginePlaywright(
request, request,
}), }),
request, request,
meta.options.timeout !== undefined timeout + meta.options.waitFor,
? defaultTimeout + meta.options.waitFor
: Infinity, // TODO: better timeout handling
); );
specialtyScrapeCheck( specialtyScrapeCheck(
@ -266,6 +268,8 @@ export async function scrapeURLWithFireEnginePlaywright(
export async function scrapeURLWithFireEngineTLSClient( export async function scrapeURLWithFireEngineTLSClient(
meta: Meta, meta: Meta,
): Promise<EngineScrapeResult> { ): Promise<EngineScrapeResult> {
const timeout = meta.options.timeout === undefined ? 30000 : Math.round(meta.options.timeout / 3);
const request: FireEngineScrapeRequestCommon & const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestTLSClient = { FireEngineScrapeRequestTLSClient = {
url: meta.url, url: meta.url,
@ -279,7 +283,7 @@ export async function scrapeURLWithFireEngineTLSClient(
geolocation: meta.options.geolocation, geolocation: meta.options.geolocation,
disableJsDom: meta.internalOptions.v0DisableJsDom, disableJsDom: meta.internalOptions.v0DisableJsDom,
timeout: meta.options.timeout === undefined ? 30000 : meta.options.timeout, // TODO: better timeout logic timeout,
}; };
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
@ -288,7 +292,7 @@ export async function scrapeURLWithFireEngineTLSClient(
request, request,
}), }),
request, request,
meta.options.timeout !== undefined ? defaultTimeout : Infinity, // TODO: better timeout handling timeout,
); );
specialtyScrapeCheck( specialtyScrapeCheck(