AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 06:29:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrapeURL/fire-engine): timeout handling

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-15 16:04:17 +01:00
parent afbd01299a
commit b4a5e1a6e9
1 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
View File

@ -121,6 +121,8 @@ export async function scrapeURLWithFireEngineChromeCDP(
...(meta.options.actions ?? []),
];
const timeout = (meta.options.timeout === undefined ? 300000 : Math.round(meta.options.timeout / 3));
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestChromeCDP = {
url: meta.url,
@ -136,7 +138,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
priority: meta.internalOptions.priority,
geolocation: meta.options.geolocation,
mobile: meta.options.mobile,
timeout: meta.options.timeout === undefined ? 300000 : meta.options.timeout, // TODO: better timeout logic
timeout, // TODO: better timeout logic
disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,
// TODO: scrollXPaths
};
@ -152,7 +154,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
request,
}),
request,
meta.options.timeout !== undefined ? defaultTimeout + totalWait : Infinity, // TODO: better timeout handling
timeout + totalWait,
);
specialtyScrapeCheck(
@ -207,6 +209,8 @@ export async function scrapeURLWithFireEngineChromeCDP(
export async function scrapeURLWithFireEnginePlaywright(
meta: Meta,
): Promise<EngineScrapeResult> {
const timeout = meta.options.timeout === undefined ? 300000 : Math.round(meta.options.timeout / 3);
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestPlaywright = {
url: meta.url,
@ -220,7 +224,7 @@ export async function scrapeURLWithFireEnginePlaywright(
wait: meta.options.waitFor,
geolocation: meta.options.geolocation,
timeout: meta.options.timeout === undefined ? 300000 : meta.options.timeout, // TODO: better timeout logic
timeout,
};
let response = await performFireEngineScrape(
@ -229,9 +233,7 @@ export async function scrapeURLWithFireEnginePlaywright(
request,
}),
request,
meta.options.timeout !== undefined
? defaultTimeout + meta.options.waitFor
: Infinity, // TODO: better timeout handling
timeout + meta.options.waitFor,
);
specialtyScrapeCheck(
@ -266,6 +268,8 @@ export async function scrapeURLWithFireEnginePlaywright(
export async function scrapeURLWithFireEngineTLSClient(
meta: Meta,
): Promise<EngineScrapeResult> {
const timeout = meta.options.timeout === undefined ? 30000 : Math.round(meta.options.timeout / 3);
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestTLSClient = {
url: meta.url,
@ -279,7 +283,7 @@ export async function scrapeURLWithFireEngineTLSClient(
geolocation: meta.options.geolocation,
disableJsDom: meta.internalOptions.v0DisableJsDom,
timeout: meta.options.timeout === undefined ? 30000 : meta.options.timeout, // TODO: better timeout logic
timeout,
};
let response = await performFireEngineScrape(
@ -288,7 +292,7 @@ export async function scrapeURLWithFireEngineTLSClient(
request,
}),
request,
meta.options.timeout !== undefined ? defaultTimeout : Infinity, // TODO: better timeout handling
timeout,
);
specialtyScrapeCheck(