AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 15:15:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrapeURL/fire-engine): timeout calculation issues

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-17 13:17:55 +01:00
parent 7f57c868be
commit 47b968fede
1 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
View File

@ -124,7 +124,12 @@ export async function scrapeURLWithFireEngineChromeCDP(
...(meta.options.actions ?? []), ...(meta.options.actions ?? []),
]; ];
const timeout = timeToRun ?? 300000; const totalWait = actions.reduce(
(a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),
0,
);
const timeout = (timeToRun ?? 300000) + totalWait;
const request: FireEngineScrapeRequestCommon & const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestChromeCDP = { FireEngineScrapeRequestChromeCDP = {
@ -146,18 +151,13 @@ export async function scrapeURLWithFireEngineChromeCDP(
// TODO: scrollXPaths // TODO: scrollXPaths
}; };
const totalWait = actions.reduce(
(a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),
0,
);
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
meta.logger.child({ meta.logger.child({
method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", method: "scrapeURLWithFireEngineChromeCDP/callFireEngine",
request, request,
}), }),
request, request,
timeout + totalWait, timeout,
); );
specialtyScrapeCheck( specialtyScrapeCheck(
@ -213,7 +213,8 @@ export async function scrapeURLWithFireEnginePlaywright(
meta: Meta, meta: Meta,
timeToRun: number | undefined, timeToRun: number | undefined,
): Promise<EngineScrapeResult> { ): Promise<EngineScrapeResult> {
const timeout = timeToRun ?? 300000; const totalWait = meta.options.waitFor;
const timeout = (timeToRun ?? 300000) + totalWait;
const request: FireEngineScrapeRequestCommon & const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestPlaywright = { FireEngineScrapeRequestPlaywright = {
@ -237,7 +238,7 @@ export async function scrapeURLWithFireEnginePlaywright(
request, request,
}), }),
request, request,
timeout + meta.options.waitFor, timeout,
); );
specialtyScrapeCheck( specialtyScrapeCheck(