AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 17:29:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrapeURL/fire-engine): timeout calculation issues

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-17 13:17:55 +01:00
parent 7f57c868be
commit 47b968fede
1 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
View File

@ -123,8 +123,13 @@ export async function scrapeURLWithFireEngineChromeCDP(
// Include specified actions
...(meta.options.actions ?? []),
];
const totalWait = actions.reduce(
(a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),
0,
);
const timeout = timeToRun ?? 300000;
const timeout = (timeToRun ?? 300000) + totalWait;
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestChromeCDP = {
@ -146,18 +151,13 @@ export async function scrapeURLWithFireEngineChromeCDP(
// TODO: scrollXPaths
};
const totalWait = actions.reduce(
(a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),
0,
);
let response = await performFireEngineScrape(
meta.logger.child({
method: "scrapeURLWithFireEngineChromeCDP/callFireEngine",
request,
}),
request,
timeout + totalWait,
timeout,
);
specialtyScrapeCheck(
@ -213,7 +213,8 @@ export async function scrapeURLWithFireEnginePlaywright(
meta: Meta,
timeToRun: number | undefined,
): Promise<EngineScrapeResult> {
const timeout = timeToRun ?? 300000;
const totalWait = meta.options.waitFor;
const timeout = (timeToRun ?? 300000) + totalWait;
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestPlaywright = {
@ -237,7 +238,7 @@ export async function scrapeURLWithFireEnginePlaywright(
request,
}),
request,
timeout + meta.options.waitFor,
timeout,
);
specialtyScrapeCheck(