AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 15:00:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(scrapeURL/fire-engine): adjust timeout for waitFor/wait actions

Browse Source
This commit is contained in:
Móricz Gergő 2024-11-11 11:43:59 +01:00
parent e97864b806
commit f097cddf23
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
View File

@ -8,7 +8,7 @@ import * as Sentry from "@sentry/node";
import { Action } from "../../../../lib/entities"; import { Action } from "../../../../lib/entities";
import { specialtyScrapeCheck } from "../utils/specialtyHandler"; import { specialtyScrapeCheck } from "../utils/specialtyHandler";
const defaultTimeout = 10000; export const defaultTimeout = 10000;
// This function does not take `Meta` on purpose. It may not access any // This function does not take `Meta` on purpose. It may not access any
// meta values to construct the request -- that must be done by the // meta values to construct the request -- that must be done by the
@ -90,9 +90,12 @@ export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise<Engi
// TODO: scrollXPaths // TODO: scrollXPaths
}; };
const totalWait = actions.reduce((a,x) => x.type === "wait" ? (x.milliseconds ?? 1000) + a : a, 0);
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }), meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),
request, request,
defaultTimeout + totalWait,
); );
specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck" }), response.responseHeaders); specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck" }), response.responseHeaders);
@ -142,6 +145,7 @@ export async function scrapeURLWithFireEnginePlaywright(meta: Meta): Promise<Eng
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }), meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),
request, request,
defaultTimeout + meta.options.waitFor
); );
specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEnginePlaywright/specialtyScrapeCheck" }), response.responseHeaders); specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEnginePlaywright/specialtyScrapeCheck" }), response.responseHeaders);