From f097cddf232536a0c970d922d4df79fd116c6026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B3ricz=20Gerg=C5=91?= Date: Mon, 11 Nov 2024 11:43:59 +0100 Subject: [PATCH] feat(scrapeURL/fire-engine): adjust timeout for waitFor/wait actions --- apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts index baf61a01..abf92fdb 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts @@ -8,7 +8,7 @@ import * as Sentry from "@sentry/node"; import { Action } from "../../../../lib/entities"; import { specialtyScrapeCheck } from "../utils/specialtyHandler"; -const defaultTimeout = 10000; +export const defaultTimeout = 10000; // This function does not take `Meta` on purpose. It may not access any // meta values to construct the request -- that must be done by the @@ -90,9 +90,12 @@ export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise x.type === "wait" ? (x.milliseconds ?? 1000) + a : a, 0); + let response = await performFireEngineScrape( meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }), request, + defaultTimeout + totalWait, ); specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck" }), response.responseHeaders); @@ -142,6 +145,7 @@ export async function scrapeURLWithFireEnginePlaywright(meta: Meta): Promise