From a9773a24a36fa1bdbfe76405b5cc465ab44d614c Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 25 Sep 2024 19:27:02 -0400 Subject: [PATCH] Nick: increased timeout for chrome-cdp due to smart wait --- apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index 295c171c..dd3577b0 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -131,7 +131,9 @@ export async function scrapWithFireEngine({ const waitTotal = (actions ?? []).filter(x => x.type === "wait").reduce((a, x) => (x as { type: "wait"; milliseconds: number; }).milliseconds + a, 0); let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`); - while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitTotal) { + + // added 5 seconds to the timeout to account for 'smart wait' + while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitTotal + 5000) { await new Promise(resolve => setTimeout(resolve, 250)); // wait 0.25 seconds checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`); }