diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index 2f7efa47..c564c471 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -591,7 +591,8 @@ export class WebScraperDataProvider { screenshot: options.pageOptions?.screenshot ?? false, useFastMode: options.pageOptions?.useFastMode ?? false, disableJsDom: options.pageOptions?.disableJsDom ?? false, - atsv: options.pageOptions?.atsv ?? false + atsv: options.pageOptions?.atsv ?? false, + actions: options.pageOptions?.actions ?? undefined, }; this.extractorOptions = options.extractorOptions ?? { mode: "markdown" }; this.replaceAllPathsWithAbsolutePaths = diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index 41c79add..deca5498 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -116,7 +116,7 @@ export async function scrapWithFireEngine({ ); }); - const waitTotal = (actions ?? []).filter(x => x.type === "wait").reduce((a, x) => x.milliseconds + a, 0); + const waitTotal = (actions ?? []).filter(x => x.type === "wait").reduce((a, x) => (x as { type: "wait"; milliseconds: number; }).milliseconds + a, 0); let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`); while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitTotal) { @@ -141,7 +141,7 @@ export async function scrapWithFireEngine({ if (checkStatusResponse.status !== 200 || checkStatusResponse.data.error) { Logger.debug( - `⛏️ Fire-Engine (${engine}): Failed to fetch url: ${url} \t status: ${checkStatusResponse.status}` + `⛏️ Fire-Engine (${engine}): Failed to fetch url: ${url} \t status: ${checkStatusResponse.status}\t ${checkStatusResponse.data.error}` ); logParams.error_message = checkStatusResponse.data?.pageError ?? checkStatusResponse.data?.error; diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 224a8db3..d61fb828 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -153,7 +153,8 @@ export async function scrapSingleUrl( onlyIncludeTags: pageOptions.onlyIncludeTags ?? [], useFastMode: pageOptions.useFastMode ?? false, disableJsDom: pageOptions.disableJsDom ?? false, - atsv: pageOptions.atsv ?? false + atsv: pageOptions.atsv ?? false, + actions: pageOptions.actions ?? undefined, } if (extractorOptions) {