diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 19bb2d81..67d66742 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -70,6 +70,14 @@ export const actionsSchema = z.array(z.union([ type: z.literal("screenshot"), fullPage: z.boolean().default(false), }), + z.object({ + type: z.literal("typeText"), + text: z.string(), + }), + z.object({ + type: z.literal("pressKey"), + key: z.string(), + }), ])); export const scrapeOptions = z.object({ diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts index cb8e5c56..1186583a 100644 --- a/apps/api/src/lib/entities.ts +++ b/apps/api/src/lib/entities.ts @@ -19,6 +19,12 @@ export type Action = { } | { type: "screenshot", fullPage?: boolean, +} | { + type: "typeText", + text: string, +} | { + type: "pressKey", + key: string, }; export type PageOptions = { diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index deca5498..ba747d07 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -21,6 +21,9 @@ import * as Sentry from "@sentry/node"; export async function scrapWithFireEngine({ url, actions, + waitFor = 0, + screenshot = false, + fullPageScreenshot = false, pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false }, fireEngineOptions = {}, headers, @@ -30,6 +33,9 @@ export async function scrapWithFireEngine({ }: { url: string; actions?: Action[]; + waitFor?: number; + screenshot?: boolean; + fullPageScreenshot?: boolean; pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean }; fireEngineOptions?: FireEngineOptions; headers?: Record; @@ -50,7 +56,10 @@ export async function scrapWithFireEngine({ try { const reqParams = await generateRequestParams(url); + let waitParam = reqParams["params"]?.wait ?? waitFor; let engineParam = reqParams["params"]?.engine ?? reqParams["params"]?.fireEngineOptions?.engine ?? fireEngineOptions?.engine ?? "chrome-cdp"; + let screenshotParam = reqParams["params"]?.screenshot ?? screenshot; + let fullPageScreenshotParam = reqParams["params"]?.fullPageScreenshot ?? fullPageScreenshot; let fireEngineOptionsParam : FireEngineOptions = reqParams["params"]?.fireEngineOptions ?? fireEngineOptions; @@ -95,6 +104,9 @@ export async function scrapWithFireEngine({ { url: url, headers: headers, + wait: waitParam, + screenshot: screenshotParam, + fullPageScreenshot: fullPageScreenshotParam, disableJsDom: pageOptions?.disableJsDom ?? false, priority, engine, diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 0a3adf5c..93fb0ce7 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -204,17 +204,23 @@ export async function scrapSingleUrl( if (process.env.FIRE_ENGINE_BETA_URL) { const response = await scrapWithFireEngine({ url, - actions: [ - ...(pageOptions.waitFor ? [{ - type: "wait" as const, - milliseconds: pageOptions.waitFor, - }] : []), - ...((pageOptions.screenshot || pageOptions.fullPageScreenshot) ? [{ - type: "screenshot" as const, - fullPage: !!pageOptions.fullPageScreenshot, - }] : []), - ...(pageOptions.actions ?? []), - ], + ...(engine === "chrome-cdp" ? ({ + actions: [ + ...(pageOptions.waitFor ? [{ + type: "wait" as const, + milliseconds: pageOptions.waitFor, + }] : []), + ...((pageOptions.screenshot || pageOptions.fullPageScreenshot) ? [{ + type: "screenshot" as const, + fullPage: !!pageOptions.fullPageScreenshot, + }] : []), + ...(pageOptions.actions ?? []), + ], + }) : ({ + waitFor: pageOptions.waitFor, + screenshot: pageOptions.screenshot, + fullPageScreenshot: pageOptions.fullPageScreenshot, + })), pageOptions: pageOptions, headers: pageOptions.headers, fireEngineOptions: { diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 4c30d5fd..23241a5d 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -93,7 +93,13 @@ export type Action = { } | { type: "screenshot", fullPage?: boolean, -}; +} | { + type: "typeText", + text: string, +} | { + type: "pressKey", + key: string, +};; export interface ScrapeParams extends CrawlScrapeOptions { extract?: {