diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 14d42025..9585175e 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -116,6 +116,7 @@ export const scrapeOptions = z.object({ timeout: z.number().int().positive().finite().safe().default(30000), waitFor: z.number().int().nonnegative().finite().safe().default(0), extract: extractOptions.optional(), + mobile: z.boolean().default(false), parsePDF: z.boolean().default(true), actions: actionsSchema.optional(), // New @@ -471,6 +472,7 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions { geolocation: x.location ?? x.geolocation, skipTlsVerification: x.skipTlsVerification, removeBase64Images: x.removeBase64Images, + mobile: x.mobile, }; } diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts index 87aa44ac..2c8f376d 100644 --- a/apps/api/src/lib/entities.ts +++ b/apps/api/src/lib/entities.ts @@ -59,6 +59,7 @@ export type PageOptions = { }; skipTlsVerification?: boolean; removeBase64Images?: boolean; + mobile?: boolean; }; export type ExtractorOptions = { diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index 615e49b4..79b312d5 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -596,6 +596,7 @@ export class WebScraperDataProvider { geolocation: options.pageOptions?.geolocation ?? undefined, skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false, removeBase64Images: options.pageOptions?.removeBase64Images ?? true, + mobile: options.pageOptions?.mobile ?? false, }; this.extractorOptions = options.extractorOptions ?? { mode: "markdown" }; this.replaceAllPathsWithAbsolutePaths = diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts index 8a37457c..7616774a 100644 --- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts +++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts @@ -28,7 +28,7 @@ export async function scrapWithFireEngine({ waitFor = 0, screenshot = false, fullPageScreenshot = false, - pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true }, + pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, removeBase64Images: true, mobile: false }, fireEngineOptions = {}, headers, options, @@ -40,7 +40,7 @@ export async function scrapWithFireEngine({ waitFor?: number; screenshot?: boolean; fullPageScreenshot?: boolean; - pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean }; + pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, removeBase64Images?: boolean, mobile?: boolean }; fireEngineOptions?: FireEngineOptions; headers?: Record; options?: any; @@ -115,6 +115,7 @@ export async function scrapWithFireEngine({ priority, engine, instantReturn: true, + mobile: pageOptions?.mobile ?? false, ...fireEngineOptionsParam, atsv: pageOptions?.atsv ?? false, scrollXPaths: pageOptions?.scrollXPaths ?? [], diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index 6f5ec541..38ec74f0 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -161,6 +161,7 @@ export async function scrapSingleUrl( geolocation: pageOptions.geolocation ?? undefined, skipTlsVerification: pageOptions.skipTlsVerification ?? false, removeBase64Images: pageOptions.removeBase64Images ?? true, + mobile: pageOptions.mobile ?? false, } if (extractorOptions) {