diff --git a/backend/functions/src/cloud-functions/crawler.ts b/backend/functions/src/cloud-functions/crawler.ts index aedb57e..f58f335 100644 --- a/backend/functions/src/cloud-functions/crawler.ts +++ b/backend/functions/src/cloud-functions/crawler.ts @@ -69,7 +69,7 @@ export class CrawlerHost extends RPCHost { // Potential privacy issue, dont cache if cookies are used return; } - if (options.injectFrameScripts?.length || options.injectPageScripts?.length) { + if (options.injectFrameScripts?.length || options.injectPageScripts?.length || options.viewport) { // Potentially mangeled content, dont cache if scripts are injected return; } @@ -725,6 +725,7 @@ export class CrawlerHost extends RPCHost { withShadowDom: opts.withShadowDom, locale: opts.locale, referer: opts.referer, + viewport: opts.viewport, }; if (opts.locale) { diff --git a/backend/functions/src/dto/scrapping-options.ts b/backend/functions/src/dto/scrapping-options.ts index 0f81cca..35a0716 100644 --- a/backend/functions/src/dto/scrapping-options.ts +++ b/backend/functions/src/dto/scrapping-options.ts @@ -16,6 +16,25 @@ const CONTENT_FORMAT_VALUES = new Set(Object.values(CONTENT_FORMAT)); export const IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'] as const; const IMAGE_RETENTION_MODE_VALUES = new Set(IMAGE_RETENTION_MODES); +class Viewport extends AutoCastable { + @Prop({ + default: 1024 + }) + width!: number; + @Prop({ + default: 1024 + }) + height!: number; + @Prop() + deviceScaleFactor?: number; + @Prop() + isMobile?: boolean; + @Prop() + isLandscape?: boolean; + @Prop() + hasTouch?: boolean; +} + @Also({ openapi: { operation: { @@ -279,6 +298,9 @@ export class CrawlerOptions extends AutoCastable { @Prop() tokenBudget?: number; + @Prop() + viewport?: Viewport; + static override from(input: any) { const instance = super.from(input) as CrawlerOptions; const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as { @@ -432,6 +454,9 @@ export class CrawlerOptions extends AutoCastable { if (this.injectFrameScript?.length || this.injectPageScript?.length) { return false; } + if (this.viewport) { + return false; + } return true; } diff --git a/backend/functions/src/services/puppeteer.ts b/backend/functions/src/services/puppeteer.ts index b532b20..8b993f9 100644 --- a/backend/functions/src/services/puppeteer.ts +++ b/backend/functions/src/services/puppeteer.ts @@ -80,6 +80,7 @@ export interface ScrappingOptions { extraHeaders?: Record; injectFrameScripts?: string[]; injectPageScripts?: string[]; + viewport?: Viewport; } @@ -863,6 +864,9 @@ export class PuppeteerControl extends AsyncService { if (options?.overrideUserAgent) { await page.setUserAgent(options.overrideUserAgent); } + if (options?.viewport) { + await page.setViewport(options.viewport); + } let nextSnapshotDeferred = Defer(); const crippleListener = () => nextSnapshotDeferred.reject(new ServiceCrashedError({ message: `Browser crashed, try again` }));