GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-18 23:25:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(crawl): viewport options

Browse Source
This commit is contained in:
yanlong.wang 2024-12-24 19:07:48 +08:00
parent d8ad1cb6a1
commit 2606c445d9
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
3 changed files with 31 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/functions/src/cloud-functions/crawler.ts
View File

@ -69,7 +69,7 @@ export class CrawlerHost extends RPCHost {
// Potential privacy issue, dont cache if cookies are used
return;
}
if (options.injectFrameScripts?.length || options.injectPageScripts?.length) {
if (options.injectFrameScripts?.length || options.injectPageScripts?.length || options.viewport) {
// Potentially mangeled content, dont cache if scripts are injected
return;
}
@ -725,6 +725,7 @@ export class CrawlerHost extends RPCHost {
withShadowDom: opts.withShadowDom,
locale: opts.locale,
referer: opts.referer,
viewport: opts.viewport,
};
if (opts.locale) {

25
backend/functions/src/dto/scrapping-options.ts
View File

@ -16,6 +16,25 @@ const CONTENT_FORMAT_VALUES = new Set<string>(Object.values(CONTENT_FORMAT));
export const IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'] as const;
const IMAGE_RETENTION_MODE_VALUES = new Set<string>(IMAGE_RETENTION_MODES);
class Viewport extends AutoCastable {
@Prop({
default: 1024
})
width!: number;
@Prop({
default: 1024
})
height!: number;
@Prop()
deviceScaleFactor?: number;
@Prop()
isMobile?: boolean;
@Prop()
isLandscape?: boolean;
@Prop()
hasTouch?: boolean;
}
@Also({
openapi: {
operation: {
@ -279,6 +298,9 @@ export class CrawlerOptions extends AutoCastable {
@Prop()
tokenBudget?: number;
@Prop()
viewport?: Viewport;
static override from(input: any) {
const instance = super.from(input) as CrawlerOptions;
const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as {
@ -432,6 +454,9 @@ export class CrawlerOptions extends AutoCastable {
if (this.injectFrameScript?.length || this.injectPageScript?.length) {
return false;
}
if (this.viewport) {
return false;
}
return true;
}

4
backend/functions/src/services/puppeteer.ts
View File

@ -80,6 +80,7 @@ export interface ScrappingOptions {
extraHeaders?: Record<string, string>;
injectFrameScripts?: string[];
injectPageScripts?: string[];
viewport?: Viewport;
}
@ -863,6 +864,9 @@ export class PuppeteerControl extends AsyncService {
if (options?.overrideUserAgent) {
await page.setUserAgent(options.overrideUserAgent);
}
if (options?.viewport) {
await page.setViewport(options.viewport);
}
let nextSnapshotDeferred = Defer();
const crippleListener = () => nextSnapshotDeferred.reject(new ServiceCrashedError({ message: `Browser crashed, try again` }));