AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-15 08:25:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: expose X-Locale parameter

Browse Source
This commit is contained in:
Zhaofeng Miao 2024-08-20 16:14:48 +08:00
parent fb5bd58ee4
commit de50c93825
3 changed files with 36 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/functions/src/cloud-functions/crawler.ts
View File

@ -1106,6 +1106,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
overrideUserAgent: opts.userAgent, overrideUserAgent: opts.userAgent,
timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined, timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined,
withIframe: opts.withIframe, withIframe: opts.withIframe,
locale: opts.locale,
}; };
return crawlOpts; return crawlOpts;

13
backend/functions/src/dto/scrapping-options.ts
View File

@ -111,6 +111,11 @@ import { parseString as parseSetCookieString } from 'set-cookie-parser';
in: 'header', in: 'header',
schema: { type: 'string' } schema: { type: 'string' }
}, },
'X-Locale': {
description: 'Specify browser locale for the page.',
in: 'header',
schema: { type: 'string' }
}
} }
} }
} }
@ -188,6 +193,9 @@ export class CrawlerOptions extends AutoCastable {
}) })
timeout?: number | null; timeout?: number | null;
@Prop()
locale?: string;
static override from(input: any) { static override from(input: any) {
const instance = super.from(input) as CrawlerOptions; const instance = super.from(input) as CrawlerOptions;
const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as { const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as {
@ -200,6 +208,11 @@ export class CrawlerOptions extends AutoCastable {
instance.respondWith = customMode; instance.respondWith = customMode;
} }
const locale = ctx?.req.get('x-locale');
if (locale !== undefined) {
instance.locale = locale;
}
const withGeneratedAlt = ctx?.req.get('x-with-generated-alt'); const withGeneratedAlt = ctx?.req.get('x-with-generated-alt');
if (withGeneratedAlt !== undefined) { if (withGeneratedAlt !== undefined) {
instance.withGeneratedAlt = Boolean(withGeneratedAlt); instance.withGeneratedAlt = Boolean(withGeneratedAlt);

22
backend/functions/src/services/puppeteer.ts
View File

@ -68,6 +68,7 @@ export interface ScrappingOptions {
minIntervalMs?: number; minIntervalMs?: number;
overrideUserAgent?: string; overrideUserAgent?: string;
timeoutMs?: number; timeoutMs?: number;
locale?: string;
} }
@ -472,6 +473,27 @@ document.addEventListener('load', handlePageLoad);
const page = await this.getNextPage(); const page = await this.getNextPage();
const sn = this.snMap.get(page); const sn = this.snMap.get(page);
this.logger.info(`Page ${sn}: Scraping ${url}`, { url }); this.logger.info(`Page ${sn}: Scraping ${url}`, { url });
this.logger.info(`Locale setting: ${options?.locale}`);
if (options?.locale) {
await page.setExtraHTTPHeaders({
'Accept-Language': options?.locale
});
await page.evaluateOnNewDocument(() => {
Object.defineProperty(navigator, "language", {
get: function() {
return options?.locale;
}
});
Object.defineProperty(navigator, "languages", {
get: function() {
return [options?.locale];
}
});
})
}
if (options?.proxyUrl) { if (options?.proxyUrl) {
await page.useProxy(options.proxyUrl); await page.useProxy(options.proxyUrl);
} }