AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-16 14:16:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add referer param

Browse Source
This commit is contained in:
Zhaofeng Miao 2024-08-22 16:48:47 +08:00
parent 080056e889
commit 7e6c2fcf48
3 changed files with 24 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/functions/src/cloud-functions/crawler.ts
View File

@ -1123,6 +1123,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined,
withIframe: opts.withIframe,
locale: opts.locale,
referer: opts.referer,
};
return crawlOpts;

13
backend/functions/src/dto/scrapping-options.ts
View File

@ -115,6 +115,11 @@ import { parseString as parseSetCookieString } from 'set-cookie-parser';
description: 'Specify browser locale for the page.',
in: 'header',
schema: { type: 'string' }
},
'X-Referer': {
description: 'Specify referer for the page.',
in: 'header',
schema: { type: 'string' }
}
}
}
@ -201,6 +206,9 @@ export class CrawlerOptions extends AutoCastable {
@Prop()
locale?: string;
@Prop()
referer?: string;
static override from(input: any) {
const instance = super.from(input) as CrawlerOptions;
const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as {
@ -218,6 +226,11 @@ export class CrawlerOptions extends AutoCastable {
instance.locale = locale;
}
const referer = ctx?.req.get('x-referer');
if (referer !== undefined) {
instance.referer = referer;
}
const withGeneratedAlt = ctx?.req.get('x-with-generated-alt');
if (withGeneratedAlt !== undefined) {
instance.withGeneratedAlt = Boolean(withGeneratedAlt);

14
backend/functions/src/services/puppeteer.ts
View File

@ -4,7 +4,7 @@ import { container, singleton } from 'tsyringe';
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency } from 'civkit';
import { Logger } from '../shared/services/logger';
import type { Browser, CookieParam, Page } from 'puppeteer';
import type { Browser, CookieParam, GoToOptions, Page } from 'puppeteer';
import puppeteer from 'puppeteer-extra';
import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources';
@ -69,6 +69,7 @@ export interface ScrappingOptions {
overrideUserAgent?: string;
timeoutMs?: number;
locale?: string;
referer?: string;
}
@ -545,11 +546,16 @@ document.addEventListener('load', handlePageLoad);
});
const timeout = options?.timeoutMs || 30_000;
const gotoPromise = page.goto(url, {
const goToOptions: GoToOptions = {
waitUntil: ['load', 'domcontentloaded', 'networkidle0'],
timeout,
})
};
if (options?.referer) {
goToOptions.referer = options.referer;
}
const gotoPromise = page.goto(url, goToOptions)
.catch((err) => {
if (err instanceof TimeoutError) {
this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err: marshalErrorLike(err) });