AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-16 04:45:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: stop early return when timeout is explicitly defined

Browse Source
This commit is contained in:
yanlong.wang 2024-07-18 11:41:58 +08:00
parent b42fe4b7be
commit 61ff011c13
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
3 changed files with 18 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
backend/functions/src/cloud-functions/crawler.ts
View File

@ -710,7 +710,9 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl); const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl);
chargeAmount = this.getChargeAmount(formatted); chargeAmount = this.getChargeAmount(formatted);
return formatted; if (crawlerOptions.timeout !== null) {
return formatted;
}
} }
if (!lastScrapped) { if (!lastScrapped) {
@ -731,14 +733,17 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl); const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl);
chargeAmount = this.getChargeAmount(formatted); chargeAmount = this.getChargeAmount(formatted);
if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
return assignTransferProtocolMeta(`${formatted}`, if (crawlerOptions.timeout !== null) {
{ code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } } if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
);
return assignTransferProtocolMeta(`${formatted}`,
{ code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } }
);
}
return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null });
} }
return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null });
} }
if (!lastScrapped) { if (!lastScrapped) {

6
backend/functions/src/dto/scrapping-options.ts
View File

@ -171,8 +171,10 @@ export class CrawlerOptions extends AutoCastable {
@Prop({ @Prop({
validate: (v: number) => v > 0 && v <= 180, validate: (v: number) => v > 0 && v <= 180,
type: Number,
nullable: true,
}) })
timeout?: number; timeout?: number | null;
static override from(input: any) { static override from(input: any) {
const instance = super.from(input) as CrawlerOptions; const instance = super.from(input) as CrawlerOptions;
@ -213,6 +215,8 @@ export class CrawlerOptions extends AutoCastable {
let timeoutSeconds = parseInt(ctx?.req.get('x-timeout') || ''); let timeoutSeconds = parseInt(ctx?.req.get('x-timeout') || '');
if (!isNaN(timeoutSeconds) && timeoutSeconds > 0 && timeoutSeconds <= 180) { if (!isNaN(timeoutSeconds) && timeoutSeconds > 0 && timeoutSeconds <= 180) {
instance.timeout = timeoutSeconds; instance.timeout = timeoutSeconds;
} else if (ctx?.req.get('x-timeout')) {
instance.timeout = null;
} }
const removeSelector = ctx?.req.get('x-remove-selector')?.split(', '); const removeSelector = ctx?.req.get('x-remove-selector')?.split(', ');

2
thinapps-shared

@ -1 +1 @@
Subproject commit f166680848c5700030389cb69181e5de1535acff Subproject commit b7fe87fca31e7fbc22db00391b95f3a32dcad997