AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-16 07:25:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: stop early return when timeout is explicitly defined

Browse Source
This commit is contained in:
yanlong.wang 2024-07-18 11:41:58 +08:00
parent b42fe4b7be
commit 61ff011c13
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
3 changed files with 18 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/functions/src/cloud-functions/crawler.ts
View File

@ -710,8 +710,10 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl);
chargeAmount = this.getChargeAmount(formatted);
if (crawlerOptions.timeout !== null) {
return formatted;
}
}
if (!lastScrapped) {
throw new AssertionFailureError(`No content available for URL ${urlToCrawl}`);
@ -731,6 +733,8 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl);
chargeAmount = this.getChargeAmount(formatted);
if (crawlerOptions.timeout !== null) {
if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
return assignTransferProtocolMeta(`${formatted}`,
@ -740,6 +744,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null });
}
}
if (!lastScrapped) {
throw new AssertionFailureError(`No content available for URL ${urlToCrawl}`);

6
backend/functions/src/dto/scrapping-options.ts
View File

@ -171,8 +171,10 @@ export class CrawlerOptions extends AutoCastable {
@Prop({
validate: (v: number) => v > 0 && v <= 180,
type: Number,
nullable: true,
})
timeout?: number;
timeout?: number | null;
static override from(input: any) {
const instance = super.from(input) as CrawlerOptions;
@ -213,6 +215,8 @@ export class CrawlerOptions extends AutoCastable {
let timeoutSeconds = parseInt(ctx?.req.get('x-timeout') || '');
if (!isNaN(timeoutSeconds) && timeoutSeconds > 0 && timeoutSeconds <= 180) {
instance.timeout = timeoutSeconds;
} else if (ctx?.req.get('x-timeout')) {
instance.timeout = null;
}
const removeSelector = ctx?.req.get('x-remove-selector')?.split(', ');

2
thinapps-shared

@ -1 +1 @@
Subproject commit f166680848c5700030389cb69181e5de1535acff
Subproject commit b7fe87fca31e7fbc22db00391b95f3a32dcad997