diff --git a/backend/functions/src/cloud-functions/crawler.ts b/backend/functions/src/cloud-functions/crawler.ts index 1b74075..cce75ff 100644 --- a/backend/functions/src/cloud-functions/crawler.ts +++ b/backend/functions/src/cloud-functions/crawler.ts @@ -710,7 +710,9 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`; const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl); chargeAmount = this.getChargeAmount(formatted); - return formatted; + if (crawlerOptions.timeout !== null) { + return formatted; + } } if (!lastScrapped) { @@ -731,14 +733,17 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`; const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl); chargeAmount = this.getChargeAmount(formatted); - if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) { - return assignTransferProtocolMeta(`${formatted}`, - { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } } - ); + if (crawlerOptions.timeout !== null) { + if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) { + + return assignTransferProtocolMeta(`${formatted}`, + { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } } + ); + } + + return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null }); } - - return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null }); } if (!lastScrapped) { diff --git a/backend/functions/src/dto/scrapping-options.ts b/backend/functions/src/dto/scrapping-options.ts index 70b8278..0eec389 100644 --- a/backend/functions/src/dto/scrapping-options.ts +++ b/backend/functions/src/dto/scrapping-options.ts @@ -171,8 +171,10 @@ export class CrawlerOptions extends AutoCastable { @Prop({ validate: (v: number) => v > 0 && v <= 180, + type: Number, + nullable: true, }) - timeout?: number; + timeout?: number | null; static override from(input: any) { const instance = super.from(input) as CrawlerOptions; @@ -213,6 +215,8 @@ export class CrawlerOptions extends AutoCastable { let timeoutSeconds = parseInt(ctx?.req.get('x-timeout') || ''); if (!isNaN(timeoutSeconds) && timeoutSeconds > 0 && timeoutSeconds <= 180) { instance.timeout = timeoutSeconds; + } else if (ctx?.req.get('x-timeout')) { + instance.timeout = null; } const removeSelector = ctx?.req.get('x-remove-selector')?.split(', '); diff --git a/thinapps-shared b/thinapps-shared index f166680..b7fe87f 160000 --- a/thinapps-shared +++ b/thinapps-shared @@ -1 +1 @@ -Subproject commit f166680848c5700030389cb69181e5de1535acff +Subproject commit b7fe87fca31e7fbc22db00391b95f3a32dcad997