AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-16 06:05:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Yanlong Wang 2024-04-12 14:17:04 +08:00
parent fdd8a8aa8d
commit 2da1b7f3a5
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/functions/src/cloud-functions/crawler.ts
View File

@ -121,8 +121,10 @@ ${this.content}
return sseStream; return sseStream;
} }
let lastScrapped;
if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) { if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
for await (const scrapped of this.puppeteerControl.scrap(urlToCrawl.toString(), noCache)) { for await (const scrapped of this.puppeteerControl.scrap(urlToCrawl.toString(), noCache)) {
lastScrapped = scrapped;
if (!scrapped?.parsed?.content) { if (!scrapped?.parsed?.content) {
continue; continue;
} }
@ -131,9 +133,12 @@ ${this.content}
return formatted; return formatted;
} }
return this.formatSnapshot(lastScrapped);
} }
for await (const scrapped of this.puppeteerControl.scrap(urlToCrawl.toString(), noCache)) { for await (const scrapped of this.puppeteerControl.scrap(urlToCrawl.toString(), noCache)) {
lastScrapped = scrapped;
if (!scrapped?.parsed?.content) { if (!scrapped?.parsed?.content) {
continue; continue;
} }
@ -143,7 +148,7 @@ ${this.content}
return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null }); return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null });
} }
throw new Error('Unreachable'); return this.formatSnapshot(lastScrapped);
} }