AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-14 22:15:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Yanlong Wang 2024-04-12 14:17:04 +08:00
parent fdd8a8aa8d
commit 2da1b7f3a5
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/functions/src/cloud-functions/crawler.ts
View File

@ -121,8 +121,10 @@ ${this.content}
return sseStream;
}
let lastScrapped;
if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
for await (const scrapped of this.puppeteerControl.scrap(urlToCrawl.toString(), noCache)) {
lastScrapped = scrapped;
if (!scrapped?.parsed?.content) {
continue;
}
@ -131,9 +133,12 @@ ${this.content}
return formatted;
}
return this.formatSnapshot(lastScrapped);
}
for await (const scrapped of this.puppeteerControl.scrap(urlToCrawl.toString(), noCache)) {
lastScrapped = scrapped;
if (!scrapped?.parsed?.content) {
continue;
}
@ -143,7 +148,7 @@ ${this.content}
return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null });
}
throw new Error('Unreachable');
return this.formatSnapshot(lastScrapped);
}