AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-18 03:35:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: cache use and edge cases

Browse Source
This commit is contained in:
yanlong.wang 2025-03-17 15:10:07 +08:00
parent b4b99f0096
commit 3a40db2590
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 63 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/api/crawler.ts
View File

@ -107,6 +107,9 @@ export class CrawlerHost extends RPCHost {
// Potentially mangeled content, dont cache if scripts are injected // Potentially mangeled content, dont cache if scripts are injected
return; return;
} }
if (snapshot.isIntermediate) {
return;
}
if (options.locale) { if (options.locale) {
Reflect.set(snapshot, 'locale', options.locale); Reflect.set(snapshot, 'locale', options.locale);
} }
@ -360,6 +363,7 @@ export class CrawlerHost extends RPCHost {
let lastScrapped; let lastScrapped;
if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) { if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {
try {
for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) { for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) {
lastScrapped = scrapped; lastScrapped = scrapped;
if (rpcReflect.signal.aborted) { if (rpcReflect.signal.aborted) {
@ -368,6 +372,9 @@ export class CrawlerHost extends RPCHost {
if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) { if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) {
continue; continue;
} }
if (!scrapped.title) {
continue;
}
const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts); const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts);
chargeAmount = this.assignChargeAmount(formatted, crawlOpts); chargeAmount = this.assignChargeAmount(formatted, crawlOpts);
@ -382,6 +389,11 @@ export class CrawlerHost extends RPCHost {
return formatted; return formatted;
} }
} catch (err) {
if (!lastScrapped) {
throw err;
}
}
if (!lastScrapped) { if (!lastScrapped) {
if (crawlOpts.targetSelector) { if (crawlOpts.targetSelector) {
@ -406,6 +418,7 @@ export class CrawlerHost extends RPCHost {
}); });
} }
try {
for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) { for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) {
lastScrapped = scrapped; lastScrapped = scrapped;
if (rpcReflect.signal.aborted) { if (rpcReflect.signal.aborted) {
@ -414,6 +427,9 @@ export class CrawlerHost extends RPCHost {
if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) { if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) {
continue; continue;
} }
if (!scrapped.title) {
continue;
}
const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts); const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts);
chargeAmount = this.assignChargeAmount(formatted, crawlOpts); chargeAmount = this.assignChargeAmount(formatted, crawlOpts);
@ -434,6 +450,11 @@ export class CrawlerHost extends RPCHost {
return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { contentType: 'text/plain; charset=utf-8', envelope: null }); return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { contentType: 'text/plain; charset=utf-8', envelope: null });
} }
} catch (err) {
if (!lastScrapped) {
throw err;
}
}
if (!lastScrapped) { if (!lastScrapped) {
if (crawlOpts.targetSelector) { if (crawlOpts.targetSelector) {
@ -733,7 +754,7 @@ export class CrawlerHost extends RPCHost {
let cache = (await cacheIt.next()).value; let cache = (await cacheIt.next()).value;
if (cache?.htmlSignificantlyModifiedByJs === false) { if (cache?.htmlSignificantlyModifiedByJs === false) {
if (crawlerOpts) { if (crawlerOpts && crawlerOpts.timeout === undefined) {
crawlerOpts.respondTiming ??= RESPOND_TIMING.HTML; crawlerOpts.respondTiming ??= RESPOND_TIMING.HTML;
} }
} }