AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-14 01:25:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: tweak default timing to be more conservative

Browse Source
This commit is contained in:
Yanlong Wang 2025-03-12 18:32:08 +08:00
parent 8121d62324
commit 6027963670
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/dto/crawler-options.ts
View File

@ -225,7 +225,8 @@ class Viewport extends AutoCastable {
`- html: unrendered HTML is enough to return\n` +
`- mutation-idle: wait for DOM mutations to settle and remain unchanged for at least 0.2s\n` +
`- resource-idle: wait for no additional resources that would affect page logic and content SUCCEEDED loading for at least 0.5s\n` +
`- media-idle: wait for no additional resources, including media resources, SUCCEEDED loading for at least 0.5s\n\n`,
`- media-idle: wait for no additional resources, including media resources, SUCCEEDED loading for at least 0.5s\n` +
`- network-idle: wait for full load of webpage, as usual.\n\n`,
in: 'header',
schema: { type: 'string' }
},
@ -583,11 +584,13 @@ export class CrawlerOptions extends AutoCastable {
if (respondTiming) {
instance.respondTiming ??= respondTiming as RESPOND_TIMING;
}
instance.respondTiming ??= (
instance.timeout ||
instance.respondWith.includes('shot') ||
instance.respondWith.includes('vlm')
) ? RESPOND_TIMING.MEDIA_IDLE : RESPOND_TIMING.MUTATION_IDLE;
if (instance.timeout) {
instance.respondTiming ??= RESPOND_TIMING.NETWORK_IDLE;
}
if (instance.respondWith.includes('shot') || instance.respondWith.includes('vlm')) {
instance.respondTiming ??= RESPOND_TIMING.MEDIA_IDLE;
}
instance.respondTiming ??= RESPOND_TIMING.RESOURCE_IDLE;
if (instance.cacheTolerance) {
instance.cacheTolerance = instance.cacheTolerance * 1000;
@ -607,7 +610,7 @@ export class CrawlerOptions extends AutoCastable {
if (this.respondTiming === RESPOND_TIMING.HTML && snapshot.html) {
return true;
}
if (this.respondTiming === RESPOND_TIMING.MEDIA_IDLE && snapshot.lastMediaResourceLoaded) {
if (this.respondTiming === RESPOND_TIMING.MEDIA_IDLE && snapshot.lastMediaResourceLoaded && snapshot.lastMutationIdle) {
const now = Date.now();
if ((Math.max(snapshot.lastMediaResourceLoaded, snapshot.lastContentResourceLoaded || 0) + 500) < now) {
return true;
@ -619,7 +622,7 @@ export class CrawlerOptions extends AutoCastable {
if ((this.respondWith.includes('vlm') || this.respondWith.includes('screenshot')) && !snapshot.screenshot) {
return false;
}
if (this.respondTiming === RESPOND_TIMING.RESOURCE_IDLE && snapshot.lastContentResourceLoaded) {
if (this.respondTiming === RESPOND_TIMING.RESOURCE_IDLE && snapshot.lastContentResourceLoaded && snapshot.lastMutationIdle) {
const now = Date.now();
if ((snapshot.lastContentResourceLoaded + 500) < now) {
return true;