AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-07-31 07:41:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: beware readerlm capacity drain

Browse Source
This commit is contained in:
Yanlong Wang 2025-03-24 10:14:53 +08:00
parent a471a6137c
commit f3654a786b
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/api/crawler.ts
View File

@ -35,7 +35,8 @@ import { AsyncLocalContext } from '../services/async-context';
import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';
import {
BudgetExceededError, InsufficientBalanceError,
SecurityCompromiseError, ServiceBadApproachError, ServiceBadAttemptError
SecurityCompromiseError, ServiceBadApproachError, ServiceBadAttemptError,
ServiceNodeResourceDrainError
} from '../services/errors';
import { countGPTToken as estimateToken } from '../shared/utils/openai';
@ -45,6 +46,7 @@ import { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';
import { RobotsTxtService } from '../services/robots-text';
import { TempFileManager } from '../services/temp-file';
import { MiscService } from '../services/misc';
import { HTTPServiceError } from 'civkit';
export interface ExtraScrappingOptions extends ScrappingOptions {
withIframe?: boolean | 'quoted';
@ -685,7 +687,14 @@ export class CrawlerHost extends RPCHost {
return;
}
yield* this.lmControl.readerLMMarkdownFromSnapshot(finalAutoSnapshot);
try {
yield* this.lmControl.readerLMMarkdownFromSnapshot(finalAutoSnapshot);
} catch (err) {
if (err instanceof HTTPServiceError && err.status === 429) {
throw new ServiceNodeResourceDrainError(`Reader LM is at capacity, please try again later.`);
}
throw err;
}
return;
}