From f3654a786b66bdcd2b1ffea0bedc04b61c81f55b Mon Sep 17 00:00:00 2001
From: Yanlong Wang <yanlong.wang@naiver.org>
Date: Mon, 24 Mar 2025 10:14:53 +0800
Subject: [PATCH] fix: beware readerlm capacity drain

---
 src/api/crawler.ts | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/api/crawler.ts b/src/api/crawler.ts
index ce1cb8f..90d1a7e 100644
--- a/src/api/crawler.ts
+++ b/src/api/crawler.ts
@@ -35,7 +35,8 @@ import { AsyncLocalContext } from '../services/async-context';
 import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';
 import {
     BudgetExceededError, InsufficientBalanceError,
-    SecurityCompromiseError, ServiceBadApproachError, ServiceBadAttemptError
+    SecurityCompromiseError, ServiceBadApproachError, ServiceBadAttemptError,
+    ServiceNodeResourceDrainError
 } from '../services/errors';
 
 import { countGPTToken as estimateToken } from '../shared/utils/openai';
@@ -45,6 +46,7 @@ import { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';
 import { RobotsTxtService } from '../services/robots-text';
 import { TempFileManager } from '../services/temp-file';
 import { MiscService } from '../services/misc';
+import { HTTPServiceError } from 'civkit';
 
 export interface ExtraScrappingOptions extends ScrappingOptions {
     withIframe?: boolean | 'quoted';
@@ -685,7 +687,14 @@ export class CrawlerHost extends RPCHost {
                 return;
             }
 
-            yield* this.lmControl.readerLMMarkdownFromSnapshot(finalAutoSnapshot);
+            try {
+                yield* this.lmControl.readerLMMarkdownFromSnapshot(finalAutoSnapshot);
+            } catch (err) {
+                if (err instanceof HTTPServiceError && err.status === 429) {
+                    throw new ServiceNodeResourceDrainError(`Reader LM is at capacity, please try again later.`);
+                }
+                throw err;
+            }
 
             return;
         }