From e529369ba6594daac2365b4101aa428faa1099aa Mon Sep 17 00:00:00 2001
From: Yanlong Wang <yanlong.wang@naiver.org>
Date: Thu, 8 Aug 2024 15:49:23 +0800
Subject: [PATCH] fix: search with failed pages

---
 backend/functions/src/cloud-functions/crawler.ts | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/backend/functions/src/cloud-functions/crawler.ts b/backend/functions/src/cloud-functions/crawler.ts
index 8baed56..c2dccdf 100644
--- a/backend/functions/src/cloud-functions/crawler.ts
+++ b/backend/functions/src/cloud-functions/crawler.ts
@@ -1036,14 +1036,18 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
         let concluded = false;
 
         const handler = async (it: AsyncGenerator<PageSnapshot | undefined>, idx: number) => {
-            for await (const x of it) {
-                results[idx] = x;
+            try {
+                for await (const x of it) {
+                    results[idx] = x;
+
+                    if (x) {
+                        nextDeferred.resolve();
+                        nextDeferred = Defer();
+                    }
 
-                if (x) {
-                    nextDeferred.resolve();
-                    nextDeferred = Defer();
                 }
-
+            } catch (err: any) {
+                this.logger.warn(`Failed to scrap ${urls[idx]}`, { err: marshalErrorLike(err) });
             }
         };