AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-19 23:19:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: search with failed pages

Browse Source
This commit is contained in:
Yanlong Wang 2024-08-08 15:49:23 +08:00
parent 0dd05b5dab
commit e529369ba6
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/functions/src/cloud-functions/crawler.ts
View File

@ -1036,6 +1036,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
let concluded = false;
const handler = async (it: AsyncGenerator<PageSnapshot | undefined>, idx: number) => {
try {
for await (const x of it) {
results[idx] = x;
@ -1045,6 +1046,9 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
}
}
} catch (err: any) {
this.logger.warn(`Failed to scrap ${urls[idx]}`, { err: marshalErrorLike(err) });
}
};
Promise.all(