AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-19 23:19:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: search with failed pages

Browse Source
This commit is contained in:
Yanlong Wang 2024-08-08 15:49:23 +08:00
parent 0dd05b5dab
commit e529369ba6
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backend/functions/src/cloud-functions/crawler.ts
View File

@ -1036,14 +1036,18 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
let concluded = false; let concluded = false;
const handler = async (it: AsyncGenerator<PageSnapshot | undefined>, idx: number) => { const handler = async (it: AsyncGenerator<PageSnapshot | undefined>, idx: number) => {
for await (const x of it) { try {
results[idx] = x; for await (const x of it) {
results[idx] = x;
if (x) {
nextDeferred.resolve();
nextDeferred = Defer();
}
if (x) {
nextDeferred.resolve();
nextDeferred = Defer();
} }
} catch (err: any) {
this.logger.warn(`Failed to scrap ${urls[idx]}`, { err: marshalErrorLike(err) });
} }
}; };