AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-15 22:05:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: fix search result amount (#1163)

Browse Source
This commit is contained in:
Aaron Ji 2025-03-10 13:38:16 +08:00 committed by GitHub
parent 8a8ae10919
commit 8ec8123ff4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/api/searcher-serper.ts
View File

@ -154,7 +154,7 @@ export class SearcherHost extends RPCHost {
const searchQuery = searchExplicitOperators.addTo(q || noSlashPath);
const r = await this.cachedWebSearch({
q: searchQuery,
num: count > 10 ? 20 : 10,
num: count > 10 ? 30 : 20,
gl,
hl,
location,
@ -176,11 +176,8 @@ export class SearcherHost extends RPCHost {
if (crawlWithoutContent || count === 0) {
const fakeResults = await this.fakeResult(crawlerOptions, organicSearchResults, !crawlWithoutContent, withFavicon);
lastScrapped = fakeResults;
if (!crawlWithoutContent) {
chargeAmount = this.assignChargeAmount(lastScrapped);
} else {
chargeAmount = 10000;
}
chargeAmount = this.assignChargeAmount(!crawlWithoutContent ? lastScrapped : [], count);
this.assignTokenUsage(lastScrapped, chargeAmount, crawlWithoutContent);
if ((!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) || count === 0) {
return lastScrapped;
@ -207,7 +204,7 @@ export class SearcherHost extends RPCHost {
break;
}
chargeAmount = this.assignChargeAmount(scrapped);
chargeAmount = this.assignChargeAmount(scrapped, count);
sseStream.write({
event: 'data',
data: scrapped,
@ -239,7 +236,7 @@ export class SearcherHost extends RPCHost {
if (!lastScrapped) {
return;
}
chargeAmount = this.assignChargeAmount(lastScrapped);
chargeAmount = this.assignChargeAmount(lastScrapped, count);
rpcReflect.return(lastScrapped);
earlyReturn = true;
}, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);
@ -259,7 +256,7 @@ export class SearcherHost extends RPCHost {
if (earlyReturnTimer) {
clearTimeout(earlyReturnTimer);
}
chargeAmount = this.assignChargeAmount(scrapped);
chargeAmount = this.assignChargeAmount(scrapped, count);
this.assignTokenUsage(scrapped, chargeAmount, crawlWithoutContent);
return scrapped;
@ -274,7 +271,7 @@ export class SearcherHost extends RPCHost {
}
if (!earlyReturn) {
chargeAmount = this.assignChargeAmount(lastScrapped);
chargeAmount = this.assignChargeAmount(lastScrapped, count);
}
this.assignTokenUsage(lastScrapped, chargeAmount, crawlWithoutContent);
@ -290,7 +287,7 @@ export class SearcherHost extends RPCHost {
if (!lastScrapped) {
return;
}
chargeAmount = this.assignChargeAmount(lastScrapped);
chargeAmount = this.assignChargeAmount(lastScrapped, count);
rpcReflect.return(assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null }));
earlyReturn = true;
}, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);
@ -313,7 +310,7 @@ export class SearcherHost extends RPCHost {
clearTimeout(earlyReturnTimer);
}
chargeAmount = this.assignChargeAmount(scrapped);
chargeAmount = this.assignChargeAmount(scrapped, count);
return assignTransferProtocolMeta(`${scrapped}`, { contentType: 'text/plain', envelope: null });
}
@ -327,7 +324,7 @@ export class SearcherHost extends RPCHost {
}
if (!earlyReturn) {
chargeAmount = this.assignChargeAmount(lastScrapped);
chargeAmount = this.assignChargeAmount(lastScrapped, count);
}
return assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null });
@ -335,7 +332,6 @@ export class SearcherHost extends RPCHost {
assignTokenUsage(result: FormattedPage[], chargeAmount: number, crawlWithoutContent: boolean) {
if (crawlWithoutContent) {
chargeAmount = 10000;
if (result) {
result.forEach((x) => {
delete x.usage;
@ -533,10 +529,15 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n')}\n` : ''}`;
return resultArray;
}
assignChargeAmount(formatted: FormattedPage[]) {
return _.sum(
assignChargeAmount(formatted: FormattedPage[], num: number) {
const countentCharge = _.sum(
formatted.map((x) => this.crawler.assignChargeAmount(x) || 0)
);
const numCharge = Math.ceil(num / 10) * 10000;
return Math.max(countentCharge, numCharge);
}
pageQualified(formattedPage: FormattedPage) {