AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-15 18:26:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: fix search result amount (#1163)

Browse Source
This commit is contained in:
Aaron Ji 2025-03-10 13:38:16 +08:00 committed by GitHub
parent 8a8ae10919
commit 8ec8123ff4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/api/searcher-serper.ts
View File

@ -154,7 +154,7 @@ export class SearcherHost extends RPCHost {
const searchQuery = searchExplicitOperators.addTo(q || noSlashPath); const searchQuery = searchExplicitOperators.addTo(q || noSlashPath);
const r = await this.cachedWebSearch({ const r = await this.cachedWebSearch({
q: searchQuery, q: searchQuery,
num: count > 10 ? 20 : 10, num: count > 10 ? 30 : 20,
gl, gl,
hl, hl,
location, location,
@ -176,11 +176,8 @@ export class SearcherHost extends RPCHost {
if (crawlWithoutContent || count === 0) { if (crawlWithoutContent || count === 0) {
const fakeResults = await this.fakeResult(crawlerOptions, organicSearchResults, !crawlWithoutContent, withFavicon); const fakeResults = await this.fakeResult(crawlerOptions, organicSearchResults, !crawlWithoutContent, withFavicon);
lastScrapped = fakeResults; lastScrapped = fakeResults;
if (!crawlWithoutContent) { chargeAmount = this.assignChargeAmount(!crawlWithoutContent ? lastScrapped : [], count);
chargeAmount = this.assignChargeAmount(lastScrapped);
} else {
chargeAmount = 10000;
}
this.assignTokenUsage(lastScrapped, chargeAmount, crawlWithoutContent); this.assignTokenUsage(lastScrapped, chargeAmount, crawlWithoutContent);
if ((!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) || count === 0) { if ((!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) || count === 0) {
return lastScrapped; return lastScrapped;
@ -207,7 +204,7 @@ export class SearcherHost extends RPCHost {
break; break;
} }
chargeAmount = this.assignChargeAmount(scrapped); chargeAmount = this.assignChargeAmount(scrapped, count);
sseStream.write({ sseStream.write({
event: 'data', event: 'data',
data: scrapped, data: scrapped,
@ -239,7 +236,7 @@ export class SearcherHost extends RPCHost {
if (!lastScrapped) { if (!lastScrapped) {
return; return;
} }
chargeAmount = this.assignChargeAmount(lastScrapped); chargeAmount = this.assignChargeAmount(lastScrapped, count);
rpcReflect.return(lastScrapped); rpcReflect.return(lastScrapped);
earlyReturn = true; earlyReturn = true;
}, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs); }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);
@ -259,7 +256,7 @@ export class SearcherHost extends RPCHost {
if (earlyReturnTimer) { if (earlyReturnTimer) {
clearTimeout(earlyReturnTimer); clearTimeout(earlyReturnTimer);
} }
chargeAmount = this.assignChargeAmount(scrapped); chargeAmount = this.assignChargeAmount(scrapped, count);
this.assignTokenUsage(scrapped, chargeAmount, crawlWithoutContent); this.assignTokenUsage(scrapped, chargeAmount, crawlWithoutContent);
return scrapped; return scrapped;
@ -274,7 +271,7 @@ export class SearcherHost extends RPCHost {
} }
if (!earlyReturn) { if (!earlyReturn) {
chargeAmount = this.assignChargeAmount(lastScrapped); chargeAmount = this.assignChargeAmount(lastScrapped, count);
} }
this.assignTokenUsage(lastScrapped, chargeAmount, crawlWithoutContent); this.assignTokenUsage(lastScrapped, chargeAmount, crawlWithoutContent);
@ -290,7 +287,7 @@ export class SearcherHost extends RPCHost {
if (!lastScrapped) { if (!lastScrapped) {
return; return;
} }
chargeAmount = this.assignChargeAmount(lastScrapped); chargeAmount = this.assignChargeAmount(lastScrapped, count);
rpcReflect.return(assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null })); rpcReflect.return(assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null }));
earlyReturn = true; earlyReturn = true;
}, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs); }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);
@ -313,7 +310,7 @@ export class SearcherHost extends RPCHost {
clearTimeout(earlyReturnTimer); clearTimeout(earlyReturnTimer);
} }
chargeAmount = this.assignChargeAmount(scrapped); chargeAmount = this.assignChargeAmount(scrapped, count);
return assignTransferProtocolMeta(`${scrapped}`, { contentType: 'text/plain', envelope: null }); return assignTransferProtocolMeta(`${scrapped}`, { contentType: 'text/plain', envelope: null });
} }
@ -327,7 +324,7 @@ export class SearcherHost extends RPCHost {
} }
if (!earlyReturn) { if (!earlyReturn) {
chargeAmount = this.assignChargeAmount(lastScrapped); chargeAmount = this.assignChargeAmount(lastScrapped, count);
} }
return assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null }); return assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null });
@ -335,7 +332,6 @@ export class SearcherHost extends RPCHost {
assignTokenUsage(result: FormattedPage[], chargeAmount: number, crawlWithoutContent: boolean) { assignTokenUsage(result: FormattedPage[], chargeAmount: number, crawlWithoutContent: boolean) {
if (crawlWithoutContent) { if (crawlWithoutContent) {
chargeAmount = 10000;
if (result) { if (result) {
result.forEach((x) => { result.forEach((x) => {
delete x.usage; delete x.usage;
@ -533,10 +529,15 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n')}\n` : ''}`;
return resultArray; return resultArray;
} }
assignChargeAmount(formatted: FormattedPage[]) { assignChargeAmount(formatted: FormattedPage[], num: number) {
return _.sum( const countentCharge = _.sum(
formatted.map((x) => this.crawler.assignChargeAmount(x) || 0) formatted.map((x) => this.crawler.assignChargeAmount(x) || 0)
); );
const numCharge = Math.ceil(num / 10) * 10000;
return Math.max(countentCharge, numCharge);
} }
pageQualified(formattedPage: FormattedPage) { pageQualified(formattedPage: FormattedPage) {