AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-07-30 08:12:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: saas issus

Browse Source
This commit is contained in:
Yanlong Wang 2025-04-15 18:40:41 +08:00
parent 75a1283a91
commit e6fdd87294
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
6 changed files with 47 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/api/searcher-serper.ts
View File

@ -6,7 +6,7 @@ import { marshalErrorLike } from 'civkit/lang';
import { objHashMd5B64Of } from 'civkit/hash';
import _ from 'lodash';
import { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit';
import { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit';
import { CrawlerHost, ExtraScrappingOptions } from './crawler';
import { SerperSearchResult } from '../db/searched';
@ -19,8 +19,16 @@ import { AsyncLocalContext } from '../services/async-context';
import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';
import { OutputServerEventStream } from '../lib/transform-server-event-stream';
import { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';
import { InsufficientBalanceError, RateLimitTriggeredError } from '../services/errors';
import { SerperImageSearchResponse, SerperNewsSearchResponse, SerperSearchQueryParams, SerperSearchResponse, SerperWebSearchResponse, WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search';
import { InsufficientBalanceError } from '../services/errors';
import {
SerperImageSearchResponse,
SerperNewsSearchResponse,
SerperSearchQueryParams,
SerperSearchResponse,
SerperWebSearchResponse,
WORLD_COUNTRIES,
WORLD_LANGUAGES
} from '../shared/3rd-party/serper-search';
import { toAsyncGenerator } from '../utils/misc';
import type { JinaEmbeddingsTokenAccount } from '../shared/db/jina-embeddings-token-account';
import { LRUCache } from 'lru-cache';
@ -218,10 +226,10 @@ export class SearcherHost extends RPCHost {
}
const now = Date.now();
let tgtDate;
if (err.retryAfter) {
tgtDate = new Date(now + err.retryAfter * 1000);
} else if (err.retryAfterDate) {
if (err.retryAfterDate) {
tgtDate = err.retryAfterDate;
} else if (err.retryAfter) {
tgtDate = new Date(now + err.retryAfter * 1000);
}
if (tgtDate) {

19
src/api/serp.ts
View File

@ -7,14 +7,14 @@ import {
import { marshalErrorLike } from 'civkit/lang';
import _ from 'lodash';
import { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit';
import { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit';
import { GlobalLogger } from '../services/logger';
import { AsyncLocalContext } from '../services/async-context';
import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';
import { OutputServerEventStream } from '../lib/transform-server-event-stream';
import { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';
import { InsufficientBalanceError, RateLimitTriggeredError } from '../services/errors';
import { InsufficientBalanceError } from '../services/errors';
import { WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search';
import { GoogleSERP } from '../services/serp/google';
import { WebSearchEntry } from '../services/serp/compat';
@ -172,10 +172,11 @@ export class SerpHost extends RPCHost {
const now = new Date();
const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf());
if (blockedTimeRemaining > 0) {
throw RateLimitTriggeredError.from({
message: `Per UID rate limit exceeded (async)`,
retryAfter: Math.ceil(blockedTimeRemaining / 1000),
});
this.logger.warn(`Rate limit triggered for ${uid}, this request should have been blocked`);
// throw RateLimitTriggeredError.from({
// message: `Per UID rate limit exceeded (async)`,
// retryAfter: Math.ceil(blockedTimeRemaining / 1000),
// });
}
}
@ -229,10 +230,10 @@ export class SerpHost extends RPCHost {
}
const now = Date.now();
let tgtDate;
if (err.retryAfter) {
tgtDate = new Date(now + err.retryAfter * 1000);
} else if (err.retryAfterDate) {
if (err.retryAfterDate) {
tgtDate = err.retryAfterDate;
} else if (err.retryAfter) {
tgtDate = new Date(now + err.retryAfter * 1000);
}
if (tgtDate) {

19
src/services/curl.ts
View File

@ -61,10 +61,23 @@ export class CurlControl extends AsyncService {
}
curlImpersonateHeader(curl: Curl, headers?: object) {
let uaPlatform = this.platform;
if (this.ua.includes('Windows')) {
uaPlatform = 'Windows';
} else if (this.ua.includes('Android')) {
uaPlatform = 'Android';
} else if (this.ua.includes('iPhone') || this.ua.includes('iPad') || this.ua.includes('iPod')) {
uaPlatform = 'iOS';
} else if (this.ua.includes('CrOS')) {
uaPlatform = 'Chrome OS';
} else if (this.ua.includes('Macintosh')) {
uaPlatform = 'macOS';
}
const mixinHeaders: Record<string, string> = {
'sch-ch-ua': `Not A(Brand";v="8", "Chromium";v="${this.chromeVersion}", "Google Chrome";v="${this.chromeVersion}"`,
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': this.platform,
'Sec-Ch-Ua': `Not A(Brand";v="8", "Chromium";v="${this.chromeVersion}", "Google Chrome";v="${this.chromeVersion}"`,
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': `"${uaPlatform}"`,
'Upgrade-Insecure-Requests': '1',
'User-Agent': this.ua,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',

30
src/services/errors.ts
View File

@ -1,4 +1,4 @@
import { ApplicationError, Prop, RPC_TRANSFER_PROTOCOL_META_SYMBOL, StatusCode } from 'civkit/civ-rpc';
import { ApplicationError, StatusCode } from 'civkit/civ-rpc';
import _ from 'lodash';
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc';
@ -46,31 +46,3 @@ export class SecurityCompromiseError extends ApplicationError { }
@StatusCode(41201)
export class BatchSizeTooLargeError extends ApplicationError { }
@StatusCode(42903)
export class RateLimitTriggeredError extends ApplicationError {
@Prop({
desc: 'Retry after seconds',
})
retryAfter?: number;
@Prop({
desc: 'Retry after date',
})
retryAfterDate?: Date;
protected override get [RPC_TRANSFER_PROTOCOL_META_SYMBOL]() {
const retryAfter = this.retryAfter || this.retryAfterDate;
if (!retryAfter) {
return super[RPC_TRANSFER_PROTOCOL_META_SYMBOL];
}
return _.merge(_.cloneDeep(super[RPC_TRANSFER_PROTOCOL_META_SYMBOL]), {
headers: {
'Retry-After': `${retryAfter instanceof Date ? dayjs(retryAfter).utc().format('ddd, DD MMM YYYY HH:mm:ss [GMT]') : retryAfter}`,
}
});
}
}

6
src/services/serper-search.ts
View File

@ -56,6 +56,7 @@ export class SerperSearchService extends AsyncService {
let maxTries = 3;
while (maxTries--) {
const t0 = Date.now();
try {
this.logger.debug(`Doing external search`, query);
let r;
@ -101,11 +102,14 @@ export class SerperSearchService extends AsyncService {
break;
}
}
const dt = Date.now() - t0;
this.blackHoleDetector.itWorked();
this.logger.debug(`External search took ${dt}ms`, { searchDt: dt, variant });
return r.parsed;
} catch (err: any) {
this.logger.error(`${variant} search failed: ${err?.message}`, { err: marshalErrorLike(err) });
const dt = Date.now() - t0;
this.logger.error(`${variant} search failed: ${err?.message}`, { searchDt: dt, err: marshalErrorLike(err) });
if (err?.status === 429) {
await delay(500 + 1000 * Math.random());
continue;

2
thinapps-shared

@ -1 +1 @@
Subproject commit f30577e73eca151f9d62198ae3b60d0172befaed
Subproject commit 580ea72e0eddaa115b85dabf29de41d079ecd2d0