From e6fdd87294214907d7d1095aae916565824d283a Mon Sep 17 00:00:00 2001 From: Yanlong Wang Date: Tue, 15 Apr 2025 18:40:41 +0800 Subject: [PATCH] fix: saas issus --- src/api/searcher-serper.ts | 20 ++++++++++++++------ src/api/serp.ts | 19 ++++++++++--------- src/services/curl.ts | 19 ++++++++++++++++--- src/services/errors.ts | 30 +----------------------------- src/services/serper-search.ts | 6 +++++- thinapps-shared | 2 +- 6 files changed, 47 insertions(+), 49 deletions(-) diff --git a/src/api/searcher-serper.ts b/src/api/searcher-serper.ts index d36ff28..462bde3 100644 --- a/src/api/searcher-serper.ts +++ b/src/api/searcher-serper.ts @@ -6,7 +6,7 @@ import { marshalErrorLike } from 'civkit/lang'; import { objHashMd5B64Of } from 'civkit/hash'; import _ from 'lodash'; -import { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit'; +import { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit'; import { CrawlerHost, ExtraScrappingOptions } from './crawler'; import { SerperSearchResult } from '../db/searched'; @@ -19,8 +19,16 @@ import { AsyncLocalContext } from '../services/async-context'; import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry'; import { OutputServerEventStream } from '../lib/transform-server-event-stream'; import { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth'; -import { InsufficientBalanceError, RateLimitTriggeredError } from '../services/errors'; -import { SerperImageSearchResponse, SerperNewsSearchResponse, SerperSearchQueryParams, SerperSearchResponse, SerperWebSearchResponse, WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search'; +import { InsufficientBalanceError } from '../services/errors'; +import { + SerperImageSearchResponse, + SerperNewsSearchResponse, + SerperSearchQueryParams, + SerperSearchResponse, + SerperWebSearchResponse, + WORLD_COUNTRIES, + WORLD_LANGUAGES +} from '../shared/3rd-party/serper-search'; import { toAsyncGenerator } from '../utils/misc'; import type { JinaEmbeddingsTokenAccount } from '../shared/db/jina-embeddings-token-account'; import { LRUCache } from 'lru-cache'; @@ -218,10 +226,10 @@ export class SearcherHost extends RPCHost { } const now = Date.now(); let tgtDate; - if (err.retryAfter) { - tgtDate = new Date(now + err.retryAfter * 1000); - } else if (err.retryAfterDate) { + if (err.retryAfterDate) { tgtDate = err.retryAfterDate; + } else if (err.retryAfter) { + tgtDate = new Date(now + err.retryAfter * 1000); } if (tgtDate) { diff --git a/src/api/serp.ts b/src/api/serp.ts index ca0edb5..74fca90 100644 --- a/src/api/serp.ts +++ b/src/api/serp.ts @@ -7,14 +7,14 @@ import { import { marshalErrorLike } from 'civkit/lang'; import _ from 'lodash'; -import { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit'; +import { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit'; import { GlobalLogger } from '../services/logger'; import { AsyncLocalContext } from '../services/async-context'; import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry'; import { OutputServerEventStream } from '../lib/transform-server-event-stream'; import { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth'; -import { InsufficientBalanceError, RateLimitTriggeredError } from '../services/errors'; +import { InsufficientBalanceError } from '../services/errors'; import { WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search'; import { GoogleSERP } from '../services/serp/google'; import { WebSearchEntry } from '../services/serp/compat'; @@ -172,10 +172,11 @@ export class SerpHost extends RPCHost { const now = new Date(); const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf()); if (blockedTimeRemaining > 0) { - throw RateLimitTriggeredError.from({ - message: `Per UID rate limit exceeded (async)`, - retryAfter: Math.ceil(blockedTimeRemaining / 1000), - }); + this.logger.warn(`Rate limit triggered for ${uid}, this request should have been blocked`); + // throw RateLimitTriggeredError.from({ + // message: `Per UID rate limit exceeded (async)`, + // retryAfter: Math.ceil(blockedTimeRemaining / 1000), + // }); } } @@ -229,10 +230,10 @@ export class SerpHost extends RPCHost { } const now = Date.now(); let tgtDate; - if (err.retryAfter) { - tgtDate = new Date(now + err.retryAfter * 1000); - } else if (err.retryAfterDate) { + if (err.retryAfterDate) { tgtDate = err.retryAfterDate; + } else if (err.retryAfter) { + tgtDate = new Date(now + err.retryAfter * 1000); } if (tgtDate) { diff --git a/src/services/curl.ts b/src/services/curl.ts index bc61661..fdb7539 100644 --- a/src/services/curl.ts +++ b/src/services/curl.ts @@ -61,10 +61,23 @@ export class CurlControl extends AsyncService { } curlImpersonateHeader(curl: Curl, headers?: object) { + let uaPlatform = this.platform; + if (this.ua.includes('Windows')) { + uaPlatform = 'Windows'; + } else if (this.ua.includes('Android')) { + uaPlatform = 'Android'; + } else if (this.ua.includes('iPhone') || this.ua.includes('iPad') || this.ua.includes('iPod')) { + uaPlatform = 'iOS'; + } else if (this.ua.includes('CrOS')) { + uaPlatform = 'Chrome OS'; + } else if (this.ua.includes('Macintosh')) { + uaPlatform = 'macOS'; + } + const mixinHeaders: Record = { - 'sch-ch-ua': `Not A(Brand";v="8", "Chromium";v="${this.chromeVersion}", "Google Chrome";v="${this.chromeVersion}"`, - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': this.platform, + 'Sec-Ch-Ua': `Not A(Brand";v="8", "Chromium";v="${this.chromeVersion}", "Google Chrome";v="${this.chromeVersion}"`, + 'Sec-Ch-Ua-Mobile': '?0', + 'Sec-Ch-Ua-Platform': `"${uaPlatform}"`, 'Upgrade-Insecure-Requests': '1', 'User-Agent': this.ua, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', diff --git a/src/services/errors.ts b/src/services/errors.ts index 44246db..51344a2 100644 --- a/src/services/errors.ts +++ b/src/services/errors.ts @@ -1,4 +1,4 @@ -import { ApplicationError, Prop, RPC_TRANSFER_PROTOCOL_META_SYMBOL, StatusCode } from 'civkit/civ-rpc'; +import { ApplicationError, StatusCode } from 'civkit/civ-rpc'; import _ from 'lodash'; import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; @@ -46,31 +46,3 @@ export class SecurityCompromiseError extends ApplicationError { } @StatusCode(41201) export class BatchSizeTooLargeError extends ApplicationError { } - - -@StatusCode(42903) -export class RateLimitTriggeredError extends ApplicationError { - - @Prop({ - desc: 'Retry after seconds', - }) - retryAfter?: number; - - @Prop({ - desc: 'Retry after date', - }) - retryAfterDate?: Date; - - protected override get [RPC_TRANSFER_PROTOCOL_META_SYMBOL]() { - const retryAfter = this.retryAfter || this.retryAfterDate; - if (!retryAfter) { - return super[RPC_TRANSFER_PROTOCOL_META_SYMBOL]; - } - - return _.merge(_.cloneDeep(super[RPC_TRANSFER_PROTOCOL_META_SYMBOL]), { - headers: { - 'Retry-After': `${retryAfter instanceof Date ? dayjs(retryAfter).utc().format('ddd, DD MMM YYYY HH:mm:ss [GMT]') : retryAfter}`, - } - }); - } -} diff --git a/src/services/serper-search.ts b/src/services/serper-search.ts index 4a6c41c..59698de 100644 --- a/src/services/serper-search.ts +++ b/src/services/serper-search.ts @@ -56,6 +56,7 @@ export class SerperSearchService extends AsyncService { let maxTries = 3; while (maxTries--) { + const t0 = Date.now(); try { this.logger.debug(`Doing external search`, query); let r; @@ -101,11 +102,14 @@ export class SerperSearchService extends AsyncService { break; } } + const dt = Date.now() - t0; this.blackHoleDetector.itWorked(); + this.logger.debug(`External search took ${dt}ms`, { searchDt: dt, variant }); return r.parsed; } catch (err: any) { - this.logger.error(`${variant} search failed: ${err?.message}`, { err: marshalErrorLike(err) }); + const dt = Date.now() - t0; + this.logger.error(`${variant} search failed: ${err?.message}`, { searchDt: dt, err: marshalErrorLike(err) }); if (err?.status === 429) { await delay(500 + 1000 * Math.random()); continue; diff --git a/thinapps-shared b/thinapps-shared index f30577e..580ea72 160000 --- a/thinapps-shared +++ b/thinapps-shared @@ -1 +1 @@ -Subproject commit f30577e73eca151f9d62198ae3b60d0172befaed +Subproject commit 580ea72e0eddaa115b85dabf29de41d079ecd2d0