AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-04-18 11:50:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

saas: tweaks

Browse Source
This commit is contained in:
yanlong.wang 2025-04-07 15:07:02 +08:00
parent efa6d9ed75
commit b7d32b9cbc
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
4 changed files with 5 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/api/crawler.ts
View File

@ -891,9 +891,6 @@ export class CrawlerHost extends RPCHost {
}
} else if (crawlOpts?.allocProxy && crawlOpts.allocProxy !== 'none' && !crawlOpts.proxyUrl) {
const proxyUrl = await this.proxyProvider.alloc(this.figureOutBestProxyCountry(crawlOpts));
if (proxyUrl.protocol === 'socks5h:') {
proxyUrl.protocol = 'socks5:';
}
crawlOpts.proxyUrl = proxyUrl.href;
}
@ -1242,7 +1239,6 @@ export class CrawlerHost extends RPCHost {
};
}
retryDet = new WeakSet<ExtraScrappingOptions>();
@retryWith((err) => {
if (err instanceof ServiceBadApproachError) {
return false;
@ -1263,12 +1259,7 @@ export class CrawlerHost extends RPCHost {
}
const proxy = await this.proxyProvider.alloc(this.figureOutBestProxyCountry(opts));
if (opts) {
if (this.retryDet.has(opts) && proxy.protocol === 'socks5h:') {
proxy.protocol = 'socks5:';
}
this.retryDet.add(opts);
}
this.logger.debug(`Proxy allocated`, { proxy: proxy.href });
const r = await this.curlControl.sideLoad(url, {
...opts,
proxyUrl: proxy.href,

2
src/api/serp.ts
View File

@ -136,7 +136,7 @@ export class SerpHost extends RPCHost {
@Param('hl', { validate: (v: string) => WORLD_LANGUAGES.some(l => l.code === v) }) hl?: string,
@Param('location') location?: string,
@Param('page') page?: number,
@Param('fallback', { default: true }) fallback?: boolean,
@Param('fallback') fallback?: boolean,
) {
const authToken = auth.bearerToken;
let highFreqKey: RateLimitCache | undefined;

10
src/services/serp/google.ts
View File

@ -16,7 +16,7 @@ import { ProxyProvider } from '../../shared/services/proxy-provider';
@singleton()
export class GoogleSERP extends AsyncService {
logger = this.globalLogger.child({ service: this.constructor.name });
googleDomain = process.env.OVERRIDE_GOOGLE_DOMAIN || 'www.google.com';
constructor(
@ -36,7 +36,6 @@ export class GoogleSERP extends AsyncService {
this.emit('ready');
}
retryDet = new WeakSet<ScrappingOptions>();
@retryWith((err) => {
if (err instanceof ServiceBadApproachError) {
return false;
@ -59,12 +58,7 @@ export class GoogleSERP extends AsyncService {
const proxy = await this.proxyProvider.alloc(
process.env.PREFERRED_PROXY_COUNTRY || 'auto'
);
if (opts) {
if (this.retryDet.has(opts) && proxy.protocol === 'socks5h:') {
proxy.protocol = 'socks5:';
}
this.retryDet.add(opts);
}
this.logger.debug(`Proxy allocated`, { proxy: proxy.href });
const r = await this.curlControl.sideLoad(url, {
...opts,
proxyUrl: proxy.href,

2
thinapps-shared

@ -1 +1 @@
Subproject commit 936ebc4158a3007b59496822ef650e995c189614
Subproject commit 424f50ca8b6277d74185e16aa67ff2b366d9f727