GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 20:29:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Yanlong Wang 2025-05-07 21:58:03 +08:00
parent a3412ef180
commit da1098f38d
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
4 changed files with 1 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/api/crawler.ts
View File

@ -340,6 +340,7 @@ export class CrawlerHost extends RPCHost {
} }
} }
const crawlOpts = await this.configure(crawlerOptions); const crawlOpts = await this.configure(crawlerOptions);
this.logger.info(`Accepting request from ${uid || ctx.ip}`, { opts: crawlerOptions });
if (crawlerOptions.robotsTxt) { if (crawlerOptions.robotsTxt) {
await this.robotsTxtService.assertAccessAllowed(targetUrl, crawlerOptions.robotsTxt); await this.robotsTxtService.assertAccessAllowed(targetUrl, crawlerOptions.robotsTxt);
} }

4
src/stand-alone/crawl.ts
View File

@ -44,10 +44,6 @@ export class CrawlStandAloneServer extends KoaServer {
const ar = new AsyncResource('HTTP2ServerRequest'); const ar = new AsyncResource('HTTP2ServerRequest');
ar.runInAsyncScope(fn, this.koaApp, req, res); ar.runInAsyncScope(fn, this.koaApp, req, res);
}); });
// HTTP2 p2p may break the load balancing
this.httpServer.updateSettings({
maxConcurrentStreams: 8,
});
// useResourceBasedDefaultTracker(); // useResourceBasedDefaultTracker();
return this; return this;

4
src/stand-alone/search.ts
View File

@ -44,10 +44,6 @@ export class SearchStandAloneServer extends KoaServer {
const ar = new AsyncResource('HTTP2ServerRequest'); const ar = new AsyncResource('HTTP2ServerRequest');
ar.runInAsyncScope(fn, this.koaApp, req, res); ar.runInAsyncScope(fn, this.koaApp, req, res);
}); });
// HTTP2 p2p may break the load balancing
this.httpServer.updateSettings({
maxConcurrentStreams: 8,
});
// useResourceBasedDefaultTracker(); // useResourceBasedDefaultTracker();
return this; return this;

4
src/stand-alone/serp.ts
View File

@ -45,10 +45,6 @@ export class SERPStandAloneServer extends KoaServer {
const ar = new AsyncResource('HTTP2ServerRequest'); const ar = new AsyncResource('HTTP2ServerRequest');
ar.runInAsyncScope(fn, this.koaApp, req, res); ar.runInAsyncScope(fn, this.koaApp, req, res);
}); });
// HTTP2 p2p may break the load balancing
this.httpServer.updateSettings({
maxConcurrentStreams: 50,
});
// useResourceBasedDefaultTracker(); // useResourceBasedDefaultTracker();
return this; return this;