AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-15 09:15:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: block suspicious requests before sideload

Browse Source
This commit is contained in:
yanlong.wang 2025-03-10 16:46:26 +08:00
parent 3b3a0265df
commit d71c89a79c
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 22 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/api/crawler.ts
View File

@ -305,11 +305,10 @@ export class CrawlerHost extends RPCHost {
} }
} }
const crawlOpts = await this.configure(crawlerOptions);
if (crawlerOptions.robotsTxt) { if (crawlerOptions.robotsTxt) {
await this.robotsTxtService.assertAccessAllowed(targetUrl, crawlerOptions.robotsTxt); await this.robotsTxtService.assertAccessAllowed(targetUrl, crawlerOptions.robotsTxt);
} }
const crawlOpts = await this.configure(crawlerOptions);
if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) { if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) {
const sseStream = new OutputServerEventStream(); const sseStream = new OutputServerEventStream();
rpcReflect.return(sseStream); rpcReflect.return(sseStream);
@ -508,7 +507,27 @@ export class CrawlerHost extends RPCHost {
}); });
} }
if (!isIP(result.hostname)) {
if (this.puppeteerControl.circuitBreakerHosts.has(result.hostname.toLowerCase())) {
throw new SecurityCompromiseError({
message: `Circular hostname: ${result.protocol}`,
path: 'url'
});
}
const isIp = isIP(result.hostname);
if (
(result.hostname === 'localhost') ||
(isIp && result.hostname.startsWith('127.'))
) {
throw new SecurityCompromiseError({
message: `Suspicious action: Request to localhost: ${result}`,
path: 'url'
});
}
if (!isIp) {
await lookup(result.hostname).catch((err) => { await lookup(result.hostname).catch((err) => {
if (err.code === 'ENOTFOUND') { if (err.code === 'ENOTFOUND') {
return Promise.reject(new ParamValidationError({ return Promise.reject(new ParamValidationError({