GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 01:55:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
yanlong.wang 2024-08-01 20:07:39 +08:00
parent f4f189c8e6
commit e658e8102c
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/functions/src/services/puppeteer.ts
View File

@ -11,6 +11,7 @@ import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources';
import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy'; import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy';
import { SecurityCompromiseError, ServiceCrashedError } from '../shared/lib/errors'; import { SecurityCompromiseError, ServiceCrashedError } from '../shared/lib/errors';
import { TimeoutError } from 'puppeteer'; import { TimeoutError } from 'puppeteer';
import { AsyncContext } from '../shared';
const tldExtract = require('tld-extract'); const tldExtract = require('tld-extract');
const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8'); const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8');
@ -214,6 +215,7 @@ export class PuppeteerControl extends AsyncService {
constructor( constructor(
protected globalLogger: Logger, protected globalLogger: Logger,
protected threadLocal: AsyncContext,
) { ) {
super(...arguments); super(...arguments);
this.setMaxListeners(2 * Math.floor(os.totalmem() / (256 * 1024 * 1024)) + 1); 148 - 95; this.setMaxListeners(2 * Math.floor(os.totalmem() / (256 * 1024 * 1024)) + 1); 148 - 95;
@ -494,8 +496,10 @@ document.addEventListener('load', handlePageLoad);
return; return;
} }
if (s?.elemCount && s.elemCount > 20_000) { if (s?.elemCount && s.elemCount > 20_000) {
page.emit('abuse', { url, page, sn, reason: `DoS attack suspected: too many DOM elements` }); if (!this.threadLocal.get('uid')) {
return; page.emit('abuse', { url, page, sn, reason: `DoS attack suspected: too many DOM elements` });
return;
}
} }
snapshot = s; snapshot = s;
nextSnapshotDeferred.resolve(s); nextSnapshotDeferred.resolve(s);