AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-15 17:35:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Yanlong Wang 2024-07-01 10:19:50 +08:00
parent 8951578ef1
commit 0a09aeb967
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
2 changed files with 24 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
backend/functions/src/services/puppeteer.ts
View File

@ -277,9 +277,14 @@ function giveSnapshot(stopActiveSnapshot) {
const domainSet = new Set<string>(); const domainSet = new Set<string>();
let reqCounter = 0; let reqCounter = 0;
const t0 = Date.now();
let halt = false;
page.on('request', (req) => { page.on('request', (req) => {
reqCounter++; reqCounter++;
if (halt) {
return req.abort('blockedbyclient', 1000);
}
const requestUrl = req.url(); const requestUrl = req.url();
if (!requestUrl.startsWith("http:") && !requestUrl.startsWith("https:") && requestUrl !== 'about:blank') { if (!requestUrl.startsWith("http:") && !requestUrl.startsWith("https:") && requestUrl !== 'about:blank') {
return req.abort('blockedbyclient', 1000); return req.abort('blockedbyclient', 1000);
@ -291,7 +296,6 @@ function giveSnapshot(stopActiveSnapshot) {
if (this.circuitBreakerHosts.has(parsedUrl.hostname.toLowerCase())) { if (this.circuitBreakerHosts.has(parsedUrl.hostname.toLowerCase())) {
page.emit('abuse', { url: requestUrl, page, sn, reason: `Abusive request: ${requestUrl}` }); page.emit('abuse', { url: requestUrl, page, sn, reason: `Abusive request: ${requestUrl}` });
return req.abort('blockedbyclient', 1000); return req.abort('blockedbyclient', 1000);
} }
@ -304,14 +308,22 @@ function giveSnapshot(stopActiveSnapshot) {
return req.abort('blockedbyclient', 1000); return req.abort('blockedbyclient', 1000);
} }
if (reqCounter > 2000) { const dt = Math.ceil((Date.now() - t0) / 1000);
page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many requests: ${reqCounter}` }); const rps = reqCounter / dt;
// console.log(`rps: ${rps}`);
return req.abort('blockedbyclient', 1000); if (reqCounter > 1000) {
if (rps > 60 || reqCounter > 2000) {
page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many requests` });
halt = true;
return req.abort('blockedbyclient', 1000);
}
} }
if (domainSet.size > 200) { if (domainSet.size > 200) {
page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many domains (${domainSet.size})` }); page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many domains` });
halt = true;
return req.abort('blockedbyclient', 1000); return req.abort('blockedbyclient', 1000);
} }
@ -329,7 +341,7 @@ const handlePageLoad = () => {
if (window.haltSnapshot) { if (window.haltSnapshot) {
return; return;
} }
if (document.readyState !== 'complete') { if (document.readyState === 'loading') {
return; return;
} }
const thisTextLength = (document.body.innerText || '').length; const thisTextLength = (document.body.innerText || '').length;
@ -503,7 +515,8 @@ document.addEventListener('load', handlePageLoad);
if (options?.minIntervalMs) { if (options?.minIntervalMs) {
ckpt.push(delay(options.minIntervalMs)); ckpt.push(delay(options.minIntervalMs));
} }
await Promise.race(ckpt); let error;
await Promise.race(ckpt).catch((err)=> error = err);
if (finalized) { if (finalized) {
yield { ...snapshot, screenshot } as PageSnapshot; yield { ...snapshot, screenshot } as PageSnapshot;
break; break;
@ -515,6 +528,9 @@ document.addEventListener('load', handlePageLoad);
if (snapshot || screenshot) { if (snapshot || screenshot) {
yield { ...snapshot, screenshot } as PageSnapshot; yield { ...snapshot, screenshot } as PageSnapshot;
} }
if (error) {
throw error;
}
} }
} finally { } finally {
gotoPromise.finally(() => { gotoPromise.finally(() => {

2
thinapps-shared

@ -1 +1 @@
Subproject commit 7a221b7a8d5fd0244f33dd5dc9d62ce551c5bded Subproject commit 24a942452fa12e622a82f05f817c6102c5e84891