From 0a09aeb967bc2fcdf129aee92834817fbf5aec5c Mon Sep 17 00:00:00 2001
From: Yanlong Wang <yanlong.wang@naiver.org>
Date: Mon, 1 Jul 2024 10:19:50 +0800
Subject: [PATCH] fix

---
 backend/functions/src/services/puppeteer.ts | 30 ++++++++++++++++-----
 thinapps-shared                             |  2 +-
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/backend/functions/src/services/puppeteer.ts b/backend/functions/src/services/puppeteer.ts
index c4cf780..69ce1ab 100644
--- a/backend/functions/src/services/puppeteer.ts
+++ b/backend/functions/src/services/puppeteer.ts
@@ -277,9 +277,14 @@ function giveSnapshot(stopActiveSnapshot) {
 
         const domainSet = new Set<string>();
         let reqCounter = 0;
+        const t0 = Date.now();
+        let halt = false;
 
         page.on('request', (req) => {
             reqCounter++;
+            if (halt) {
+                return req.abort('blockedbyclient', 1000);
+            }
             const requestUrl = req.url();
             if (!requestUrl.startsWith("http:") && !requestUrl.startsWith("https:") && requestUrl !== 'about:blank') {
                 return req.abort('blockedbyclient', 1000);
@@ -291,7 +296,6 @@ function giveSnapshot(stopActiveSnapshot) {
 
             if (this.circuitBreakerHosts.has(parsedUrl.hostname.toLowerCase())) {
                 page.emit('abuse', { url: requestUrl, page, sn, reason: `Abusive request: ${requestUrl}` });
-
                 return req.abort('blockedbyclient', 1000);
             }
 
@@ -304,14 +308,22 @@ function giveSnapshot(stopActiveSnapshot) {
                 return req.abort('blockedbyclient', 1000);
             }
 
-            if (reqCounter > 2000) {
-                page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many requests: ${reqCounter}` });
+            const dt = Math.ceil((Date.now() - t0) / 1000);
+            const rps = reqCounter / dt;
+            // console.log(`rps: ${rps}`);
 
-                return req.abort('blockedbyclient', 1000);
+            if (reqCounter > 1000) {
+                if (rps > 60 || reqCounter > 2000) {
+                    page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many requests` });
+                    halt = true;
+
+                    return req.abort('blockedbyclient', 1000);
+                }
             }
 
             if (domainSet.size > 200) {
-                page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many domains (${domainSet.size})` });
+                page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many domains` });
+                halt = true;
 
                 return req.abort('blockedbyclient', 1000);
             }
@@ -329,7 +341,7 @@ const handlePageLoad = () => {
     if (window.haltSnapshot) {
         return;
     }
-    if (document.readyState !== 'complete') {
+    if (document.readyState === 'loading') {
         return;
     }
     const thisTextLength = (document.body.innerText || '').length;
@@ -503,7 +515,8 @@ document.addEventListener('load', handlePageLoad);
                 if (options?.minIntervalMs) {
                     ckpt.push(delay(options.minIntervalMs));
                 }
-                await Promise.race(ckpt);
+                let error;
+                await Promise.race(ckpt).catch((err)=> error = err);
                 if (finalized) {
                     yield { ...snapshot, screenshot } as PageSnapshot;
                     break;
@@ -515,6 +528,9 @@ document.addEventListener('load', handlePageLoad);
                 if (snapshot || screenshot) {
                     yield { ...snapshot, screenshot } as PageSnapshot;
                 }
+                if (error) {
+                    throw error;
+                }
             }
         } finally {
             gotoPromise.finally(() => {
diff --git a/thinapps-shared b/thinapps-shared
index 7a221b7..24a9424 160000
--- a/thinapps-shared
+++ b/thinapps-shared
@@ -1 +1 @@
-Subproject commit 7a221b7a8d5fd0244f33dd5dc9d62ce551c5bded
+Subproject commit 24a942452fa12e622a82f05f817c6102c5e84891