From c5abdf85705123d2fab943dcf273a560ec23becd Mon Sep 17 00:00:00 2001
From: Yanlong Wang <yanlong.wang@naiver.org>
Date: Sun, 8 Sep 2024 16:29:45 +0800
Subject: [PATCH] tweak: try alleviate oom killed issue

---
 backend/functions/src/cloud-functions/crawler.ts |  6 +++---
 backend/functions/src/services/puppeteer.ts      | 10 ++++++++--
 thinapps-shared                                  |  2 +-
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/backend/functions/src/cloud-functions/crawler.ts b/backend/functions/src/cloud-functions/crawler.ts
index ba3dfb0..295b5b8 100644
--- a/backend/functions/src/cloud-functions/crawler.ts
+++ b/backend/functions/src/cloud-functions/crawler.ts
@@ -130,11 +130,11 @@ export class CrawlerHost extends RPCHost {
     })
     @CloudHTTPv2({
         runtime: {
-            memory: '8GiB',
+            memory: '4GiB',
             cpu: 4,
             timeoutSeconds: 300,
-            concurrency: 22,
-            maxInstances: 455,
+            concurrency: 8,
+            maxInstances: 1250,
             minInstances: 1,
         },
         tags: ['Crawler'],
diff --git a/backend/functions/src/services/puppeteer.ts b/backend/functions/src/services/puppeteer.ts
index 04dbd06..96184b5 100644
--- a/backend/functions/src/services/puppeteer.ts
+++ b/backend/functions/src/services/puppeteer.ts
@@ -9,7 +9,7 @@ import puppeteer from 'puppeteer-extra';
 
 import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources';
 import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy';
-import { SecurityCompromiseError, ServiceCrashedError } from '../shared/lib/errors';
+import { SecurityCompromiseError, ServiceCrashedError, ServiceNodeResourceDrainError } from '../shared/lib/errors';
 import { TimeoutError } from 'puppeteer';
 import _ from 'lodash';
 const tldExtract = require('tld-extract');
@@ -285,7 +285,13 @@ export class PuppeteerControl extends AsyncService {
         await this.serviceReady();
         const dedicatedContext = await this.browser.createBrowserContext();
         const sn = this._sn++;
-        const page = await dedicatedContext.newPage();
+        let page
+        try {
+            page = await dedicatedContext.newPage();
+        } catch (err: any) {
+            this.logger.warn(`Failed to create page ${sn}`, { err: marshalErrorLike(err) });
+            throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`);
+        }
         const preparations = [];
 
         // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));
diff --git a/thinapps-shared b/thinapps-shared
index d2b0fbf..2ce99a9 160000
--- a/thinapps-shared
+++ b/thinapps-shared
@@ -1 +1 @@
-Subproject commit d2b0fbf184b4c77e80e8d1dd36b3f4d1807e0e09
+Subproject commit 2ce99a95bff6ea07f48e3388e270b2126d752d0c