From 5ed3f90b9c322c2cb76c90778055b45461b640ee Mon Sep 17 00:00:00 2001 From: Yanlong Wang Date: Sat, 13 Apr 2024 07:53:58 +0800 Subject: [PATCH] fix --- backend/functions/.puppeteerrc.cjs | 2 +- backend/functions/src/cloud-functions/crawler.ts | 4 ++-- backend/functions/src/services/puppeteer.ts | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/backend/functions/.puppeteerrc.cjs b/backend/functions/.puppeteerrc.cjs index 64284be..ea22710 100644 --- a/backend/functions/.puppeteerrc.cjs +++ b/backend/functions/.puppeteerrc.cjs @@ -4,7 +4,7 @@ let config = {}; if (!process.env.FUNCTIONS_EMULATOR) { config = { // Changes the cache location for Puppeteer. - cacheDirectory: join(__dirname, '.cache', 'puppeteer'), + cacheDirectory: join(__dirname, 'puppeteer'), }; } diff --git a/backend/functions/src/cloud-functions/crawler.ts b/backend/functions/src/cloud-functions/crawler.ts index 4ab4c11..c331d95 100644 --- a/backend/functions/src/cloud-functions/crawler.ts +++ b/backend/functions/src/cloud-functions/crawler.ts @@ -65,9 +65,9 @@ ${this.content} }) @CloudHTTPv2({ runtime: { - memory: '4GiB', + memory: '8GiB', timeoutSeconds: 540, - concurrency: 4, + concurrency: 16, }, httpMethod: ['get', 'post'], returnType: [String, OutputServerEventStream], diff --git a/backend/functions/src/services/puppeteer.ts b/backend/functions/src/services/puppeteer.ts index 542c136..0d3dcb2 100644 --- a/backend/functions/src/services/puppeteer.ts +++ b/backend/functions/src/services/puppeteer.ts @@ -49,11 +49,12 @@ export class PuppeteerControl extends AsyncService { return page.browser().connected && !page.isClosed(); } }, { - max: Math.max(1 + Math.floor(os.freemem() / 1024 * 1024 * 1024), 4), + max: Math.max(1 + Math.floor(os.freemem() / 1024 * 1024 * 1024), 16), min: 1, acquireTimeoutMillis: 60_000, testOnBorrow: true, testOnReturn: true, + autostart: false, }); constructor(protected globalLogger: Logger) { @@ -63,6 +64,8 @@ export class PuppeteerControl extends AsyncService { override async init() { await this.dependencyReady(); + this.pagePool.start(); + if (this.browser) { if (this.browser.connected) { await this.browser.close(); @@ -72,7 +75,13 @@ export class PuppeteerControl extends AsyncService { } this.browser = await puppeteer.launch({ headless: true, - timeout: 45_000 + timeout: 10_000 + }).catch((err) => { + this.logger.error(`Unknown firebase issue, just die fast, quitting process.`, { err }); + process.nextTick(()=> { + process.exit(1); + }); + return Promise.reject(err); }); this.browser.once('disconnected', () => { this.logger.warn(`Browser disconnected`); @@ -90,7 +99,7 @@ export class PuppeteerControl extends AsyncService { const page = await dedicatedContext.newPage(); const preparations = []; - preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`)); + // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`)); preparations.push(page.setBypassCSP(true)); preparations.push(page.setViewport({ width: 1920, height: 1080 })); preparations.push(page.exposeFunction('reportSnapshot', (snapshot: any) => {