diff --git a/backend/functions/package-lock.json b/backend/functions/package-lock.json index 58ef8d0..38f0027 100644 --- a/backend/functions/package-lock.json +++ b/backend/functions/package-lock.json @@ -14,7 +14,7 @@ "archiver": "^6.0.1", "axios": "^1.3.3", "bcrypt": "^5.1.0", - "civkit": "^0.6.5-79b1e2c", + "civkit": "^0.6.5-7a4ba56", "cors": "^2.8.5", "dayjs": "^1.11.9", "express": "^4.19.2", @@ -3674,9 +3674,9 @@ } }, "node_modules/civkit": { - "version": "0.6.5-79b1e2c", - "resolved": "https://registry.npmjs.org/civkit/-/civkit-0.6.5-79b1e2c.tgz", - "integrity": "sha512-JwuDgfo6YMopniSHmYXwtzSOUs/i8FA+GNLhPUivh1AtfMp7nO6C354xNalVP+nP8TqZE2HgmLL2aiyxrX51sQ==", + "version": "0.6.5-7a4ba56", + "resolved": "https://registry.npmjs.org/civkit/-/civkit-0.6.5-7a4ba56.tgz", + "integrity": "sha512-WAKnZn7DwuHkjEaH/bGXN4ZSYFvzM06ky1S9LjzHd1Ud+fMd3sEJR0b68BprzqXdeBNB5LyPHO4Gikf1z7J1bA==", "dependencies": { "lodash": "^4.17.21", "tslib": "^2.5.0" diff --git a/backend/functions/package.json b/backend/functions/package.json index 80c4c88..acc0546 100644 --- a/backend/functions/package.json +++ b/backend/functions/package.json @@ -34,7 +34,7 @@ "archiver": "^6.0.1", "axios": "^1.3.3", "bcrypt": "^5.1.0", - "civkit": "^0.6.5-79b1e2c", + "civkit": "^0.6.5-7a4ba56", "cors": "^2.8.5", "dayjs": "^1.11.9", "express": "^4.19.2", diff --git a/backend/functions/src/services/puppeteer.ts b/backend/functions/src/services/puppeteer.ts index a2caad6..d13c761 100644 --- a/backend/functions/src/services/puppeteer.ts +++ b/backend/functions/src/services/puppeteer.ts @@ -2,7 +2,7 @@ import os from 'os'; import fs from 'fs'; import { container, singleton } from 'tsyringe'; import genericPool from 'generic-pool'; -import { AsyncService, Defer, marshalErrorLike, AssertionFailureError } from 'civkit'; +import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency } from 'civkit'; import { Logger } from '../shared/services/logger'; import type { Browser, CookieParam, Page } from 'puppeteer'; @@ -82,8 +82,16 @@ export class PuppeteerControl extends AsyncService { return page; }, destroy: async (page) => { - await page.removeExposedFunction('reportSnapshot'); - await page.browserContext().close(); + await Promise.race([ + (async () => { + const ctx = page.browserContext(); + await page.removeExposedFunction('reportSnapshot'); + await page.close(); + await ctx.close(); + })(), delay(5000) + ]).catch((err) => { + this.logger.error(`Failed to destroy page`, { err: marshalErrorLike(err) }); + }); }, validate: async (page) => { return page.browser().connected && !page.isClosed(); @@ -95,13 +103,20 @@ export class PuppeteerControl extends AsyncService { testOnBorrow: true, testOnReturn: true, autostart: false, + priorityRange: 3 }); + private __healthCheckInterval?: NodeJS.Timeout; + constructor(protected globalLogger: Logger) { super(...arguments); } override async init() { + if (this.__healthCheckInterval) { + clearInterval(this.__healthCheckInterval); + this.__healthCheckInterval = undefined; + } await this.dependencyReady(); this.logger.info(`PuppeteerControl initializing with pool size ${this.pagePool.max}`, { poolSize: this.pagePool.max }); this.pagePool.start(); @@ -110,7 +125,7 @@ export class PuppeteerControl extends AsyncService { if (this.browser.connected) { await this.browser.close(); } else { - this.browser.process()?.kill(); + this.browser.process()?.kill('SIGKILL'); } } this.browser = await puppeteer.launch({ @@ -130,6 +145,27 @@ export class PuppeteerControl extends AsyncService { this.logger.info(`Browser launched: ${this.browser.process()?.pid}`); this.emit('ready'); + + this.__healthCheckInterval = setInterval(() => this.healthCheck(), 30_000); + } + + @maxConcurrency(1) + async healthCheck() { + const healthyPage = await Promise.race([this.pagePool.acquire(3), delay(60_000).then(() => null)]).catch((err) => { + this.logger.error(`Health check failed`, { err: marshalErrorLike(err) }); + return null; + }); + + if (healthyPage) { + this.pagePool.release(healthyPage); + return; + } + + this.logger.warn(`Health check failed, trying to clean up.`); + await this.pagePool.clear(); + this.browser.process()?.kill('SIGKILL'); + Reflect.deleteProperty(this, 'browser'); + this.emit('crippled'); } async newPage() { diff --git a/thinapps-shared b/thinapps-shared index e681cf8..e2a1d58 160000 --- a/thinapps-shared +++ b/thinapps-shared @@ -1 +1 @@ -Subproject commit e681cf89bd21d77469dd286b2348e4cf5fce76e7 +Subproject commit e2a1d586063f8e8d663c013fa2febe9f621f9f8e