AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-14 20:15:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

tweak: try alleviate oom killed issue

Browse Source
This commit is contained in:
Yanlong Wang 2024-09-08 16:29:45 +08:00
parent e324c4667f
commit c5abdf8570
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
3 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/functions/src/cloud-functions/crawler.ts
View File

@ -130,11 +130,11 @@ export class CrawlerHost extends RPCHost {
}) })
@CloudHTTPv2({ @CloudHTTPv2({
runtime: { runtime: {
memory: '8GiB', memory: '4GiB',
cpu: 4, cpu: 4,
timeoutSeconds: 300, timeoutSeconds: 300,
concurrency: 22, concurrency: 8,
maxInstances: 455, maxInstances: 1250,
minInstances: 1, minInstances: 1,
}, },
tags: ['Crawler'], tags: ['Crawler'],

10
backend/functions/src/services/puppeteer.ts
View File

@ -9,7 +9,7 @@ import puppeteer from 'puppeteer-extra';
import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources'; import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources';
import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy'; import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy';
import { SecurityCompromiseError, ServiceCrashedError } from '../shared/lib/errors'; import { SecurityCompromiseError, ServiceCrashedError, ServiceNodeResourceDrainError } from '../shared/lib/errors';
import { TimeoutError } from 'puppeteer'; import { TimeoutError } from 'puppeteer';
import _ from 'lodash'; import _ from 'lodash';
const tldExtract = require('tld-extract'); const tldExtract = require('tld-extract');
@ -285,7 +285,13 @@ export class PuppeteerControl extends AsyncService {
await this.serviceReady(); await this.serviceReady();
const dedicatedContext = await this.browser.createBrowserContext(); const dedicatedContext = await this.browser.createBrowserContext();
const sn = this._sn++; const sn = this._sn++;
const page = await dedicatedContext.newPage(); let page
try {
page = await dedicatedContext.newPage();
} catch (err: any) {
this.logger.warn(`Failed to create page ${sn}`, { err: marshalErrorLike(err) });
throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`);
}
const preparations = []; const preparations = [];
// preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`)); // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));

2
thinapps-shared

@ -1 +1 @@
Subproject commit d2b0fbf184b4c77e80e8d1dd36b3f4d1807e0e09 Subproject commit 2ce99a95bff6ea07f48e3388e270b2126d752d0c