GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-18 21:25:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Yanlong Wang 2024-10-01 12:55:06 +08:00
parent a66791d85f
commit f0c3a9b70e
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
4 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/functions/package-lock.json generated
View File

@ -38,7 +38,7 @@
"robots-parser": "^3.0.1",
"set-cookie-parser": "^2.6.0",
"stripe": "^11.11.0",
"tiktoken": "^1.0.10",
"tiktoken": "^1.0.16",
"tld-extract": "^2.1.0",
"turndown": "^7.1.3",
"turndown-plugin-gfm": "^1.0.2",
@ -11218,9 +11218,9 @@
}
},
"node_modules/tiktoken": {
"version": "1.0.13",
"resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.13.tgz",
"integrity": "sha512-JaL9ZnvTbGFMDIBeGdVkLt4qWTeCPw+n7Ock+wceAGRenuHA6nOOvMJFliNDyXsjg2osGKJWsXtO2xc74VxyDw=="
"version": "1.0.16",
"resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.16.tgz",
"integrity": "sha512-hRcORIGF2YlAgWx3nzrGJOrKSJwLoc81HpXmMQk89632XAgURc7IeV2FgQ2iXo9z/J96fCvpsHg2kWoHcbj9fg=="
},
"node_modules/tiny-lru": {
"version": "11.2.5",

2
backend/functions/package.json
View File

@ -58,7 +58,7 @@
"robots-parser": "^3.0.1",
"set-cookie-parser": "^2.6.0",
"stripe": "^11.11.0",
"tiktoken": "^1.0.10",
"tiktoken": "^1.0.16",
"tld-extract": "^2.1.0",
"turndown": "^7.1.3",
"turndown-plugin-gfm": "^1.0.2",

13
backend/functions/src/services/puppeteer.ts
View File

@ -225,9 +225,17 @@ export class PuppeteerControl extends AsyncService {
super(...arguments);
this.setMaxListeners(2 * Math.floor(os.totalmem() / (256 * 1024 * 1024)) + 1); 148 - 95;
let crippledTimes = 0;
this.on('crippled', () => {
crippledTimes += 1;
this.__loadedPage.length = 0;
this.livePages.clear();
if (crippledTimes > 5) {
process.nextTick(() => {
this.emit('error', new Error('Browser crashed too many times, quitting...'));
// process.exit(1);
});
}
});
}
@ -257,7 +265,9 @@ export class PuppeteerControl extends AsyncService {
});
this.browser.once('disconnected', () => {
this.logger.warn(`Browser disconnected`);
if (this.browser) {
this.emit('crippled');
}
process.nextTick(() => this.serviceReady());
});
this.logger.info(`Browser launched: ${this.browser.process()?.pid}`);
@ -287,13 +297,14 @@ export class PuppeteerControl extends AsyncService {
async newPage() {
await this.serviceReady();
const dedicatedContext = await this.browser.createBrowserContext();
const sn = this._sn++;
let page;
try {
const dedicatedContext = await this.browser.createBrowserContext();
page = await dedicatedContext.newPage();
} catch (err: any) {
this.logger.warn(`Failed to create page ${sn}`, { err: marshalErrorLike(err) });
this.browser.process()?.kill('SIGKILL');
throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`);
}
const preparations = [];

2
thinapps-shared

@ -1 +1 @@
Subproject commit c2a3af9607d6a0ada64639d4f4e73bfc9c3e024f
Subproject commit d91b3aab5924c4090cff0d9d6cde9c5c0a4dc8bd