GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 04:05:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Yanlong Wang 2024-10-01 12:55:06 +08:00
parent a66791d85f
commit f0c3a9b70e
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
4 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/functions/package-lock.json generated
View File

@ -38,7 +38,7 @@
"robots-parser": "^3.0.1", "robots-parser": "^3.0.1",
"set-cookie-parser": "^2.6.0", "set-cookie-parser": "^2.6.0",
"stripe": "^11.11.0", "stripe": "^11.11.0",
"tiktoken": "^1.0.10", "tiktoken": "^1.0.16",
"tld-extract": "^2.1.0", "tld-extract": "^2.1.0",
"turndown": "^7.1.3", "turndown": "^7.1.3",
"turndown-plugin-gfm": "^1.0.2", "turndown-plugin-gfm": "^1.0.2",
@ -11218,9 +11218,9 @@
} }
}, },
"node_modules/tiktoken": { "node_modules/tiktoken": {
"version": "1.0.13", "version": "1.0.16",
"resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.13.tgz", "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.16.tgz",
"integrity": "sha512-JaL9ZnvTbGFMDIBeGdVkLt4qWTeCPw+n7Ock+wceAGRenuHA6nOOvMJFliNDyXsjg2osGKJWsXtO2xc74VxyDw==" "integrity": "sha512-hRcORIGF2YlAgWx3nzrGJOrKSJwLoc81HpXmMQk89632XAgURc7IeV2FgQ2iXo9z/J96fCvpsHg2kWoHcbj9fg=="
}, },
"node_modules/tiny-lru": { "node_modules/tiny-lru": {
"version": "11.2.5", "version": "11.2.5",

2
backend/functions/package.json
View File

@ -58,7 +58,7 @@
"robots-parser": "^3.0.1", "robots-parser": "^3.0.1",
"set-cookie-parser": "^2.6.0", "set-cookie-parser": "^2.6.0",
"stripe": "^11.11.0", "stripe": "^11.11.0",
"tiktoken": "^1.0.10", "tiktoken": "^1.0.16",
"tld-extract": "^2.1.0", "tld-extract": "^2.1.0",
"turndown": "^7.1.3", "turndown": "^7.1.3",
"turndown-plugin-gfm": "^1.0.2", "turndown-plugin-gfm": "^1.0.2",

13
backend/functions/src/services/puppeteer.ts
View File

@ -225,9 +225,17 @@ export class PuppeteerControl extends AsyncService {
super(...arguments); super(...arguments);
this.setMaxListeners(2 * Math.floor(os.totalmem() / (256 * 1024 * 1024)) + 1); 148 - 95; this.setMaxListeners(2 * Math.floor(os.totalmem() / (256 * 1024 * 1024)) + 1); 148 - 95;
let crippledTimes = 0;
this.on('crippled', () => { this.on('crippled', () => {
crippledTimes += 1;
this.__loadedPage.length = 0; this.__loadedPage.length = 0;
this.livePages.clear(); this.livePages.clear();
if (crippledTimes > 5) {
process.nextTick(() => {
this.emit('error', new Error('Browser crashed too many times, quitting...'));
// process.exit(1);
});
}
}); });
} }
@ -257,7 +265,9 @@ export class PuppeteerControl extends AsyncService {
}); });
this.browser.once('disconnected', () => { this.browser.once('disconnected', () => {
this.logger.warn(`Browser disconnected`); this.logger.warn(`Browser disconnected`);
if (this.browser) {
this.emit('crippled'); this.emit('crippled');
}
process.nextTick(() => this.serviceReady()); process.nextTick(() => this.serviceReady());
}); });
this.logger.info(`Browser launched: ${this.browser.process()?.pid}`); this.logger.info(`Browser launched: ${this.browser.process()?.pid}`);
@ -287,13 +297,14 @@ export class PuppeteerControl extends AsyncService {
async newPage() { async newPage() {
await this.serviceReady(); await this.serviceReady();
const dedicatedContext = await this.browser.createBrowserContext();
const sn = this._sn++; const sn = this._sn++;
let page; let page;
try { try {
const dedicatedContext = await this.browser.createBrowserContext();
page = await dedicatedContext.newPage(); page = await dedicatedContext.newPage();
} catch (err: any) { } catch (err: any) {
this.logger.warn(`Failed to create page ${sn}`, { err: marshalErrorLike(err) }); this.logger.warn(`Failed to create page ${sn}`, { err: marshalErrorLike(err) });
this.browser.process()?.kill('SIGKILL');
throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`); throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`);
} }
const preparations = []; const preparations = [];

2
thinapps-shared

@ -1 +1 @@
Subproject commit c2a3af9607d6a0ada64639d4f4e73bfc9c3e024f Subproject commit d91b3aab5924c4090cff0d9d6cde9c5c0a4dc8bd