GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-20 01:29:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: block media resources to improve speed

Browse Source
This commit is contained in:
yanlong.wang 2024-04-18 15:06:28 +08:00
parent cbc13ecbbd
commit abc817e960
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
3 changed files with 26 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
backend/functions/package-lock.json generated
View File

@ -28,6 +28,7 @@
"openai": "^4.20.0", "openai": "^4.20.0",
"puppeteer": "^22.6.3", "puppeteer": "^22.6.3",
"puppeteer-extra": "^3.3.6", "puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-block-resources": "^2.4.3",
"puppeteer-extra-plugin-stealth": "^2.11.2", "puppeteer-extra-plugin-stealth": "^2.11.2",
"stripe": "^11.11.0", "stripe": "^11.11.0",
"tiktoken": "^1.0.10", "tiktoken": "^1.0.10",
@ -9912,6 +9913,26 @@
} }
} }
}, },
"node_modules/puppeteer-extra-plugin-block-resources": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-block-resources/-/puppeteer-extra-plugin-block-resources-2.4.3.tgz",
"integrity": "sha512-WLxkytm3Nn6D+3N8lFMIi9lL9lLA1mTRHZAflR+zDs9/d1oOYVy52pWPaiQaetOObgfHDpPlHBmzHE2Yhr3ZnQ==",
"dependencies": {
"debug": "^4.1.1",
"puppeteer-extra-plugin": "^3.2.3"
},
"engines": {
"node": ">=8"
},
"peerDependencies": {
"puppeteer-extra": "*"
},
"peerDependenciesMeta": {
"puppeteer-extra": {
"optional": true
}
}
},
"node_modules/puppeteer-extra-plugin-stealth": { "node_modules/puppeteer-extra-plugin-stealth": {
"version": "2.11.2", "version": "2.11.2",
"resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz", "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz",

1
backend/functions/package.json
View File

@ -48,6 +48,7 @@
"openai": "^4.20.0", "openai": "^4.20.0",
"puppeteer": "^22.6.3", "puppeteer": "^22.6.3",
"puppeteer-extra": "^3.3.6", "puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-block-resources": "^2.4.3",
"puppeteer-extra-plugin-stealth": "^2.11.2", "puppeteer-extra-plugin-stealth": "^2.11.2",
"stripe": "^11.11.0", "stripe": "^11.11.0",
"tiktoken": "^1.0.10", "tiktoken": "^1.0.10",

4
backend/functions/src/services/puppeteer.ts
View File

@ -51,6 +51,10 @@ puppeteer.use(puppeteerStealth());
// userAgent: `Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`, // userAgent: `Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`,
// platform: `Linux`, // platform: `Linux`,
// })) // }))
const puppeteerBlockResources = require('puppeteer-extra-plugin-block-resources');
puppeteer.use(puppeteerBlockResources({
blockedTypes: new Set(['media']),
}));
@singleton() @singleton()
export class PuppeteerControl extends AsyncService { export class PuppeteerControl extends AsyncService {