AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-18 07:25:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: req cap issues

Browse Source
This commit is contained in:
yanlong.wang 2024-09-02 14:30:23 +08:00
parent 405fe6372e
commit 5171e5f94b
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
backend/functions/src/services/puppeteer.ts
View File

@ -1,7 +1,7 @@
import os from 'os'; import os from 'os';
import fs from 'fs'; import fs from 'fs';
import { container, singleton } from 'tsyringe'; import { container, singleton } from 'tsyringe';
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency, Deferred } from 'civkit'; import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency, Deferred, perNextTick } from 'civkit';
import { Logger } from '../shared/services/logger'; import { Logger } from '../shared/services/logger';
import type { Browser, CookieParam, GoToOptions, Page } from 'puppeteer'; import type { Browser, CookieParam, GoToOptions, Page } from 'puppeteer';
@ -310,11 +310,14 @@ export class PuppeteerControl extends AsyncService {
this.logger.warn(`Browser killed`); this.logger.warn(`Browser killed`);
} }
@perNextTick()
reqCapRoutine() { reqCapRoutine() {
const now = Date.now(); const now = Date.now();
const numToPass = Math.round((now - this.lastReqSentAt) / 1000 * this.rpsCap); const numToPass = Math.round((now - this.lastReqSentAt) / 1000 * this.rpsCap);
this.requestDeferredQueue.splice(0, numToPass).forEach((x) => x.resolve(true)); this.requestDeferredQueue.splice(0, numToPass).forEach((x) => x.resolve(true));
if (numToPass) {
this.lastReqSentAt = now; this.lastReqSentAt = now;
}
if (!this.requestDeferredQueue.length) { if (!this.requestDeferredQueue.length) {
if (this.__reqCapInterval) { if (this.__reqCapInterval) {
clearInterval(this.__reqCapInterval); clearInterval(this.__reqCapInterval);
@ -403,10 +406,12 @@ export class PuppeteerControl extends AsyncService {
return req.abort('blockedbyclient', 1000); return req.abort('blockedbyclient', 1000);
} }
if (requestUrl.startsWith('http')) {
const d = Defer(); const d = Defer();
this.requestDeferredQueue.push(d); this.requestDeferredQueue.push(d);
process.nextTick(() => this.reqCapRoutine()); this.reqCapRoutine();
await d.promise; await d.promise;
}
if (req.isInterceptResolutionHandled()) { if (req.isInterceptResolutionHandled()) {
return; return;