AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-18 03:55:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: req cap issues

Browse Source
This commit is contained in:
yanlong.wang 2024-09-02 14:30:23 +08:00
parent 405fe6372e
commit 5171e5f94b
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
backend/functions/src/services/puppeteer.ts
View File

@ -1,7 +1,7 @@
import os from 'os';
import fs from 'fs';
import { container, singleton } from 'tsyringe';
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency, Deferred } from 'civkit';
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency, Deferred, perNextTick } from 'civkit';
import { Logger } from '../shared/services/logger';
import type { Browser, CookieParam, GoToOptions, Page } from 'puppeteer';
@ -310,11 +310,14 @@ export class PuppeteerControl extends AsyncService {
this.logger.warn(`Browser killed`);
}
@perNextTick()
reqCapRoutine() {
const now = Date.now();
const numToPass = Math.round((now - this.lastReqSentAt) / 1000 * this.rpsCap);
this.requestDeferredQueue.splice(0, numToPass).forEach((x) => x.resolve(true));
if (numToPass) {
this.lastReqSentAt = now;
}
if (!this.requestDeferredQueue.length) {
if (this.__reqCapInterval) {
clearInterval(this.__reqCapInterval);
@ -403,10 +406,12 @@ export class PuppeteerControl extends AsyncService {
return req.abort('blockedbyclient', 1000);
}
if (requestUrl.startsWith('http')) {
const d = Defer();
this.requestDeferredQueue.push(d);
process.nextTick(() => this.reqCapRoutine());
this.reqCapRoutine();
await d.promise;
}
if (req.isInterceptResolutionHandled()) {
return;