AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 19:46:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick: got rid of job interval sleep and math.min

Browse Source
This commit is contained in:
Nicolas 2024-10-01 16:11:12 -03:00
parent 18f9cd09e1
commit c6717fecaa
2 changed files with 4 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/controllers/v1/concurrency-check.ts
View File

@ -12,7 +12,7 @@ export async function concurrencyCheckController(
req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>,
res: Response<ConcurrencyCheckResponse>
) {
const concurrencyLimiterKey = "concurrency-limiter:" + req.params.teamId;
const concurrencyLimiterKey = "concurrency-limiter:" + req.auth.team_id;
const now = Date.now();
const activeJobsOfTeam = await redisConnection.zrangebyscore(
concurrencyLimiterKey,

10
apps/api/src/services/queue-worker.ts
View File

@ -151,12 +151,8 @@ const workerFun = async (
await job.moveToFailed(new Error("Concurrency limit hit"), token, false);
// Remove the job from the queue
await job.remove();
// Increment the priority of the job exponentially by 5%
let newJobPriority = Math.round((job.opts.priority ?? 10) * 1.05);
// Max priority is 200k, limit is 2 million
if(newJobPriority > 200000) {
newJobPriority = 200000;
}
// Increment the priority of the job exponentially by 5%, Note: max bull priority is 2 million
const newJobPriority = Math.min(Math.round((job.opts.priority ?? 10) * 1.05), 20000);
// Add the job back to the queue with the new priority
await queue.add(job.name, {
...job.data,
@ -167,7 +163,7 @@ const workerFun = async (
priority: newJobPriority, // exponential backoff for stuck jobs
});
await sleep(gotJobInterval);
// await sleep(gotJobInterval);
continue;
} else {
// If we are not throttled, add the job back to the queue with the new priority