fix(crawl): make request 0 of crawl jobs higher priority

This commit is contained in:
Gergő Móricz 2024-08-14 19:34:18 +02:00
parent f4466f6bb0
commit 2ca1017fc3
2 changed files with 3 additions and 1 deletions

View File

@ -139,6 +139,8 @@ export async function crawlController(req: Request, res: Response) {
pageOptions: pageOptions,
origin: req.body.origin ?? defaultOrigin,
crawl_id: id,
}, {
priority: 1, // prioritize request 0 of crawl jobs same as scrape jobs
});
await addCrawlJob(id, job.id);
}

View File

@ -9,9 +9,9 @@ export async function addScrapeJob(
jobId: string = uuidv4(),
): Promise<Job> {
return await getScrapeQueue().add(jobId, webScraperOptions, {
priority: webScraperOptions.crawl_id ? 2 : 1,
...options,
jobId,
priority: webScraperOptions.crawl_id ? 2 : 1,
});
}