AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-15 20:15:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(sentry): add queue instrumentation to

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-22 19:17:51 +02:00
parent 7265ab7c67
commit dd737f1235
2 changed files with 13 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/src/controllers/search.ts
View File

@ -11,6 +11,7 @@ import { v4 as uuidv4 } from "uuid";
import { Logger } from "../lib/logger"; import { Logger } from "../lib/logger";
import { getScrapeQueue, scrapeQueueEvents } from "../services/queue-service"; import { getScrapeQueue, scrapeQueueEvents } from "../services/queue-service";
import * as Sentry from "@sentry/node"; import * as Sentry from "@sentry/node";
import { addScrapeJob } from "../services/queue-jobs";
export async function searchHelper( export async function searchHelper(
jobId: string, jobId: string,
@ -96,7 +97,16 @@ export async function searchHelper(
}; };
}) })
const jobs = await getScrapeQueue().addBulk(jobDatas); let jobs = [];
if (Sentry.isInitialized()) {
for (const job of jobDatas) {
// add with sentry instrumentation
jobs.push(await addScrapeJob(job.data as any, {}, job.opts.jobId));
}
} else {
jobs = await getScrapeQueue().addBulk(jobDatas);
await getScrapeQueue().addBulk(jobs);
}
const docs = (await Promise.all(jobs.map(x => x.waitUntilFinished(scrapeQueueEvents, 60000)))).map(x => x[0]); const docs = (await Promise.all(jobs.map(x => x.waitUntilFinished(scrapeQueueEvents, 60000)))).map(x => x[0]);

2
apps/api/src/services/queue-jobs.ts
View File

@ -10,8 +10,8 @@ async function addScrapeJobRaw(
jobId: string, jobId: string,
): Promise<Job> { ): Promise<Job> {
return await getScrapeQueue().add(jobId, webScraperOptions, { return await getScrapeQueue().add(jobId, webScraperOptions, {
priority: webScraperOptions.crawl_id ? 20 : 10,
...options, ...options,
priority: webScraperOptions.crawl_id ? 20 : 10,
jobId, jobId,
}); });
} }