diff --git a/apps/api/src/controllers/search.ts b/apps/api/src/controllers/search.ts index 8a04a978..304176a3 100644 --- a/apps/api/src/controllers/search.ts +++ b/apps/api/src/controllers/search.ts @@ -11,6 +11,7 @@ import { v4 as uuidv4 } from "uuid"; import { Logger } from "../lib/logger"; import { getScrapeQueue, scrapeQueueEvents } from "../services/queue-service"; import * as Sentry from "@sentry/node"; +import { addScrapeJob } from "../services/queue-jobs"; export async function searchHelper( jobId: string, @@ -95,8 +96,17 @@ export async function searchHelper( } }; }) - - const jobs = await getScrapeQueue().addBulk(jobDatas); + + let jobs = []; + if (Sentry.isInitialized()) { + for (const job of jobDatas) { + // add with sentry instrumentation + jobs.push(await addScrapeJob(job.data as any, {}, job.opts.jobId)); + } + } else { + jobs = await getScrapeQueue().addBulk(jobDatas); + await getScrapeQueue().addBulk(jobs); + } const docs = (await Promise.all(jobs.map(x => x.waitUntilFinished(scrapeQueueEvents, 60000)))).map(x => x[0]); diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts index 33997890..888cdefc 100644 --- a/apps/api/src/services/queue-jobs.ts +++ b/apps/api/src/services/queue-jobs.ts @@ -10,8 +10,8 @@ async function addScrapeJobRaw( jobId: string, ): Promise { return await getScrapeQueue().add(jobId, webScraperOptions, { - priority: webScraperOptions.crawl_id ? 20 : 10, ...options, + priority: webScraperOptions.crawl_id ? 20 : 10, jobId, }); }