From 95d1cd2f78967c65644094394422ca9b7f8ce8c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Mon, 2 Jun 2025 21:39:35 +0200
Subject: [PATCH] feat(search): ignore concurrency limit for search (temp)

---
 apps/api/src/controllers/v1/search.ts | 2 +-
 apps/api/src/services/queue-jobs.ts   | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 8a34c438..7cbe6e64 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -102,11 +102,11 @@ async function scrapeSearchResult(
         internalOptions: { teamId: options.teamId, useCache: true },
         origin: options.origin,
         is_scrape: true,
-        
       },
       {},
       jobId,
       jobPriority,
+      true,
     );
 
     const doc: Document = await waitForJob(jobId, options.timeout);
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 3456c82c..3ea7de22 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -97,6 +97,7 @@ async function addScrapeJobRaw(
   options: any,
   jobId: string,
   jobPriority: number,
+  directToBullMQ: boolean = false,
 ) {
   const hasCrawlDelay = webScraperOptions.crawl_id && webScraperOptions.crawlerOptions?.delay;
 
@@ -127,7 +128,7 @@ async function addScrapeJobRaw(
 
   const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(webScraperOptions.team_id);
 
-  if (concurrencyLimited) {
+  if (concurrencyLimited && !directToBullMQ) {
     // Detect if they hit their concurrent limit
     // If above by 2x, send them an email
     // No need to 2x as if there are more than the max concurrency in the concurrency queue, it is already 2x
@@ -161,6 +162,7 @@ export async function addScrapeJob(
   options: any = {},
   jobId: string = uuidv4(),
   jobPriority: number = 10,
+  directToBullMQ: boolean = false,
 ) {
   if (Sentry.isInitialized()) {
     const size = JSON.stringify(webScraperOptions).length;
@@ -187,6 +189,7 @@ export async function addScrapeJob(
           options,
           jobId,
           jobPriority,
+          directToBullMQ,
         );
       },
     );